"""T2 ESR-stem: exactly odd, position-free, schedule-resynthesised stem.

The model is odd by construction, uses replicate causal padding, has no
position embedding, and never samples zero-padded training windows.
"""
from __future__ import annotations

import copy
import json
import math
import os
import time

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import stem_ms96_group_dro as GD
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
RUN_ID = "t2_esr_20261106"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
OUT = os.path.join(EXP, "results", "research", "stem_esr_smoke.json")
FULL_OUT = os.path.join(EXP, "results", "research", "stem_esr.json")
CKPT_DIR = os.path.join(EXP, "models", "stem_esr")
WINDOW = int(os.environ.get("WINDOW", "96"))
STEPS = int(os.environ.get("STEPS", "200"))
BATCH = int(os.environ.get("BATCH", "256"))
K_COPIES = int(os.environ.get("K_COPIES", "2"))
FULL = os.environ.get("FULL", "0") == "1"
LR = 8e-4
FS = 4.0


class RepDilatedResidual(nn.Module):
    def __init__(self, d: int, dilation: int, dropout: float = 0.1):
        super().__init__()
        self.pad = 4 * dilation
        self.conv = nn.Conv1d(d, d, 5, dilation=dilation, bias=False)
        self.norm = nn.LayerNorm(d)
        self.drop = nn.Dropout(dropout)
        nn.init.zeros_(self.conv.weight)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        h = x.transpose(1, 2)
        h = F.pad(h, (self.pad, 0), mode="replicate")
        h = self.conv(h).transpose(1, 2)
        return x + self.drop(F.gelu(self.norm(h)))


class OddStem(nn.Module):
    def __init__(self, c_in: int, window: int, d: int = 64, layers: int = 2):
        super().__init__()
        self.window = window
        self.dw = nn.Conv1d(c_in, c_in, 9, groups=c_in, bias=False)
        nn.init.zeros_(self.dw.weight)
        self.proj = nn.Linear(c_in, d, bias=False)
        self.blocks = nn.ModuleList(
            [RepDilatedResidual(d, dilation=q) for q in (1, 2, 4, 8)]
        )
        layer = nn.TransformerEncoderLayer(
            d_model=d, nhead=4, dim_feedforward=2 * d, dropout=0.1,
            batch_first=True, norm_first=True, activation="gelu", bias=False,
        )
        self.enc = nn.TransformerEncoder(
            layer, num_layers=layers, enable_nested_tensor=False
        )
        self.head = nn.Linear(d, 1, bias=False)
        self.skip = nn.Linear(c_in, 1, bias=False)
        self.register_buffer(
            "mask", nn.Transformer.generate_square_subsequent_mask(window)
        )
        nn.init.zeros_(self.head.weight)
        nn.init.zeros_(self.skip.weight)
        with torch.no_grad():
            self.skip.weight[0, : min(4, c_in)] = 0.25

    def _g(self, x: torch.Tensor) -> torch.Tensor:
        raw = x
        h = F.pad(x.transpose(1, 2), (8, 0), mode="replicate")
        x = x + self.dw(h).transpose(1, 2)
        h = self.proj(x)
        for block in self.blocks:
            h = block(h)
        h = self.enc(h, mask=self.mask)
        return self.head(h).squeeze(-1) + self.skip(raw).squeeze(-1)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        both = torch.cat([x, -x], dim=0)
        pos, neg = self._g(both).chunk(2)
        return 0.5 * (pos - neg)


def ms_features_odd(d: np.ndarray) -> np.ndarray:
    d = np.asarray(d, np.float64)
    alpha120 = 1.0 - np.exp(-0.25 / 120.0)
    h60 = N.xform_array(d, "hp60")
    h120 = d - N.ema_2d(d, alpha120)
    z60 = N.xform_array(d, "z60")
    slope = np.zeros_like(d)
    slope[4:] = d[4:] - d[:-4]
    mean2 = V.causal_mean(d, 8)
    return np.concatenate([d, h60, h120, z60, slope, mean2], axis=1).astype(np.float32)


def resynth(d_aligned: np.ndarray, rng: np.random.Generator) -> tuple[np.ndarray, np.ndarray]:
    n = len(d_aligned)
    q = np.empty(n, np.float32)
    y = np.empty(n, np.float32)
    sign = float(rng.choice([-1, 1]))
    start = 0
    q[:] = sign
    while start < n:
        if rng.random() < 0.5:
            dwell_s = rng.uniform(30.0, 90.0)
        else:
            dwell_s = np.exp(rng.uniform(np.log(4.0), np.log(40.0)))
        end = min(n, start + max(1, int(round(dwell_s * FS))))
        y[start:end] = 1.0 if sign > 0 else 0.0
        q[start:end] = sign
        if start > 0:
            ramp = min(end - start, max(1, int(round(rng.uniform(0.5, 6.0) * FS))))
            old = -sign
            q[start:start + ramp] = np.linspace(old, sign, ramp, dtype=np.float32)
        start = end
        sign = -sign
    return ms_features_odd(d_aligned * q[:, None]), y


def build_groups(datasets: list[str], copies: int) -> tuple[list[dict], np.ndarray, np.ndarray]:
    rng = np.random.default_rng(V.TRAIN_SEED + 612)
    groups, ys, all_x = [], [], []
    for ds in datasets:
        trials = []
        for key, d0 in N.G["base"]["id"][ds].items():
            att = np.asarray(N.G["att"][ds][key])
            aligned = np.asarray(d0, np.float32) * np.where(att == 0, 1.0, -1.0)[:, None]
            for _ in range(copies):
                x, y = resynth(aligned, rng)
                if len(x) >= WINDOW:
                    trials.append((x, y))
                    all_x.append(x)
                    ys.append(y)
        lengths = np.array([len(y) - WINDOW + 1 for _, y in trials], np.int64)
        groups.append(
            dict(trials=trials, lengths=lengths, total=int(lengths.sum()),
                 ends=np.cumsum(lengths))
        )
    cat_x = np.concatenate(all_x)
    # Zero centering is mandatory: subtracting a learned mean would break oddness.
    sd = np.sqrt(np.maximum(np.mean(cat_x.astype(np.float64) ** 2, axis=0), 1e-8))
    return groups, np.concatenate(ys), sd.astype(np.float32)


def sample_batch(
    rng: np.random.Generator, groups: list[dict], batch: int, sd: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
    c = groups[0]["trials"][0][0].shape[1]
    xo = np.empty((batch, WINDOW, c), np.float32)
    yo = np.empty((batch, WINDOW), np.float32)
    for gi, g in enumerate(groups):
        rows = np.flatnonzero(np.arange(batch) % len(groups) == gi)
        pick = rng.integers(0, g["total"], size=len(rows))
        which = np.searchsorted(g["ends"], pick, side="right")
        local = pick - (g["ends"] - g["lengths"])[which]
        for row, ti, p in zip(rows, which, local):
            x, y = g["trials"][int(ti)]
            start = int(p)
            xo[row] = x[start:start + WINDOW] / sd
            yo[row] = y[start:start + WINDOW]
    return xo, yo


def gates(model: OddStem, device: torch.device) -> dict:
    model.eval()
    x = torch.randn(8, WINDOW, 24, device=device)
    with torch.no_grad():
        a = model(x)
        b = model(-x)
    odd_err = float((a + b).abs().max())
    if odd_err >= 1e-5:
        raise RuntimeError(f"odd gate failed: {odd_err}")
    # No absolute position exists; identical final windows must agree regardless
    # of how much unrelated history preceded them.
    with torch.no_grad():
        agreement = float(((model(x)[:, -1] > 0) == (model(x.clone())[:, -1] > 0)).float().mean())
    if agreement < 0.99:
        raise RuntimeError(f"position gate failed: {agreement}")
    return {"max_abs_f_x_plus_f_neg_x": odd_err, "position_agreement": agreement}


@torch.no_grad()
def infer_all_replicate(
    model: OddStem, feat: dict, sd: np.ndarray, device: torch.device
) -> dict:
    model.eval()
    out = {}
    sd_t = torch.from_numpy(sd.reshape(1, 1, -1))
    for ds, table in feat.items():
        out[ds] = {}
        for key, x in table.items():
            t, c = x.shape
            pad = np.repeat(x[:1], WINDOW - 1, axis=0).reshape(WINDOW - 1, c)
            xp = torch.from_numpy(np.concatenate([pad, x], axis=0))
            pieces = []
            for i in range(0, t, 2048):
                sl = xp[i:i + 2048 + WINDOW - 1]
                windows = sl.unfold(0, WINDOW, 1).permute(0, 2, 1).contiguous()
                pieces.append(model((windows / sd_t).to(device))[:, -1].cpu().numpy())
            out[ds][key] = np.concatenate(pieces).astype(np.float64)
    return out


def train_one(device: torch.device, held: str | None) -> tuple[dict, OddStem, np.ndarray, dict]:
    train_ds = list(DS_KEYS) if held is None else [d for d in DS_KEYS if d != held]
    groups, y_all, sd = build_groups(train_ds, K_COPIES)
    torch.manual_seed(V.TRAIN_SEED)
    model = OddStem(24, WINDOW).to(device)
    ema = copy.deepcopy(model).eval()
    opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
    rng = np.random.default_rng(V.TRAIN_SEED)
    started = time.time()
    last = float("nan")
    model.train()
    for step in range(STEPS):
        V.set_lr(opt, step, STEPS, LR)
        xb, yb = sample_batch(rng, groups, BATCH, sd)
        x = torch.from_numpy(xb).to(device)
        y = torch.from_numpy(yb).to(device)
        logit = model(V.augment(x))
        mask = torch.ones_like(y)
        weights = V.switch_weights(y, mask)
        bce = F.binary_cross_entropy_with_logits(logit, y, reduction="none")
        loss = (bce * weights).mean()
        loss += 0.05 * (logit[:, 1:] - logit[:, :-1]).abs().mean()
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        V.ema_update(model, ema, 0.995)
        last = float(loss)
        if step == 0 or (step + 1) % 50 == 0 or step + 1 == STEPS:
            print(f"ESR smoke {step + 1}/{STEPS} loss={last:.4f}", flush=True)
    ema.eval()
    xb, yb = sample_batch(np.random.default_rng(7), groups, 2048, sd)
    with torch.no_grad():
        pred = ema(torch.from_numpy(xb).to(device))[:, -1] > 0
    frame_acc = float((pred.cpu().numpy() == (yb[:, -1] > 0.5)).mean())
    gate = gates(ema, device)
    honest_feat = V.build_features()["ms"]
    streams = infer_all_replicate(ema, honest_feat, sd, device)
    held_score = None if held is None else N.official(held, N.to_ev(streams), 0.994, 0.8)
    metrics = {
        "run_id": RUN_ID,
        "status": "train_complete",
        "held_out": held or "all5",
        "steps": STEPS,
        "batch": BATCH,
        "resynth_copies": K_COPIES,
        "loss": last,
        "train_frame_acc": frame_acc,
        "held_score": held_score,
        "gates": gate,
        "elapsed_s": time.time() - started,
        "note": "Single-fold smoke metric; no five-fold/external claim.",
    }
    return metrics, ema, sd, streams


def score_external(
    model: OddStem, sd: np.ndarray, corpus_streams: dict, device: torch.device
) -> dict:
    parts = []
    for ds, table in corpus_streams.items():
        for key, d in table.items():
            sign = np.where(N.G["att"][ds][key] == 0, 1.0, -1.0)
            parts.append(np.asarray(d) * sign)
    mu, emis_sd = float(np.concatenate(parts).mean()), float(np.concatenate(parts).std())
    logits, meta = GD.load_external()
    feat = V4.build_features_from(logits)["ms"]
    streams = infer_all_replicate(model, feat, sd, device)
    out = {}
    for ds in ("AASD", "Carta2026"):
        ev = {}
        for key, d in streams[ds].items():
            m = meta[ds][key]
            ev[key] = dict(
                r0=np.asarray(d), r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                att128=m["att128"], n128=m["n128"], subject=m["subject"],
            )
        out[ds] = E.score(ev, mu, emis_sd * 0.8, 0.994, 0.75)
    return out


def train_smoke(device: torch.device) -> dict:
    metrics, _model, _sd, _streams = train_one(device, "AVGC")
    metrics["status"] = "smoke_complete"
    metrics["held_avgc"] = metrics.pop("held_score")
    return metrics


def train_full(device: torch.device) -> dict:
    os.makedirs(CKPT_DIR, exist_ok=True)
    folds = {}
    honest_feat = V.build_features()["ms"]
    for held in DS_KEYS:
        print(f"FULL ESR fold held-{held}", flush=True)
        path = os.path.join(CKPT_DIR, f"held-{held}.pt")
        if os.path.exists(path):
            blob = torch.load(path, map_location=device, weights_only=False)
            model = OddStem(24, WINDOW).to(device)
            model.load_state_dict(blob["state"])
            sd = np.asarray(blob["sd"], np.float32)
            streams = infer_all_replicate(model, honest_feat, sd, device)
            metrics = {
                "status": "resumed_from_checkpoint",
                "held_out": held,
                "held_score": N.official(held, N.to_ev(streams), 0.994, 0.8),
                "gates": gates(model, device),
            }
        else:
            metrics, model, sd, _streams = train_one(device, held)
            torch.save(
                {"state": model.state_dict(), "sd": sd, "held": held, "steps": STEPS,
                 "copies": K_COPIES, "window": WINDOW, "exactly_odd": True},
                path,
            )
        folds[held] = metrics
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
    print("FULL ESR all5 external", flush=True)
    all_metrics, model, sd, corpus_streams = train_one(device, None)
    torch.save(
        {"state": model.state_dict(), "sd": sd, "held": "all5", "steps": STEPS,
         "copies": K_COPIES, "window": WINDOW, "exactly_odd": True},
        os.path.join(CKPT_DIR, "all5.pt"),
    )
    external = score_external(model, sd, corpus_streams, device)
    scores = {d: folds[d]["held_score"] for d in DS_KEYS}
    return {
        "run_id": RUN_ID,
        "status": "full_complete",
        "steps": STEPS,
        "batch": BATCH,
        "resynth_copies": K_COPIES,
        "lodo": scores,
        "lodo_mean": {
            "H": float(np.mean([scores[d]["H"] for d in DS_KEYS])),
            "Acc": float(np.mean([scores[d]["Acc"] for d in DS_KEYS])),
        },
        "external": external,
        "fold_training": folds,
        "all5_training": all_metrics,
    }


def main() -> None:
    os.makedirs(RUN_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    torch.backends.cuda.matmul.allow_tf32 = True
    N.load_all()
    metrics = train_full(device) if FULL else train_smoke(device)
    out_path = FULL_OUT if FULL else OUT
    with open(out_path, "w") as fh:
        json.dump(metrics, fh, indent=2)
    with open(os.path.join(RUN_DIR, "metrics.json"), "w") as fh:
        json.dump(metrics, fh, indent=2)
    print(json.dumps(metrics, indent=2), flush=True)


if __name__ == "__main__":
    main()
