"""Causal transformer on the four direct-classifier logit streams. No decoder.

The network sees only a past window of the per-seed logits (and, in one family, a causal
120 s high-pass of those logits). It never reads the held-out dataset during its own
training. HMM p_stay and emission scale are chosen the same way.

Selection is per held-out dataset D, using only inner datasets. When inner dataset A is
scored, the network and the emission mean/sd are fit on the three datasets that are
neither D nor A. D is scored once afterwards with score.score_dataset, after refitting
on all four non-D datasets.

Rules, fixed before any held-out score is computed:
  acc74  PRIMARY. Among configs with mean inner Acc >= 74, maximise inner H.
         Ties: higher Acc, then lower SDT. If none reach 74, take the highest inner Acc,
         then H, then lower SDT.
  acc73  same threshold at 73. Reported separately. Not used to pick the primary result.

Train seed is 20270101. One fixed step budget, no early stopping on a new subject split.
usage: python ensemble_clf_transformer.py
writes results/research/ensemble_clf_transformer.csv and ensemble_clf_transformer_inner.csv
checkpoints: models/logit_tf/
"""
import os
import time

import numpy as np
import pandas as pd
import torch
import torch.nn as nn

import ensemble_clf_nested as N
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(EXP, "results", "research", "ensemble_clf_transformer.csv")
INNER = os.path.join(EXP, "results", "research", "ensemble_clf_transformer_inner.csv")
CKPT = os.path.join(EXP, "models", "logit_tf")

TRAIN_SEED = 20270101
STEPS = 800
BATCH = 512
P_STAY = (0.996, 0.998, 0.999)
SCALES = (0.9, 1.0, 1.15)

# name, feature family, window (steps of 0.25 s), layers
MODELS = (
    ("tf_raw_w16", "raw", 16, 1),
    ("tf_raw_w48", "raw", 48, 1),
    ("tf_hp_w32", "hp", 32, 1),
    ("tf_hp_w48", "hp", 48, 2),
)


class CausalLogitTransformer(nn.Module):
    """Readout is the last frame of a window that contains only times <= t."""

    def __init__(self, c_in, window, d_model=32, nhead=4, nlayers=1, dropout=0.1):
        super().__init__()
        self.window = window
        self.proj = nn.Linear(c_in, d_model)
        self.pos = nn.Parameter(torch.randn(1, window, d_model) * 0.02)
        layer = nn.TransformerEncoderLayer(
            d_model=d_model, nhead=nhead, dim_feedforward=d_model * 2,
            dropout=dropout, batch_first=True, norm_first=True, activation="gelu")
        self.enc = nn.TransformerEncoder(layer, num_layers=nlayers)
        self.head = nn.Linear(d_model, 1)
        mask = nn.Transformer.generate_square_subsequent_mask(window)
        self.register_buffer("mask", mask)

    def forward(self, x):
        h = self.proj(x) + self.pos[:, : x.size(1)]
        h = self.enc(h, mask=self.mask)
        return self.head(h[:, -1]).squeeze(-1)


def hp120(D):
    alpha = 1.0 - np.exp(-0.25 / 120.0)
    return D - N.ema_2d(D, alpha)


def build_features():
    raw, hp = {}, {}
    for ds in DS_KEYS:
        raw[ds], hp[ds] = {}, {}
        for key, D in N.G["base"]["id"][ds].items():
            D = np.asarray(D, np.float32)
            H = hp120(D.astype(np.float64)).astype(np.float32)
            raw[ds][key] = D
            hp[ds][key] = np.concatenate([D, H], 1)
    return {"raw": raw, "hp": hp}


def norm_stats(feat, datasets):
    acc_n = 0
    acc_s = None
    acc_sq = None
    for ds in datasets:
        for X in feat[ds].values():
            acc_n += len(X)
            s = X.sum(0).astype(np.float64)
            sq = np.square(X, dtype=np.float64).sum(0)
            acc_s = s if acc_s is None else acc_s + s
            acc_sq = sq if acc_sq is None else acc_sq + sq
    mu = acc_s / acc_n
    var = acc_sq / acc_n - mu ** 2
    sd = np.sqrt(np.maximum(var, 1e-8))
    return mu.astype(np.float32), sd.astype(np.float32)


def trial_list(feat, datasets):
    out = []
    for ds in datasets:
        att = N.G["att"][ds]
        for key, X in feat[ds].items():
            y = (att[key] == 0).astype(np.float32)
            out.append((X, y))
    return out


def sample_batch(rng, trials, lengths, total, W, batch, mu, sd):
    C = trials[0][0].shape[1]
    Xo = np.zeros((batch, W, C), np.float32)
    yo = np.empty(batch, np.float32)
    pick = rng.integers(0, total, size=batch)
    ends = np.cumsum(lengths)
    which = np.searchsorted(ends, pick, side="right")
    starts = ends - lengths
    for i in range(batch):
        ti = int(which[i])
        t = int(pick[i] - starts[ti])
        x, y = trials[ti]
        a = t - W + 1
        if a >= 0:
            Xo[i] = x[a:t + 1]
        else:
            Xo[i, -a:] = x[:t + 1]
        yo[i] = y[t]
    Xo = (Xo - mu) / sd
    return Xo, yo


def train_model(cfg, datasets, feat, device):
    name, family, window, nlayers = cfg
    trials = trial_list(feat[family], datasets)
    mu, sd = norm_stats(feat[family], datasets)
    lengths = np.array([len(y) for _, y in trials], np.int64)
    total = int(lengths.sum())
    y_all = np.concatenate([y for _, y in trials])
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    c_in = trials[0][0].shape[1]
    g = torch.Generator(device="cpu")
    g.manual_seed(TRAIN_SEED)
    torch.manual_seed(TRAIN_SEED)
    model = CausalLogitTransformer(c_in, window, nlayers=nlayers).to(device)
    opt = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)
    loss_fn = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
    rng = np.random.default_rng(TRAIN_SEED)
    model.train()
    t0 = time.time()
    last = None
    for step in range(STEPS):
        xb, yb = sample_batch(rng, trials, lengths, total, window, BATCH, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        opt.zero_grad(set_to_none=True)
        logit = model(x)
        loss = loss_fn(logit, y)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss at step {step} for {name}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        last = float(loss.item())
    model.eval()
    acc = train_frame_acc(model, trials, lengths, total, window, mu, sd, device)
    print(f"  trained {name} on {'+'.join(datasets)}  loss {last:.3f}  frame-acc {acc:.3f}  ({time.time() - t0:.0f}s)",
          flush=True)
    return model, mu, sd


def train_frame_acc(model, trials, lengths, total, window, mu, sd, device):
    rng = np.random.default_rng(0)
    xb, yb = sample_batch(rng, trials, lengths, total, window, 2048, mu, sd)
    with torch.no_grad():
        logit = model(torch.from_numpy(xb).to(device))
    pred = (logit > 0).cpu().numpy()
    return float((pred == (yb > 0.5)).mean())


@torch.no_grad()
def infer_all(model, feat_ds, mu, sd, window, device):
    model.eval()
    out = {}
    mu_t = mu.reshape(1, 1, -1)
    sd_t = sd.reshape(1, 1, -1)
    for ds, table in feat_ds.items():
        out[ds] = {}
        for key, X in table.items():
            T, C = X.shape
            pad = np.zeros((window - 1, C), np.float32)
            xp = torch.from_numpy(np.concatenate([pad, X], 0))
            w = xp.unfold(0, window, 1).permute(0, 2, 1).contiguous()
            w = (w - torch.from_numpy(mu_t)) / torch.from_numpy(sd_t)
            pieces = []
            for i in range(0, T, 2048):
                pieces.append(model(w[i:i + 2048].to(device)).cpu().numpy())
            pred = np.concatenate(pieces).astype(np.float64)
            if len(pred) != T:
                raise RuntimeError(f"length {len(pred)} != {T} for {key}")
            out[ds][key] = pred
    return out


def assert_causal(device):
    torch.manual_seed(0)
    model = CausalLogitTransformer(4, 16, nlayers=1).to(device).eval()
    x = torch.randn(1, 30, 4, device=device)
    full = infer_tensor(model, x)
    pref = infer_tensor(model, x[:, :18])
    if not torch.allclose(full[:18], pref, atol=1e-5):
        raise RuntimeError("transformer readout is not causal")
    print("causality check passed", flush=True)


@torch.no_grad()
def infer_tensor(model, x):
    """x (1, T, C) already on device. Returns (T,) logits."""
    T = x.size(1)
    W = model.window
    C = x.size(2)
    pad = torch.zeros(1, W - 1, C, device=x.device)
    xp = torch.cat([pad, x], 1).squeeze(0)
    w = xp.unfold(0, W, 1).permute(0, 2, 1).contiguous()
    return model(w)


def triplet_key(excluded):
    return "ex-" + "-".join(excluded)


def select_rule(m, thr):
    picks = []
    n_ok = 0
    for _, g in m.groupby("outer"):
        ok = g[g.Acc >= thr]
        if len(ok):
            n_ok += 1
            picks.append(ok.sort_values(["H", "Acc", "SDT"], ascending=[False, False, True]).iloc[0])
        else:
            picks.append(g.sort_values(["Acc", "H", "SDT"], ascending=[False, False, True]).iloc[0])
    return pd.DataFrame(picks), n_ok


def score_pick(streams, ds, p_stay, scale):
    ev = N.to_ev(streams)
    return N.official(ds, ev, float(p_stay), float(scale))


def main():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(f"device {device}", flush=True)
    torch.backends.cuda.matmul.allow_tf32 = True
    N.load_all()
    checks = N.check_fast_matches_official()
    base_H = float(np.mean([c[1]["H"] for c in checks]))
    base_Acc = float(np.mean([c[1]["Acc"] for c in checks]))
    print(f"reproduced lda4 @ 0.998  H {base_H:.4f}  Acc {base_Acc:.2f}", flush=True)
    if abs(base_H - 0.7965) > 5e-4 or abs(base_Acc - 73.01) > 0.02:
        raise RuntimeError("baseline reproduction does not match the published LDA numbers")
    assert_causal(device)
    feat = build_features()
    os.makedirs(CKPT, exist_ok=True)

    # Train one network per (architecture, legal triplet). A triplet is every set of three
    # datasets; it is the train set when the other two are (outer, inner) in either order.
    cache = {}
    excluded_pairs = []
    for i, a in enumerate(DS_KEYS):
        for b in DS_KEYS[i + 1:]:
            excluded_pairs.append((a, b))
    for cfg in MODELS:
        name, family, window, nlayers = cfg
        for a, b in excluded_pairs:
            train_ds = [d for d in DS_KEYS if d not in (a, b)]
            model, mu, sd = train_model(cfg, train_ds, feat, device)
            path = os.path.join(CKPT, f"{name}__{triplet_key((a, b))}.pt")
            torch.save({"state": model.state_dict(), "mu": mu, "sd": sd,
                        "window": window, "nlayers": nlayers, "family": family,
                        "train": train_ds}, path)
            streams = infer_all(model, feat[family], mu, sd, window, device)
            cache[(name, a, b)] = streams
            cache[(name, b, a)] = streams
            del model
            torch.cuda.empty_cache()

    rows = []
    for outer in DS_KEYS:
        for inner in DS_KEYS:
            if inner == outer:
                continue
            train_ds = [d for d in DS_KEYS if d not in (outer, inner)]
            for cfg in MODELS:
                name = cfg[0]
                streams = cache[(name, outer, inner)]
                # emissions only on the three legal datasets, not on outer and not on inner
                mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
                for p in P_STAY:
                    for scale in SCALES:
                        r = N.evaluate(inner, streams[inner], mu, sd * scale, p)
                        rows.append(dict(outer=outer, inner=inner, model=name, p_stay=p, scale=scale, **r))
            print(f"scored inners of {outer} except held {inner}", flush=True)
    inner = pd.DataFrame(rows)
    os.makedirs(os.path.dirname(INNER), exist_ok=True)
    inner.to_csv(INNER, index=False)
    print(f"wrote {INNER}", flush=True)
    grouped = inner.groupby(["outer", "model", "p_stay", "scale"])[["H", "Acc", "SDT", "L", "acc1"]].mean().reset_index()

    rules = {"acc74": 74.0, "acc73": 73.0}
    chosen = {}
    for rule, thr in rules.items():
        picks, n_ok = select_rule(grouped, thr)
        chosen[rule] = picks
        print(f"select {rule}: {n_ok}/5 folds reached inner Acc>={thr:g}  "
              f"inner H {picks.H.mean():.4f} Acc {picks.Acc.mean():.2f}", flush=True)
        for _, b in picks.iterrows():
            print(f"  {b.outer} {b.model} p {b.p_stay} sc {b.scale}  inner H {b.H:.4f} Acc {b.Acc:.2f}", flush=True)

    # Refit on the four non-held-out datasets only after the rule has been applied.
    # Cache by (model, held-out) so acc74 and acc73 share networks.
    refit = {}
    out_rows = []
    for rule, picks in chosen.items():
        for _, b in picks.iterrows():
            key = (b.model, b.outer)
            if key not in refit:
                cfg = next(c for c in MODELS if c[0] == b.model)
                train_ds = [d for d in DS_KEYS if d != b.outer]
                model, mu, sd = train_model(cfg, train_ds, feat, device)
                path = os.path.join(CKPT, f"{b.model}__held-{b.outer}.pt")
                torch.save({"state": model.state_dict(), "mu": mu, "sd": sd,
                            "window": cfg[2], "nlayers": cfg[3], "family": cfg[1],
                            "train": train_ds, "rule_context": "refit-four"}, path)
                refit[key] = infer_all(model, feat[cfg[1]], mu, sd, cfg[2], device)
                del model
                torch.cuda.empty_cache()
            off = score_pick(refit[key], b.outer, b.p_stay, b.scale)
            out_rows.append(dict(role=rule, ds=b.outer, model=b.model, p_stay=float(b.p_stay),
                                 scale=float(b.scale), H=off["H"], Acc=off["Acc"], SDT=off["SDT"],
                                 L=off["L"], acc1=off["acc1"], inner_H=float(b.H), inner_Acc=float(b.Acc),
                                 scorer="official"))
            print(f"{rule} {b.outer} {b.model} p {b.p_stay} sc {b.scale}  "
                  f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f}", flush=True)

    # Per-architecture nested selection under the primary rule, still from inner scores only.
    for name, family, window, nlayers in MODELS:
        sub = grouped[grouped.model == name]
        picks, n_ok = select_rule(sub, 74.0)
        print(f"family {name}: {n_ok}/5 inner Acc>=74  inner H {picks.H.mean():.4f} Acc {picks.Acc.mean():.2f}", flush=True)
        for _, b in picks.iterrows():
            key = (b.model, b.outer)
            if key not in refit:
                cfg = (name, family, window, nlayers)
                train_ds = [d for d in DS_KEYS if d != b.outer]
                model, mu, sd = train_model(cfg, train_ds, feat, device)
                path = os.path.join(CKPT, f"{name}__held-{b.outer}.pt")
                torch.save({"state": model.state_dict(), "mu": mu, "sd": sd,
                            "window": window, "nlayers": nlayers, "family": family, "train": train_ds}, path)
                refit[key] = infer_all(model, feat[family], mu, sd, window, device)
                del model
                torch.cuda.empty_cache()
            off = score_pick(refit[key], b.outer, b.p_stay, b.scale)
            out_rows.append(dict(role=f"family_{name}", ds=b.outer, model=name, p_stay=float(b.p_stay),
                                 scale=float(b.scale), H=off["H"], Acc=off["Acc"], SDT=off["SDT"],
                                 L=off["L"], acc1=off["acc1"], inner_H=float(b.H), inner_Acc=float(b.Acc),
                                 scorer="official"))
            print(f"family {name} {b.outer}  H {off['H']:.4f} Acc {off['Acc']:.2f}", flush=True)

    df = pd.DataFrame(out_rows)
    df.to_csv(OUT, index=False)
    print("\nmeans:")
    for role, sub in df.groupby("role"):
        m = sub[["H", "Acc", "SDT", "L", "acc1"]].mean()
        print(f"  {role:16s}  H {m.H:.4f}  Acc {m.Acc:.2f}  SDT {m.SDT:.2f}  L {m.L:.3f}  acc1 {m.acc1:.3f}")
    prim = df[df.role == "acc74"]
    print(f"\nPRIMARY acc74  H {prim.H.mean():.4f}  Acc {prim.Acc.mean():.2f}")
    print(f"goal H>0.81 and Acc>74: {bool(prim.H.mean() > 0.81 and prim.Acc.mean() > 74)}")
    print(f"wrote {OUT}")


if __name__ == "__main__":
    main()
