"""Causal local-attention logit model, fourth round.

Same metrics as ensemble_clf_transformer_v2.py. H is the official harmonic mean of
subject-mean sample accuracy (reported in percent as Acc) and switch score L.
Accuracy is that official Acc, not balanced accuracy and not acc1.

What was compared with stem_ms96:
  linear_hp30  no new weights. Causal one-pole high-pass (tau 30 s) of the four
               seed logits, then their mean. This is the external winner below.
  rel_ms       local causal attention (relative bias, 6.25 s), dilated stem,
               stochastic depth. Skip starts as the mean of the four logits.
  rel_hp       same block on high-passed logits, with a penalty toward that mean.

Measured at the pre-specified HMM pair (p_stay 0.994, scale 0.8), emissions fit on
the five corpus sets only: linear_hp30 AASD H 0.642 Acc 53.12, Carta H 0.652 Acc 60.54.
rel_ms and rel_hp were worse on both external sets. On the five-dataset LODO score,
linear_hp30 is H 0.790 Acc 69.21, below stem_ms96 (H 0.868 Acc 82.42).

Train seed 20270101. The external headline uses the pre-specified HMM pair from the
v2 acc74 rule (p_stay 0.994, emission scale 0.8), chosen without AASD or Carta labels.

usage:
  MODE=external python ensemble_clf_transformer_v4.py
  MODE=lodo_hp python ensemble_clf_transformer_v4.py
"""
import os
import time

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from scipy.signal import lfilter

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v3 as V3
import eval_external as E
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CKPT = os.path.join(EXP, "models", "logit_tf_v4")
LOG_EXT = os.path.join(EXP, "experiments", "transformer_v4_external.log")
OUT_EXT = os.path.join(EXP, "results", "research", "ensemble_clf_transformer_v4_external.csv")
LOG_HP = os.path.join(EXP, "experiments", "transformer_v4_hp_lodo.log")
OUT_HP = os.path.join(EXP, "results", "research", "ensemble_clf_transformer_v4_hp_lodo.csv")

CACHE = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/eeg-thao_unzipped/eeg-thao/cache/logit_v2"
EVID = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research"
TRAIN_NAMES = ("AVGC", "DTU", "KUL", "UHD", "USTC", "BRO", "FUG")
CORPUS = ("AVGC", "DTU", "KUL", "UHD", "USTC")
TEST_NAMES = ("AASD", "Carta2026")
PRESET = (0.994, 0.8)
HP_TAU = 30.0
LOOKAHEAD = 0.75

# name, family, window, d_model, layers, lr, dropout, drop_path, anchor
CONFIGS = (
    ("rel_ms", "ms", 96, 48, 2, 8e-4, 0.2, 0.1, 0.0),
    ("rel_hp", "hp30", 96, 48, 2, 8e-4, 0.25, 0.15, 0.25),
)


def log_line(path, text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "a") as fh:
        fh.write(text + "\n")


def highpass(D, tau=HP_TAU, hop=0.25):
    """Causal one-pole high-pass. y = x - EMA(x), EMA uses only the present and the past."""
    alpha = 1.0 - np.exp(-hop / tau)
    ema = lfilter([alpha], [1.0, -(1.0 - alpha)], np.asarray(D, np.float64), axis=0)
    return (np.asarray(D, np.float64) - ema).astype(np.float32)


def build_features_from(base_id):
    """ms matches v2. hp30 high-passes the four logits first, then builds the same features."""
    ms, hp30 = {}, {}
    alpha120 = 1.0 - np.exp(-0.25 / 120.0)
    for ds, table in base_id.items():
        ms[ds], hp30[ds] = {}, {}
        for key, D0 in table.items():
            D = np.asarray(D0, np.float64)
            ms[ds][key] = _ms(D, alpha120)
            hp30[ds][key] = _ms(highpass(D), alpha120)
    return {"ms": ms, "hp30": hp30}


def _ms(D, alpha120):
    D = np.asarray(D, np.float64)
    h60 = N.xform_array(D, "hp60")
    h120 = D - N.ema_2d(D, alpha120)
    z = N.xform_array(D, "z60")
    slope = np.zeros_like(D)
    if len(D) > 4:
        slope[4:] = D[4:] - D[:-4]
    mean = V.causal_mean(D, 8)
    return np.concatenate([D, h60, h120, z, slope, mean], 1).astype(np.float32)


class RelLocal(nn.Module):
    """Dilated causal stem and local causal attention. No absolute positions.

    The linear skip is initialised to the mean of the four logit channels and the
    attention/FFN output projections start at zero, so the network starts as that mean.
    """

    def __init__(self, c_in, d_model=48, nhead=4, nlayers=2, dropout=0.2, span=25, drop_path=0.1):
        super().__init__()
        self.drop_path = drop_path
        self.proj = nn.Linear(c_in, d_model)
        self.blocks = nn.ModuleList([
            V.DilatedResidual(d_model, k=5, dilation=dil, dropout=dropout) for dil in (1, 2, 4)
        ])
        self.local = nn.ModuleList([
            V3.LocalCausalBlock(d_model, nhead, span, dropout) for _ in range(nlayers)
        ])
        self.head = nn.Linear(d_model, 1)
        self.skip = nn.Linear(c_in, 1, bias=False)
        nn.init.zeros_(self.head.weight)
        nn.init.zeros_(self.head.bias)
        nn.init.zeros_(self.skip.weight)
        with torch.no_grad():
            self.skip.weight[0, : min(4, c_in)] = 1.0 / 4.0

    def forward(self, x):
        raw = x
        h = self.proj(x)
        for block in self.blocks:
            if self.training and self.drop_path > 0 and torch.rand((), device=x.device) < self.drop_path:
                continue
            h = block(h)
        for layer in self.local:
            if self.training and self.drop_path > 0 and torch.rand((), device=x.device) < self.drop_path:
                continue
            h = layer(h)
        return self.head(h).squeeze(-1) + self.skip(raw).squeeze(-1)


def assert_causal(device):
    torch.manual_seed(0)
    model = RelLocal(8, d_model=32, nlayers=1, drop_path=0.0).to(device).eval()
    x = torch.randn(2, 32, 8, device=device)
    with torch.no_grad():
        base = model(x)
        late = x.clone()
        late[:, -1] = torch.randn(2, 8, device=device)
        y_late = model(late)
        mid = x.clone()
        mid[:, 10] = torch.randn(2, 8, device=device)
        y_mid = model(mid)
    if not torch.allclose(base[:, :-1], y_late[:, :-1], atol=1e-5):
        raise RuntimeError("rel-local readout sees the future")
    if not torch.allclose(base[:, :10], y_mid[:, :10], atol=1e-5):
        raise RuntimeError("rel-local readout is not causal at mid-window")
    print("causality check passed", flush=True)


def train_model(cfg, datasets, feat, att, device, steps):
    name, family, window, d_model, nlayers, lr, dropout, drop_path, anchor = cfg
    N.G["att"] = att
    groups, y_all = V.trial_groups(feat[family], datasets)
    mu, sd = V.norm_stats(feat[family], datasets)
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    c_in = groups[0]["trials"][0][0].shape[1]
    torch.manual_seed(V.TRAIN_SEED)
    if device.type == "cuda":
        torch.cuda.manual_seed_all(V.TRAIN_SEED)
    model = RelLocal(c_in, d_model=d_model, nlayers=nlayers, dropout=dropout, drop_path=drop_path).to(device)
    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)
    rng = np.random.default_rng(V.TRAIN_SEED)
    model.train()
    t0 = time.time()
    last = None
    for step in range(steps):
        V.set_lr(opt, step, steps, lr)
        xb, yb, mb = V.sample_batch(rng, groups, window, V.BATCH, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        m = torch.from_numpy(mb).to(device, non_blocking=True)
        x = V.augment(x)
        opt.zero_grad(set_to_none=True)
        logit = model(x)
        sw = V.switch_weights(y, m)
        bce = F.binary_cross_entropy_with_logits(logit, y, pos_weight=pos_weight, reduction="none")
        denom = (sw * m).sum().clamp_min(1.0)
        loss = (bce * sw * m).sum() / denom
        if anchor:
            with torch.no_grad():
                anchored = model.skip(x).squeeze(-1)
            loss = loss + anchor * ((logit - anchored) ** 2 * m).sum() / m.sum().clamp_min(1.0)
        dv = (logit[:, 1:] - logit[:, :-1]).abs()
        m2 = m[:, 1:] * m[:, :-1]
        loss = loss + 0.05 * (dv * m2).sum() / m2.sum().clamp_min(1.0)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss at step {step} for {name}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        last = float(loss.item())
        if step == 0 or (step + 1) % 200 == 0:
            print(f"    {name} step {step + 1}/{steps} loss {last:.3f}", flush=True)
    model.eval()
    acc = V.train_frame_acc(model, groups, window, mu, sd, device)
    print(f"  trained {name} on {'+'.join(datasets)}  loss {last:.3f}  frame-acc {acc:.3f}  "
          f"({time.time() - t0:.0f}s)", flush=True)
    return model, mu, sd


def load_train():
    logits, att = {}, {}
    for name in TRAIN_NAMES:
        with open(os.path.join(CACHE, name + ".pkl"), "rb") as fh:
            blob = pickle_load(fh)
        logits[name] = blob["logits"]
        att[name] = blob["att"]
    return logits, att


def pickle_load(fh):
    import pickle
    return pickle.load(fh)


def load_test():
    import pickle
    logits, meta = {}, {}
    for ds in TEST_NAMES:
        evs = []
        for i in range(4):
            path = os.path.join(EVID, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(att=np.asarray(e0["att"]), t_end=np.asarray(e0["t_end"], np.float64),
                                 att128=np.asarray(e0["att128"]), n128=int(e0["n128"]), subject=int(e0["subject"]))
        print(f"  test {ds}: {len(logits[ds])} trials", flush=True)
    return logits, meta


def emissions(streams, att):
    parts = []
    for ds, table in streams.items():
        labels = att[ds]
        for key, d in table.items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def pack_ev(stream, meta):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return ev


def score_external(name, ds, stream, meta, mu, sd, log):
    rows = []
    ev = pack_ev(stream, meta)
    plain = E.plain(ev)
    for p, scale, preset in (
        (PRESET[0], PRESET[1], True),
        (0.998, 1.0, False),
    ):
        t0 = time.time()
        r = E.score(ev, mu, sd * float(scale), float(p), LOOKAHEAD)
        rec = dict(model=name, ds=ds, p_stay=float(p), scale=float(scale), preset=preset,
                   acc1=plain["acc1"], snr1=plain["snr1"], seconds=time.time() - t0, **r)
        rows.append(rec)
        log_line(log, f"TEST {ds} {name} p {p} sc {scale}  H {r['H']:.4f} Acc {r['Acc']:.2f} "
                      f"SDT {r['SDT']:.2f} L {r['L']:.3f} acc1 {plain['acc1']:.3f}")
    return rows


def linear_streams(logits, kind):
    """Decision streams with no learned model: mean of the four seeds, raw or high-passed."""
    out = {}
    for ds, table in logits.items():
        out[ds] = {}
        for key, D in table.items():
            X = highpass(D) if kind == "hp30" else np.asarray(D, np.float32)
            out[ds][key] = X.mean(1).astype(np.float64)
    return out


def run_external(device):
    os.makedirs(os.path.dirname(LOG_EXT), exist_ok=True)
    open(LOG_EXT, "w").close()
    steps = int(os.environ.get("STEPS", "800"))
    log_line(LOG_EXT, f"device {device}  steps {steps}  batch {V.BATCH}  preset p {PRESET[0]} sc {PRESET[1]}")
    torch.backends.cuda.matmul.allow_tf32 = True
    assert_causal(device)
    log_line(LOG_EXT, "loading logit tables")
    train_logits, train_att = load_train()
    test_logits, test_meta = load_test()
    base = {}
    base.update(train_logits)
    base.update(test_logits)
    log_line(LOG_EXT, "building causal features")
    feat = build_features_from(base)
    rows = []
    # linear references, emissions from the five corpus sets only (no test labels, no BRO/FUG)
    for kind in ("mean", "hp30"):
        streams = linear_streams(base, kind)
        mu, sd = emissions({ds: streams[ds] for ds in CORPUS}, {ds: train_att[ds] for ds in CORPUS})
        log_line(LOG_EXT, f"linear {kind} corpus emis mu {mu:.4f} sd {sd:.4f}")
        for ds in TEST_NAMES:
            rows.extend(score_external(f"linear_{kind}", ds, streams[ds], test_meta[ds], mu, sd, LOG_EXT))
    os.makedirs(CKPT, exist_ok=True)
    for cfg in CONFIGS:
        name, family, window = cfg[0], cfg[1], cfg[2]
        model, mu, sd = train_model(cfg, list(TRAIN_NAMES), feat, train_att, device, steps)
        path = os.path.join(CKPT, f"{name}.pt")
        torch.save({"state": model.state_dict(), "mu": mu, "sd": sd, "cfg": cfg,
                    "train": list(TRAIN_NAMES), "steps": steps}, path)
        train_feat = {ds: feat[family][ds] for ds in CORPUS}
        train_streams = V.infer_all(model, train_feat, mu, sd, window, device)
        mu_e, sd_e = emissions(train_streams, {ds: train_att[ds] for ds in CORPUS})
        log_line(LOG_EXT, f"  saved {path}  emis mu {mu_e:.4f} sd {sd_e:.4f}")
        test_feat = {ds: feat[family][ds] for ds in TEST_NAMES}
        test_streams = V.infer_all(model, test_feat, mu, sd, window, device)
        for ds in TEST_NAMES:
            rows.extend(score_external(name, ds, test_streams[ds], test_meta[ds], mu_e, sd_e, LOG_EXT))
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
    df = pd.DataFrame(rows)
    os.makedirs(os.path.dirname(OUT_EXT), exist_ok=True)
    df.to_csv(OUT_EXT, index=False)
    log_line(LOG_EXT, f"wrote {OUT_EXT}")
    preset = df[df.preset]
    for model, sub in preset.groupby("model"):
        log_line(LOG_EXT, f"  preset {model:16s}  " + "  ".join(
            f"{r.ds} H {r.H:.4f} Acc {r.Acc:.2f}" for r in sub.itertuples()))


def _mean_and_hp_from_folds():
    """Equal mean of the four LODO direct-classifier seeds, and its causal 30 s high-pass."""
    import score as S
    import ensemble_clf as EC
    evs, lookahead = [], None
    for _seed, exp in EC.SEEDS:
        ev, info = S.load_folds(exp, "causal")
        evs.append(ev)
        for ds in DS_KEYS:
            la = float(info[ds]["lookahead_s"])
            lookahead = la if lookahead is None else lookahead
    mean_ev, hp_ev = {}, {}
    for ds in DS_KEYS:
        mean_ev[ds], hp_ev[ds] = {}, {}
        keys = evs[0][ds].keys()
        for k in keys:
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            D = np.stack(cols, 1)
            e0 = evs[0][ds][k]
            base = dict(att=e0["att"], t_end=e0["t_end"], pre_switched=True)
            m = D.mean(1)
            mean_ev[ds][k] = dict(r0=m, r1=np.zeros_like(m), **base)
            h = highpass(D).mean(1).astype(np.float64)
            hp_ev[ds][k] = dict(r0=h, r1=np.zeros_like(h), **base)
    return mean_ev, hp_ev, lookahead


def _install_scorer_state(ev, lookahead):
    """Fill the fields N.evaluate / N.official read. One synthetic schedule, shared by both decision streams."""
    import score as S
    from e1_switch import synth_trial
    from common import SEED
    att, tend, prep = {}, {}, {}
    for ds in DS_KEYS:
        att[ds], tend[ds], prep[ds] = {}, {}, []
        rng = np.random.default_rng(SEED + DS_KEYS.index(ds))
        for _, r in S.rows_of(ds).iterrows():
            k = r["key"]
            e = ev[ds][k]
            att[ds][k] = e["att"]
            tend[ds][k] = np.asarray(e["t_end"], np.float64)
            T = int(r["n"])
            st, _, _ = synth_trial(k, T, e["r0"], tend[ds][k], rng)
            guess = int(rng.integers(0, 2))
            lab = np.load(os.path.join(S.EDA_CACHE, "cnd", f"{k}.npz"))
            raw_att = lab["att_col"][:T].astype(int)
            gt = np.where(st, 1 - raw_att, raw_att)
            prep[ds].append(dict(key=k, subj=r["subj_uid"], T=T, t_end=tend[ds][k], guess=guess, gt=gt))
    N.G["lookahead"] = lookahead
    N.G["att"] = att
    N.G["tend"] = tend
    N.G["prep"] = prep


def run_lodo_hp():
    """Nested p_stay / scale selection for the linear high-pass, then one official score per dataset.

    Inner scores use the fast scorer (N.evaluate), which the nested ensemble checks against
    score.score_dataset. Emissions for an inner dataset exclude that dataset and the outer
    dataset. The reported number is score.score_dataset on the held-out dataset.
    """
    import score as S
    os.makedirs(os.path.dirname(LOG_HP), exist_ok=True)
    open(LOG_HP, "w").close()
    log_line(LOG_HP, "loading four LODO classifier folds")
    mean_ev, hp_ev, lookahead = _mean_and_hp_from_folds()
    _install_scorer_state(mean_ev, lookahead)
    log_line(LOG_HP, f"lookahead {lookahead}")
    sanity = N.official("AVGC", mean_ev, 0.998, 1.0)
    log_line(LOG_HP, f"sanity AVGC mean p0.998 H {sanity['H']:.4f} Acc {sanity['Acc']:.2f} "
                     f"(published equal ensemble H 0.7268 Acc 68.90)")
    if abs(sanity["H"] - 0.7268389576132155) > 2e-3 or abs(sanity["Acc"] - 68.8990503853785) > 0.05:
        log_line(LOG_HP, "WARNING sanity does not match the published equal ensemble; continuing")
    rows = []
    streams = {
        "linear_mean": {ds: {k: e["r0"] for k, e in mean_ev[ds].items()} for ds in DS_KEYS},
        "linear_hp30": {ds: {k: e["r0"] for k, e in hp_ev[ds].items()} for ds in DS_KEYS},
    }
    evs = {"linear_mean": mean_ev, "linear_hp30": hp_ev}
    for name in ("linear_mean", "linear_hp30"):
        t0 = time.time()
        dmap = streams[name]
        picks = []
        for outer in DS_KEYS:
            inner_rows = []
            for inner in DS_KEYS:
                if inner == outer:
                    continue
                train = [d for d in DS_KEYS if d not in (outer, inner)]
                mu, sd = N.emis(dmap, train)
                for p in V.P_STAY:
                    for scale in V.SCALES:
                        r = N.evaluate(inner, dmap[inner], mu, sd * scale, p)
                        inner_rows.append(dict(inner=inner, p_stay=p, scale=scale, **r))
            inn = pd.DataFrame(inner_rows)
            grouped = inn.groupby(["p_stay", "scale"])[["H", "Acc", "SDT"]].mean().reset_index()
            ok = grouped[grouped.Acc >= V.THR]
            pool = ok if len(ok) else grouped
            best = pool.sort_values(["H", "Acc", "SDT"], ascending=[False, False, True]).iloc[0]
            off = N.official(outer, evs[name], float(best.p_stay), float(best.scale))
            rec = dict(model=name, ds=outer, p_stay=float(best.p_stay), scale=float(best.scale),
                       H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"], acc1=off["acc1"],
                       inner_H=float(best.H), inner_Acc=float(best.Acc), scorer="official")
            picks.append(rec)
            rows.append(rec)
            log_line(LOG_HP, f"{name} {outer} p {best.p_stay} sc {best.scale}  "
                             f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f} "
                             f"inner H {best.H:.4f} Acc {best.Acc:.2f}")
        sub = pd.DataFrame(picks)
        log_line(LOG_HP, f"MEAN {name}  H {sub.H.mean():.4f}  Acc {sub.Acc.mean():.2f}  "
                         f"SDT {sub.SDT.mean():.2f}  L {sub.L.mean():.3f}  ({time.time() - t0:.0f}s)")
    df = pd.DataFrame(rows)
    os.makedirs(os.path.dirname(OUT_HP), exist_ok=True)
    df.to_csv(OUT_HP, index=False)
    log_line(LOG_HP, f"wrote {OUT_HP}")


def main():
    mode = os.environ.get("MODE", "external").strip().lower()
    if mode == "lodo_hp":
        run_lodo_hp()
        return
    if mode != "external":
        raise SystemExit(f"unknown MODE {mode}")
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    run_external(device)


if __name__ == "__main__":
    main()
