"""Front-end: robust per-frame median re-reference of EEG, then short SW2 FT.

y_c(t) = x_c(t) − median_c'(x_c'(t))  (spatial; causal in time only via per-sample).
Distinct from prep CAR (mean), cm_off on logits, EEG causal demean (temporal).

usage:
  GLIMPSE_THR=norm SCREEN=1 flock multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u median_reref_sw2.py
  FULL=1 ...
"""
from __future__ import annotations

import json
import os
import pickle
import shutil
from types import SimpleNamespace

os.environ.setdefault("GLIMPSE_THR", "norm")

import numpy as np
import torch
import torch.nn.functional as F

import decoders
import direct_classifier as DC
import eval_external as E
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, fold_file, load_trial, resolve_model, rows_of, save_pickle
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times
from run_experiment import load_external

PREP = "eaP_n5_t60"
MAX_EPOCHS = int(os.environ.get("MAX_EPOCHS", "4"))
MIN_EPOCHS = 2
BATCH, FT_LR, STEPS = 64, 2.5e-4, 250
P_SWITCH, P_SWITCH2 = 0.5, 0.5
P_STAY, SCALE, LOOKAHEAD = 0.994, 0.8, 0.75
CHANCE_LOSS = 0.685
SCREEN = os.environ.get("SCREEN", "0") == "1"
FULL = os.environ.get("FULL", "0") == "1"
SEEDS = [int(x) for x in os.environ.get("SEEDS", "0").split(",") if x.strip()]
TAG = "_w10b3_median_reref"
CKPT_ROOT = os.path.join(EXP, "models", "median_reref_sw2")
LOG = os.path.join(EXP, "experiments", "median_reref_sw2.log")
OUT = os.path.join(EXP, "results", "research", "median_reref_sw2.json")
MD = os.path.join(EXP, "multi_agent_opt", "results_median_reref_sw2.md")
EXT_DIR = os.path.join(EXP, "results", "research")


def log(m):
    print(m, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(m + "\n")


def median_reref(x):
    """x (T,C) → subtract per-time median across channels (present=nonzero)."""
    x = np.asarray(x, np.float32)
    # mask all-zero channels (missing)
    present = np.any(np.abs(x) > 0, axis=0)
    if present.sum() < 2:
        return x
    med = np.median(x[:, present], axis=1, keepdims=True)
    out = x.copy()
    out[:, present] = x[:, present] - med
    return out


def prep_trial(tr):
    tr = dict(tr)
    tr["eeg"] = median_reref(tr["eeg"])
    return tr


def pub_fold(seed, held):
    if seed == 0:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts__{held}.pt"
    else:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s{seed}_aug-sw2_wts__{held}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    return path if os.path.exists(path) else resolve_model(os.path.join("models", name))


def pub_all5(seed):
    name = f"all5_direct_clf_w10b3sw2P_s{seed}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    return path if os.path.exists(path) else resolve_model(os.path.join("models", name))


def configure(seed):
    DC.W = int(10 * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS
    D["targets"] = ["env", "onset", "dom"]
    D["augment"] = {**CFG["dnn_augment"], **CFG["aug_variants"]["cluster"], "clf_p_switch2": P_SWITCH2}
    D["train_seed"] = seed
    D["model_blocks"] = 3
    D["max_epochs"] = MAX_EPOCHS
    D["steps_per_epoch"] = STEPS
    D["batch_size"] = BATCH
    D["patience"] = 6
    D["lr"] = FT_LR
    D["val_windows"] = 512
    D["weight_decay"] = float(CFG.get("weight_decay", 5e-4))
    D["lr_patience"] = int(CFG.get("lr_patience", 2))
    decoders.apply_overrides(["model_blocks=3", f"lr={FT_LR}"], [f"clf_p_switch2={P_SWITCH2}"])


def fit_net(train, val, init_from, seed):
    Fs = train[0]["_S"].shape[-1]
    torch.manual_seed(seed)
    net = DC.DirectNet(58, Fs, "vlaai_ws").to(DEV)
    net.load_state_dict(torch.load(resolve_model(init_from), map_location=DEV, weights_only=False), strict=True)
    log(f"  FT-init {init_from} median-reref")
    aug = decoders.DNNDec.__new__(decoders.DNNDec)
    aug.W, aug.fut, aug.hop, aug.C = DC.W, DC.FUT, DC.HOP, 58
    rs = np.random.default_rng(seed)
    Xv, Sv, yv = DC.windows(val, D["val_windows"], rs, None, P_SWITCH, False)
    opt = torch.optim.AdamW(net.parameters(), lr=D["lr"], weight_decay=D["weight_decay"])
    best, bad, state = np.inf, 0, None
    early = []
    for ep in range(D["max_epochs"]):
        tl = []
        net.train()
        for si in range(D["steps_per_epoch"]):
            X, S_, y = [z.to(DEV) for z in DC.windows(train, BATCH, rs, aug, P_SWITCH, False)]
            loss = F.binary_cross_entropy_with_logits(net(X, S_), y)
            opt.zero_grad(); loss.backward()
            torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0); opt.step()
            tl.append(float(loss.detach()))
            if ep == 0 and si < 50:
                early.append(float(loss.detach()))
        if ep == 0:
            me = float(np.mean(early))
            log(f"  early@50={me:.4f}")
            if me >= CHANCE_LOSS:
                raise RuntimeError(f"ABORT_CHANCE {me}")
        net.eval()
        with torch.no_grad():
            lv = torch.cat([net(Xv[i:i+256].to(DEV), Sv[i:i+256].to(DEV)).cpu()
                            for i in range(0, len(Xv), 256)])
        vl = float(F.binary_cross_entropy_with_logits(lv, yv))
        va = float(((lv > 0).float() == yv).float().mean())
        log(f"    ep {ep:02d} train {np.mean(tl):.4f} val {vl:.4f} acc {va:.3f}")
        if vl < best - 1e-4:
            best, bad, state = vl, 0, {k: v.detach().clone() for k, v in net.state_dict().items()}
        elif ep >= MIN_EPOCHS:
            bad += 1
            if bad >= D["patience"]:
                break
    net.load_state_dict(state); net.eval()
    return net, best


def dump_ev(net, held):
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev = {}
    for _, r in rows_of(held).iterrows():
        tr = prep_trial(load_trial(PREP, r["key"]))
        Sfeat, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128); srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur; cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], Sfeat[:, ::-1], Sfeat)
        lg, starts = DC.logits(net, x, Ssw)
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        n10 = len(lg) // 40
        ev[r["key"]] = dict(
            r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            r0_10=lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            r1_10=-lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            att10=(truth[:n10 * 40].reshape(n10, 40).mean(1) > 0.5).astype(np.int8),
            pre_switched=True,
        )
    return ev


def exp_name(seed):
    return f"direct_clf__{PREP}_median_reref_s{seed}"


def run_screen():
    open(LOG, "w").close()
    log("=== SCREEN median-reref SW2 held-AVGC ===")
    configure(0); held = "AVGC"
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
    train, val = [], []
    for ds in DS_KEYS:
        if ds == held: continue
        rows = rows_of(ds); vs = rng.choice(sorted(rows.subject.unique()))
        for _, r in rows.iterrows():
            tr = prep_trial(load_trial(PREP, r["key"])); tr["_S"] = _targets(tr)
            (val if r["subject"] == vs else train).append(tr)
    train += [prep_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
    for tr in train:
        if "_S" not in tr: tr["_S"] = _targets(tr)
    try:
        net, best = fit_net(train, val, pub_fold(0, held), 0)
    except RuntimeError as e:
        log(f"SCREEN FAIL {e}"); return 1
    os.makedirs(os.path.join(CKPT_ROOT, "s0"), exist_ok=True)
    torch.save(net.state_dict(), os.path.join(CKPT_ROOT, "s0", "held-AVGC.pt"))
    ev = dump_ev(net, held)
    A = np.concatenate([(e["r0"]-e["r1"])*np.where(e["att"]==0,1.0,-1.0) for e in ev.values()])
    stub = {"r0":np.zeros(8,np.float32),"r1":np.zeros(8,np.float32),"att":np.zeros(8,np.int8),
            "t_end":np.arange(1,9,dtype=np.float32)*0.25,"pre_switched":True}
    pack = {held: ev}
    for ds in DS_KEYS:
        if ds != held: pack[ds] = {"__stub__": dict(stub)}
    r = S.score_dataset(held, pack, LOOKAHEAD, emis=(float(A.mean()), float(A.std())*SCALE), p_stay=P_STAY)
    log(f"SCREEN AVGC H={r['H']:.4f} Acc={r['Acc']:.2f}")
    ok = r["H"] > 0.70 and r["Acc"] > 65
    with open(OUT, "w") as fh:
        json.dump(dict(screen=True, avgc=dict(H=float(r["H"]), Acc=float(r["Acc"])), best_val=best), fh, indent=1)
    if not ok:
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
    log(f"SCREEN {'PASS' if ok else 'FAIL'}")
    return 0 if ok else 1


def run_full():
    seed = SEEDS[0]
    log(f"=== FULL median-reref seed={seed} ===")
    configure(seed)
    for held in DS_KEYS:
        log(f"fold {held}")
        ckpt = os.path.join(CKPT_ROOT, f"s{seed}", f"held-{held}.pt")
        os.makedirs(os.path.dirname(ckpt), exist_ok=True)
        if os.path.exists(ckpt):
            net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
            net.load_state_dict(torch.load(ckpt, map_location=DEV, weights_only=False)); net.eval()
            best = float("nan")
        else:
            rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held) + 17 * seed)
            train, val = [], []
            for ds in DS_KEYS:
                if ds == held: continue
                rows = rows_of(ds); vs = rng.choice(sorted(rows.subject.unique()))
                for _, r in rows.iterrows():
                    tr = prep_trial(load_trial(PREP, r["key"])); tr["_S"] = _targets(tr)
                    (val if r["subject"] == vs else train).append(tr)
            train += [prep_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
            for tr in train:
                if "_S" not in tr: tr["_S"] = _targets(tr)
            net, best = fit_net(train, val, pub_fold(seed, held), seed)
            torch.save(net.state_dict(), ckpt); del train, val
        ev = dump_ev(net, held)
        save_pickle(dict(evidence={"causal": ev}, info=dict(
            decoder="direct_clf", prep=PREP, held_out=held, best_val=best,
            lookahead_s=(DC.FUT+DC.HOP)/FS, front_end="median_reref")),
            fold_file(exp_name(seed), held))
        del net
        if DEV.type == "cuda": torch.cuda.empty_cache()

    all5 = os.path.join(CKPT_ROOT, f"s{seed}", "all5.pt")
    if os.path.exists(all5):
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        net.load_state_dict(torch.load(all5, map_location=DEV, weights_only=False)); net.eval()
    else:
        rng = np.random.default_rng(CFG["seed"] + 99 + 13 * seed)
        train, val = [], []
        for ds in DS_KEYS:
            rows = rows_of(ds); vs = rng.choice(sorted(rows.subject.unique()))
            for _, r in rows.iterrows():
                tr = prep_trial(load_trial(PREP, r["key"])); tr["_S"] = _targets(tr)
                (val if r["subject"] == vs else train).append(tr)
        train += [prep_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
        for tr in train:
            if "_S" not in tr: tr["_S"] = _targets(tr)
        net, best = fit_net(train, val, pub_all5(seed), seed)
        torch.save(net.state_dict(), all5); del train, val
    paths = {}
    for ds in ("AASD", "Carta2026"):
        path = os.path.join(EXT_DIR, f"ext_clf_{ds}{TAG}_evidence_s{seed}.pkl")
        paths[ds] = path
        if os.path.exists(path): continue
        EV = {}
        for i, tr in enumerate(E.ext_trials(ds)):
            tr_d = prep_trial(tr)
            lg, starts = DC.logits(net, tr_d["eeg"], _targets(tr_d))
            truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
            EV[tr["key"]] = dict(r0=lg/2, r1=-lg/2, att=truth.astype(np.int8),
                t_end=(starts/FS+tr["t0"]).astype(np.float32),
                att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
        with open(path, "wb") as fh: pickle.dump(EV, fh)
        log(f"  wrote {path}")
    del net

    la = (DC.FUT + DC.HOP) / FS
    ev, _ = S.load_folds(exp_name(seed), "causal")
    grid = []
    for p, sc in ((0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)):
        per = {}
        for ds in DS_KEYS:
            A = np.concatenate([(ee["r0"]-ee["r1"])*np.where(ee["att"]==0,1.,-1.)
                                for d in DS_KEYS if d != ds for ee in ev[d].values()])
            per[ds] = S.score_dataset(ds, ev, la, emis=(float(A.mean()), float(A.std())*sc), p_stay=p)
        mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        grid.append(dict(p=p, sc=sc, H=mH, Acc=mA))
        log(f"  LODO p={p} sc={sc} H={mH:.4f} Acc={mA:.2f}")
    mild = next(r for r in grid if r["p"]==0.996 and r["sc"]==0.8)
    ok = [r for r in grid if r["H"]>0.85 and r["Acc"]>75]
    best = mild if mild["H"]>0.85 and mild["Acc"]>75 else (
        sorted(ok, key=lambda r: r["H"], reverse=True)[0] if ok else
        sorted(grid, key=lambda r: (r["Acc"], r["H"]), reverse=True)[0])
    A = np.concatenate([(e["r0"]-e["r1"])*np.where(e["att"]==0,1.,-1.)
                        for ds in DS_KEYS for e in ev[ds].values()])
    mu, sd = float(A.mean()), float(A.std())
    ext = {}
    for ds, path in paths.items():
        with open(path, "rb") as fh: EV = pickle.load(fh)
        r = E.score(EV, mu, sd*best["sc"], best["p"], LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H","Acc","L","SDT") if k in r}
        log(f"  EXT {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")
    gate = best["H"]>0.85 and best["Acc"]>75 and ext["AASD"]["H"]>=0.65 and ext["Carta2026"]["H"]>=0.65
    out = dict(method="median_reref_sw2", selected=best, grid=grid, external=ext, pass_joint=gate)
    with open(OUT, "w") as fh: json.dump(out, fh, indent=1, default=float)
    __import__("pathlib").Path(MD).write_text(
        f"# results_median_reref_sw2\n\nLODO {best['H']:.4f}/{best['Acc']:.2f}; "
        f"AASD {ext['AASD']['H']:.4f}; Carta {ext['Carta2026']['H']:.4f}; pass={gate}\n")
    log(f"DONE pass={gate}")
    if not gate:
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
    return 0 if gate else 1


if __name__ == "__main__":
    raise SystemExit(run_full() if FULL else run_screen())
