"""Front-end: causal EEG demean (τ≈30s) then short FT of published SW2.

Born-debiased EEG (slow drift stripped) before vlaai_ws — distinct from
prep 1–9 Hz Butterworth, running-z `*_rz`, and logit-level linear_hp30 /
hp30_center_ft.

Screen: seed0 held-AVGC chance-abort + LODO-AVGC H/Acc; expand if promising.
Infer with demeaned EEG; score mean seed + optional post logit-hp30.

usage:
  SCREEN=1 flock multi_agent_opt/gpu.lock python -u eeg_causal_demean_sw2.py
  FULL=1 SEEDS=0,1,2,3 flock multi_agent_opt/gpu.lock python -u eeg_causal_demean_sw2.py
"""
from __future__ import annotations

import json
import os
import shutil
import time
from types import SimpleNamespace

os.environ.setdefault("GLIMPSE_THR", "norm")

import numpy as np
import torch
import torch.nn.functional as F
from scipy.signal import lfilter

import decoders
import direct_classifier as DC
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, load_trial, resolve_model, rows_of
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times
from run_experiment import load_external

PREP = "eaP_n5_t60"
TAU = float(os.environ.get("EEG_TAU", "30"))
MAX_EPOCHS = int(os.environ.get("MAX_EPOCHS", "4"))
MIN_EPOCHS = int(os.environ.get("MIN_EPOCHS", "2"))
BATCH = int(os.environ.get("BATCH", "64"))
FT_LR = float(os.environ.get("FT_LR", "2.5e-4"))
STEPS = int(os.environ.get("STEPS_PER_EPOCH", "250"))
P_SWITCH = 0.5
P_SWITCH2 = 0.5
SEEDS = [int(x) for x in os.environ.get("SEEDS", "0").split(",") if x.strip() != ""]
SCREEN = os.environ.get("SCREEN", "0") == "1"
FULL = os.environ.get("FULL", "0") == "1"
P_STAY, SCALE = 0.994, 0.8
LOOKAHEAD = 0.75
CHANCE_LOSS = 0.685

TAG = f"_w10b3_ptSW2_eegdemean_t{int(TAU)}"
CKPT_ROOT = os.path.join(EXP, "models", "eeg_causal_demean_sw2")
LOG_PATH = os.path.join(EXP, "experiments", "eeg_causal_demean_sw2.log")
OUT_JSON = os.path.join(EXP, "results", "research", "eeg_causal_demean_sw2.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_eeg_causal_demean_sw2.md")
EXT_DIR = os.path.join(EXP, "results", "research")


def log(msg):
    print(msg, flush=True)
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
    with open(LOG_PATH, "a") as fh:
        fh.write(msg + "\n")


def causal_demean(x, tau_s=TAU, fs=FS):
    """Causal one-pole HP (= demean via EMA) along time. x (T, C)."""
    alpha = 1.0 - np.exp(-1.0 / (tau_s * fs))
    ema = lfilter([alpha], [1.0, -(1.0 - alpha)], np.asarray(x, np.float64), axis=0)
    return (np.asarray(x, np.float64) - ema).astype(np.float32)


def demean_trial(tr):
    tr = dict(tr)
    tr["eeg"] = causal_demean(tr["eeg"])
    return tr


def pub_fold_ckpt(seed, held):
    if seed == 0:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts__{held}.pt"
    else:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s{seed}_aug-sw2_wts__{held}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    if not os.path.exists(path):
        raise FileNotFoundError(name)
    return path


def pub_all5_ckpt(seed):
    name = f"all5_direct_clf_w10b3sw2P_s{seed}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    if not os.path.exists(path):
        raise FileNotFoundError(name)
    return path


def configure(seed):
    DC.W = int(10 * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS
    D["targets"] = ["env", "onset", "dom"]
    D["augment"] = {
        **CFG["dnn_augment"],
        **CFG["aug_variants"]["cluster"],
        "clf_p_switch2": P_SWITCH2,
    }
    D["train_seed"] = int(seed)
    D["model_blocks"] = 3
    D["max_epochs"] = MAX_EPOCHS
    D["steps_per_epoch"] = STEPS
    D["batch_size"] = BATCH
    D["patience"] = 6
    D["lr"] = FT_LR
    D["val_windows"] = 512
    D["weight_decay"] = CFG.get("weight_decay", 5e-4)
    D["lr_patience"] = CFG.get("lr_patience", 2)
    decoders.apply_overrides(["model_blocks=3", f"lr={FT_LR}"], [f"clf_p_switch2={P_SWITCH2}"])


def fit_net(train, val, a, seed):
    Fs = train[0]["_S"].shape[-1]
    torch.manual_seed(seed)
    net = DC.DirectNet(58, Fs, "vlaai_ws").to(DEV)
    sd = torch.load(resolve_model(a.init_from), map_location=DEV, weights_only=False)
    net.load_state_dict(sd, strict=True)
    log(f"  FT-init {a.init_from} EEG_τ={TAU}s")
    aug = decoders.DNNDec.__new__(decoders.DNNDec)
    aug.W, aug.fut, aug.hop, aug.C = DC.W, DC.FUT, DC.HOP, 58
    rs = np.random.default_rng(seed)
    Xv, Sv, yv = DC.windows(val, D["val_windows"], rs, None, a.p_switch, False)
    opt = torch.optim.AdamW(net.parameters(), lr=D["lr"], weight_decay=D["weight_decay"])
    sch = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode="min", factor=0.5, patience=D["lr_patience"])
    best, bad, state = np.inf, 0, None
    early = []
    for ep in range(D["max_epochs"]):
        tl = []
        net.train()
        for step_i in range(D["steps_per_epoch"]):
            X, S_, y = [z.to(DEV) for z in DC.windows(train, D["batch_size"], rs, aug, a.p_switch, False)]
            loss = F.binary_cross_entropy_with_logits(net(X, S_), y)
            opt.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0)
            opt.step()
            tl.append(float(loss.detach()))
            if ep == 0 and step_i < 50:
                early.append(float(loss.detach()))
        if ep == 0 and early:
            mean_e = float(np.mean(early))
            log(f"  early mean_loss@50={mean_e:.4f}")
            if mean_e >= CHANCE_LOSS:
                raise RuntimeError(f"ABORT_CHANCE: {mean_e:.4f} >= {CHANCE_LOSS}")
        net.eval()
        with torch.no_grad():
            lv = torch.cat([
                net(Xv[i:i + 256].to(DEV), Sv[i:i + 256].to(DEV)).cpu()
                for i in range(0, len(Xv), 256)
            ])
        vl = float(F.binary_cross_entropy_with_logits(lv, yv))
        va = float(((lv > 0).float() == yv).float().mean())
        if ep >= a.min_epochs:
            sch.step(vl)
        log(f"    epoch {ep:02d} train {np.mean(tl):.4f} val BCE {vl:.4f} acc {va:.3f}")
        if vl < best - 1e-4:
            best, bad, state = vl, 0, {k: v.detach().clone() for k, v in net.state_dict().items()}
        elif ep >= a.min_epochs:
            bad += 1
            if bad >= D["patience"]:
                break
    if state is None:
        state = {k: v.detach().clone() for k, v in net.state_dict().items()}
    net.load_state_dict(state)
    net.eval()
    return net, best


def dump_fold_evidence(net, held, seed, a):
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev = {}
    for _, r in rows_of(held).iterrows():
        tr = demean_trial(load_trial(a.prep, r["key"]))
        Sfeat, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128)
        srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur
            cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(
            T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], Sfeat[:, ::-1], Sfeat)
        lg, starts = DC.logits(net, x, Ssw)
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        n10 = len(lg) // 40
        ev[r["key"]] = dict(
            r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            r0_10=lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            r1_10=-lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            att10=(truth[:n10 * 40].reshape(n10, 40).mean(1) > 0.5).astype(np.int8),
            pre_switched=True,
        )
    return ev


def run_screen():
    """seed0 held-AVGC only."""
    open(LOG_PATH, "w").close()
    log(f"=== SCREEN eeg-demean τ={TAU} seed0 held-AVGC ===")
    seed, held = 0, "AVGC"
    configure(seed)
    a = SimpleNamespace(
        backbone="vlaai_ws", p_switch=P_SWITCH, zero_eeg=False,
        min_epochs=MIN_EPOCHS, init_from=pub_fold_ckpt(seed, held), prep=PREP, tag=TAG,
    )
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
    train, val = [], []
    for ds in DS_KEYS:
        if ds == held:
            continue
        rows = rows_of(ds)
        vs = rng.choice(sorted(rows.subject.unique()))
        for _, r in rows.iterrows():
            tr = demean_trial(load_trial(PREP, r["key"]))
            tr["_S"] = _targets(tr)
            (val if r["subject"] == vs else train).append(tr)
    train += [demean_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
    for tr in train:
        if "_S" not in tr:
            tr["_S"] = _targets(tr)
    log(f"  train {len(train)} val {len(val)}")
    try:
        net, best = fit_net(train, val, a, seed)
    except RuntimeError as e:
        log(f"SCREEN FAIL: {e}")
        return 1
    os.makedirs(os.path.join(CKPT_ROOT, "s0"), exist_ok=True)
    torch.save(net.state_dict(), os.path.join(CKPT_ROOT, "s0", "held-AVGC.pt"))
    held_ev = dump_fold_evidence(net, held, seed, a)
    A = np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0) for e in held_ev.values()
    ])
    # score_dataset always iterates other DS for emis fit; stub them when overriding emis.
    stub = {
        "r0": np.zeros(8, np.float32), "r1": np.zeros(8, np.float32),
        "att": np.zeros(8, np.int8), "t_end": np.arange(1, 9, dtype=np.float32) * 0.25,
        "pre_switched": True,
    }
    pack = {held: held_ev}
    for ds in DS_KEYS:
        if ds != held:
            pack[ds] = {"__stub__": dict(stub)}
    r = S.score_dataset(
        held, pack, LOOKAHEAD,
        emis=(float(A.mean()), float(A.std()) * SCALE), p_stay=P_STAY,
    )
    log(f"SCREEN AVGC (self-emis) H={r['H']:.4f} Acc={r['Acc']:.2f}")
    out = dict(screen=True, avgc=r, best_val=best, tau=TAU)
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    # pass screen if H > 0.70 (pub ~0.726) and Acc > 65
    ok = r["H"] > 0.70 and r["Acc"] > 65
    log(f"SCREEN {'PASS' if ok else 'FAIL'} criteria H>0.70 Acc>65")
    if not ok:
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
    return 0 if ok else 1


def exp_name(seed):
    return f"direct_clf__{PREP}_w10b3_eegdemean_t{int(TAU)}_s{seed}"


def run_fold(held, seed, a):
    from common_exp import save_pickle, fold_file as ff
    configure(seed)
    a = SimpleNamespace(
        backbone="vlaai_ws", p_switch=P_SWITCH, zero_eeg=False,
        min_epochs=MIN_EPOCHS, init_from=pub_fold_ckpt(seed, held), prep=PREP, tag=TAG,
    )
    ckpt = os.path.join(CKPT_ROOT, f"s{seed}", f"held-{held}.pt")
    os.makedirs(os.path.dirname(ckpt), exist_ok=True)
    if os.path.exists(ckpt):
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        net.load_state_dict(torch.load(ckpt, map_location=DEV, weights_only=False))
        net.eval()
        log(f"  cache-hit {ckpt}")
        best = float("nan")
    else:
        rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held) + 17 * seed)
        train, val = [], []
        for ds in DS_KEYS:
            if ds == held:
                continue
            rows = rows_of(ds)
            vs = rng.choice(sorted(rows.subject.unique()))
            for _, r in rows.iterrows():
                tr = demean_trial(load_trial(PREP, r["key"]))
                tr["_S"] = _targets(tr)
                (val if r["subject"] == vs else train).append(tr)
        train += [demean_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
        for tr in train:
            if "_S" not in tr:
                tr["_S"] = _targets(tr)
        net, best = fit_net(train, val, a, seed)
        torch.save(net.state_dict(), ckpt)
        del train, val
    ev = dump_fold_evidence(net, held, seed, a)
    info = dict(decoder="direct_clf", prep=PREP, held_out=held, best_val=best,
                lookahead_s=(DC.FUT + DC.HOP) / FS, eeg_tau=TAU, ft_from="published_sw2_demean")
    save_pickle(dict(evidence={"causal": ev}, info=info), ff(exp_name(seed), held))
    del net
    if DEV.type == "cuda":
        torch.cuda.empty_cache()
    log(f"  fold {held} done best_val={best}")


def train_all5_ext(seed):
    import pickle
    import eval_external as EE
    configure(seed)
    a = SimpleNamespace(
        backbone="vlaai_ws", p_switch=P_SWITCH, zero_eeg=False,
        min_epochs=MIN_EPOCHS, init_from=pub_all5_ckpt(seed), prep=PREP, tag=TAG,
    )
    all5_pt = os.path.join(CKPT_ROOT, f"s{seed}", "all5.pt")
    os.makedirs(os.path.dirname(all5_pt), exist_ok=True)
    if os.path.exists(all5_pt):
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        net.load_state_dict(torch.load(all5_pt, map_location=DEV, weights_only=False))
        net.eval()
        log(f"  loaded {all5_pt}")
    else:
        rng = np.random.default_rng(CFG["seed"] + 99 + 13 * seed)
        train, val = [], []
        for ds in DS_KEYS:
            rows = rows_of(ds)
            vs = rng.choice(sorted(rows.subject.unique()))
            for _, r in rows.iterrows():
                tr = demean_trial(load_trial(PREP, r["key"]))
                tr["_S"] = _targets(tr)
                (val if r["subject"] == vs else train).append(tr)
        train += [demean_trial(t) for t in load_external(["Fuglsang2020"], PREP, log)]
        for tr in train:
            if "_S" not in tr:
                tr["_S"] = _targets(tr)
        net, best = fit_net(train, val, a, seed)
        torch.save(net.state_dict(), all5_pt)
        log(f"  saved {all5_pt} best={best:.4f}")
        del train, val
    paths = {}
    for ds in ("AASD", "Carta2026"):
        path = os.path.join(EXT_DIR, f"ext_clf_{ds}{TAG}_evidence_s{seed}.pkl")
        paths[ds] = path
        if os.path.exists(path):
            continue
        EV = {}
        trials = EE.ext_trials(ds)
        log(f"  external {ds}: {len(trials)} trials")
        for i, tr in enumerate(trials):
            tr_d = demean_trial(tr)
            lg, starts = DC.logits(net, tr_d["eeg"], _targets(tr_d))
            truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
            EV[tr["key"]] = dict(
                r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
                t_end=(starts / FS + tr["t0"]).astype(np.float32),
                att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
            if (i + 1) % 50 == 0 or i + 1 == len(trials):
                log(f"    {ds} {i+1}/{len(trials)}")
        with open(path, "wb") as fh:
            pickle.dump(EV, fh)
    del net
    if DEV.type == "cuda":
        torch.cuda.empty_cache()
    return paths


def soft_hp_ev(ev, tau=30.0):
    import ensemble_clf_transformer_v4 as V4
    out = {}
    for ds in ev:
        out[ds] = {}
        for k, e in ev[ds].items():
            d = (np.asarray(e["r0"], float) - np.asarray(e["r1"], float))
            h = V4.highpass(d[:, None], tau=tau)[:, 0]
            s = 0.5 * d + 0.5 * h
            out[ds][k] = {**e, "r0": s, "r1": np.zeros_like(s)}
    return out


def score_full_lodo(exp):
    la = (DC.FUT + DC.HOP) / FS
    ev, _ = S.load_folds(exp, "causal")
    grid = {}
    for name, e in (("raw", ev), ("soft_hp50", soft_hp_ev(ev, 50)), ("soft_hp30", soft_hp_ev(ev, 30))):
        rows = []
        for p, sc in ((0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)):
            per = {}
            for ds in DS_KEYS:
                A = np.concatenate([
                    (ee["r0"] - ee["r1"]) * np.where(ee["att"] == 0, 1.0, -1.0)
                    for d in DS_KEYS if d != ds for ee in e[d].values()
                ])
                mu, sd = float(A.mean()), float(A.std())
                per[ds] = S.score_dataset(ds, e, la, emis=(mu, sd * sc), p_stay=p)
            mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
            mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
            rows.append(dict(p=p, sc=sc, H=mH, Acc=mA,
                             per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"])) for d in DS_KEYS}))
            log(f"  LODO {name} p={p} sc={sc} H={mH:.4f} Acc={mA:.2f}")
        grid[name] = rows
    return grid, ev


def score_full_ext(ev_lodo, paths, name, p, sc):
    import pickle
    import eval_external as EE
    A = np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev_lodo[ds].values()
    ])
    mu, sd = float(A.mean()), float(A.std())
    ext = {}
    for ds, path in paths.items():
        with open(path, "rb") as fh:
            EV = pickle.load(fh)
        if name.startswith("soft_hp"):
            tau = 50.0 if "50" in name else 30.0
            tmp = soft_hp_ev({"_": EV}, tau)["_"]
            EV = tmp
        r = EE.score(EV, mu, sd * sc, p, LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        log(f"  EXT {name} {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")
    return ext


def run_full():
    open(LOG_PATH, "a").write("\n=== FULL eeg-demean ===\n")
    log(f"=== FULL eeg-demean τ={TAU} seeds={SEEDS} ===")
    seed = SEEDS[0]
    configure(seed)
    for held in DS_KEYS:
        log(f"fold held-out={held}")
        run_fold(held, seed, None)
    paths = train_all5_ext(seed)
    grid, ev = score_full_lodo(exp_name(seed))
    # select Acc>75 then H among variants
    cands = []
    for name, rows in grid.items():
        ok = [r for r in rows if r["H"] > 0.85 and r["Acc"] > 75]
        mild = next((r for r in rows if r["p"] == 0.996 and r["sc"] == 0.8), rows[0])
        best = mild if mild in ok or (mild["H"] > 0.85 and mild["Acc"] > 75) else (
            sorted(ok, key=lambda r: r["H"], reverse=True)[0] if ok else
            sorted(rows, key=lambda r: (r["Acc"], r["H"]), reverse=True)[0]
        )
        cands.append((name, best))
    cands.sort(key=lambda x: (1 if x[1]["H"] > 0.85 and x[1]["Acc"] > 75 else 0, x[1]["H"]), reverse=True)
    vname, best = cands[0]
    log(f"SELECTED {vname} p={best['p']} sc={best['sc']} H={best['H']:.4f} Acc={best['Acc']:.2f}")
    # external for selected + raw @ mild
    e_sel = soft_hp_ev(ev, 50) if vname == "soft_hp50" else (
        soft_hp_ev(ev, 30) if vname == "soft_hp30" else ev)
    ext = score_full_ext(e_sel, paths, vname, best["p"], best["sc"])
    # also tip-like raw @ 0.996 for Assd/Carta comparison
    ext_raw996 = score_full_ext(ev, paths, "raw", 0.996, 0.8)
    gate = (
        best["H"] > 0.85 and best["Acc"] > 75
        and ext["AASD"]["H"] >= 0.65 and ext["Carta2026"]["H"] >= 0.65
    )
    out = dict(
        method="eeg_causal_demean_sw2", tau=TAU, seed=seed,
        selected=dict(variant=vname, **best), lodo_grid=grid,
        external=ext, external_raw996=ext_raw996, pass_joint=gate,
    )
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    Path = __import__("pathlib").Path
    Path(RESULTS_MD).write_text(
        "# results_eeg_causal_demean_sw2\n\n"
        f"Causal EEG demean τ={TAU}s + SW2 FT. Selected {vname} "
        f"LODO {best['H']:.4f}/{best['Acc']:.2f}; "
        f"AASD {ext['AASD']['H']:.4f}; Carta {ext['Carta2026']['H']:.4f}; pass={gate}\n"
    )
    log(f"DONE pass={gate}")
    if not gate:
        # keep screen ckpts? delete new full folds if fail
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
        log(f"deleted failing ckpts {CKPT_ROOT}")
        # also wipe fold pickles? leave evidence for analysis
    return 0 if gate else 1


def main():
    if FULL:
        return run_full()
    return run_screen()


if __name__ == "__main__":
    raise SystemExit(main())
