"""Train-time KD: distill soft_hp50_accfuse_final teacher into SW2 tip student.

Teacher = AccFuse↔hp30 tip × soft_hp50(stem) w=0.15 (frozen signed streams).
Student = published SW2 DirectNet, short FT with
  loss = BCE(student, y) + λ · Huber(student_logit, teacher_aligned)

At test: student alone (NO stem / tip↔stem readout mix). Goal: bake LODO Acc of
the near-miss teacher into a tip-like front-end that keeps Assd/Carta H≥0.65.

≠ tip↔stem soft/AC/σ/|Δ| mixes at readout. ≠ hp30_center_ft. ≠ EEG demean alone.

usage:
  SCREEN=1 GLIMPSE_THR=norm flock multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u kd_softhp50_into_sw2.py
  FULL=1 ...
"""
from __future__ import annotations

import json
import os
import pickle
import shutil
import time
from types import SimpleNamespace

os.environ.setdefault("GLIMPSE_THR", "norm")

import numpy as np
import torch
import torch.nn.functional as F

import decoders
import direct_classifier as DC
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, fold_file, load_trial, resolve_model, rows_of, save_pickle
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times
from run_experiment import load_external
from accfuse_hp30_hselect import align_hp, load_accfuse
from stem_hp30_accfuse import TIP_ALPHA, blend_signed, ev_to_signed, load_stem_honest
from dual_hp30_sw2_bmms import blend_ev
from multitau_stem_hp_accfuse import soft_hp

PREP = "eaP_n5_t60"
LAM = float(os.environ.get("LAM", "0.5"))
MAX_EPOCHS = int(os.environ.get("MAX_EPOCHS", "4"))
MIN_EPOCHS = int(os.environ.get("MIN_EPOCHS", "2"))
BATCH = 64
FT_LR = float(os.environ.get("FT_LR", "2.5e-4"))
STEPS = int(os.environ.get("STEPS_PER_EPOCH", "250"))
P_SWITCH, P_SWITCH2 = 0.5, 0.5
P_STAY, SCALE = 0.994, 0.8
LOOKAHEAD = 0.75
CHANCE_LOSS = 0.685
W_STEM = 0.15
SCREEN = os.environ.get("SCREEN", "0") == "1"
FULL = os.environ.get("FULL", "0") == "1"
SEEDS = [int(x) for x in os.environ.get("SEEDS", "0").split(",") if x.strip()]

TAG = f"_w10b3_kdsofthp50_l{LAM}"
CKPT_ROOT = os.path.join(EXP, "models", "kd_softhp50_into_sw2")
LOG_PATH = os.path.join(EXP, "experiments", "kd_softhp50_into_sw2.log")
OUT_JSON = os.path.join(EXP, "results", "research", "kd_softhp50_into_sw2.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_kd_softhp50_into_sw2.md")
EXT_DIR = os.path.join(EXP, "results", "research")


def log(msg):
    print(msg, flush=True)
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
    with open(LOG_PATH, "a") as fh:
        fh.write(msg + "\n")


def build_teacher_signed():
    """Frozen soft_hp50_accfuse_final LODO signed streams {ds:{key:d}}."""
    af = load_accfuse()
    _, lin_hp, _ = V4._mean_and_hp_from_folds()
    lin_al = align_hp(af, lin_hp)
    tip_ev = {ds: blend_ev(af[ds], lin_al[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    soft50 = soft_hp(load_stem_honest(), 50)
    # blend_signed returns evidence-like; get signed
    from stem_hp30_accfuse import blend_signed as bs
    # bs returns {ds:{k: dict r0,r1,...}} or signed? Check
    ev = bs(tip_s, soft50, W_STEM, af)
    out = {}
    for ds in DS_KEYS:
        out[ds] = {}
        for k, e in ev[ds].items():
            out[ds][k] = np.asarray(e["r0"] - e["r1"] if "r1" in e else e, np.float64).reshape(-1)
    return out


def teacher_at_window(teacher_ds, key, start, W=None):
    """Mean teacher signed over hops covered by window [start, start+W)."""
    W = W or DC.W
    d = teacher_ds.get(key)
    if d is None or len(d) == 0:
        return 0.0
    # logits hop every HOP samples; window covers starts in [start, start+W)
    # Approximate: map window mid to hop index
    mid = start + W // 2
    hop = DC.HOP
    idx = int(np.clip(mid // hop, 0, len(d) - 1))
    return float(d[idx])


def pub_fold(seed, held):
    if seed == 0:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts__{held}.pt"
    else:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s{seed}_aug-sw2_wts__{held}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    return path


def pub_all5(seed):
    name = f"all5_direct_clf_w10b3sw2P_s{seed}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    return path


def configure(seed):
    DC.W = int(10 * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS
    D["targets"] = ["env", "onset", "dom"]
    D["augment"] = {**CFG["dnn_augment"], **CFG["aug_variants"]["cluster"], "clf_p_switch2": P_SWITCH2}
    D["train_seed"] = seed
    D["model_blocks"] = 3
    D["max_epochs"] = MAX_EPOCHS
    D["steps_per_epoch"] = STEPS
    D["batch_size"] = BATCH
    D["patience"] = 6
    D["lr"] = FT_LR
    D["val_windows"] = 512
    D["weight_decay"] = float(CFG.get("weight_decay", 5e-4))
    D["lr_patience"] = int(CFG.get("lr_patience", 2))
    decoders.apply_overrides(["model_blocks=3", f"lr={FT_LR}"], [f"clf_p_switch2={P_SWITCH2}"])


def windows_with_teacher(trials, n, rng, aug, teacher_by_key):
    """Like DC.windows but also returns teacher signed targets for each window."""
    X, S, y = DC.windows(trials, n, rng, aug, P_SWITCH, False)
    ti = DC.windows.last_ti
    starts = DC.windows.last_start
    tch = np.zeros(n, np.float32)
    for j in range(n):
        tr = trials[int(ti[j])]
        tch[j] = teacher_at_window(teacher_by_key, tr["key"], int(starts[j]))
    return X, S, y, torch.from_numpy(tch)


def fit_kd(train, val, init_from, seed, teacher_by_key):
    Fs = train[0]["_S"].shape[-1]
    torch.manual_seed(seed)
    net = DC.DirectNet(58, Fs, "vlaai_ws").to(DEV)
    net.load_state_dict(torch.load(resolve_model(init_from), map_location=DEV, weights_only=False), strict=True)
    log(f"  KD-init {init_from} λ={LAM}")
    aug = decoders.DNNDec.__new__(decoders.DNNDec)
    aug.W, aug.fut, aug.hop, aug.C = DC.W, DC.FUT, DC.HOP, 58
    rs = np.random.default_rng(seed)
    Xv, Sv, yv, tv = windows_with_teacher(val, D["val_windows"], rs, None, teacher_by_key)
    opt = torch.optim.AdamW(net.parameters(), lr=D["lr"], weight_decay=D["weight_decay"])
    best, bad, state = np.inf, 0, None
    early = []
    for ep in range(D["max_epochs"]):
        tl = []
        net.train()
        for si in range(D["steps_per_epoch"]):
            X, S_, y, tch = windows_with_teacher(train, BATCH, rs, aug, teacher_by_key)
            X, S_, y, tch = X.to(DEV), S_.to(DEV), y.to(DEV), tch.to(DEV)
            logit = net(X, S_)
            bce = F.binary_cross_entropy_with_logits(logit, y)
            # scale teacher to logit-ish range: teacher is signed evidence ~ N(μ,σ)
            kd = F.smooth_l1_loss(logit, tch)
            loss = bce + LAM * kd
            opt.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0)
            opt.step()
            tl.append(float(loss.detach()))
            if ep == 0 and si < 50:
                early.append(float(loss.detach()))
        if ep == 0:
            me = float(np.mean(early))
            log(f"  early@50={me:.4f}")
            if me >= CHANCE_LOSS + 0.5:  # KD can inflate loss; softer abort
                raise RuntimeError(f"ABORT_CHANCE {me}")
        net.eval()
        with torch.no_grad():
            lv = torch.cat([net(Xv[i:i + 256].to(DEV), Sv[i:i + 256].to(DEV)).cpu()
                            for i in range(0, len(Xv), 256)])
        vl = float(F.binary_cross_entropy_with_logits(lv, yv) + LAM * F.smooth_l1_loss(lv, tv))
        va = float(((lv > 0).float() == yv).float().mean())
        log(f"    ep {ep:02d} train {np.mean(tl):.4f} val {vl:.4f} acc {va:.3f}")
        if vl < best - 1e-4:
            best, bad, state = vl, 0, {k: v.detach().clone() for k, v in net.state_dict().items()}
        elif ep >= MIN_EPOCHS:
            bad += 1
            if bad >= D["patience"]:
                break
    net.load_state_dict(state)
    net.eval()
    return net, best


def dump_ev(net, held):
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev = {}
    for _, r in rows_of(held).iterrows():
        tr = load_trial(PREP, r["key"])
        Sfeat, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128)
        srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur
            cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(
            T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], Sfeat[:, ::-1], Sfeat)
        lg, starts = DC.logits(net, x, Ssw)
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        n10 = len(lg) // 40
        ev[r["key"]] = dict(
            r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            r0_10=lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            r1_10=-lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            att10=(truth[:n10 * 40].reshape(n10, 40).mean(1) > 0.5).astype(np.int8),
            pre_switched=True,
        )
    return ev


def run_screen():
    open(LOG_PATH, "w").close()
    log(f"=== SCREEN KD soft_hp50→SW2 λ={LAM} held-AVGC ===")
    configure(0)
    held = "AVGC"
    log("building teacher soft_hp50_accfuse_final...")
    teacher = build_teacher_signed()
    # merge all train keys into one lookup
    tch_all = {}
    for ds in DS_KEYS:
        if ds == held:
            continue
        tch_all.update(teacher[ds])
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
    train, val = [], []
    for ds in DS_KEYS:
        if ds == held:
            continue
        rows = rows_of(ds)
        vs = rng.choice(sorted(rows.subject.unique()))
        for _, r in rows.iterrows():
            tr = load_trial(PREP, r["key"])
            tr["_S"] = _targets(tr)
            (val if r["subject"] == vs else train).append(tr)
    train += load_external(["Fuglsang2020"], PREP, log)
    for tr in train:
        if "_S" not in tr:
            tr["_S"] = _targets(tr)
        # Fuglsang may lack teacher — leave 0
    try:
        net, best = fit_kd(train, val, pub_fold(0, held), 0, tch_all)
    except RuntimeError as e:
        log(f"SCREEN FAIL {e}")
        return 1
    os.makedirs(os.path.join(CKPT_ROOT, "s0"), exist_ok=True)
    torch.save(net.state_dict(), os.path.join(CKPT_ROOT, "s0", "held-AVGC.pt"))
    ev = dump_ev(net, held)
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0) for e in ev.values()])
    stub = {
        "r0": np.zeros(8, np.float32), "r1": np.zeros(8, np.float32),
        "att": np.zeros(8, np.int8), "t_end": np.arange(1, 9, dtype=np.float32) * 0.25,
        "pre_switched": True,
    }
    pack = {held: ev}
    for ds in DS_KEYS:
        if ds != held:
            pack[ds] = {"__stub__": dict(stub)}
    r = S.score_dataset(held, pack, LOOKAHEAD, emis=(float(A.mean()), float(A.std()) * SCALE), p_stay=P_STAY)
    log(f"SCREEN AVGC H={r['H']:.4f} Acc={r['Acc']:.2f}")
    ok = r["H"] > 0.70 and r["Acc"] > 65
    out = dict(screen=True, avgc=dict(H=float(r["H"]), Acc=float(r["Acc"])), best_val=best, lam=LAM)
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log(f"SCREEN {'PASS' if ok else 'FAIL'}")
    if not ok:
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
    return 0 if ok else 1


def exp_name(seed):
    return f"direct_clf__{PREP}_kdsofthp50_l{LAM}_s{seed}"


def run_full():
    open(LOG_PATH, "a").write("\n=== FULL KD ===\n")
    seed = SEEDS[0]
    log(f"=== FULL KD soft_hp50→SW2 λ={LAM} seed={seed} ===")
    configure(seed)
    teacher = build_teacher_signed()
    for held in DS_KEYS:
        log(f"fold held-out={held}")
        ckpt = os.path.join(CKPT_ROOT, f"s{seed}", f"held-{held}.pt")
        os.makedirs(os.path.dirname(ckpt), exist_ok=True)
        tch = {}
        for ds in DS_KEYS:
            if ds != held:
                tch.update(teacher[ds])
        if os.path.exists(ckpt):
            net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
            net.load_state_dict(torch.load(ckpt, map_location=DEV, weights_only=False))
            net.eval()
            log(f"  cache {ckpt}")
            best = float("nan")
        else:
            rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held) + 17 * seed)
            train, val = [], []
            for ds in DS_KEYS:
                if ds == held:
                    continue
                rows = rows_of(ds)
                vs = rng.choice(sorted(rows.subject.unique()))
                for _, r in rows.iterrows():
                    tr = load_trial(PREP, r["key"])
                    tr["_S"] = _targets(tr)
                    (val if r["subject"] == vs else train).append(tr)
            train += load_external(["Fuglsang2020"], PREP, log)
            for tr in train:
                if "_S" not in tr:
                    tr["_S"] = _targets(tr)
            net, best = fit_kd(train, val, pub_fold(seed, held), seed, tch)
            torch.save(net.state_dict(), ckpt)
            del train, val
        ev = dump_ev(net, held)
        save_pickle(
            dict(evidence={"causal": ev}, info=dict(
                decoder="direct_clf", prep=PREP, held_out=held, best_val=best,
                lookahead_s=(DC.FUT + DC.HOP) / FS, kd_lam=LAM, teacher="soft_hp50_accfuse_final")),
            fold_file(exp_name(seed), held),
        )
        del net
        if DEV.type == "cuda":
            torch.cuda.empty_cache()

    # all5 + external
    all5_pt = os.path.join(CKPT_ROOT, f"s{seed}", "all5.pt")
    tch_all = {}
    for ds in DS_KEYS:
        tch_all.update(teacher[ds])
    if os.path.exists(all5_pt):
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        net.load_state_dict(torch.load(all5_pt, map_location=DEV, weights_only=False))
        net.eval()
    else:
        rng = np.random.default_rng(CFG["seed"] + 99 + 13 * seed)
        train, val = [], []
        for ds in DS_KEYS:
            rows = rows_of(ds)
            vs = rng.choice(sorted(rows.subject.unique()))
            for _, r in rows.iterrows():
                tr = load_trial(PREP, r["key"])
                tr["_S"] = _targets(tr)
                (val if r["subject"] == vs else train).append(tr)
        train += load_external(["Fuglsang2020"], PREP, log)
        for tr in train:
            if "_S" not in tr:
                tr["_S"] = _targets(tr)
        net, best = fit_kd(train, val, pub_all5(seed), seed, tch_all)
        torch.save(net.state_dict(), all5_pt)
        del train, val
    paths = {}
    for ds in ("AASD", "Carta2026"):
        path = os.path.join(EXT_DIR, f"ext_clf_{ds}{TAG}_evidence_s{seed}.pkl")
        paths[ds] = path
        if os.path.exists(path):
            continue
        EV = {}
        trials = E.ext_trials(ds)
        log(f"  external {ds}: {len(trials)}")
        for i, tr in enumerate(trials):
            lg, starts = DC.logits(net, tr["eeg"], _targets(tr))
            truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
            EV[tr["key"]] = dict(
                r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
                t_end=(starts / FS + tr["t0"]).astype(np.float32),
                att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
        with open(path, "wb") as fh:
            pickle.dump(EV, fh)
        log(f"  wrote {path}")
    del net

    la = (DC.FUT + DC.HOP) / FS
    ev, _ = S.load_folds(exp_name(seed), "causal")
    grid = []
    for p, sc in ((0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)):
        per = {}
        for ds in DS_KEYS:
            A = np.concatenate([
                (ee["r0"] - ee["r1"]) * np.where(ee["att"] == 0, 1.0, -1.0)
                for d in DS_KEYS if d != ds for ee in ev[d].values()
            ])
            mu, sd = float(A.mean()), float(A.std())
            per[ds] = S.score_dataset(ds, ev, la, emis=(mu, sd * sc), p_stay=p)
        mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        grid.append(dict(p=p, sc=sc, H=mH, Acc=mA))
        log(f"  LODO p={p} sc={sc} H={mH:.4f} Acc={mA:.2f}")
    mild = next(r for r in grid if r["p"] == 0.996 and r["sc"] == 0.8)
    ok = [r for r in grid if r["H"] > 0.85 and r["Acc"] > 75]
    best = mild if mild["H"] > 0.85 and mild["Acc"] > 75 else (
        sorted(ok, key=lambda r: r["H"], reverse=True)[0] if ok else
        sorted(grid, key=lambda r: (r["Acc"], r["H"]), reverse=True)[0]
    )
    log(f"SELECTED p={best['p']} sc={best['sc']} H={best['H']:.4f} Acc={best['Acc']:.2f}")
    A = np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev[ds].values()
    ])
    mu, sd = float(A.mean()), float(A.std())
    ext = {}
    for ds, path in paths.items():
        with open(path, "rb") as fh:
            EV = pickle.load(fh)
        r = E.score(EV, mu, sd * best["sc"], best["p"], LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        log(f"  EXT {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")
    gate = best["H"] > 0.85 and best["Acc"] > 75 and ext["AASD"]["H"] >= 0.65 and ext["Carta2026"]["H"] >= 0.65
    out = dict(method="kd_softhp50_into_sw2", lam=LAM, selected=best, lodo_grid=grid,
               external=ext, pass_joint=gate)
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    __import__("pathlib").Path(RESULTS_MD).write_text(
        f"# results_kd_softhp50_into_sw2\n\n"
        f"KD soft_hp50→SW2 λ={LAM}. LODO {best['H']:.4f}/{best['Acc']:.2f}; "
        f"AASD {ext['AASD']['H']:.4f}; Carta {ext['Carta2026']['H']:.4f}; pass={gate}\n"
    )
    log(f"DONE pass={gate}")
    if not gate:
        shutil.rmtree(CKPT_ROOT, ignore_errors=True)
        log(f"deleted {CKPT_ROOT}")
    return 0 if gate else 1


def main():
    if FULL:
        return run_full()
    return run_screen()


if __name__ == "__main__":
    raise SystemExit(main())
