"""T7 Weight soup of published SW2 classifiers + linear_hp30 readout.

NEW vs ALREADY_TRIED:
  - Parameter averaging across seeds (≠ logit equal-mean / inv-var ensemble)
  - ≠ SWA on stem_ms96, ≠ seed-reliability / power-trim on logits
  - Inherits linear_hp30: causal τ=30 one-pole HP on the soup evidence for
    train→test A/B shift robustness (same as v4 external winner)

Protocol:
  LODO fold X: average the 4 held-X seed fold weights → one soup net →
    synthetic-switch evidence (same draw_switch_times schedule as training).
  External: average 4 all5 SW2 weights → soup → AASD/Carta.
  HMM preset p_stay=0.994, emission scale=0.8. Emissions from LODO only.
  Never uses AASD/Carta labels for selection.

Primary report: soup_hp30 (HP on soup). Also report soup_raw for reference.
Gate (T7): soup LODO acc1 within 0.5 pt of 4-seed equal-mean, else flag drop.

usage:
  GLIMPSE_THR=onorm flock multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u weight_soup_hp30.py
"""
from __future__ import annotations

import json
import os
import pickle
import time
from pathlib import Path

import numpy as np
import torch

os.environ.setdefault("GLIMPSE_THR", "onorm")

import direct_classifier as DC
import ensemble_clf as EC
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, load_trial, rows_of, save_pickle
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times

RUN_ID = "t7_weight_soup_hp30_20261107"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
LOG = os.path.join(RUN_DIR, "run.log")
OUT_JSON = os.path.join(EXP, "results", "research", "weight_soup_hp30.json")
OUT_MD = os.path.join(EXP, "multi_agent_opt", "results_weight_soup_hp30.md")
CKPT_DIR = os.path.join(EXP, "models", "weight_soup_hp30")
FOLD_EXP = "direct_clf__eaP_n5_t60_w10b3_ptSPK_soup_aug-sw2"

P_STAY, SCALE = 0.994, 0.8
LOOKAHEAD = 0.75
WINDOW_S = 10.0
HP_TAU = 30.0
TEST_NAMES = ("AASD", "Carta2026")

# Seed 0 uses tag without _s0; seeds 1–3 use _s{k}
SEED_TAGS = [
    ("0", "direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts"),
    ("1", "direct_clf__eaP_n5_t60_w10b3_ptSPK_s1_aug-sw2_wts"),
    ("2", "direct_clf__eaP_n5_t60_w10b3_ptSPK_s2_aug-sw2_wts"),
    ("3", "direct_clf__eaP_n5_t60_w10b3_ptSPK_s3_aug-sw2_wts"),
]
ALL5_NAMES = [f"all5_direct_clf_w10b3sw2P_s{i}.pt" for i in range(4)]


def log(msg: str) -> None:
    print(f"{time.strftime('%F %T')} {msg}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(msg + "\n")


def configure() -> None:
    D["window_s"] = WINDOW_S
    D["model_blocks"] = 3
    D["targets"] = ["env", "onset", "dom"]
    D["causal_future_s"] = float(CFG["dnn"].get("causal_future_s", 0.5))
    DC.HOP = int(D.get("infer_hop_s", 0.25) * FS)
    DC.FUT = int(D["causal_future_s"] * FS)
    DC.W = int(WINDOW_S * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS


def model_path(name: str) -> str:
    for root in (os.path.join(DATA_EXP, "models"), os.path.join(EXP, "models")):
        p = os.path.join(root, name)
        if os.path.exists(p):
            return p
    raise FileNotFoundError(name)


def fold_weight_path(tag: str, held: str) -> str:
    return model_path(f"fold_{tag}__{held}.pt")


def average_state_dicts(paths):
    states = [torch.load(p, map_location="cpu", weights_only=False) for p in paths]
    keys = states[0].keys()
    out = {}
    for k in keys:
        stacked = torch.stack([s[k].float() for s in states], 0)
        out[k] = stacked.mean(0)
    return out


def build_net(state):
    net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
    net.load_state_dict(state)
    net.eval()
    return net


def held_evidence(net, held: str) -> dict:
    """Replay score.py synthetic-switch schedule on held-out trials (same as run_fold)."""
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev = {}
    for _, r in rows_of(held).iterrows():
        tr = load_trial("eaP_n5_t60", r["key"])
        S, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128)
        srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur
            cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], S[:, ::-1], S)
        lg, starts = DC.logits(net, x, Ssw)
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        n10 = len(lg) // 40
        ev[r["key"]] = dict(
            r0=lg / 2,
            r1=-lg / 2,
            att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            r0_10=lg[: n10 * 40].reshape(n10, 40).mean(1) / 2,
            r1_10=-lg[: n10 * 40].reshape(n10, 40).mean(1) / 2,
            att10=(truth[: n10 * 40].reshape(n10, 40).mean(1) > 0.5).astype(np.int8),
            pre_switched=True,
        )
    return ev


def ext_evidence(net, ds: str) -> dict:
    EV = {}
    trials = E.ext_trials(ds)
    log(f"  external {ds}: {len(trials)} trials")
    for i, tr in enumerate(trials):
        lg, starts = DC.logits(net, tr["eeg"], _targets(tr))
        truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
        EV[tr["key"]] = dict(
            r0=lg / 2,
            r1=-lg / 2,
            att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            att128=tr["att128"],
            n128=tr["n128"],
            subject=tr["subject"],
        )
        if (i + 1) % 40 == 0 or i + 1 == len(trials):
            log(f"    {ds} {i + 1}/{len(trials)}")
    return EV


def apply_hp30_ev(ev_by_ds):
    """Causal τ=30 HP on signed logit, store as symmetric r0/r1."""
    out = {}
    for ds, table in ev_by_ds.items():
        out[ds] = {}
        for k, e in table.items():
            d = (np.asarray(e["r0"], np.float64) - np.asarray(e["r1"], np.float64))
            hp = V4.highpass(d[:, None], tau=HP_TAU, hop=0.25)[:, 0]
            ne = dict(e)
            ne["r0"] = (hp / 2).astype(np.float32)
            ne["r1"] = (-hp / 2).astype(np.float32)
            out[ds][k] = ne
    return out


def apply_hp30_ext(EV):
    out = {}
    for k, e in EV.items():
        d = (np.asarray(e["r0"], np.float64) - np.asarray(e["r1"], np.float64))
        hp = V4.highpass(d[:, None], tau=HP_TAU, hop=0.25)[:, 0]
        ne = dict(e)
        ne["r0"] = (hp / 2).astype(np.float32)
        ne["r1"] = (-hp / 2).astype(np.float32)
        out[k] = ne
    return out


def score_lodo(ev):
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate([
            (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
            for d in DS_KEYS if d != ds for e in ev[d].values()
        ])
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, LOOKAHEAD, emis=(mu, sd * SCALE), p_stay=P_STAY)
        keys = ("H", "Acc", "L", "SDT")
        per[ds] = {k: float(r[k]) for k in keys}
        # frame accuracy proxy from signed evidence vs att (not official Acc)
        parts = []
        for e in ev[ds].values():
            d = (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
            parts.append((d > 0).astype(np.float64))
        per[ds]["frame_acc"] = float(np.mean(np.concatenate(parts)))
        log(f"  LODO {ds}  H {per[ds]['H']:.4f}  Acc {per[ds]['Acc']:.2f}  "
            f"L {per[ds]['L']:.3f}  frame_acc {per[ds]['frame_acc']:.4f}")
    mean = {k: float(np.mean([per[d][k] for d in DS_KEYS]))
            for k in ("H", "Acc", "L", "SDT", "frame_acc")}
    log(f"  LODO MEAN  H {mean['H']:.4f}  Acc {mean['Acc']:.2f}  frame_acc {mean['frame_acc']:.4f}")
    return per, mean


def lodo_emis(ev):
    A = np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev[ds].values()
    ])
    return float(A.mean()), float(A.std())


def baseline_equal_mean_frame_acc():
    """4-seed equal-mean LODO frame_acc for T7 gate (reuse published folds)."""
    evs = []
    for _seed, exp in EC.SEEDS:
        ev, _ = S.load_folds(exp, "causal")
        evs.append(ev)
    comb = EC.combine(evs, np.ones(4) / 4.0)
    accs = []
    for ds in DS_KEYS:
        parts = []
        for e in comb[ds].values():
            d = (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
            parts.append((d > 0).astype(np.float64))
        accs.append(float(np.mean(np.concatenate(parts))))
    return float(np.mean(accs))


def write_results(payload):
    os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
    with open(OUT_JSON, "w") as fh:
        json.dump(payload, fh, indent=2)
    md = [
        "# results_weight_soup_hp30",
        "",
        "T7 parameter soup of published SW2 vlaai_ws 10s seeds + causal τ=30 HP readout.",
        f"HMM p_stay={P_STAY} scale={SCALE}. Emissions from LODO only.",
        "",
        "| Stream | LODO H | LODO Acc | AASD H/Acc | Carta H/Acc |",
        "|---|---:|---:|---|---|",
    ]
    for name in ("soup_raw", "soup_hp30"):
        m = payload[name]
        md.append(
            f"| {name} | {m['lodo']['mean']['H']:.4f} | {m['lodo']['mean']['Acc']:.2f} | "
            f"{m['ext']['AASD']['H']:.4f}/{m['ext']['AASD']['Acc']:.2f} | "
            f"{m['ext']['Carta2026']['H']:.4f}/{m['ext']['Carta2026']['Acc']:.2f} |"
        )
    md += [
        "",
        f"T7 gate (soup frame_acc vs equal-mean): soup={payload['soup_raw']['lodo']['mean']['frame_acc']:.4f} "
        f"baseline={payload['baseline_equal_mean_acc1']:.4f} "
        f"Δ={payload['acc1_delta_pt']:.3f} pt  pass={payload['t7_gate_pass']}",
        "",
        f"Targets: LODO H>0.80 Acc>70; AASD/Carta H>0.60 Acc>60.",
        f"Verdict: {payload['verdict']}",
        "",
        f"Artifacts: `{OUT_JSON}`, `{CKPT_DIR}/`, `{RUN_DIR}/`.",
    ]
    with open(OUT_MD, "w") as fh:
        fh.write("\n".join(md) + "\n")
    log(f"wrote {OUT_MD}")


def main():
    t0 = time.time()
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(os.path.join(EXP, "folds", FOLD_EXP), exist_ok=True)
    configure()
    log(f"=== T7 weight soup + hp30  device={DEV} LA={LOOKAHEAD}s ===")

    # ---- LODO soups (resume from fold pickles if present) ----
    lodo_raw = {}
    resume = os.environ.get("RESUME", "1") == "1"
    for held in DS_KEYS:
        fold_pkl = os.path.join(EXP, "folds", FOLD_EXP, f"{held}.pkl")
        if resume and os.path.exists(fold_pkl):
            with open(fold_pkl, "rb") as fh:
                lodo_raw[held] = pickle.load(fh)["evidence"]["causal"]
            log(f"resume LODO held={held} from {fold_pkl}")
            continue
        paths = [fold_weight_path(tag, held) for _sid, tag in SEED_TAGS]
        for p in paths:
            if not os.path.exists(p):
                raise FileNotFoundError(p)
        log(f"soup LODO held={held} from {len(paths)} fold weights")
        state = average_state_dicts(paths)
        soup_pt = os.path.join(CKPT_DIR, f"soup_held-{held}.pt")
        torch.save(state, soup_pt)
        net = build_net(state)
        ev = held_evidence(net, held)
        save_pickle(
            dict(
                evidence={"causal": ev},
                info=dict(
                    decoder="direct_clf_soup",
                    prep="eaP_n5_t60",
                    held_out=held,
                    lookahead_s=LOOKAHEAD,
                    method="weight_soup_4seed",
                ),
            ),
            fold_pkl,
        )
        lodo_raw[held] = ev
        del net
        torch.cuda.empty_cache()
        log(f"  held={held} done ({time.time() - t0:.0f}s)")

    lodo_hp = apply_hp30_ev(lodo_raw)
    log("--- score soup_raw LODO ---")
    per_raw, mean_raw = score_lodo(lodo_raw)
    log("--- score soup_hp30 LODO ---")
    per_hp, mean_hp = score_lodo(lodo_hp)

    base_fa = baseline_equal_mean_frame_acc()
    acc1_delta_pt = (mean_raw["frame_acc"] - base_fa) * 100.0
    t7_gate = abs(acc1_delta_pt) <= 0.5
    log(f"T7 gate: soup frame_acc={mean_raw['frame_acc']:.4f} equal-mean={base_fa:.4f} "
        f"Δ={acc1_delta_pt:.3f} pt pass={t7_gate}")

    # ---- External all5 soup ----
    all5_paths = [model_path(n) for n in ALL5_NAMES]
    log(f"soup external from {len(all5_paths)} all5 weights")
    state_ext = average_state_dicts(all5_paths)
    torch.save(state_ext, os.path.join(CKPT_DIR, "soup_all5.pt"))
    net = build_net(state_ext)
    E.CONF.update(prep="eaP_n5_t60", targets=list(D["targets"]))
    ext_raw, ext_hp_scores = {}, {}
    mu_r, sd_r = lodo_emis(lodo_raw)
    mu_h, sd_h = lodo_emis(lodo_hp)
    log(f"emis raw mu={mu_r:.4f} sd={sd_r:.4f}; hp30 mu={mu_h:.4f} sd={sd_h:.4f}")
    for ds in TEST_NAMES:
        EV = ext_evidence(net, ds)
        path = os.path.join(EXP, "results", "research", f"ext_clf_{ds}_w10b3soupP_evidence.pkl")
        with open(path, "wb") as fh:
            pickle.dump(EV, fh)
        r_raw = E.score(EV, mu_r, sd_r * SCALE, P_STAY, LOOKAHEAD)
        EV_hp = apply_hp30_ext(EV)
        r_hp = E.score(EV_hp, mu_h, sd_h * SCALE, P_STAY, LOOKAHEAD)
        ext_raw[ds] = {k: float(r_raw[k]) for k in ("H", "Acc", "L", "SDT")}
        ext_hp_scores[ds] = {k: float(r_hp[k]) for k in ("H", "Acc", "L", "SDT")}
        log(f"  EXT raw  {ds}  H {ext_raw[ds]['H']:.4f}  Acc {ext_raw[ds]['Acc']:.2f}")
        log(f"  EXT hp30 {ds}  H {ext_hp_scores[ds]['H']:.4f}  Acc {ext_hp_scores[ds]['Acc']:.2f}")
    del net
    torch.cuda.empty_cache()

    def targets_ok(lodo_mean, ext):
        ok_lodo = lodo_mean["H"] > 0.80 and lodo_mean["Acc"] > 70.0
        ok_ext = all(
            ext[ds]["H"] > 0.60 and ext[ds]["Acc"] > 60.0 for ds in TEST_NAMES
        )
        return ok_lodo and ok_ext

    # Primary = soup_hp30 (inherits linear_hp30)
    primary = "soup_hp30"
    verdict_pass = targets_ok(mean_hp, ext_hp_scores)
    payload = {
        "method": "weight_soup_hp30",
        "run_id": RUN_ID,
        "primary": primary,
        "hmm": {"p_stay": P_STAY, "scale": SCALE, "lookahead": LOOKAHEAD},
        "inherited_from_linear_hp30": "causal one-pole HP tau=30s on soup evidence",
        "baseline_equal_mean_acc1": base_fa,
        "acc1_delta_pt": acc1_delta_pt,
        "t7_gate_pass": t7_gate,
        "soup_raw": {"lodo": {"per": per_raw, "mean": mean_raw}, "ext": ext_raw},
        "soup_hp30": {"lodo": {"per": per_hp, "mean": mean_hp}, "ext": ext_hp_scores},
        "verdict": "PASS" if verdict_pass else "FAIL",
        "elapsed_s": time.time() - t0,
        "ckpt_dir": CKPT_DIR,
    }
    write_results(payload)
    log(f"VERDICT {payload['verdict']}  elapsed={payload['elapsed_s']:.0f}s")
    if not verdict_pass:
        log("FAIL vs targets — keep ckpts until ALREADY_TRIED archive; cleanup deferred to caller")
    return 0 if verdict_pass else 1


if __name__ == "__main__":
    raise SystemExit(main())
