"""IRM ↔ stem-stochdepth logit blend at HMM time (no retrain).

Load frozen IRM and stem_ms96_stochdepth checkpoints (both abs-pos LogitTransformer
via F.load_ckpt pattern), infer once, blend
  stream = w * IRM + (1 - w) * stoch
with w in {0, 0.25, 0.5, 0.75, 1.0}.

Weight selection (no held-out leak for headline):
  - primary: fixed w=0.5
  - global_w: for each outer held H, score each w on the four INNER datasets
    (official emissions from the other four excl. that inner ds); pick w that
    maximises mean inner H across all (held, inner) cells; apply that one w
    to all LODO held-outs and to external.
  - oracle_w: per held-out best w using held-out scores (analysis only).

HMM preset p_stay 0.994 scale 0.8. Also causal tau-30 high-pass of blended
evidence (honest; corpus emissions). External uses all5 IRM + all5 stoch.

Distinct from IRM↔synth, IRM↔SWA, IRM↔RoPE (new second stream: stochdepth).
Never use AASD/Carta for w selection.

usage:
  flock multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u \\
    stem_ms96_irm_stoch_fuse.py
"""
import json
import os
import pickle
import time

import numpy as np
import torch

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
IRM_DIR = os.path.join(EXP, "models", "stem_ms96_irm")
STOCH_DIR = os.path.join(EXP, "models", "stem_ms96_stochdepth")
LOG = os.path.join(EXP, "experiments", "stem_ms96_irm_stoch_fuse.log")
OUT = os.path.join(EXP, "results", "research", "stem_ms96_irm_stoch_fuse.json")
EVID = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research"
TEST_NAMES = ("AASD", "Carta2026")

WINDOW = 96
P_STAY = 0.994
SCALE = 0.8
WEIGHTS = (0.0, 0.25, 0.5, 0.75, 1.0)
# blend = w * IRM + (1-w) * stoch
LINEAR_HP30_BASELINE = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
STOCH_LODO = dict(AVGC=dict(H=0.8149, Acc=75.38), DTU=dict(H=0.9013, Acc=89.35),
                  KUL=dict(H=0.8690, Acc=81.99), UHD=dict(H=0.8528, Acc=80.43),
                  USTC=dict(H=0.9206, Acc=86.32))
IRM_LODO = dict(AVGC=dict(H=0.8095, Acc=75.12), DTU=dict(H=0.8999, Acc=88.88),
                KUL=dict(H=0.8702, Acc=81.84), UHD=dict(H=0.8530, Acc=79.96),
                USTC=dict(H=0.9148, Acc=85.96))
IRM_SYNTH_FUSE_W05 = dict(mean=dict(H=0.8790, Acc=83.11),
                          AVGC=dict(H=0.8192, Acc=75.52),
                          UHD=dict(H=0.8614, Acc=80.54))
IRM_SWA_FUSE_W05 = dict(mean=dict(H=0.8725, Acc=82.58),
                        AVGC=dict(H=0.8100, Acc=75.14),
                        UHD=dict(H=0.8580, Acc=80.10))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def hp30_stream(stream):
    out = {}
    for key, d in stream.items():
        out[key] = V4.highpass(np.asarray(d, np.float64)[:, None]).reshape(-1).astype(np.float64)
    return out


def hp30_all(streams):
    return {ds: hp30_stream(streams[ds]) for ds in streams}


def blend_streams(irm, stoch, w):
    """w * IRM + (1-w) * stoch; keys must match."""
    out = {}
    for ds in irm:
        out[ds] = {}
        for key in irm[ds]:
            a = np.asarray(irm[ds][key], np.float64)
            b = np.asarray(stoch[ds][key], np.float64)
            n = min(len(a), len(b))
            out[ds][key] = (w * a[:n] + (1.0 - w) * b[:n]).astype(np.float64)
    return out


def blend_one(irm_stream, stoch_stream, w):
    out = {}
    for key in irm_stream:
        a = np.asarray(irm_stream[key], np.float64)
        b = np.asarray(stoch_stream[key], np.float64)
        n = min(len(a), len(b))
        out[key] = (w * a[:n] + (1.0 - w) * b[:n]).astype(np.float64)
    return out


def score_lodo_held(streams, held):
    return N.official(held, N.to_ev(streams), P_STAY, SCALE)


def emissions_from(streams, att, datasets=None):
    datasets = list(datasets) if datasets is not None else list(streams)
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, 0.75)


def load_external():
    E.CONF.update(prep="eaP_n5_t60", targets=["env", "onset", "dom"])
    logits, meta = {}, {}
    for ds in TEST_NAMES:
        evs = []
        for i in range(4):
            path = os.path.join(EVID, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]), t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]), n128=int(e0["n128"]), subject=int(e0["subject"]))
        log_line(f"  external {ds}: {len(logits[ds])} trials")
    return logits, meta


def infer_pair(irm_path, stoch_path, feat, device, tag):
    log_line(f"infer {tag} IRM={irm_path} stoch={stoch_path}")
    t0 = time.time()
    irm_net, irm_mu, irm_sd = F.load_ckpt(irm_path, device)
    irm_st = V.infer_all(irm_net, feat, irm_mu, irm_sd, WINDOW, device)
    del irm_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    stoch_net, stoch_mu, stoch_sd = F.load_ckpt(stoch_path, device)
    stoch_st = V.infer_all(stoch_net, feat, stoch_mu, stoch_sd, WINDOW, device)
    del stoch_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    log_line(f"  infer done ({time.time() - t0:.0f}s)")
    return irm_st, stoch_st


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    log_line(f"device {device} weights {WEIGHTS} p_stay {P_STAY} scale {SCALE}")
    log_line("blend = w*IRM + (1-w)*stoch; primary w=0.5; global_w by mean inner H")
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    V.assert_causal(device)
    N.load_all()
    feat_all = V.build_features()["ms"]

    # --- LODO: infer once per held-out, score all w ---
    lodo_by_w = {w: {} for w in WEIGHTS}
    inner_by_w = {w: [] for w in WEIGHTS}  # list of (held, inner_ds, H, Acc)

    for held in DS_KEYS:
        irm_path = os.path.join(IRM_DIR, f"held-{held}.pt")
        stoch_path = os.path.join(STOCH_DIR, f"held-{held}.pt")
        if not os.path.isfile(irm_path) or not os.path.isfile(stoch_path):
            raise FileNotFoundError(f"missing ckpt for held={held}")
        irm_st, stoch_st = infer_pair(irm_path, stoch_path, feat_all, device, f"held-{held}")

        for w in WEIGHTS:
            blended = blend_streams(irm_st, stoch_st, w)
            off = score_lodo_held(blended, held)
            off_hp = score_lodo_held(hp30_all(blended), held)
            lodo_by_w[w][held] = dict(blend=off, hp30=off_hp)
            log_line(f"LODO w={w:.2f} held={held}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"hp30 H {off_hp['H']:.4f} Acc {off_hp['Acc']:.2f}")
            for inner in DS_KEYS:
                if inner == held:
                    continue
                inn = score_lodo_held(blended, inner)
                inner_by_w[w].append(dict(held=held, inner=inner, H=inn["H"], Acc=inn["Acc"]))

        del irm_st, stoch_st
        if device.type == "cuda":
            torch.cuda.empty_cache()

    # global w from mean of all inner cells — H then Acc
    global_summary = {}
    for w in WEIGHTS:
        hs = [r["H"] for r in inner_by_w[w]]
        ac = [r["Acc"] for r in inner_by_w[w]]
        global_summary[w] = dict(mean_inner_H=float(np.mean(hs)), mean_inner_Acc=float(np.mean(ac)),
                                 n=len(hs))
        log_line(f"INNER mean w={w:.2f}  H {global_summary[w]['mean_inner_H']:.4f} "
                 f"Acc {global_summary[w]['mean_inner_Acc']:.2f} n={len(hs)}")
    global_w = max(WEIGHTS, key=lambda w: (global_summary[w]["mean_inner_H"],
                                           global_summary[w]["mean_inner_Acc"]))
    log_line(f"GLOBAL_W selected {global_w} (max mean inner H then Acc)")

    heldout_mean = {}
    for w in WEIGHTS:
        hs = [lodo_by_w[w][held]["blend"]["H"] for held in DS_KEYS]
        ac = [lodo_by_w[w][held]["blend"]["Acc"] for held in DS_KEYS]
        heldout_mean[w] = dict(H=float(np.mean(hs)), Acc=float(np.mean(ac)))
        log_line(f"HELDOUT mean w={w:.2f}  H {heldout_mean[w]['H']:.4f} "
                 f"Acc {heldout_mean[w]['Acc']:.2f}")
    heldout_pick_w = max(WEIGHTS, key=lambda w: (heldout_mean[w]["H"], heldout_mean[w]["Acc"]))
    log_line(f"HELDOUT_PICK_W {heldout_pick_w} (analysis; not primary — uses held-out)")

    oracle = {}
    for held in DS_KEYS:
        best_w = max(WEIGHTS, key=lambda w: (lodo_by_w[w][held]["blend"]["H"],
                                             lodo_by_w[w][held]["blend"]["Acc"]))
        oracle[held] = dict(w=best_w, **lodo_by_w[best_w][held]["blend"])
        log_line(f"ORACLE held={held} w={best_w} H {oracle[held]['H']:.4f} Acc {oracle[held]['Acc']:.2f}")

    def lodo_table(w):
        rows = {held: lodo_by_w[w][held] for held in DS_KEYS}
        mean_h = float(np.mean([rows[d]["blend"]["H"] for d in DS_KEYS]))
        mean_acc = float(np.mean([rows[d]["blend"]["Acc"] for d in DS_KEYS]))
        return dict(per=rows, mean=dict(H=mean_h, Acc=mean_acc))

    primary = lodo_table(0.5)
    selected = lodo_table(global_w)
    log_line(f"PRIMARY w=0.5 LODO mean H {primary['mean']['H']:.4f} Acc {primary['mean']['Acc']:.2f}")
    log_line(f"GLOBAL  w={global_w} LODO mean H {selected['mean']['H']:.4f} Acc {selected['mean']['Acc']:.2f}")
    for held in DS_KEYS:
        p = primary["per"][held]["blend"]
        log_line(f"  primary {held} H {p['H']:.4f} Acc {p['Acc']:.2f} "
                 f"(stoch {STOCH_LODO[held]['H']:.4f}/{STOCH_LODO[held]['Acc']:.2f} "
                 f"IRM {IRM_LODO[held]['H']:.4f}/{IRM_LODO[held]['Acc']:.2f})")

    # --- External: all5 IRM + all5 stoch ---
    log_line("=== external all5 ===")
    irm_path = os.path.join(IRM_DIR, "all5.pt")
    stoch_path = os.path.join(STOCH_DIR, "all5.pt")
    irm_corp, stoch_corp = infer_pair(irm_path, stoch_path, feat_all, device, "all5-corpus")
    ext_logits, ext_meta = load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    irm_ext_net, irm_mu, irm_sd = F.load_ckpt(irm_path, device)
    irm_ext = V.infer_all(irm_ext_net, ext_feat, irm_mu, irm_sd, WINDOW, device)
    del irm_ext_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    stoch_ext_net, stoch_mu, stoch_sd = F.load_ckpt(stoch_path, device)
    stoch_ext = V.infer_all(stoch_ext_net, ext_feat, stoch_mu, stoch_sd, WINDOW, device)
    del stoch_ext_net
    if device.type == "cuda":
        torch.cuda.empty_cache()

    att = N.G["att"]
    external_by_w = {}
    for w in WEIGHTS:
        corp = blend_streams(irm_corp, stoch_corp, w)
        mu_e, sd_e = emissions_from(corp, att)
        corp_hp = hp30_all(corp)
        mu_h, sd_h = emissions_from(corp_hp, att)
        rows = {}
        for ds in TEST_NAMES:
            st = blend_one(irm_ext[ds], stoch_ext[ds], w)
            r = score_external(st, ext_meta[ds], mu_e, sd_e)
            r_hp = score_external(hp30_stream(st), ext_meta[ds], mu_h, sd_h)
            rows[ds] = dict(blend=r, hp30=r_hp)
            log_line(f"EXT w={w:.2f} {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"hp30 H {r_hp['H']:.4f} Acc {r_hp['Acc']:.2f}  "
                     f"(linear_hp30 {LINEAR_HP30_BASELINE[ds]['H']:.4f}/"
                     f"{LINEAR_HP30_BASELINE[ds]['Acc']:.2f})")
        external_by_w[w] = rows

    out = dict(
        p_stay=P_STAY, scale=SCALE, weights=list(WEIGHTS),
        blend_def="w*IRM + (1-w)*stoch",
        primary_w=0.5, global_w=global_w, heldout_pick_w=heldout_pick_w,
        global_inner_summary={str(k): v for k, v in global_summary.items()},
        heldout_mean={str(k): v for k, v in heldout_mean.items()},
        lodo_by_w={str(w): {held: lodo_by_w[w][held] for held in DS_KEYS} for w in WEIGHTS},
        primary=primary, selected=selected, oracle=oracle,
        external_by_w={str(w): external_by_w[w] for w in WEIGHTS},
        external_primary=external_by_w[0.5],
        external_selected=external_by_w[global_w],
        baselines=dict(linear_hp30=LINEAR_HP30_BASELINE, stoch_lodo=STOCH_LODO,
                       irm_lodo=IRM_LODO, irm_synth_fuse_w05=IRM_SYNTH_FUSE_W05,
                       irm_swa_fuse_w05=IRM_SWA_FUSE_W05),
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT}")


if __name__ == "__main__":
    main()
