"""IRM ↔ RoPE logit blend at HMM time (no retrain).

Load frozen IRM (abs-pos stem_ms96) and RoPE (rotary Q/K) checkpoints, infer
once, blend
  stream = w * IRM + (1 - w) * RoPE
with w in {0, 0.25, 0.5, 0.75, 1.0}.

Weight selection (no AASD/Carta labels; no held-out leak for headline):
  - primary: fixed w=0.5 (same as irm_swa_fuse)
  - global_w: for each outer held H, score each w on the four INNER datasets
    (official emissions from the other four excl. that inner ds); pick w that
    maximises mean inner Acc, then H; apply that one w to all LODO held-outs
    and to external.
  - oracle_w: per held-out best w using held-out scores (analysis only).

HMM preset p_stay 0.994 scale 0.8. Also causal tau-30 high-pass of blended
evidence (honest; corpus emissions). External uses all5 IRM + all5 RoPE.

Distinct from IRM↔SWA fuse.

usage:
  python stem_ms96_irm_rope_fuse.py
"""
import json
import os
import pickle
import time

import numpy as np
import torch

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import stem_ms96_rope as R
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
IRM_DIR = os.path.join(EXP, "models", "stem_ms96_irm")
ROPE_DIR = os.path.join(EXP, "models", "stem_ms96_rope")
LOG = os.path.join(EXP, "experiments", "stem_ms96_irm_rope_fuse.log")
OUT = os.path.join(EXP, "results", "research", "stem_ms96_irm_rope_fuse.json")
EVID = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research"
TEST_NAMES = ("AASD", "Carta2026")

WINDOW = 96
P_STAY = 0.994
SCALE = 0.8
WEIGHTS = (0.0, 0.25, 0.5, 0.75, 1.0)
# blend = w * IRM + (1-w) * RoPE
LINEAR_HP30_BASELINE = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
ROPE_LODO = dict(AVGC=dict(H=0.8124, Acc=75.66), DTU=dict(H=0.8969, Acc=89.15),
                 KUL=dict(H=0.8678, Acc=81.88), UHD=dict(H=0.8541, Acc=80.11),
                 USTC=dict(H=0.9245, Acc=86.59))
IRM_LODO = dict(AVGC=dict(H=0.8095, Acc=75.12), DTU=dict(H=0.8999, Acc=88.88),
                KUL=dict(H=0.8702, Acc=81.84), UHD=dict(H=0.8530, Acc=79.96),
                USTC=dict(H=0.9148, Acc=85.96))
IRM_SWA_FUSE_W05 = dict(mean=dict(H=0.8725, Acc=82.58),
                        AVGC=dict(H=0.8100, Acc=75.14))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def load_irm_ckpt(path, device):
    blob = torch.load(path, map_location="cpu", weights_only=False)
    name, _family, window, d_model, nlayers, stem, _lr = blob["cfg"]
    if name != "stem_ms96" or not stem or window != WINDOW:
        raise RuntimeError(f"unexpected IRM cfg {blob['cfg']} in {path}")
    c_in = int(np.asarray(blob["mu"]).shape[0])
    net = V.LogitTransformer(c_in, window, d_model=d_model, nlayers=nlayers, stem=True)
    good_mask = net.mask.detach().clone()
    net.load_state_dict(blob["state"])
    net.mask.copy_(good_mask)
    if torch.isnan(net.mask).any():
        raise RuntimeError(f"causal mask has nan in {path}")
    net = net.to(device).eval()
    for p in net.parameters():
        p.requires_grad_(False)
    mu = np.asarray(blob["mu"], np.float32)
    sd = np.asarray(blob["sd"], np.float32)
    return net, mu, sd


def load_rope_ckpt(path, device):
    blob = torch.load(path, map_location="cpu", weights_only=False)
    name, _family, window, d_model, nlayers, stem, _lr = blob["cfg"]
    if name != "stem_ms96_rope" or not stem or int(window) != WINDOW:
        raise RuntimeError(f"unexpected RoPE cfg {blob['cfg']} in {path}")
    if int(d_model) != 64 or int(nlayers) != 2:
        raise RuntimeError(f"unexpected RoPE dims {blob['cfg']} in {path}")
    c_in = int(np.asarray(blob["mu"]).shape[0])
    net = R.LogitTransformerRoPE(
        c_in, WINDOW, d_model=d_model, nhead=R.NHEAD, nlayers=nlayers, stem=True)
    good_mask = net.mask.detach().clone()
    good_cos = net.rope_cos.detach().clone()
    good_sin = net.rope_sin.detach().clone()
    missing, unexpected = net.load_state_dict(blob["state"], strict=False)
    miss_ok = {m for m in missing if not m.startswith(("rope_cos", "rope_sin", "mask"))}
    if miss_ok:
        raise RuntimeError(f"missing RoPE keys in {path}: {sorted(miss_ok)}")
    if unexpected:
        log_line(f"  warn unexpected RoPE keys: {sorted(unexpected)}")
    net.mask.copy_(good_mask)
    net.rope_cos.copy_(good_cos)
    net.rope_sin.copy_(good_sin)
    if torch.isnan(net.mask).any():
        raise RuntimeError(f"causal mask has nan in {path}")
    net = net.to(device).eval()
    for p in net.parameters():
        p.requires_grad_(False)
    mu = np.asarray(blob["mu"], np.float32)
    sd = np.asarray(blob["sd"], np.float32)
    return net, mu, sd


def hp30_stream(stream):
    out = {}
    for key, d in stream.items():
        out[key] = V4.highpass(np.asarray(d, np.float64)[:, None]).reshape(-1).astype(np.float64)
    return out


def hp30_all(streams):
    return {ds: hp30_stream(streams[ds]) for ds in streams}


def blend_streams(irm, rope, w):
    """w * IRM + (1-w) * RoPE; keys must match."""
    out = {}
    for ds in irm:
        out[ds] = {}
        for key in irm[ds]:
            a = np.asarray(irm[ds][key], np.float64)
            b = np.asarray(rope[ds][key], np.float64)
            n = min(len(a), len(b))
            out[ds][key] = (w * a[:n] + (1.0 - w) * b[:n]).astype(np.float64)
    return out


def blend_one(irm_stream, rope_stream, w):
    out = {}
    for key in irm_stream:
        a = np.asarray(irm_stream[key], np.float64)
        b = np.asarray(rope_stream[key], np.float64)
        n = min(len(a), len(b))
        out[key] = (w * a[:n] + (1.0 - w) * b[:n]).astype(np.float64)
    return out


def score_lodo_held(streams, held):
    return N.official(held, N.to_ev(streams), P_STAY, SCALE)


def emissions_from(streams, att, datasets=None):
    datasets = list(datasets) if datasets is not None else list(streams)
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, 0.75)


def load_external():
    E.CONF.update(prep="eaP_n5_t60", targets=["env", "onset", "dom"])
    logits, meta = {}, {}
    for ds in TEST_NAMES:
        evs = []
        for i in range(4):
            path = os.path.join(EVID, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]), t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]), n128=int(e0["n128"]), subject=int(e0["subject"]))
        log_line(f"  external {ds}: {len(logits[ds])} trials")
    return logits, meta


def infer_pair(irm_path, rope_path, feat, device, tag):
    log_line(f"infer {tag} IRM={irm_path} RoPE={rope_path}")
    t0 = time.time()
    irm_net, irm_mu, irm_sd = load_irm_ckpt(irm_path, device)
    irm_st = V.infer_all(irm_net, feat, irm_mu, irm_sd, WINDOW, device)
    del irm_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    rope_net, rope_mu, rope_sd = load_rope_ckpt(rope_path, device)
    rope_st = V.infer_all(rope_net, feat, rope_mu, rope_sd, WINDOW, device)
    del rope_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    log_line(f"  infer done ({time.time() - t0:.0f}s)")
    return irm_st, rope_st


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    log_line(f"device {device} weights {WEIGHTS} p_stay {P_STAY} scale {SCALE}")
    log_line("blend = w*IRM + (1-w)*RoPE; primary w=0.5; global by mean inner Acc then H")
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    V.assert_causal(device)
    R.assert_causal_rope(device)
    N.load_all()
    feat_all = V.build_features()["ms"]

    # --- LODO: infer once per held-out, score all w ---
    lodo_by_w = {w: {} for w in WEIGHTS}
    inner_by_w = {w: [] for w in WEIGHTS}  # list of (held, inner_ds, H, Acc)

    for held in DS_KEYS:
        irm_path = os.path.join(IRM_DIR, f"held-{held}.pt")
        rope_path = os.path.join(ROPE_DIR, f"held-{held}.pt")
        if not os.path.isfile(irm_path) or not os.path.isfile(rope_path):
            raise FileNotFoundError(f"missing ckpt for held={held}")
        irm_st, rope_st = infer_pair(irm_path, rope_path, feat_all, device, f"held-{held}")

        for w in WEIGHTS:
            blended = blend_streams(irm_st, rope_st, w)
            off = score_lodo_held(blended, held)
            off_hp = score_lodo_held(hp30_all(blended), held)
            lodo_by_w[w][held] = dict(blend=off, hp30=off_hp)
            log_line(f"LODO w={w:.2f} held={held}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"hp30 H {off_hp['H']:.4f} Acc {off_hp['Acc']:.2f}")
            for inner in DS_KEYS:
                if inner == held:
                    continue
                inn = score_lodo_held(blended, inner)
                inner_by_w[w].append(dict(held=held, inner=inner, H=inn["H"], Acc=inn["Acc"]))

        del irm_st, rope_st
        if device.type == "cuda":
            torch.cuda.empty_cache()

    # global w from mean of all inner cells — Acc then H (user rule)
    global_summary = {}
    for w in WEIGHTS:
        hs = [r["H"] for r in inner_by_w[w]]
        ac = [r["Acc"] for r in inner_by_w[w]]
        global_summary[w] = dict(mean_inner_H=float(np.mean(hs)), mean_inner_Acc=float(np.mean(ac)),
                                 n=len(hs))
        log_line(f"INNER mean w={w:.2f}  Acc {global_summary[w]['mean_inner_Acc']:.2f} "
                 f"H {global_summary[w]['mean_inner_H']:.4f} n={len(hs)}")
    global_w = max(WEIGHTS, key=lambda w: (global_summary[w]["mean_inner_Acc"],
                                           global_summary[w]["mean_inner_H"]))
    log_line(f"GLOBAL_W selected {global_w} (max mean inner Acc then H)")

    # also report held-out mean Acc then H among w (analysis / optional selector note)
    heldout_mean = {}
    for w in WEIGHTS:
        hs = [lodo_by_w[w][held]["blend"]["H"] for held in DS_KEYS]
        ac = [lodo_by_w[w][held]["blend"]["Acc"] for held in DS_KEYS]
        heldout_mean[w] = dict(H=float(np.mean(hs)), Acc=float(np.mean(ac)))
        log_line(f"HELDOUT mean w={w:.2f}  Acc {heldout_mean[w]['Acc']:.2f} "
                 f"H {heldout_mean[w]['H']:.4f}")
    heldout_pick_w = max(WEIGHTS, key=lambda w: (heldout_mean[w]["Acc"], heldout_mean[w]["H"]))
    log_line(f"HELDOUT_PICK_W {heldout_pick_w} (analysis; not primary — uses held-out)")

    oracle = {}
    for held in DS_KEYS:
        best_w = max(WEIGHTS, key=lambda w: (lodo_by_w[w][held]["blend"]["Acc"],
                                             lodo_by_w[w][held]["blend"]["H"]))
        oracle[held] = dict(w=best_w, **lodo_by_w[best_w][held]["blend"])
        log_line(f"ORACLE held={held} w={best_w} H {oracle[held]['H']:.4f} Acc {oracle[held]['Acc']:.2f}")

    def lodo_table(w):
        rows = {held: lodo_by_w[w][held] for held in DS_KEYS}
        mean_h = float(np.mean([rows[d]["blend"]["H"] for d in DS_KEYS]))
        mean_acc = float(np.mean([rows[d]["blend"]["Acc"] for d in DS_KEYS]))
        return dict(per=rows, mean=dict(H=mean_h, Acc=mean_acc))

    primary = lodo_table(0.5)
    selected = lodo_table(global_w)
    log_line(f"PRIMARY w=0.5 LODO mean H {primary['mean']['H']:.4f} Acc {primary['mean']['Acc']:.2f}")
    log_line(f"GLOBAL  w={global_w} LODO mean H {selected['mean']['H']:.4f} Acc {selected['mean']['Acc']:.2f}")
    for held in DS_KEYS:
        p = primary["per"][held]["blend"]
        log_line(f"  primary {held} H {p['H']:.4f} Acc {p['Acc']:.2f} "
                 f"(RoPE {ROPE_LODO[held]['H']:.4f}/{ROPE_LODO[held]['Acc']:.2f} "
                 f"IRM {IRM_LODO[held]['H']:.4f}/{IRM_LODO[held]['Acc']:.2f})")

    # --- External: all5 IRM + all5 RoPE ---
    log_line("=== external all5 ===")
    irm_path = os.path.join(IRM_DIR, "all5.pt")
    rope_path = os.path.join(ROPE_DIR, "all5.pt")
    irm_corp, rope_corp = infer_pair(irm_path, rope_path, feat_all, device, "all5-corpus")
    ext_logits, ext_meta = load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    irm_ext_net, irm_mu, irm_sd = load_irm_ckpt(irm_path, device)
    irm_ext = V.infer_all(irm_ext_net, ext_feat, irm_mu, irm_sd, WINDOW, device)
    del irm_ext_net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    rope_ext_net, rope_mu, rope_sd = load_rope_ckpt(rope_path, device)
    rope_ext = V.infer_all(rope_ext_net, ext_feat, rope_mu, rope_sd, WINDOW, device)
    del rope_ext_net
    if device.type == "cuda":
        torch.cuda.empty_cache()

    att = N.G["att"]
    external_by_w = {}
    for w in WEIGHTS:
        corp = blend_streams(irm_corp, rope_corp, w)
        mu_e, sd_e = emissions_from(corp, att)
        corp_hp = hp30_all(corp)
        mu_h, sd_h = emissions_from(corp_hp, att)
        rows = {}
        for ds in TEST_NAMES:
            st = blend_one(irm_ext[ds], rope_ext[ds], w)
            r = score_external(st, ext_meta[ds], mu_e, sd_e)
            r_hp = score_external(hp30_stream(st), ext_meta[ds], mu_h, sd_h)
            rows[ds] = dict(blend=r, hp30=r_hp)
            log_line(f"EXT w={w:.2f} {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"hp30 H {r_hp['H']:.4f} Acc {r_hp['Acc']:.2f}  "
                     f"(linear_hp30 {LINEAR_HP30_BASELINE[ds]['H']:.4f}/"
                     f"{LINEAR_HP30_BASELINE[ds]['Acc']:.2f})")
        external_by_w[w] = rows

    out = dict(
        p_stay=P_STAY, scale=SCALE, weights=list(WEIGHTS),
        blend_def="w*IRM + (1-w)*RoPE",
        primary_w=0.5, global_w=global_w, heldout_pick_w=heldout_pick_w,
        global_inner_summary={str(k): v for k, v in global_summary.items()},
        heldout_mean={str(k): v for k, v in heldout_mean.items()},
        lodo_by_w={str(w): {held: lodo_by_w[w][held] for held in DS_KEYS} for w in WEIGHTS},
        primary=primary, selected=selected, oracle=oracle,
        external_by_w={str(w): external_by_w[w] for w in WEIGHTS},
        external_primary=external_by_w[0.5],
        external_selected=external_by_w[global_w],
        baselines=dict(linear_hp30=LINEAR_HP30_BASELINE, rope_lodo=ROPE_LODO,
                       irm_lodo=IRM_LODO, irm_swa_fuse_w05=IRM_SWA_FUSE_W05),
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT}")


if __name__ == "__main__":
    main()
