"""H2 residual: existing T2 ESR-stem logits + causal τ=30 HP readout.

Does NOT retrain ESR (exact T2 recipe is ALREADY_TRIED / CLOSED).
Inference-only on published models/stem_esr/{held-*,all5}.pt.

Selection: LODO Acc then H @ HMM p_stay∈{0.994,0.998} scale 0.8 — never AASD/Carta.
Priority B: LODO H>0.80 Acc>70; Carta H>0.60 Acc>60; AASD H>0.60 (Acc soft).
"""
from __future__ import annotations

import json
import os
import time

import numpy as np
import torch

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
import stem_esr as ESR
import stem_ms96_group_dro as GD
from common import DS_KEYS
from common_exp import EXP

RUN_ID = "esr_hp30_20261107"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
LOG = os.path.join(RUN_DIR, "run.log")
OUT_JSON = os.path.join(EXP, "results", "research", "esr_hp30.json")
OUT_MD = os.path.join(EXP, "multi_agent_opt", "results_esr_hp30.md")
CKPT_DIR = os.path.join(EXP, "models", "stem_esr")

SCALE = 0.8
LOOKAHEAD = 0.75
TEST_NAMES = ("AASD", "Carta2026")
P_GRID = (0.994, 0.998)


def log(msg: str) -> None:
    print(f"{time.strftime('%F %T')} {msg}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(msg + "\n")


def apply_hp30_stream(streams: dict) -> dict:
    out = {}
    for ds, table in streams.items():
        out[ds] = {}
        for k, d in table.items():
            h = V4.highpass(np.asarray(d, np.float64)[:, None], tau=30.0, hop=0.25)[:, 0]
            out[ds][k] = h.astype(np.float64)
    return out


def to_ev(streams: dict) -> dict:
    """Match N.to_ev shape: r0=d, r1=0, att/t_end from N.G when available."""
    ev = {}
    for ds, table in streams.items():
        ev[ds] = {}
        for k, d in table.items():
            d = np.asarray(d, np.float64)
            base = dict(r0=d, r1=np.zeros_like(d), pre_switched=True)
            if "att" in N.G and ds in N.G["att"] and k in N.G["att"][ds]:
                base["att"] = N.G["att"][ds][k]
                base["t_end"] = N.G["tend"][ds][k]
            ev[ds][k] = base
    return ev


def load_fold_model(held: str, device):
    path = os.path.join(CKPT_DIR, f"held-{held}.pt")
    blob = torch.load(path, map_location=device, weights_only=False)
    win = int(blob.get("window", 96))
    ESR.WINDOW = win
    model = ESR.OddStem(24, win).to(device)
    model.load_state_dict(blob["state"])
    model.eval()
    sd = np.asarray(blob["sd"], np.float32)
    return model, sd, win


def build_lodo_hp_ev(device) -> dict:
    honest_feat = V.build_features()["ms"]
    # Install scorer state from mean of 4-seed classifier so att/t_end exist
    mean_ev, _hp, la = V4._mean_and_hp_from_folds()
    V4._install_scorer_state(mean_ev, la)
    hp_ev = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        log(f"infer ESR held-{held} + hp30")
        model, sd, _win = load_fold_model(held, device)
        streams = ESR.infer_all_replicate(model, {held: honest_feat[held]}, sd, device)
        streams_hp = apply_hp30_stream(streams)
        for k, d in streams_hp[held].items():
            e0 = mean_ev[held][k]
            hp_ev[held][k] = dict(
                r0=np.asarray(d, np.float64),
                r1=np.zeros_like(d, dtype=np.float64),
                att=e0["att"],
                t_end=e0["t_end"],
                pre_switched=True,
            )
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
    return hp_ev


def score_lodo(ev, p_stay):
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate(
            [
                (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                for d in DS_KEYS
                if d != ds
                for e in ev[d].values()
            ]
        )
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, LOOKAHEAD, emis=(mu, sd * SCALE), p_stay=p_stay)
        per[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT")}
    mean = {k: float(np.mean([per[d][k] for d in DS_KEYS])) for k in ("H", "Acc", "L", "SDT")}
    log(f"  p_stay={p_stay} LODO MEAN H={mean['H']:.4f} Acc={mean['Acc']:.2f}")
    return per, mean


def lodo_emis(ev):
    A = np.concatenate(
        [
            (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
            for ds in DS_KEYS
            for e in ev[ds].values()
        ]
    )
    return float(A.mean()), float(A.std())


def score_ext_hp30(device, mu, sd_emis, p_stay):
    path = os.path.join(CKPT_DIR, "all5.pt")
    blob = torch.load(path, map_location=device, weights_only=False)
    win = int(blob.get("window", 96))
    ESR.WINDOW = win
    model = ESR.OddStem(24, win).to(device)
    model.load_state_dict(blob["state"])
    model.eval()
    sd = np.asarray(blob["sd"], np.float32)
    logits, meta = GD.load_external()
    feat = V4.build_features_from(logits)["ms"]
    streams = ESR.infer_all_replicate(model, feat, sd, device)
    streams_hp = apply_hp30_stream(streams)
    out = {}
    for ds in TEST_NAMES:
        ev = {}
        for key, d in streams_hp[ds].items():
            m = meta[ds][key]
            ev[key] = dict(
                r0=np.asarray(d, np.float64),
                r1=np.zeros(len(d), np.float64),
                att=m["att"],
                t_end=m["t_end"],
                att128=m["att128"],
                n128=m["n128"],
                subject=m["subject"],
            )
        r = E.score(ev, mu, sd_emis * SCALE, p_stay, LOOKAHEAD)
        out[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT")}
        log(f"  EXT esr_hp30 {ds} H={out[ds]['H']:.4f} Acc={out[ds]['Acc']:.2f}")
    del model
    if device.type == "cuda":
        torch.cuda.empty_cache()
    return out


def priority_b_ok(mean, ext):
    return (
        mean["H"] > 0.80
        and mean["Acc"] > 70.0
        and ext["Carta2026"]["H"] > 0.60
        and ext["Carta2026"]["Acc"] > 60.0
        and ext["AASD"]["H"] > 0.60
    )


def scrub(o):
    if isinstance(o, dict):
        return {k: scrub(v) for k, v in o.items()}
    if isinstance(o, (list, tuple)):
        return [scrub(v) for v in o]
    if isinstance(o, (np.floating, np.integer)):
        return float(o)
    return o


def main():
    t0 = time.time()
    os.makedirs(RUN_DIR, exist_ok=True)
    open(LOG, "w").close()
    log("H2 esr_hp30 residual start")
    for name in list(DS_KEYS) + ["all5"]:
        p = os.path.join(CKPT_DIR, f"held-{name}.pt" if name != "all5" else "all5.pt")
        if not os.path.exists(p):
            log(f"MISSING {p} — skip H2")
            payload = dict(
                method="esr_hp30",
                run_id=RUN_ID,
                verdict="SKIP",
                reason=f"missing {p}",
            )
            with open(OUT_JSON, "w") as fh:
                json.dump(payload, fh, indent=2)
            with open(OUT_MD, "w") as fh:
                fh.write("# results_esr_hp30\n\nSKIP: missing ESR checkpoints.\n")
            return 2

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    N.load_all()
    hp_ev = build_lodo_hp_ev(device)

    rows = []
    for p in P_GRID:
        per, mean = score_lodo(hp_ev, p)
        rows.append(dict(p_stay=p, mean=mean, per=per))
    rows.sort(key=lambda r: (r["mean"]["Acc"], r["mean"]["H"]), reverse=True)
    best = rows[0]
    log(f"SELECTED p_stay={best['p_stay']} LODO H={best['mean']['H']:.4f} Acc={best['mean']['Acc']:.2f}")

    mu, sd = lodo_emis(hp_ev)
    ext = score_ext_hp30(device, mu, sd, best["p_stay"])
    ok = priority_b_ok(best["mean"], ext)
    payload = dict(
        method="esr_hp30",
        run_id=RUN_ID,
        selected_p_stay=best["p_stay"],
        lodo={"mean": best["mean"], "per": best["per"]},
        grid=[dict(p_stay=r["p_stay"], H=r["mean"]["H"], Acc=r["mean"]["Acc"]) for r in rows],
        ext=ext,
        verdict="PASS" if ok else "FAIL",
        elapsed_s=time.time() - t0,
        note="Inference-only ESR ckpts + τ=30 HP; no ESR retrain",
    )
    with open(OUT_JSON, "w") as fh:
        json.dump(scrub(payload), fh, indent=2)
    lines = [
        "# results_esr_hp30",
        "",
        f"Selected p_stay={best['p_stay']}",
        "",
        "| Stream | LODO H | LODO Acc | AASD H/Acc | Carta H/Acc |",
        "|---|---:|---:|---|---|",
        (
            f"| esr_hp30 | {best['mean']['H']:.4f} | {best['mean']['Acc']:.2f} | "
            f"{ext['AASD']['H']:.4f}/{ext['AASD']['Acc']:.2f} | "
            f"{ext['Carta2026']['H']:.4f}/{ext['Carta2026']['Acc']:.2f} |"
        ),
        "",
        f"Verdict: **{payload['verdict']}** (priority B).",
        f"Artifacts: `{OUT_JSON}`.",
    ]
    with open(OUT_MD, "w") as fh:
        fh.write("\n".join(lines) + "\n")
    log(f"VERDICT {payload['verdict']} elapsed={payload['elapsed_s']:.0f}s")
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())
