"""Novel: causal τ=30 HP on IRM↔synth stem fuse, then soft-blend with AccFuse↔linear_hp30 tip.

Root-cause extension of AccFuse↔hp30 tip (α=0.50): stem transformers win LODO
(H~0.88/Acc~83) but carry slow signed bias that kills real-switch H; τ=30 HP
strips that bias for 4-seed logits (linear_hp30). Never previously: apply the
same HP to IRM↔synth fuse w=0.5, then AccFuse-tip ↔ stem_hp soft-w.

Also screens:
  - AccFuse_raw ↔ stem_hp
  - tip ↔ stem_raw (capped, for contrast)
  - AccFuse_hp30 (HP of AccFuse alone) ↔ linear_hp30  [≠ dual SW2-hp↔BMMS-hp]

Distinct from: MoE fuse↔linear_hp30, IRM↔SWA blend+hp30 (soft-mix with 4-seed
hp30 stream), esr-hp30, dual_hp30_sw2_bmms, agree-gate-dual-hp30.

Selection: among Acc>75 pick max H; else Acc>70 max H; else Acc then H.
Global p_stay grid on winner. Never AASD/Carta labels for selection.
"""
from __future__ import annotations

import json
import os
import pickle
import time
from pathlib import Path

import numpy as np

import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS
from common_exp import EXP
from dual_hp30_sw2_bmms import (
    LOOKAHEAD,
    TEST_NAMES,
    as_sym,
    blend_ev,
    load_ext_mean4,
    score_lodo,
    signed,
)
from accfuse_hp30_hselect import align_hp, load_accfuse

RUN_ID = "stem_hp30_accfuse_20261108"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
OUT_JSON = os.path.join(EXP, "results", "research", "stem_hp30_accfuse.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_stem_hp30_accfuse.md")
CACHE = os.path.join(EXP, "models", "fuse_conf_resid", "cache")
TIP_ALPHA = 0.50
WS = [0.0, 0.05, 0.10, 0.15, 0.20, 0.25, 0.35, 0.50, 0.75, 1.0]
P_GRID = [(0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)]


def log(m):
    print(f"{time.strftime('%F %T')} {m}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(os.path.join(RUN_DIR, "run.log"), "a") as fh:
        fh.write(m + "\n")


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def load_stem_honest():
    """Per-dataset LODO-honest IRM↔synth w=0.5 signed streams."""
    out = {}
    for ds in DS_KEYS:
        blob = load_pkl(os.path.join(CACHE, f"irm_synth_w05__held-{ds}.pkl"))
        out[ds] = {k: np.asarray(v, np.float64) for k, v in blob[ds].items()}
    return out


def load_stem_ext():
    blob = load_pkl(os.path.join(CACHE, "irm_synth_w05__all5_external.pkl"))
    return {ds: {k: np.asarray(v, np.float64) for k, v in blob[ds].items()} for ds in TEST_NAMES}


def apply_hp30_signed(streams, tau=30.0):
    out = {}
    for ds, table in streams.items():
        out[ds] = {}
        for k, d in table.items():
            h = V4.highpass(np.asarray(d, np.float64)[:, None], tau=tau, hop=0.25)[:, 0]
            out[ds][k] = h.astype(np.float64)
    return out


def streams_to_ev(streams, template_ev):
    """Wrap signed 1-D into r0/r1 evidence using template att/t_end."""
    out = {}
    for ds in streams:
        out[ds] = {}
        tmpl = template_ev[ds]
        for k, s in streams[ds].items():
            if k not in tmpl:
                continue
            n = min(len(s), len(tmpl[k]["att"]))
            out[ds][k] = as_sym(tmpl[k], np.asarray(s, np.float64)[:n])
    return out


def ev_to_signed(ev):
    return {ds: {k: signed(e) for k, e in table.items()} for ds, table in ev.items()}


def blend_signed(a, b, w, template_ev):
    """(1-w)*a + w*b → evidence."""
    out = {}
    for ds in a:
        out[ds] = {}
        for k in a[ds]:
            if k not in b[ds] or k not in template_ev[ds]:
                continue
            sa, sb = a[ds][k], b[ds][k]
            n = min(len(sa), len(sb), len(template_ev[ds][k]["att"]))
            s = (1.0 - w) * sa[:n] + w * sb[:n]
            out[ds][k] = as_sym(template_ev[ds][k], s)
    return out


def select(grid, acc_hi=75.0, acc_lo=70.0):
    ok_hi = [r for r in grid if r["mean_Acc"] > acc_hi]
    if ok_hi:
        return sorted(ok_hi, key=lambda r: (r["mean_H"], r["mean_Acc"]), reverse=True)[0]
    ok_lo = [r for r in grid if r["mean_Acc"] > acc_lo]
    if ok_lo:
        return sorted(ok_lo, key=lambda r: (r["mean_H"], r["mean_Acc"]), reverse=True)[0]
    return sorted(grid, key=lambda r: (r["mean_Acc"], r["mean_H"]), reverse=True)[0]


def score_ext_ev(ev_ext, mu, sd, p, sc):
    ext = {}
    for ds in TEST_NAMES:
        r = E.score(ev_ext[ds], mu, sd * sc, p, LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        log(f"  EXT {ds} H {ext[ds]['H']:.4f} Acc {ext[ds]['Acc']:.2f}")
    return ext


def emis_from(ev):
    A = np.concatenate([
        signed(e) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev[ds].values()
    ])
    return float(A.mean()), float(A.std())


def run_family(name, stream_a, stream_b, template, weights):
    log(f"=== {name} ===")
    grid = []
    for w in weights:
        ev = blend_signed(stream_a, stream_b, w, template)
        per, mH, mA = score_lodo(ev)
        row = dict(
            family=name, w=w, mean_H=mH, mean_Acc=mA,
            per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"])) for d in DS_KEYS},
        )
        grid.append(row)
        log(f"  w={w:.2f} LODO H={mH:.4f} Acc={mA:.2f}")
    best = select(grid)
    log(f"  SELECTED w={best['w']:.2f} H={best['mean_H']:.4f} Acc={best['mean_Acc']:.2f}")
    return grid, best


def main():
    t0 = time.time()
    os.makedirs(RUN_DIR, exist_ok=True)
    open(os.path.join(RUN_DIR, "run.log"), "w").close()
    log("=== stem_hp30 ↔ AccFuse tip ===")

    # Tip streams (AccFuse raw ↔ linear_hp30 α=0.50)
    af = load_accfuse()
    mean_ev, lin_hp, _ = V4._mean_and_hp_from_folds()
    lin_al = align_hp(af, lin_hp)
    tip_ev = {ds: blend_ev(af[ds], lin_al[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    af_s = ev_to_signed(af)
    lin_s = ev_to_signed(lin_al)

    # AccFuse_hp30 = HP of AccFuse signed
    af_hp_s = apply_hp30_signed(af_s)

    # Stem honest + HP
    stem_s = load_stem_honest()
    stem_hp_s = apply_hp30_signed(stem_s)
    log(f"stem keys AVGC={len(stem_s['AVGC'])} tip={len(tip_s['AVGC'])}")

    # Template with att/t_end: AccFuse evidence
    tmpl = af

    # Sanity tip @0.996/0.8
    per, mH, mA = score_lodo(tip_ev, p_stay=0.996, scale=0.8)
    log(f"SANITY tip@0.996/0.8 LODO H={mH:.4f} Acc={mA:.2f} (expect ~0.8011/71.43)")

    families = {}
    all_grids = []

    # Baselines alone
    for name, stream in (
        ("tip_alone", tip_s),
        ("stem_raw_alone", stem_s),
        ("stem_hp_alone", stem_hp_s),
        ("af_hp_alone", af_hp_s),
        ("lin_hp_alone", lin_s),
    ):
        ev = streams_to_ev(stream, tmpl)
        per, mH, mA = score_lodo(ev)
        row = dict(family=name, w=None, mean_H=mH, mean_Acc=mA,
                   per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"])) for d in DS_KEYS})
        all_grids.append(row)
        log(f"BASE {name} LODO H={mH:.4f} Acc={mA:.2f}")

    # Soft-blend families (w = weight on second stream)
    for name, a, b in (
        ("tip_x_stem_hp", tip_s, stem_hp_s),
        ("af_raw_x_stem_hp", af_s, stem_hp_s),
        ("tip_x_stem_raw", tip_s, stem_s),
        ("af_hp_x_lin_hp", af_hp_s, lin_s),
        ("stem_hp_x_lin_hp", stem_hp_s, lin_s),
    ):
        grid, best = run_family(name, a, b, tmpl, WS)
        families[name] = dict(grid=grid, best=best)
        all_grids.extend(grid)

    # Global select across families (prefer Acc>75 then H)
    candidates = [families[n]["best"] for n in families]
    # also include baselines with Acc>70
    for row in all_grids:
        if row.get("w") is None and row["mean_Acc"] > 70:
            candidates.append(row)
    winner = select(candidates)
    log(f"WINNER family={winner.get('family')} w={winner.get('w')} "
        f"H={winner['mean_H']:.4f} Acc={winner['mean_Acc']:.2f}")

    # Rebuild winner evidence + p_stay grid
    fam = winner["family"]
    w = winner.get("w")
    if fam == "tip_alone" or (fam.startswith("tip") and w == 0.0):
        win_ev = tip_ev
    elif fam == "stem_hp_alone" or (fam.endswith("stem_hp") and w == 1.0 and "tip" not in fam and "af" not in fam):
        win_ev = streams_to_ev(stem_hp_s, tmpl)
    elif fam in families and w is not None:
        a_map = {
            "tip_x_stem_hp": (tip_s, stem_hp_s),
            "af_raw_x_stem_hp": (af_s, stem_hp_s),
            "tip_x_stem_raw": (tip_s, stem_s),
            "af_hp_x_lin_hp": (af_hp_s, lin_s),
            "stem_hp_x_lin_hp": (stem_hp_s, lin_s),
        }
        a, b = a_map[fam]
        win_ev = blend_signed(a, b, w, tmpl)
    else:
        # baseline alone
        alone = {
            "tip_alone": tip_s,
            "stem_raw_alone": stem_s,
            "stem_hp_alone": stem_hp_s,
            "af_hp_alone": af_hp_s,
            "lin_hp_alone": lin_s,
        }
        win_ev = streams_to_ev(alone[fam], tmpl)

    pgrid = []
    for p, sc in P_GRID:
        per, mH, mA = score_lodo(win_ev, p_stay=p, scale=sc)
        pgrid.append(dict(p=p, sc=sc, H=mH, Acc=mA,
                          per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"])) for d in DS_KEYS}))
        log(f"  p={p} sc={sc} LODO H={mH:.4f} Acc={mA:.2f}")
    # Prefer Acc>75 then H; else Acc>70 then H
    ok75 = [r for r in pgrid if r["Acc"] > 75]
    ok70 = [r for r in pgrid if r["Acc"] > 70]
    best_p = sorted(ok75 or ok70 or pgrid, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
    log(f"SELECTED HMM p={best_p['p']} sc={best_p['sc']} H={best_p['H']:.4f} Acc={best_p['Acc']:.2f}")

    mu, sd = emis_from(win_ev)

    # External tip components
    sw2e = load_ext_mean4("w10b3sw2P")
    bmmse = load_ext_mean4("w10bmms3enhP")
    af_ext = {ds: blend_ev(sw2e[ds], bmmse[ds], 0.5) for ds in TEST_NAMES}
    train_logits, _ = V4.load_train()
    test_logits, _ = V4.load_test()
    base = {}
    base.update(train_logits)
    base.update(test_logits)
    streams = V4.linear_streams(base, "hp30")
    hp_ext = {}
    for ds in TEST_NAMES:
        hp_ext[ds] = {}
        for k, d in streams[ds].items():
            if k not in af_ext[ds]:
                continue
            n = min(len(d), len(af_ext[ds][k]["r0"]))
            hp_ext[ds][k] = as_sym(af_ext[ds][k], np.asarray(d, float)[:n])
    tip_ext = {ds: blend_ev(af_ext[ds], hp_ext[ds], TIP_ALPHA) for ds in TEST_NAMES}
    tip_ext_s = ev_to_signed(tip_ext)
    af_ext_s = ev_to_signed(af_ext)
    lin_ext_s = ev_to_signed(hp_ext)
    af_ext_hp_s = apply_hp30_signed(af_ext_s)

    stem_ext_s = load_stem_ext()
    stem_ext_hp_s = apply_hp30_signed(stem_ext_s)

    # Align stem ext keys to tip template
    def rebuild_ext(fam, w):
        if fam == "tip_alone" or (isinstance(w, float) and w == 0.0 and fam.startswith("tip")):
            return tip_ext
        a_map = {
            "tip_x_stem_hp": (tip_ext_s, stem_ext_hp_s),
            "af_raw_x_stem_hp": (af_ext_s, stem_ext_hp_s),
            "tip_x_stem_raw": (tip_ext_s, stem_ext_s),
            "af_hp_x_lin_hp": (af_ext_hp_s, lin_ext_s),
            "stem_hp_x_lin_hp": (stem_ext_hp_s, lin_ext_s),
            "stem_hp_alone": (None, stem_ext_hp_s),
            "stem_raw_alone": (None, stem_ext_s),
            "af_hp_alone": (None, af_ext_hp_s),
            "lin_hp_alone": (None, lin_ext_s),
        }
        if fam in ("stem_hp_alone", "stem_raw_alone", "af_hp_alone", "lin_hp_alone"):
            return streams_to_ev(a_map[fam][1], tip_ext)
        if fam in a_map and w is not None:
            a, b = a_map[fam]
            return blend_signed(a, b, w, tip_ext)
        return tip_ext

    ext_ev = rebuild_ext(fam, w)
    ext_sel = score_ext_ev(ext_ev, mu, sd, best_p["p"], best_p["sc"])
    ext_preset = score_ext_ev(ext_ev, mu, sd, 0.994, 0.8)
    # Also mild tip HMM for comparison if winner is tip-like
    ext_996 = score_ext_ev(ext_ev, mu, sd, 0.996, 0.8)

    def gate(lodo_H, lodo_A, ext):
        return (
            lodo_H > 0.85
            and lodo_A > 75.0
            and (ext["Carta2026"]["H"] >= 0.65 or ext["AASD"]["H"] >= 0.65)
        )

    ok = gate(best_p["H"], best_p["Acc"], ext_sel) or gate(best_p["H"], best_p["Acc"], ext_996)

    out = dict(
        method="stem_hp30_accfuse",
        tip_alpha=TIP_ALPHA,
        baselines=[r for r in all_grids if r.get("w") is None],
        families={k: dict(best=v["best"], grid=v["grid"]) for k, v in families.items()},
        winner=winner,
        pgrid=pgrid,
        selected_hmm=best_p,
        external_at_selected=ext_sel,
        external_at_preset=ext_preset,
        external_at_996=ext_996,
        emissions=dict(mu=mu, sd=sd),
        targets=dict(
            pass_new=ok,
            lodo_H=best_p["H"],
            lodo_Acc=best_p["Acc"],
            carta_H=ext_sel["Carta2026"]["H"],
            aasd_H=ext_sel["AASD"]["H"],
        ),
        vs_tip=dict(lodo="0.8011/71.43@0.996", carta="0.652", aasd="0.642"),
        elapsed_s=time.time() - t0,
    )
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)

    Path(RESULTS_MD).write_text(
        "# results_stem_hp30_accfuse\n\n"
        "Causal τ=30 HP on IRM↔synth fuse w=0.5, soft-blended with AccFuse↔linear_hp30 tip.\n\n"
        f"**Winner:** `{winner.get('family')}` w={winner.get('w')} "
        f"@ p={best_p['p']}/sc={best_p['sc']}\n\n"
        f"| Setting | LODO H/Acc | AASD H/Acc | Carta H/Acc |\n"
        f"|---|---:|---:|---:|\n"
        f"| selected | {best_p['H']:.4f}/{best_p['Acc']:.2f} | "
        f"{ext_sel['AASD']['H']:.4f}/{ext_sel['AASD']['Acc']:.2f} | "
        f"{ext_sel['Carta2026']['H']:.4f}/{ext_sel['Carta2026']['Acc']:.2f} |\n"
        f"| @0.996/0.8 | — | "
        f"{ext_996['AASD']['H']:.4f}/{ext_996['AASD']['Acc']:.2f} | "
        f"{ext_996['Carta2026']['H']:.4f}/{ext_996['Carta2026']['Acc']:.2f} |\n"
        f"| pass_new (H>0.85 Acc>75 & test H≥0.65) | **{ok}** |\n"
    )
    log(f"DONE elapsed={time.time()-t0:.1f}s pass={ok} wrote {OUT_JSON}")


if __name__ == "__main__":
    main()
