"""Causal agree-gate on frozen IRM ↔ stem-synth streams (no retrain).

On each frame:
  if sign(IRM) == sign(synth): emit mean  (= w=0.5 fuse)
  else (disagree): emit one of
    - max-|logit| stream
    - IRM-only
    - synth-only
    - soft |logit|-weighted mix  w=|IRM|/(|IRM|+|synth|)

CPU-only: reuse IRM/synth pickles under models/fuse_conf_resid/cache/
(_tmp_irm__*, _tmp_synth__*). Distinct from fixed-w IRM↔synth, MoE fuse↔hp30,
and fuse-conf-resid.

Select by mean LODO Acc then H @ HMM 0.994/0.8. Never AASD/Carta for selection.
Score full LODO5 + AASD + Carta; report AVGC carefully vs tip 0.8192/75.52.

usage:
  CUDA_VISIBLE_DEVICES= \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u \\
    irm_synth_agree_gate.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_synth_fuse as SF
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "fuse_conf_resid", "cache")
LOG = os.path.join(EXP, "experiments", "irm_synth_agree_gate.log")
OUT = os.path.join(EXP, "results", "research", "irm_synth_agree_gate.json")
CKPT = os.path.join(EXP, "models", "irm_synth_agree_gate")

P_STAY = SF.P_STAY
SCALE = SF.SCALE
TEST_NAMES = SF.TEST_NAMES
EPS = 1e-8

FUSE_TIP = dict(
    mean=dict(H=0.8790, Acc=83.11),
    AVGC=dict(H=0.8192, Acc=75.52),
    AASD=dict(H=0.6187, Acc=50.92),
    Carta2026=dict(H=0.5497, Acc=49.09),
)
LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_SWMIX_TIP = dict(AASD=dict(Acc=53.53), Carta2026=dict(Acc=59.71))

# (name, mode, selectable)
# mode: mean | irm | synth | maxabs | soft_abs | agree_maxabs | agree_irm | agree_synth | agree_soft_abs
VARIANT_SPECS = [
    ("fuse_w05_mean", "mean", False),          # always mean = tip reference
    ("pure_irm", "irm", False),
    ("pure_synth", "synth", False),
    ("always_maxabs", "maxabs", False),
    ("always_soft_abs", "soft_abs", False),
    ("agree_maxabs", "agree_maxabs", True),
    ("agree_irm", "agree_irm", True),
    ("agree_synth", "agree_synth", True),
    ("agree_soft_abs", "agree_soft_abs", True),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def require_cache(path):
    if not os.path.isfile(path):
        raise FileNotFoundError(f"missing cache (CPU-only run expects hit): {path}")
    log_line(f"cache hit {path}")
    return load_pkl(path)


def load_held_pair(held):
    irm = require_cache(os.path.join(CACHE_DIR, f"_tmp_irm__held-{held}.pkl"))
    syn = require_cache(os.path.join(CACHE_DIR, f"_tmp_synth__held-{held}.pkl"))
    return irm, syn


def load_all5_corpus_pair():
    irm = require_cache(os.path.join(CACHE_DIR, "_tmp_irm__all5_corpus.pkl"))
    syn = require_cache(os.path.join(CACHE_DIR, "_tmp_synth__all5_corpus.pkl"))
    return irm, syn


def load_all5_external_pair():
    irm = require_cache(os.path.join(CACHE_DIR, "_tmp_irm__all5_external.pkl"))
    syn = require_cache(os.path.join(CACHE_DIR, "_tmp_synth__all5_external.pkl"))
    return irm, syn


def gate_1d(a, b, mode):
    """Apply agree-gate / fuse policy to aligned 1-d logit streams."""
    a = np.asarray(a, np.float64).reshape(-1)
    b = np.asarray(b, np.float64).reshape(-1)
    n = min(len(a), len(b))
    a, b = a[:n], b[:n]
    mean = 0.5 * (a + b)
    abs_a = np.abs(a)
    abs_b = np.abs(b)
    maxabs = np.where(abs_a >= abs_b, a, b)
    w = abs_a / (abs_a + abs_b + EPS)
    soft = w * a + (1.0 - w) * b

    if mode == "mean":
        return mean
    if mode == "irm":
        return a.copy()
    if mode == "synth":
        return b.copy()
    if mode == "maxabs":
        return maxabs
    if mode == "soft_abs":
        return soft

    # agree-* : mean on same sign; disagree policy otherwise
    # np.sign(0)==0 → 0 agrees with 0; 0 vs nonzero counts as disagree
    agree = np.sign(a) == np.sign(b)
    if mode == "agree_maxabs":
        alt = maxabs
    elif mode == "agree_irm":
        alt = a
    elif mode == "agree_synth":
        alt = b
    elif mode == "agree_soft_abs":
        alt = soft
    else:
        raise ValueError(mode)
    return np.where(agree, mean, alt).astype(np.float64)


def gate_stream(irm_st, syn_st, mode):
    out = {}
    for key in irm_st:
        if key not in syn_st:
            continue
        out[key] = gate_1d(irm_st[key], syn_st[key], mode)
    return out


def gate_all(irm_all, syn_all, mode):
    return {ds: gate_stream(irm_all[ds], syn_all[ds], mode)
            for ds in irm_all if ds in syn_all}


def agree_stats(irm_st, syn_st):
    """Fraction of frames with sign agreement (diagnostic)."""
    n_agree, n_tot = 0, 0
    for key in irm_st:
        if key not in syn_st:
            continue
        a = np.asarray(irm_st[key], np.float64).reshape(-1)
        b = np.asarray(syn_st[key], np.float64).reshape(-1)
        n = min(len(a), len(b))
        if n == 0:
            continue
        n_agree += int(np.sum(np.sign(a[:n]) == np.sign(b[:n])))
        n_tot += n
    return float(n_agree / n_tot) if n_tot else 0.0, n_tot


def pick_variant(lodo_means, candidates):
    best_name, best_key = None, None
    for name in candidates:
        m = lodo_means[name]
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def main():
    os.makedirs(CKPT, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"irm-synth-agree-gate CPU  p_stay {P_STAY} scale {SCALE}")
    log_line("agree → mean; disagree → maxabs|irm|synth|soft_|logit|-mix")
    log_line("select by mean LODO Acc then H; never AASD/Carta for selection")
    log_line("distinct from fixed-w fuse / MoE fuse↔hp30 / fuse-conf-resid")

    N.load_all()
    att = N.G["att"]

    log_line("=== load frozen IRM + synth caches ===")
    held_pairs = {}
    for held in DS_KEYS:
        held_pairs[held] = load_held_pair(held)

    # Sanity: mean == tip fuse on AVGC
    san_st = gate_all(*held_pairs["AVGC"], "mean")
    san = M.score_lodo(san_st, "AVGC")
    log_line(f"sanity mean(=fuse w0.5) AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} "
             f"(expect ~0.8192/75.52)")
    frac_a, n_a = agree_stats(held_pairs["AVGC"][0]["AVGC"], held_pairs["AVGC"][1]["AVGC"])
    log_line(f"sanity AVGC sign-agree frac {frac_a:.4f} n={n_a}")

    results = dict(
        method=("causal agree-gate on frozen IRM↔synth: same-sign→mean; "
                "disagree→max-|logit| / IRM / synth / soft |logit| mix"),
        p_stay=P_STAY, scale=SCALE,
        honesty=("CPU caches only. Select by mean LODO Acc then H. "
                 "No AASD/Carta for selection. ≠ fixed-w / MoE / conf-resid."),
        baselines=dict(irm_synth_fuse_w05=FUSE_TIP, linear_hp30_ext=LINEAR_HP30_EXT,
                       sw2_swmix_tip=SW2_SWMIX_TIP),
        variants={},
    )

    log_line("=== LODO ===")
    lodo_means = {}
    for name, mode, selectable in VARIANT_SPECS:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            irm, syn = held_pairs[held]
            gated = gate_all(irm, syn, mode)
            off = M.score_lodo(gated, held)
            # agree frac on held-out ds only
            frac, nfr = agree_stats(irm[held], syn[held])
            per[held] = dict(H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"),
                             L=off.get("L"), acc1=off.get("acc1"),
                             agree_frac=frac, n_frames=nfr)
            log_line(f"LODO {name:18s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"agree {frac:.3f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:18s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            mode=mode, selectable=selectable,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    gated_names = [n for n, _, sel in VARIANT_SPECS if sel]
    selected = pick_variant(lodo_means, gated_names)
    ranked = sorted(gated_names, key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
                    reverse=True)
    log_line(f"SELECTED (LODO Acc then H, agree-* only): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    log_line(f"ranked: {ranked}")
    results["selection"] = dict(
        criterion="mean LODO Acc then H (agree-* variants only; never AASD/Carta)",
        ranked=ranked, selected=selected,
        excluded_from_select=[n for n, _, sel in VARIANT_SPECS if not sel],
    )
    results["selected"] = selected

    log_line("=== external (all5 corpus emissions + external streams) ===")
    irm_corp, syn_corp = load_all5_corpus_pair()
    irm_ext, syn_ext = load_all5_external_pair()
    ext_logits, ext_meta = SF.load_external()
    del ext_logits  # meta only; streams from cache

    for name, mode, _sel in VARIANT_SPECS:
        gated_train = gate_all(irm_corp, syn_corp, mode)
        gated_test = {
            ds: gate_stream(irm_ext[ds], syn_ext[ds], mode) for ds in irm_ext
        }
        mu_e, sd_e = SF.emissions_from(gated_train, att, datasets=list(V4.CORPUS))
        ext = {}
        for ds in TEST_NAMES:
            r = SF.score_external(gated_test[ds], ext_meta[ds], mu_e, sd_e)
            ext[ds] = r
            d_fuse = r["Acc"] - FUSE_TIP[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            log_line(f"EXT {name:18s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs fuse {d_fuse:+.2f}  vs hp30 {d_hp:+.2f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    avgc = prim["lodo"]["AVGC"]
    fuse_avgc = results["variants"]["fuse_w05_mean"]["lodo"]["AVGC"]
    d_tip = dict(H=avgc["H"] - FUSE_TIP["AVGC"]["H"], Acc=avgc["Acc"] - FUSE_TIP["AVGC"]["Acc"])
    d_sanity = dict(H=fuse_avgc["H"] - FUSE_TIP["AVGC"]["H"],
                    Acc=fuse_avgc["Acc"] - FUSE_TIP["AVGC"]["Acc"])
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=avgc,
        fuse_w05_AVGC=fuse_avgc,
        AVGC_delta_vs_fuse_tip=d_tip,
        fuse_w05_AVGC_delta_vs_tip=d_sanity,
        AASD_Acc_delta_vs_fuse=aasd["Acc"] - FUSE_TIP["AASD"]["Acc"],
        AASD_Acc_delta_vs_linear_hp30=aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"],
        AASD_Acc_delta_vs_sw2_swmix=aasd["Acc"] - SW2_SWMIX_TIP["AASD"]["Acc"],
        targets=dict(
            AVGC_ge_085_78=bool(avgc["H"] >= 0.85 and avgc["Acc"] >= 78),
            AASD_Acc_gt_60=bool(aasd["Acc"] > 60),
        ),
    )
    log_line(f"HEADLINE {selected} AVGC H {avgc['H']:.4f} Acc {avgc['Acc']:.2f} "
             f"(Δtip {d_tip['H']:+.4f}/{d_tip['Acc']:+.2f})  "
             f"AASD Acc {aasd['Acc']:.2f}  Carta Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}")
    log_line(f"fuse_w05 sanity AVGC H {fuse_avgc['H']:.4f} Acc {fuse_avgc['Acc']:.2f} "
             f"(Δtip {d_sanity['H']:+.4f}/{d_sanity['Acc']:+.2f})")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "summary.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       headline=results["headline"]), fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
