"""Confidence-gated cross-stream: fuse→hp30 σ/residual only when EWMA |fuse| low.

Distinct from always-on cross_stream_scale (regressed) and from MoE soft mix of
raw fuse↔hp30 streams. Here the *adapted* stream is fuse scaled toward train
hp30 σ; the gate chooses adapted vs raw fuse — never raw hp30.

Variants (pre-specified; select by mean LODO Acc then H among gated only):
  hard_xσ_t*     — if EWMA_|fuse| < thresh → fuse_xσ else raw fuse
  soft_xσ_t*_s4  — g=expit(s*(ewma_abs−t)); out = g·raw + (1−g)·fuse_xσ
  hard_resid_*   — if low conf → fuse + α(fuse_xσ−fuse) else raw
  soft_resid_*   — fuse + (1−g)·α·(fuse_xσ−fuse)

Also score fuse_w05 (sanity) and always_xσ (always-on reference; not selectable).

HMM preset p_stay 0.994 scale 0.8. CPU on causal_moe_fuse caches + hp30 loaders.
No GPU / no gpu.lock.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u conf_gate_xstream.py
"""
from __future__ import annotations

import json
import os
import re
import time

import numpy as np
from scipy.signal import lfilter
from scipy.special import expit

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = M.CACHE_DIR
LOG = os.path.join(EXP, "experiments", "conf_gate_xstream.log")
OUT = os.path.join(EXP, "results", "research", "conf_gate_xstream.json")
CKPT = os.path.join(EXP, "models", "conf_gate_xstream")

P_STAY = M.P_STAY
SCALE = M.SCALE
TEST_NAMES = M.TEST_NAMES
HOP = 0.25
EPS = 1e-6
TAU_MOMENT = 60.0
TAU_GATE = 4.0
SCALE_CLIP = (0.5, 2.0)

FUSE_BASELINE = dict(
    AASD=dict(H=0.6179, Acc=50.94),
    Carta2026=dict(H=0.5461, Acc=48.38),
    LODO_mean=dict(H=0.8725, Acc=82.58),
    AVGC=dict(H=0.8100, Acc=75.14),
)
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT
OMF_SELECTED = dict(  # online-moment-fuse trial_scale_t60
    AASD=dict(H=0.6183, Acc=50.97),
    AVGC=dict(H=0.8168, Acc=75.70),
    LODO_mean=dict(H=0.8769, Acc=83.01),
)
XSS_SELECTED = dict(  # cross-stream-scale fuse_xσ_t60 (always-on, regressed)
    AASD=dict(H=0.6147, Acc=50.29),
    AVGC=dict(H=0.7703, Acc=73.40),
    LODO_mean=dict(H=0.8139, Acc=82.13),
)

# (name, kind, thresh, resid_alpha, soft_sharp, selectable)
# kind: None | "always_xσ" | "hard_xσ" | "soft_xσ" | "hard_resid" | "soft_resid"
VARIANT_SPECS = [
    ("fuse_w05", None, None, None, None, False),
    ("always_xσ", "always_xσ", None, None, None, False),
    ("hard_xσ_t0.5", "hard_xσ", 0.5, None, None, True),
    ("hard_xσ_t1.0", "hard_xσ", 1.0, None, None, True),
    ("hard_xσ_t1.5", "hard_xσ", 1.5, None, None, True),
    ("soft_xσ_t0.5_s4", "soft_xσ", 0.5, None, 4.0, True),
    ("soft_xσ_t1.0_s4", "soft_xσ", 1.0, None, 4.0, True),
    ("hard_resid_a0.5_t0.5", "hard_resid", 0.5, 0.5, None, True),
    ("hard_resid_a0.5_t1.0", "hard_resid", 1.0, 0.5, None, True),
    ("hard_resid_a1.0_t0.5", "hard_resid", 0.5, 1.0, None, True),
    ("soft_resid_a0.5_t0.5_s4", "soft_resid", 0.5, 0.5, 4.0, True),
    ("soft_resid_a1.0_t1.0_s4", "soft_resid", 1.0, 1.0, 4.0, True),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def subject_of(key, meta_entry=None):
    if meta_entry is not None and "subject" in meta_entry:
        return int(meta_entry["subject"])
    m = re.search(r"subject[-_]?(\d+)", str(key), re.I)
    if m:
        return int(m.group(1))
    m = re.search(r"_S(\d+)_", str(key), re.I)
    if m:
        return int(m.group(1))
    return hash(str(key)) % 10_000_007


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def train_moments(streams, datasets):
    parts = []
    for ds in datasets:
        for _k, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    x = np.concatenate(parts)
    return float(x.mean()), float(x.std()) + EPS


def _ema_past(x, alpha, init):
    x = np.asarray(x, np.float64).reshape(-1)
    a = float(alpha)
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x, zi=np.asarray([init * (1.0 - a)], np.float64))
    past = np.empty_like(x)
    past[0] = init
    if len(x) > 1:
        past[1:] = y[:-1]
    return past, float(y[-1]) if len(y) else float(init)


def adapt_1d_xσ(z, mu_tgt, sd_tgt, alpha, init_mu=None, init_second=None):
    """Past-only scale fuse so online σ → sd_tgt (keep running mean)."""
    z = np.asarray(z, np.float64).reshape(-1)
    if len(z) == 0:
        return z.copy(), (mu_tgt if init_mu is None else init_mu), (
            mu_tgt * mu_tgt + sd_tgt * sd_tgt if init_second is None else init_second)
    mu0 = float(mu_tgt if init_mu is None else init_mu)
    sec0 = float((mu_tgt * mu_tgt + sd_tgt * sd_tgt) if init_second is None else init_second)
    mu_past, mu_final = _ema_past(z, alpha, mu0)
    sec_past, sec_final = _ema_past(z * z, alpha, sec0)
    var = np.maximum(sec_past - mu_past * mu_past, EPS)
    sd = np.sqrt(var)
    lo, hi = SCALE_CLIP
    scale = np.clip(sd_tgt / sd, lo, hi)
    out = (z - mu_past) * scale + mu_past
    return out, mu_final, sec_final


def adapt_streams_xσ(streams, meta_by_ds, mu_tgt, sd_tgt, alpha, persist_subject=False):
    out = {}
    for ds, table in streams.items():
        meta = (meta_by_ds or {}).get(ds) or {}
        by_sub = {}
        for key in table:
            sid = subject_of(key, meta.get(key))
            by_sub.setdefault(sid, []).append(key)
        adapted = {}
        for _sid, keys in by_sub.items():
            keys = sorted(keys)
            state_mu, state_second = None, None
            for key in keys:
                if not persist_subject:
                    state_mu, state_second = None, None
                y, state_mu, state_second = adapt_1d_xσ(
                    table[key], mu_tgt, sd_tgt, alpha,
                    init_mu=state_mu, init_second=state_second,
                )
                adapted[key] = y
        out[ds] = adapted
    return out


def ewma_abs_feat(f, tau_s=TAU_GATE):
    """Causal EWMA of |fuse| (same family as MoE ewma_abs_fuse; inclusive)."""
    return M.ewma_1d(np.abs(np.asarray(f, np.float64)), tau_s=tau_s, hop=HOP)


def gate_weight(f, thresh, sharp):
    """g high → trust raw fuse; low → apply xσ/residual. Soft sigmoid on EWMA |fuse|."""
    feat = ewma_abs_feat(f)
    return expit(float(sharp) * (feat - float(thresh)))


def combine_trial(raw, adapted, kind, thresh, resid_alpha, soft_sharp):
    raw = np.asarray(raw, np.float64).reshape(-1)
    adapted = np.asarray(adapted, np.float64).reshape(-1)
    n = min(len(raw), len(adapted))
    raw, adapted = raw[:n], adapted[:n]
    if kind is None:
        return raw.copy()
    if kind == "always_xσ":
        return adapted.copy()
    feat = ewma_abs_feat(raw)
    if kind == "hard_xσ":
        use_adapt = feat < float(thresh)
        out = np.where(use_adapt, adapted, raw)
        return out.astype(np.float64)
    if kind == "soft_xσ":
        g = expit(float(soft_sharp) * (feat - float(thresh)))
        return (g * raw + (1.0 - g) * adapted).astype(np.float64)
    if kind == "hard_resid":
        a = float(resid_alpha)
        blended = raw + a * (adapted - raw)
        use_adapt = feat < float(thresh)
        return np.where(use_adapt, blended, raw).astype(np.float64)
    if kind == "soft_resid":
        a = float(resid_alpha)
        g = expit(float(soft_sharp) * (feat - float(thresh)))
        return (raw + (1.0 - g) * a * (adapted - raw)).astype(np.float64)
    raise ValueError(f"unknown kind {kind}")


def apply_variant(fuse_st, adapted_st, kind, thresh, resid_alpha, soft_sharp):
    if kind is None:
        return {ds: {k: np.asarray(d, np.float64) for k, d in table.items()}
                for ds, table in fuse_st.items()}
    out = {}
    for ds, table in fuse_st.items():
        ad = adapted_st[ds]
        mixed = {}
        for key, raw in table.items():
            if key not in ad:
                continue
            mixed[key] = combine_trial(
                raw, ad[key], kind, thresh, resid_alpha, soft_sharp,
            )
        out[ds] = mixed
    return out


def pick_variant(lodo_means, candidates):
    best_name, best_key = None, None
    for name in candidates:
        m = lodo_means[name]
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def load_held_fuse(held):
    path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
    if not os.path.isfile(path):
        raise FileNotFoundError(f"missing fuse cache {path}")
    return M.load_pkl(path)


def frac_adapted(fuse_st, kind, thresh, soft_sharp):
    """Mean fraction of frames using adapted path (hard) or mean (1-g) soft."""
    if kind is None:
        return 0.0
    if kind == "always_xσ":
        return 1.0
    ws, tot = 0.0, 0
    for table in fuse_st.values():
        for raw in table.values():
            f = np.asarray(raw, np.float64).reshape(-1)
            if len(f) == 0:
                continue
            feat = ewma_abs_feat(f)
            if kind in ("hard_xσ", "hard_resid"):
                ws += float(np.sum(feat < float(thresh)))
            else:
                g = expit(float(soft_sharp) * (feat - float(thresh)))
                ws += float(np.sum(1.0 - g))
            tot += len(f)
    return float(ws / tot) if tot else 0.0


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    alpha_m = alpha_from_tau(TAU_MOMENT)
    log_line(f"conf-gate-xstream CPU  p_stay {P_STAY} scale {SCALE}  "
             f"moment_tau {TAU_MOMENT}s alpha {alpha_m:.6g}  gate_tau {TAU_GATE}s")
    log_line("honesty: fuse→train-hp30 σ only when EWMA |fuse| low; else raw fuse; "
             "no labels; no entropy-min; no raw-hp30 mix")
    log_line("distinct from always-on cross-stream-scale and MoE fuse↔hp30 mix")

    N.load_all()
    att = N.G["att"]

    log_line("loading LODO fuse caches + hp30")
    fuse_by_held = {}
    for held in DS_KEYS:
        fuse_by_held[held] = load_held_fuse(held)
        log_line(f"cache hit fuse_w05__held-{held}.pkl")
    hp30, _mean4, _la = M.load_lodo_hp30()

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity raw fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")
    san_hp = M.score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    mu0, sd0 = train_moments(hp30, [d for d in DS_KEYS if d != "AVGC"])
    mu_f0, sd_f0 = train_moments(fuse_by_held["AVGC"], [d for d in DS_KEYS if d != "AVGC"])
    log_line(f"ref train moments (held AVGC): fuse μ/σ {mu_f0:.4f}/{sd_f0:.4f}  "
             f"hp30 μ/σ {mu0:.4f}/{sd0:.4f}")

    results = dict(
        method=("confidence-gated cross-stream: apply fuse→train-hp30 σ / residual "
                "only when EWMA |fuse| is low; else raw fuse"),
        p_stay=P_STAY, scale=SCALE,
        tau_moment=TAU_MOMENT, alpha_moment=alpha_m,
        tau_gate=TAU_GATE, scale_clip=list(SCALE_CLIP),
        honesty=("Past-only EWMA for fuse→hp30 σ; causal EWMA |fuse| gate. "
                 "No labels, no entropy-min. Distinct from always-on xσ and MoE mix."),
        baselines=dict(
            fuse=FUSE_BASELINE,
            linear_hp30_ext=LINEAR_HP30_EXT,
            online_moment_fuse_selected=OMF_SELECTED,
            cross_stream_scale_selected=XSS_SELECTED,
        ),
        variants={},
    )

    log_line("=== LODO (fixed HMM; selection among gated variants) ===")
    lodo_means = {}
    for name, kind, thresh, resid_a, soft_s, selectable in VARIANT_SPECS:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            fuse = fuse_by_held[held]
            train_ds = [d for d in DS_KEYS if d != held]
            mu_hp, sd_hp = train_moments(hp30, train_ds)
            if kind is None:
                adapted_x = fuse
            else:
                adapted_x = adapt_streams_xσ(fuse, None, mu_hp, sd_hp, alpha_m)
            gated = apply_variant(fuse, adapted_x, kind, thresh, resid_a, soft_s)
            off = M.score_lodo(gated, held)
            frac = frac_adapted(fuse, kind, thresh, soft_s) if kind else 0.0
            per[held] = dict(H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"),
                             L=off.get("L"), acc1=off.get("acc1"),
                             train_hp30_mu=mu_hp, train_hp30_sd=sd_hp,
                             frac_adapted=frac)
            log_line(f"LODO {name:28s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"frac_ad {frac:.3f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:28s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            kind=kind, thresh=thresh, resid_alpha=resid_a, soft_sharp=soft_s,
            selectable=selectable,
            tau_moment=TAU_MOMENT if kind else None,
            tau_gate=TAU_GATE if kind and kind != "always_xσ" else None,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    gated_names = [n for n, k, *_, sel in VARIANT_SPECS if sel]
    selected = pick_variant(lodo_means, gated_names)
    ranked = sorted(gated_names, key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
                    reverse=True)
    log_line(f"SELECTED (LODO Acc then H, gated only): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    log_line(f"ranked: {ranked}")
    results["selection"] = dict(
        criterion="mean LODO Acc then H (gated variants only; never AASD/Carta)",
        ranked=ranked, selected=selected,
        excluded_from_select=["fuse_w05", "always_xσ"],
    )
    results["selected"] = selected

    log_line("=== external (all5 corpus + external fuse + hp30) ===")
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))
    corp_hp, ext_hp, train_att, test_meta = M.load_ext_hp30()
    mu_corp_hp, sd_corp_hp = train_moments(corp_hp, list(V4.CORPUS))
    mu_corp_f, sd_corp_f = train_moments(corp_fuse, list(V4.CORPUS))
    log_line(f"corpus fuse μ/σ {mu_corp_f:.6f}/{sd_corp_f:.6f}  "
             f"hp30 μ/σ {mu_corp_hp:.6f}/{sd_corp_hp:.6f}")
    results["train_moments_corpus"] = dict(
        fuse=dict(mu=mu_corp_f, sd=sd_corp_f),
        hp30=dict(mu=mu_corp_hp, sd=sd_corp_hp),
    )

    for name, kind, thresh, resid_a, soft_s, _sel in VARIANT_SPECS:
        if kind is None:
            adapted_train = corp_fuse
            adapted_test = ext_fuse
        else:
            adapted_train = adapt_streams_xσ(
                corp_fuse, None, mu_corp_hp, sd_corp_hp, alpha_m,
            )
            adapted_test = adapt_streams_xσ(
                ext_fuse, test_meta, mu_corp_hp, sd_corp_hp, alpha_m,
            )
        gated_train = apply_variant(corp_fuse, adapted_train, kind, thresh, resid_a, soft_s)
        mu_e, sd_e = F.emissions_from(gated_train, train_att, datasets=list(V4.CORPUS))
        gated_test = apply_variant(ext_fuse, adapted_test, kind, thresh, resid_a, soft_s)
        frac = frac_adapted(ext_fuse, kind, thresh, soft_s) if kind else 0.0
        ext = {}
        for ds in TEST_NAMES:
            r = F.score_external(gated_test[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            d_fuse = r["Acc"] - FUSE_BASELINE[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            log_line(f"EXT {name:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs fuse {d_fuse:+.2f}  vs hp30 {d_hp:+.2f}  frac_ad {frac:.3f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)
        results["variants"][name]["frac_adapted_ext"] = frac

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_fuse = aasd["Acc"] - FUSE_BASELINE["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    d_omf = aasd["Acc"] - OMF_SELECTED["AASD"]["Acc"]
    d_xss = aasd["Acc"] - XSS_SELECTED["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_fuse=d_fuse,
        AASD_Acc_delta_vs_linear_hp30=d_hp,
        AASD_Acc_delta_vs_online_moment_fuse=d_omf,
        AASD_Acc_delta_vs_cross_stream_scale=d_xss,
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=prim["lodo"]["AVGC"],
        frac_adapted_ext=prim.get("frac_adapted_ext"),
        baseline_fuse_ext=FUSE_BASELINE,
        baseline_linear_hp30_ext=LINEAR_HP30_EXT,
        baseline_omf=OMF_SELECTED,
        baseline_xss=XSS_SELECTED,
    )
    log_line(f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δfuse {d_fuse:+.2f}, Δhp30 {d_hp:+.2f}, Δomf {d_omf:+.2f}, Δxss {d_xss:+.2f})  "
             f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}  "
             f"AVGC H {prim['lodo']['AVGC']['H']:.4f} Acc {prim['lodo']['AVGC']['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "summary.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       train_moments_corpus=results["train_moments_corpus"],
                       headline=results["headline"]), fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
