"""Seed-disagree gate on 4-seed classifier logits (NOT fuse↔hp30 assignment).

CPU-only. Past-only / causal. No new network. No AASD/Carta labels for selection.

At each frame, pairwise seed disagreement:
  disagree[t] = mean_{i<j} |s_i[t] - s_j[t]|
optionally EWMA(disagree) with tau in {0=instant,1,4,8}s at hop 0.25 s.

Soft gate:  e' = e * sigmoid(alpha * (thr - disagree))
Hard gate:  if EWMA disagree > thr → hold last accepted e, or emit 0.

Apply to: raw mean-of-4, linear_hp30 mean, median-of-4 (raw or hp30).
Disagree computed on the same 4-seed stack (raw or after causal hp30).

Select by mean LODO Acc then H at HMM 0.994/0.8. Score AASD + Carta.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u seed_disagree_gate.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np
from scipy.signal import lfilter
from scipy.special import expit

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "seed_disagree_gate.log")
OUT = os.path.join(EXP, "results", "research", "seed_disagree_gate.json")
CKPT = os.path.join(EXP, "models", "seed_disagree_gate")

CORPUS = ("AVGC", "DTU", "KUL", "UHD", "USTC")
TEST_NAMES = ("AASD", "Carta2026")
P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = 0.75
HOP = 0.25

LINEAR_HP30_BASELINE = dict(
    AASD=dict(H=0.6422, Acc=53.12),
    Carta2026=dict(H=0.6520, Acc=60.54),
    LODO_mean=dict(H=0.7899, Acc=69.21),
)

# Soft: (name_prefix, stream, tau_s|None, alpha)  thr filled from train p50/p75
SOFT_BASE = []
for stream in ("hp30",):
    for tau in (None, 1.0, 4.0, 8.0):
        for alpha in (1.0, 2.0, 4.0):
            tlab = "t0" if tau is None else f"t{int(tau)}"
            SOFT_BASE.append((f"soft_{stream}_{tlab}_a{int(alpha)}", stream, tau, alpha))
for stream in ("raw_mean", "med4_hp30"):
    for alpha in (2.0,):
        for tau in (4.0,):
            tlab = f"t{int(tau)}"
            SOFT_BASE.append((f"soft_{stream}_{tlab}_a{int(alpha)}", stream, tau, alpha))

# Hard: (name_prefix, stream, tau_s, mode) mode in {hold, zero}
HARD_BASE = []
for stream in ("hp30", "raw_mean"):
    for tau in (1.0, 4.0, 8.0):
        for mode in ("hold", "zero"):
            HARD_BASE.append((f"hard_{mode}_{stream}_t{int(tau)}", stream, tau, mode))

BASELINES = (
    ("raw_mean", "raw_mean"),
    ("linear_hp30", "hp30"),
    ("med4", "med4"),
    ("med4_hp30", "med4_hp30"),
)


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def ewma_causal(x, tau):
    """Past-inclusive causal EWMA (uses x[0..t] at emit t). Instant if tau is None."""
    x = np.asarray(x, np.float64).reshape(-1)
    if tau is None or float(tau) <= 0:
        return x.copy()
    a = alpha_from_tau(tau)
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x, zi=np.asarray([x[0] * (1.0 - a)], np.float64))
    return y


def pairwise_disagree(D):
    """mean_{i<j} |s_i - s_j| over 4 seed columns. D: (T,4)."""
    D = np.asarray(D, np.float64)
    if D.ndim != 2 or D.shape[1] != 4:
        raise ValueError(f"expected (T,4) got {D.shape}")
    parts = []
    for i in range(4):
        for j in range(i + 1, 4):
            parts.append(np.abs(D[:, i] - D[:, j]))
    return np.mean(np.stack(parts, 0), axis=0)


def seed_stack(D, use_hp):
    D = np.asarray(D, np.float64)
    if use_hp:
        D = V4.highpass(D).astype(np.float64)
    return D


def evidence_from(D, stream):
    """Build 1-D evidence and the 4-seed stack used for disagree."""
    use_hp = stream in ("hp30", "med4_hp30")
    X = seed_stack(D, use_hp)
    if stream in ("raw_mean", "hp30"):
        e = X.mean(1)
    elif stream in ("med4", "med4_hp30"):
        e = np.median(X, axis=1)
    else:
        raise ValueError(stream)
    return e.astype(np.float64), X


def soft_gate(e, disagree, alpha, thr):
    g = expit(float(alpha) * (float(thr) - disagree))
    return e * g


def hard_gate(e, disagree, thr, mode):
    e = np.asarray(e, np.float64).reshape(-1)
    d = np.asarray(disagree, np.float64).reshape(-1)
    out = np.empty_like(e)
    last = 0.0
    thr = float(thr)
    for t in range(len(e)):
        if d[t] <= thr:
            last = e[t]
            out[t] = last
        else:
            out[t] = last if mode == "hold" else 0.0
    return out


def apply_variant(logits_4d, stream, gate, tau, alpha, thr, hard_mode=None):
    """gate: none | soft | hard. Returns {ds: {key: 1d}}."""
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {}
        for k, D in table.items():
            e, X = evidence_from(D, stream)
            if gate == "none":
                out[ds][k] = e
                continue
            dis = ewma_causal(pairwise_disagree(X), tau)
            if gate == "soft":
                out[ds][k] = soft_gate(e, dis, alpha, thr)
            elif gate == "hard":
                out[ds][k] = hard_gate(e, dis, thr, hard_mode)
            else:
                raise ValueError(gate)
    return out


def train_disagree_percentiles(logits_4d, datasets, stream, tau, qs=(50.0, 75.0)):
    parts = []
    for ds in datasets:
        for _k, D in logits_4d[ds].items():
            _e, X = evidence_from(D, stream)
            parts.append(ewma_causal(pairwise_disagree(X), tau))
    x = np.concatenate(parts)
    return {int(q): float(np.percentile(x, q)) for q in qs}


def emissions_from(streams, att, datasets):
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, LOOKAHEAD)


def load_lodo_4d():
    import score as S
    evs = []
    for _seed, exp in EC.SEEDS:
        ev, _info = S.load_folds(exp, "causal")
        evs.append(ev)
    logits = {}
    for ds in DS_KEYS:
        logits[ds] = {}
        for k in evs[0][ds].keys():
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            logits[ds][k] = np.stack(cols, 1).astype(np.float64)
    return logits


def pick_variant(lodo_means):
    best_name, best_key = None, None
    for name, m in lodo_means.items():
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def build_variant_list(thr_tables):
    """thr_tables[(stream, tau_key)] = {50:.., 75:..}."""
    variants = []
    for name, stream in BASELINES:
        variants.append(dict(
            name=name, stream=stream, gate="none", tau=None, alpha=None,
            thr=None, thr_q=None, hard_mode=None, selectable=True,
        ))
    for prefix, stream, tau, alpha in SOFT_BASE:
        tau_key = "t0" if tau is None else f"t{int(tau)}"
        # thr percentiles from matching stream; for tau=None use instant table
        tab = thr_tables[(stream, tau_key)]
        for q in (50, 75):
            thr = tab[q]
            variants.append(dict(
                name=f"{prefix}_p{q}", stream=stream, gate="soft", tau=tau,
                alpha=alpha, thr=thr, thr_q=q, hard_mode=None, selectable=True,
            ))
    for prefix, stream, tau, mode in HARD_BASE:
        tau_key = f"t{int(tau)}"
        tab = thr_tables[(stream, tau_key)]
        for q in (50, 75):
            thr = tab[q]
            variants.append(dict(
                name=f"{prefix}_p{q}", stream=stream, gate="hard", tau=tau,
                alpha=None, thr=thr, thr_q=q, hard_mode=mode, selectable=True,
            ))
    return variants


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"seed-disagree-gate CPU-only  p_stay {P_STAY} scale {SCALE}")
    log_line("honesty: pairwise seed |Δ|; soft/hard gate evidence; select LODO Acc then H; no AASD/Carta labels")
    log_line("NOT fuse↔hp30 MoE / emis-LL / trial-MMD / cross-stream-scale")

    N.load_all()

    log_line("loading CACHE train + external 4-seed evidence")
    cache_logits, cache_att = V4.load_train()
    src_cache = {ds: cache_logits[ds] for ds in CORPUS}
    train_att = {ds: cache_att[ds] for ds in CORPUS}
    test_logits, test_meta = V4.load_test()

    log_line("loading LODO fold 4-seed logits")
    lodo_4d = load_lodo_4d()

    # Train disagreement percentiles (CORPUS cache only) for thr grid
    thr_tables = {}
    need = set()
    for _p, stream, tau, _a in SOFT_BASE:
        need.add((stream, "t0" if tau is None else f"t{int(tau)}", tau))
    for _p, stream, tau, _m in HARD_BASE:
        need.add((stream, f"t{int(tau)}", tau))
    # also need med4 stream percentiles if any soft uses med4_hp30
    log_line("=== train disagree percentiles (CORPUS) ===")
    for stream, tau_key, tau in sorted(need, key=lambda z: (z[0], z[1])):
        pct = train_disagree_percentiles(src_cache, list(CORPUS), stream, tau)
        thr_tables[(stream, tau_key)] = pct
        log_line(f"  {stream:12s} {tau_key:4s}  p50 {pct[50]:.4f}  p75 {pct[75]:.4f}")

    variants = build_variant_list(thr_tables)
    log_line(f"n_variants {len(variants)}")

    results = dict(
        method="seed-disagree-gate on 4-seed logits (soft/hard; raw|hp30|median)",
        p_stay=P_STAY, scale=SCALE,
        honesty="pairwise |seed_i-seed_j|; gate evidence; LODO Acc then H; no AASD/Carta labels",
        baselines=LINEAR_HP30_BASELINE,
        thr_tables={f"{s}|{t}": v for (s, t), v in thr_tables.items()},
        variants={},
    )

    log_line("=== LODO (fixed HMM; selection) ===")
    lodo_means = {}
    for v in variants:
        name = v["name"]
        t0 = time.time()
        adapted = apply_variant(
            lodo_4d, v["stream"], v["gate"], v["tau"], v["alpha"], v["thr"],
            hard_mode=v["hard_mode"],
        )
        per = {}
        for held in DS_KEYS:
            off = N.official(held, N.to_ev(adapted), P_STAY, SCALE)
            per[held] = off
            log_line(f"LODO {name:36s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:36s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            stream=v["stream"], gate=v["gate"], tau=v["tau"], alpha=v["alpha"],
            thr=v["thr"], thr_q=v["thr_q"], hard_mode=v["hard_mode"],
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    selected = pick_variant(lodo_means)
    log_line(f"SELECTED (LODO Acc then H): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    results["selected"] = selected

    log_line("=== external ===")
    for v in variants:
        name = v["name"]
        adapted_train = apply_variant(
            src_cache, v["stream"], v["gate"], v["tau"], v["alpha"], v["thr"],
            hard_mode=v["hard_mode"],
        )
        mu_e, sd_e = emissions_from(adapted_train, train_att, list(CORPUS))
        adapted_test = apply_variant(
            test_logits, v["stream"], v["gate"], v["tau"], v["alpha"], v["thr"],
            hard_mode=v["hard_mode"],
        )
        ext = {}
        for ds in TEST_NAMES:
            r = score_external(adapted_test[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            base_acc = LINEAR_HP30_BASELINE[ds]["Acc"]
            dacc = r["Acc"] - base_acc
            log_line(f"EXT {name:36s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs linear_hp30 {dacc:+.2f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=prim["lodo_mean"],
        AVGC=prim["lodo"].get("AVGC"),
    )
    log_line(f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δ {d_aasd:+.2f} vs 53.12)  Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}")
    if "AVGC" in prim["lodo"]:
        av = prim["lodo"]["AVGC"]
        log_line(f"HEADLINE AVGC H {av['H']:.4f} Acc {av['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "selection.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       thr_tables=results["thr_tables"]), fh, indent=1)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
