"""Causal disagree-gate on published SW2 decoder ↔ clf evidence streams.

On frozen fuse_general SW2 stack (4 dec + 4 clf10-sw2), per frame:
  if sign(z_dec) == sign(z_clf): emit fuse_general a ∈ {0.5, 0.75}
  else (disagree): emit one of
    - clf-only (a=1)
    - decoder-only (a=0)
    - max-|logit| of (dec, clf)
    - soft |logit|-weighted mix  w=|dec|/(|dec|+|clf|)

DISTINCT from fuse-general-a-grid (fixed a only), SW2↔swmix soft-w,
tip Platt/domain-emis, IRM↔synth agree-gate.

Reuse fuse_general_a_grid / sw2 caches. CPU only.
Select by mean LODO Acc then H @ HMM 0.994/0.8 (never AASD/Carta).
Score AASD + Carta vs tip Acc 53.53.

usage:
  /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u \\
    dec_clf_disagree_gate.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np

import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import sw2_swmix_blend as SM
from common import DS_KEYS
from fuse_evidence import LA, OFF

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "dec_clf_disagree_gate", "cache")
LOG = os.path.join(EXP, "experiments", "dec_clf_disagree_gate.log")
OUT = os.path.join(EXP, "results", "research", "dec_clf_disagree_gate.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_dec_clf_disagree_gate.md")
AGRID_CACHE = os.path.join(EXP, "models", "fuse_general_a_grid", "cache", "sw2_external_agrid.pkl")

P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = float(LA)
EPS = 1e-8

LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_ALONE_EXT = dict(AASD=dict(H=0.6439, Acc=53.46), Carta2026=dict(H=0.6470, Acc=59.60))
TIP_EXT = dict(AASD=dict(H=0.6424, Acc=53.53), Carta2026=dict(H=0.6460, Acc=59.71))

# (name, agree_a, disagree_mode, selectable)
# disagree_mode: clf | dec | maxabs | soft_abs
# baseline modes (no agree): fuse | clf | dec | maxabs | soft_abs  with agree_a unused
VARIANT_SPECS = [
    ("fuse_a05", 0.5, "fuse", False),
    ("fuse_a075", 0.75, "fuse", False),
    ("pure_dec", 0.0, "dec", False),
    ("pure_clf", 1.0, "clf", False),
    ("always_maxabs", None, "maxabs", False),
    ("always_soft_abs", None, "soft_abs", False),
    ("agree_a05_clf", 0.5, "clf", True),
    ("agree_a05_dec", 0.5, "dec", True),
    ("agree_a05_maxabs", 0.5, "maxabs", True),
    ("agree_a05_soft_abs", 0.5, "soft_abs", True),
    ("agree_a075_clf", 0.75, "clf", True),
    ("agree_a075_dec", 0.75, "dec", True),
    ("agree_a075_maxabs", 0.75, "maxabs", True),
    ("agree_a075_soft_abs", 0.75, "soft_abs", True),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def _json_default(o):
    if isinstance(o, (np.floating, np.integer)):
        return float(o)
    if isinstance(o, np.ndarray):
        return o.tolist()
    raise TypeError(type(o))


def gate_1d(zd, zc, agree_a, mode):
    """Framewise dec↔clf policy.

    agree_a None → always apply mode (clf|dec|maxabs|soft_abs).
    mode fuse → fixed fuse_general mix with weight agree_a.
    else agree-* → same-sign fuse(agree_a); disagree → mode.
    """
    a = np.asarray(zd, np.float64).reshape(-1)
    b = np.asarray(zc, np.float64).reshape(-1)
    n = min(len(a), len(b))
    a, b = a[:n], b[:n]
    abs_a = np.abs(a)
    abs_b = np.abs(b)
    maxabs = np.where(abs_a >= abs_b, a, b)
    w = abs_a / (abs_a + abs_b + EPS)
    soft = w * a + (1.0 - w) * b

    def alt_for(m):
        if m == "clf":
            return b
        if m == "dec":
            return a
        if m == "maxabs":
            return maxabs
        if m == "soft_abs":
            return soft
        raise ValueError(m)

    if mode == "fuse":
        aa = float(agree_a)
        return ((1.0 - aa) * a + aa * b).astype(np.float64)
    if agree_a is None:
        return np.asarray(alt_for(mode), np.float64).copy()

    aa = float(agree_a)
    fused = (1.0 - aa) * a + aa * b
    agree = np.sign(a) == np.sign(b)
    return np.where(agree, fused, alt_for(mode)).astype(np.float64)


def call_args(name, agree_a, mode):
    """Normalize VARIANT_SPECS → (agree_a, mode) for gate_1d."""
    if name.startswith("fuse_"):
        return agree_a, "fuse"
    if name.startswith(("pure_", "always_")):
        return None, mode
    return agree_a, mode


def gate_streams(zd_all, zc_all, agree_a, mode):
    out = {}
    for ds in zd_all:
        if ds not in zc_all:
            continue
        out[ds] = {}
        for k in zd_all[ds]:
            if k not in zc_all[ds]:
                continue
            out[ds][k] = gate_1d(zd_all[ds][k], zc_all[ds][k], agree_a, mode)
    return out


def gate_held(Z, held, agree_a, mode):
    zd, zc = Z[held]
    return gate_streams(zd, zc, agree_a, mode)


def agree_stats(zd_st, zc_st):
    n_agree, n_tot = 0, 0
    for key in zd_st:
        if key not in zc_st:
            continue
        a = np.asarray(zd_st[key], np.float64).reshape(-1)
        b = np.asarray(zc_st[key], np.float64).reshape(-1)
        n = min(len(a), len(b))
        if n == 0:
            continue
        n_agree += int(np.sum(np.sign(a[:n]) == np.sign(b[:n])))
        n_tot += n
    return float(n_agree / n_tot) if n_tot else 0.0, n_tot


def mean_lodo(Z, meta, agree_a, mode):
    hs, acs, per = [], [], {}
    for held in DS_KEYS:
        stream = gate_held(Z, held, agree_a, mode)
        r = SM.score_lodo_held(stream, meta, held)
        zd, zc = Z[held]
        frac, nfr = agree_stats(zd[held], zc[held])
        hs.append(r["H"])
        acs.append(r["Acc"])
        per[held] = dict(
            H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r.get("L"),
            agree_frac=frac, n_frames=nfr,
        )
    return float(np.mean(hs)), float(np.mean(acs)), per


def corpus_emis(Z, meta, agree_a, mode):
    acc = {ds: {} for ds in DS_KEYS}
    counts = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        stream = gate_held(Z, held, agree_a, mode)
        for ds in DS_KEYS:
            for k, d in stream[ds].items():
                if k not in acc[ds]:
                    acc[ds][k] = np.zeros_like(d, np.float64)
                    counts[ds][k] = 0
                n = min(len(acc[ds][k]), len(d))
                acc[ds][k][:n] += d[:n]
                counts[ds][k] += 1
    stream = {}
    for ds in DS_KEYS:
        stream[ds] = {k: acc[ds][k] / max(counts[ds][k], 1) for k in acc[ds]}
    parts = []
    for ds in DS_KEYS:
        for k, d in stream[ds].items():
            att = meta[ds][k][0]
            n = min(len(d), len(att))
            parts.append(d[:n] * np.where(np.asarray(att)[:n] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def gate_ext_ev(ev_dec, ev_clf, agree_a, mode):
    """ev_dec / ev_clf are external evidence dicts (a=0 / a=1)."""
    out = {}
    for k, e in ev_dec.items():
        if k not in ev_clf:
            continue
        d = gate_1d(e["r0"], ev_clf[k]["r0"], agree_a, mode)
        n = len(d)
        out[k] = dict(
            r0=d,
            r1=np.zeros(n),
            att=e["att"][:n],
            t_end=e["t_end"][:n],
            att128=e["att128"],
            n128=e["n128"],
            subject=e["subject"],
        )
    return out


def resolve_a_key(ev_ds, a):
    if a in ev_ds:
        return a
    return min(ev_ds.keys(), key=lambda x: abs(float(x) - float(a)))


def write_md_abandon(reason):
    text = f"""# results_dec_clf_disagree_gate

**Status: abandoned**

Could not load SW2 fuse_general / a-grid caches.

Reason:
```
{reason}
```

## NEXT IDEA

Restore `models/sw2_hp30_blend/cache/` and `models/fuse_general_a_grid/cache/` before retrying.
Do not retrain SW2 decoder/clf for a disagree-gate.
"""
    with open(RESULTS_MD, "w") as fh:
        fh.write(text)


def write_md(summary):
    sel = summary["selected"]
    ext = summary["external"]
    lines = []
    lines.append("# results_dec_clf_disagree_gate\n")
    lines.append(
        "**Claim:** `dec-clf-disagree-gate` — causal disagree-gate on published SW2 "
        "**decoder ↔ clf** evidence (same streams as fuse_general).\n"
    )
    lines.append(
        "**Rule:** same-sign → fuse_general a∈{0.5, 0.75}; disagree → clf-only / "
        "decoder-only / max-|logit| / soft `|logit|`-weighted mix.\n"
    )
    lines.append(
        "**Distinct from** fuse-general-a-grid (fixed a), SW2↔swmix soft-w, "
        "tip Platt/domain-emis, IRM↔synth agree-gate.\n"
    )
    lines.append(
        f"**Honesty:** CPU caches only. Select by mean LODO Acc then H @ HMM "
        f"**{P_STAY}/{SCALE}**. Never AASD/Carta for selection. "
        f"Code: `dec_clf_disagree_gate.py`. JSON: `results/research/dec_clf_disagree_gate.json`.\n"
    )
    lines.append("## Selection\n")
    lines.append(
        f"**SELECTED `{sel['name']}`** — mean LODO H **{sel['mean_LODO_H']:.4f}** "
        f"Acc **{sel['mean_LODO_Acc']:.2f}**\n"
    )
    lines.append("| Held-out | H | Acc % | agree frac |")
    lines.append("|---|---:|---:|---:|")
    for ds in DS_KEYS:
        p = sel["per_ds"][ds]
        lines.append(
            f"| {ds} | {p['H']:.4f} | {p['Acc']:.2f} | {p.get('agree_frac', float('nan')):.3f} |"
        )
    lines.append(
        f"| **mean** | **{sel['mean_LODO_H']:.4f}** | **{sel['mean_LODO_Acc']:.2f}** | |\n"
    )
    lines.append("### Variant grid\n")
    lines.append("| Rule | mean H | mean Acc | selectable |")
    lines.append("|---|---:|---:|---|")
    for r in summary["lodo_grid"]:
        mark = " **←**" if r["name"] == sel["name"] else ""
        lines.append(
            f"| `{r['name']}`{mark} | {r['mean_LODO_H']:.4f} | {r['mean_LODO_Acc']:.2f} | "
            f"{'yes' if r['selectable'] else 'no'} |"
        )
    lines.append("\n## External (preset)\n")
    lines.append("| Set | selected H/Acc | tip Acc leader | SW2 a=0.5 | linear_hp30 |")
    lines.append("|---|---|---|---|---|")
    for ds in ("AASD", "Carta2026"):
        s = ext[ds]["selected"]
        tip = TIP_EXT[ds]
        swb = SW2_ALONE_EXT[ds]
        b = LINEAR_HP30_EXT[ds]
        lines.append(
            f"| {ds} | {s['H']:.4f} / **{s['Acc']:.2f}** | {tip['H']:.4f}/{tip['Acc']:.2f} | "
            f"{swb['H']:.4f}/{swb['Acc']:.2f} | {b['H']:.4f}/{b['Acc']:.2f} |"
        )
    aasd = ext["AASD"]["selected"]
    carta = ext["Carta2026"]["selected"]
    d_tip = aasd["Acc"] - TIP_EXT["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    d_carta = carta["Acc"] - LINEAR_HP30_EXT["Carta2026"]["Acc"]
    lines.append("\n## Targets\n")
    lines.append("| Target | Result |")
    lines.append("|---|---|")
    lines.append(
        f"| AASD Acc vs tip 53.53 / hp30 53.12 | Acc **{aasd['Acc']:.2f}** "
        f"(Δtip {d_tip:+.2f}; Δhp {d_hp:+.2f}); H {aasd['H']:.4f} |"
    )
    lines.append(
        f"| Carta vs hp30 60.54 | Acc **{carta['Acc']:.2f}** (Δ {d_carta:+.2f}); "
        f"H {carta['H']:.4f} |"
    )
    lines.append(
        f"| LODO mean | H {sel['mean_LODO_H']:.4f} Acc {sel['mean_LODO_Acc']:.2f} |"
    )
    lines.append(f"| AASD Acc>60? | {'YES' if aasd['Acc'] > 60 else 'NO'} |\n")
    lines.append("## Verdict\n")
    if aasd["Acc"] > 60:
        lines.append("Selected dec↔clf disagree-gate clears AASD Acc>60.\n")
    else:
        lines.append(
            "Honest LODO Acc/H selector over dec↔clf same-sign→fuse / disagree policies "
            "does not create AASD Acc>60. Sign-gated mix of published SW2 streams stays "
            "in the Acc mid-50s portfolio.\n"
        )
    lines.append("## Do not repeat\n")
    lines.append(
        "Do not re-grid SW2 decoder↔clf same-sign→a∈{0.5,0.75} / disagree "
        "clf|dec|maxabs|soft_|logit| at this HMM under LODO Acc/H.\n"
    )
    lines.append("## NEXT IDEA\n")
    lines.append(summary.get("next_idea", "") + "\n")
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(
        f"dec-clf-disagree-gate start  p_stay={P_STAY} scale={SCALE} OFF={OFF}"
    )
    log_line("agree → fuse a∈{0.5,0.75}; disagree → clf|dec|maxabs|soft_|logit|")
    log_line("select by mean LODO Acc then H; never AASD/Carta for selection")

    try:
        Z, meta = SM.ensure_sw2_lodo()
    except FileNotFoundError as e:
        log_line(f"ABORT: {e}")
        summary = dict(status="abandoned", reason=str(e))
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2)
        write_md_abandon(str(e))
        return

    # sanity: fuse a=0.5 on AVGC
    r05 = SM.score_lodo_held(gate_held(Z, "AVGC", 0.5, "fuse"), meta, "AVGC")
    log_line(
        f"sanity AVGC fuse_a05 H {r05['H']:.4f} Acc {r05['Acc']:.2f} "
        f"(expect ~0.71 / ~68 from sw2_swmix_blend)"
    )
    zd, zc = Z["AVGC"]
    frac_a, n_a = agree_stats(zd["AVGC"], zc["AVGC"])
    log_line(f"sanity AVGC sign-agree frac {frac_a:.4f} n={n_a}")

    rows = []
    best = None
    log_line(f"scoring {len(VARIANT_SPECS)} variants on mean LODO Acc then H")
    for name, agree_a, mode, selectable in VARIANT_SPECS:
        t0 = time.time()
        aa, md = call_args(name, agree_a, mode)
        mean_h, mean_acc, per = mean_lodo(Z, meta, aa, md)
        row = dict(
            name=name,
            agree_a=agree_a,
            mode=mode,
            selectable=selectable,
            mean_LODO_H=mean_h,
            mean_LODO_Acc=mean_acc,
            per_ds=per,
        )
        rows.append(row)
        log_line(
            f"  LODO {name:22s} H {mean_h:.4f} Acc {mean_acc:.2f} "
            f"({'sel' if selectable else 'ref'}) ({time.time() - t0:.0f}s)"
        )
        if selectable:
            key = (mean_acc, mean_h)
            if best is None or key > best[0]:
                best = (key, row)

    selected = best[1]
    ranked = sorted(
        [r for r in rows if r["selectable"]],
        key=lambda r: (r["mean_LODO_Acc"], r["mean_LODO_H"]),
        reverse=True,
    )
    log_line(
        f"SELECTED {selected['name']} mean LODO H {selected['mean_LODO_H']:.4f} "
        f"Acc {selected['mean_LODO_Acc']:.2f}"
    )
    log_line(f"ranked: {[r['name'] for r in ranked]}")

    if not os.path.isfile(AGRID_CACHE):
        msg = f"missing a-grid external cache: {AGRID_CACHE}"
        log_line(f"ABORT external: {msg}")
        summary = dict(
            status="partial",
            selected=selected,
            lodo_grid=rows,
            reason=msg,
        )
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2, default=_json_default)
        write_md(
            dict(
                selected=selected,
                lodo_grid=rows,
                external={
                    ds: {"selected": dict(H=float("nan"), Acc=float("nan"))}
                    for ds in ("AASD", "Carta2026")
                },
                next_idea="Restore fuse_general_a_grid external cache; retry AASD/Carta only.",
            )
        )
        return

    log_line(f"cache hit {AGRID_CACHE}")
    EV, emis_agrid = SM.load_pkl(AGRID_CACHE)

    def ext_for_variant(name, agree_a, mode):
        aa, md = call_args(name, agree_a, mode)
        mu, sd = corpus_emis(Z, meta, aa, md)
        out = {}
        for ds in ("AASD", "Carta2026"):
            k0 = resolve_a_key(EV[ds], 0.0)
            k1 = resolve_a_key(EV[ds], 1.0)
            ev_use = gate_ext_ev(EV[ds][k0], EV[ds][k1], aa, md)
            r = E.score(ev_use, mu, sd * SCALE, P_STAY, LOOKAHEAD)
            pl = E.plain(ev_use)
            out[ds] = dict(
                H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r["L"],
                acc1=pl["acc1"], nSwitches=r["nSwitches"], nUndetected=r["nUndetected"],
                emis_mu=mu, emis_sd=sd,
            )
        return out

    saa, smd = call_args(selected["name"], selected["agree_a"], selected["mode"])
    mu_c, sd_c = corpus_emis(Z, meta, saa, smd)
    log_line(f"corpus gated emis ({selected['name']}) mu={mu_c:.4f} sd={sd_c:.4f}")

    # score all variants externally (CPU; reuse Z/EV)
    ext_rows = {}
    for name, agree_a, mode, _sel in VARIANT_SPECS:
        t0 = time.time()
        ex = ext_for_variant(name, agree_a, mode)
        for ds in ("AASD", "Carta2026"):
            r = ex[ds]
            log_line(
                f"EXT {name:22s} {ds:10s} H {r['H']:.4f} Acc {r['Acc']:.2f} "
                f"({time.time()-t0:.0f}s)"
            )
        if name == selected["name"]:
            ext_rows = {ds: dict(selected=ex[ds]) for ds in ("AASD", "Carta2026")}
        for rrow in rows:
            if rrow["name"] == name:
                rrow["external"] = ex

    aasd_acc = ext_rows["AASD"]["selected"]["Acc"]
    next_idea = (
        "Do not re-grid SW2 decoder↔clf disagree-gates under LODO Acc/H — Acc stays mid-50s. "
        "Prefer a qualitatively new Acc front-end (clf denser switch FT already in flight) "
        "or Carta-side path that preserves hp30 Acc 60.54 without collapsing AASD. "
        "Keep tip Acc 53.53 and linear_hp30 Carta 60.54 as portfolio leaders."
    )
    if aasd_acc > 60:
        next_idea = (
            "dec↔clf disagree-gate clears Acc>60 — freeze rule and probe Carta stability; "
            "avoid soft-w Acc re-grids."
        )

    summary = dict(
        status="done",
        method="dec-clf-disagree-gate",
        selected=selected,
        p_stay=P_STAY,
        scale=SCALE,
        OFF=OFF,
        lodo_grid=rows,
        ranked=[r["name"] for r in ranked],
        external=ext_rows,
        baselines=dict(
            linear_hp30_ext=LINEAR_HP30_EXT,
            sw2_alone_ext=SW2_ALONE_EXT,
            tip_ext=TIP_EXT,
        ),
        sanity=dict(AVGC_fuse_a05=r05, AVGC_agree_frac=frac_a),
        next_idea=next_idea,
        elapsed_s=time.time() - t_all,
    )
    with open(OUT, "w") as fh:
        json.dump(summary, fh, indent=2, default=_json_default)
    write_md(summary)
    log_line(f"done in {time.time() - t_all:.0f}s → {OUT}")


if __name__ == "__main__":
    main()
