"""Grid fuse_general mix weight a on published SW2 decoder↔clf evidence.

On frozen SW2 stack (4 dec + 4 clf10-sw2):
  d = (1-a) z_dec + a z_clf,  a ∈ {0, 0.25, 0.5, 0.75, 1.0}

Optionally also score each a then Acc-tip hard_abs_swmix|t1 (fixed tip rule from
sw2_swmix_blend) — do NOT re-grid soft-w SW2↔swmix/hp30.

Select by mean LODO Acc then H at HMM p_stay=0.994 scale=0.8 (never AASD/Carta).
Score AASD + Carta vs tip Acc 53.53 / Carta hp30 60.54.

DISTINCT from sw2-swmix-blend soft-w, tri-acc, tip Platt/domain-emis, IRM↔synth.
Reuse sw2_swmix_blend / fuse_general caches. CPU only.

usage:
  /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u fuse_general_a_grid.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np

import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import fuse_external as FE
import sw2_hp30_blend as SW2
import sw2_swmix_blend as SM
from common import DS_KEYS
from fuse_evidence import LA, OFF

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "fuse_general_a_grid", "cache")
LOG = os.path.join(EXP, "experiments", "fuse_general_a_grid.log")
OUT = os.path.join(EXP, "results", "research", "fuse_general_a_grid.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_fuse_general_a_grid.md")

P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = float(LA)

A_GRID = (0.0, 0.25, 0.5, 0.75, 1.0)
# Acc tip hard rule from sw2_swmix_blend — fixed, not re-gridded
TIP_KIND, TIP_THR, TIP_SHARP = "hard_abs_swmix", 1.0, 1.0

LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_ALONE_EXT = dict(AASD=dict(H=0.6439, Acc=53.46), Carta2026=dict(H=0.6470, Acc=59.60))
TIP_EXT = dict(AASD=dict(H=0.6424, Acc=53.53), Carta2026=dict(H=0.6460, Acc=59.71))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def _json_default(o):
    if isinstance(o, (np.floating, np.integer)):
        return float(o)
    if isinstance(o, np.ndarray):
        return o.tolist()
    raise TypeError(type(o))


def cand_key(a, with_tip):
    base = f"a={a:g}"
    if with_tip:
        return f"{base}+{TIP_KIND}|t{TIP_THR:g}|s{TIP_SHARP:g}"
    return base


def mean_lodo_for_a(Z, meta, swmix_full, a, with_tip):
    hs, acs, per = [], [], {}
    for held in DS_KEYS:
        stream = SM.fuse_sw2_held(Z, held, a=a)
        if with_tip:
            sm = SM.align_swmix_to_sw2(swmix_full, stream)
            stream = SM.apply_gate_all(stream, sm, TIP_KIND, TIP_THR, TIP_SHARP)
        r = SM.score_lodo_held(stream, meta, held)
        hs.append(r["H"])
        acs.append(r["Acc"])
        per[held] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r.get("L"))
    return float(np.mean(hs)), float(np.mean(acs)), per


def ensure_sw2_external_agrid():
    """External EV for all a in A_GRID (reuse pickles; cache locally)."""
    path = os.path.join(CACHE_DIR, "sw2_external_agrid.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return SM.load_pkl(path)
    # Prefer extending a=0.5 cache by rebuilding once for full grid
    log_line(f"building SW2 external fuse for a∈{list(A_GRID)} (dec-tag _bestP) …")
    t0 = time.time()
    decs = [x.split(":") for x in SW2.EXT_DECS]
    clfs = [x.split(":") for x in SW2.EXT_CLFS]
    res = FE.RES
    missing = []
    for d in decs:
        for ds_tag in ("aasd_c1_bestP", "ext_c1_Carta2026_bestP"):
            p = os.path.join(res, f"{ds_tag}_evidence_s{d[1]}.pkl")
            if not os.path.exists(p):
                missing.append(p)
    for c in clfs:
        for ds in ("AASD", "Carta2026"):
            p = os.path.join(res, f"ext_clf_{ds}{c[1]}_evidence_s{c[2]}.pkl")
            if not os.path.exists(p):
                missing.append(p)
    if missing:
        raise FileNotFoundError(
            "SW2 external evidence pickles missing:\n" + "\n".join(missing[:12])
        )
    EV, emis = FE.build_external(decs, clfs, list(A_GRID), ["AASD", "Carta2026"], "_bestP")
    SM.save_pkl(path, (EV, emis))
    log_line(f"SW2 external a-grid cache wrote ({time.time() - t0:.0f}s)")
    return EV, emis


def corpus_emis_for_a(Z, meta, swmix_full, a, with_tip):
    """Signed mean/std on corpus blended streams (mean over held fits)."""
    acc = {ds: {} for ds in DS_KEYS}
    counts = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        stream = SM.fuse_sw2_held(Z, held, a=a)
        if with_tip:
            sm = SM.align_swmix_to_sw2(swmix_full, stream)
            stream = SM.apply_gate_all(stream, sm, TIP_KIND, TIP_THR, TIP_SHARP)
        for ds in DS_KEYS:
            for k, d in stream[ds].items():
                if k not in acc[ds]:
                    acc[ds][k] = np.zeros_like(d, np.float64)
                    counts[ds][k] = 0
                n = min(len(acc[ds][k]), len(d))
                acc[ds][k][:n] += d[:n]
                counts[ds][k] += 1
    stream = {}
    for ds in DS_KEYS:
        stream[ds] = {k: acc[ds][k] / max(counts[ds][k], 1) for k in acc[ds]}
    parts = []
    for ds in DS_KEYS:
        for k, d in stream[ds].items():
            att = meta[ds][k][0]
            n = min(len(d), len(att))
            parts.append(d[:n] * np.where(np.asarray(att)[:n] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def write_md_abandon(reason):
    text = f"""# results_fuse_general_a_grid

**Status: abandoned**

Could not load SW2 fuse_general and/or swmix caches.

Reason:
```
{reason}
```

## NEXT IDEA

Restore `models/sw2_hp30_blend/cache/` and `models/sw2_swmix_blend/cache/` before retrying.
Do not retrain SW2 decoder/clf for an a-grid.
"""
    with open(RESULTS_MD, "w") as fh:
        fh.write(text)


def write_md(summary):
    sel = summary["selected"]
    ext = summary["external"]
    lines = []
    lines.append("# results_fuse_general_a_grid\n")
    lines.append(
        "Grid **`fuse_general` mix weight a** on published SW2 decoder↔clf evidence streams "
        f"(4 dec + 4 clf10-sw2): `d=(1-a)z_dec + a·z_clf`, a∈{{{', '.join(str(x) for x in A_GRID)}}}. "
        f"Optional: each a then fixed Acc-tip `{TIP_KIND}|t{TIP_THR:g}|s{TIP_SHARP:g}` "
        "(from sw2_swmix_blend). **Not** soft-w SW2↔swmix/hp30 re-grid, ≠ tip Platt/domain-emis, "
        "≠ IRM↔synth post-process.\n"
    )
    lines.append(
        f"HMM preset **{P_STAY} / {SCALE}**. Code: `fuse_general_a_grid.py`. "
        f"JSON: `results/research/fuse_general_a_grid.json`.\n"
    )
    lines.append("## Selection (mean LODO Acc then H; never AASD/Carta)\n")
    lines.append(
        f"**SELECTED `{sel['key']}`** — mean LODO H **{sel['mean_LODO_H']:.4f}** "
        f"Acc **{sel['mean_LODO_Acc']:.2f}**\n"
    )
    lines.append("| Held-out | H | Acc % |")
    lines.append("|---|---:|---:|")
    for ds in DS_KEYS:
        p = sel["per_ds"][ds]
        lines.append(f"| {ds} | {p['H']:.4f} | {p['Acc']:.2f} |")
    lines.append(
        f"| **mean** | **{sel['mean_LODO_H']:.4f}** | **{sel['mean_LODO_Acc']:.2f}** |\n"
    )
    lines.append("### a-grid (+ optional tip)\n")
    lines.append("| Rule | mean H | mean Acc |")
    lines.append("|---|---:|---:|")
    for r in summary["lodo_grid"]:
        mark = " **←**" if r["key"] == sel["key"] else ""
        lines.append(
            f"| `{r['key']}`{mark} | {r['mean_LODO_H']:.4f} | {r['mean_LODO_Acc']:.2f} |"
        )
    lines.append("\n## External (preset)\n")
    lines.append("| Set | selected H/Acc | tip Acc leader | SW2 a=0.5 | linear_hp30 |")
    lines.append("|---|---|---|---|---|")
    for ds in ("AASD", "Carta2026"):
        s = ext[ds]["selected"]
        tip = TIP_EXT[ds]
        swb = SW2_ALONE_EXT[ds]
        b = LINEAR_HP30_EXT[ds]
        lines.append(
            f"| {ds} | {s['H']:.4f} / **{s['Acc']:.2f}** | {tip['H']:.4f}/{tip['Acc']:.2f} | "
            f"{swb['H']:.4f}/{swb['Acc']:.2f} | {b['H']:.4f}/{b['Acc']:.2f} |"
        )
    aasd = ext["AASD"]["selected"]
    carta = ext["Carta2026"]["selected"]
    d_tip = aasd["Acc"] - TIP_EXT["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    d_carta = carta["Acc"] - LINEAR_HP30_EXT["Carta2026"]["Acc"]
    lines.append("\n## Targets\n")
    lines.append("| Target | Result |")
    lines.append("|---|---|")
    lines.append(
        f"| AASD Acc vs tip 53.53 / hp30 53.12 | Acc **{aasd['Acc']:.2f}** "
        f"(Δtip {d_tip:+.2f}; Δhp {d_hp:+.2f}); H {aasd['H']:.4f} |"
    )
    lines.append(
        f"| Carta vs hp30 60.54 | Acc **{carta['Acc']:.2f}** (Δ {d_carta:+.2f}); "
        f"H {carta['H']:.4f} |"
    )
    lines.append(
        f"| LODO mean | H {sel['mean_LODO_H']:.4f} Acc {sel['mean_LODO_Acc']:.2f} |"
    )
    lines.append(f"| AASD Acc>60? | {'YES' if aasd['Acc'] > 60 else 'NO'} |\n")
    lines.append("## Verdict\n")
    if aasd["Acc"] > 60:
        lines.append("Selected fuse_general a clears AASD Acc>60.\n")
    else:
        lines.append(
            "Honest LODO Acc/H selector over fuse_general a (and a+tip hard_abs) does not "
            "create AASD Acc>60. Mix weight a alone stays in the Acc mid-50s portfolio.\n"
        )
    lines.append("## NEXT IDEA\n")
    lines.append(summary.get("next_idea", "") + "\n")
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(
        f"fuse-general-a-grid start  p_stay={P_STAY} scale={SCALE} "
        f"a_grid={list(A_GRID)} tip={TIP_KIND}|t{TIP_THR}|s{TIP_SHARP} OFF={OFF}"
    )

    try:
        Z, meta = SM.ensure_sw2_lodo()
        swmix_full, la_sm = SM.ensure_swmix_lodo()
        log_line(f"lookahead LA={LOOKAHEAD} swmix_fold_la={la_sm}")
    except FileNotFoundError as e:
        log_line(f"ABORT: {e}")
        summary = dict(
            status="abandoned",
            reason=str(e),
            next_idea="Restore SW2 / swmix caches; do not rebuild decoder LODO for a-grid.",
        )
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2)
        write_md_abandon(str(e))
        return

    # sanity: published a=0.5 on AVGC
    r05 = SM.score_lodo_held(SM.fuse_sw2_held(Z, "AVGC", a=0.5), meta, "AVGC")
    log_line(
        f"sanity AVGC a=0.5 H {r05['H']:.4f} Acc {r05['Acc']:.2f} "
        f"(expect ~0.71 / ~68 from sw2_swmix_blend)"
    )

    candidates = []
    for a in A_GRID:
        candidates.append((a, False))
    for a in A_GRID:
        candidates.append((a, True))

    log_line(f"scoring {len(candidates)} candidates on mean LODO Acc then H")
    rows = []
    best = None
    for a, with_tip in candidates:
        t0 = time.time()
        mean_h, mean_a, per = mean_lodo_for_a(Z, meta, swmix_full, a, with_tip)
        key = cand_key(a, with_tip)
        row = dict(
            a=a,
            with_tip=with_tip,
            key=key,
            mean_LODO_H=mean_h,
            mean_LODO_Acc=mean_a,
            per_ds=per,
        )
        rows.append(row)
        log_line(
            f"  LODO {key:40s} H {mean_h:.4f} Acc {mean_a:.2f} ({time.time() - t0:.0f}s)"
        )
        sel = (mean_a, mean_h)
        if best is None or sel > best[0]:
            best = (sel, row)

    selected = best[1]
    log_line(
        f"SELECTED {selected['key']} mean LODO H {selected['mean_LODO_H']:.4f} "
        f"Acc {selected['mean_LODO_Acc']:.2f}"
    )

    try:
        EV, emis = ensure_sw2_external_agrid()
        sm_ext = SM.ensure_swmix_external()
    except FileNotFoundError as e:
        log_line(f"ABORT external: {e}")
        summary = dict(
            status="partial",
            selected=selected,
            lodo_grid=rows,
            reason=str(e),
            next_idea="Restore SW2 external evidence pickles under results/research/.",
        )
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2, default=_json_default)
        write_md(
            dict(
                selected=selected,
                lodo_grid=rows,
                external={
                    ds: {
                        "selected": dict(H=float("nan"), Acc=float("nan")),
                    }
                    for ds in ("AASD", "Carta2026")
                },
                next_idea="Restore external pickles; retry AASD/Carta score only.",
            )
        )
        return

    a_sel = float(selected["a"])
    with_tip = bool(selected["with_tip"])
    if with_tip:
        mu_c, sd_c = corpus_emis_for_a(Z, meta, swmix_full, a_sel, True)
        log_line(f"corpus tip-blend emis a={a_sel:g} mu={mu_c:.4f} sd={sd_c:.4f}")
    else:
        mu_c, sd_c = emis[a_sel]
        log_line(f"corpus FE emis a={a_sel:g} mu={mu_c:.4f} sd={sd_c:.4f}")

    ext_rows = {}
    for ds in ("AASD", "Carta2026"):
        # float keys from pickle may be exact; tolerate tiny float mismatch
        if a_sel in EV[ds]:
            sw2_ev = EV[ds][a_sel]
        else:
            key_a = min(EV[ds].keys(), key=lambda x: abs(float(x) - a_sel))
            sw2_ev = EV[ds][key_a]
        if with_tip:
            sm_al = SM.align_ext_swmix_to_sw2(sm_ext[ds], sw2_ev)
            ev_use = SM.blend_ext_ev(sw2_ev, sm_al, TIP_KIND, TIP_THR, TIP_SHARP)
            mu, sd = mu_c, sd_c
        else:
            ev_use = sw2_ev
            mu, sd = mu_c, sd_c
        r = E.score(ev_use, mu, sd * SCALE, P_STAY, LOOKAHEAD)
        pl = E.plain(ev_use)
        ext_rows[ds] = dict(
            selected=dict(
                H=r["H"],
                Acc=r["Acc"],
                SDT=r["SDT"],
                L=r["L"],
                acc1=pl["acc1"],
                nSwitches=r["nSwitches"],
                nUndetected=r["nUndetected"],
            )
        )
        log_line(
            f"EXT {ds} selected H {r['H']:.4f} Acc {r['Acc']:.2f} "
            f"(tip {TIP_EXT[ds]['Acc']:.2f}; SW2 {SW2_ALONE_EXT[ds]['Acc']:.2f}; "
            f"hp30 {LINEAR_HP30_EXT[ds]['Acc']:.2f})"
        )

    aasd_acc = ext_rows["AASD"]["selected"]["Acc"]
    next_idea = (
        "Do not re-grid fuse_general a on published SW2 under LODO Acc/H — Acc stays mid-50s. "
        "Prefer a qualitatively new Acc front-end (clf denser switch FT already in flight) "
        "or Carta-side path that preserves hp30 Acc 60.54 without collapsing AASD. "
        "Keep tip Acc 53.53 and linear_hp30 Carta 60.54 as portfolio leaders."
    )
    if aasd_acc > 60:
        next_idea = (
            "fuse_general a selected clears Acc>60 — freeze mix weight and probe Carta "
            "stability; avoid soft-w Acc re-grids."
        )

    summary = dict(
        status="done",
        method="fuse-general-a-grid",
        selected=selected,
        p_stay=P_STAY,
        scale=SCALE,
        a_grid=list(A_GRID),
        tip_rule=dict(kind=TIP_KIND, thresh=TIP_THR, sharp=TIP_SHARP),
        OFF=OFF,
        lodo_grid=rows,
        external=ext_rows,
        baselines=dict(
            linear_hp30_ext=LINEAR_HP30_EXT,
            sw2_alone_ext=SW2_ALONE_EXT,
            tip_ext=TIP_EXT,
        ),
        next_idea=next_idea,
        elapsed_s=time.time() - t_all,
    )
    with open(OUT, "w") as fh:
        json.dump(summary, fh, indent=2, default=_json_default)
    write_md(summary)
    log_line(f"done in {time.time() - t_all:.0f}s → {OUT}")


if __name__ == "__main__":
    main()
