"""SW2 as a 5th raw logit seed alongside 4 direct-classifier seeds.

DISTINCT from sw2_hp30_blend (soft blend of SW2 ↔ already-formed linear_hp30).
Here SW2 fuse_general a=0.5 evidence is an extra channel BEFORE aggregation:

  mean5        equal mean of [s0,s1,s2,s3,sw2] (DEC_OFFSET-aligned)
  hp30_mean5   causal one-pole HP τ=30 on mean5
  mean_hp5     per-channel HP τ=30 then equal mean of 5
  soft_mean4_sw2_w∈{0..1}  (1-w)·mean4 + w·sw2   (raw mean4 ↔ SW2; NOT ↔ hp30)

Selector: mean LODO Acc then H at HMM p_stay=0.994 scale=0.8. Never AASD/Carta labels.
Score 5 LODO held-outs + AASD + Carta2026. CPU only.

Reuses SW2 LODO/external caches from sw2_hp30_blend.py (DEC_OFFSET / OFF).

usage:
  /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u sw2_5th_seed.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
import sw2_hp30_blend as SW2
from common import DS_KEYS
from fuse_evidence import LA, OFF

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "sw2_5th_seed", "cache")
LOG = os.path.join(EXP, "experiments", "sw2_5th_seed.log")
OUT = os.path.join(EXP, "results", "research", "sw2_5th_seed.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_sw2_5th_seed.md")

P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = float(LA)
A_SW2 = SW2.A_SW2
HOP = 0.25
TAU = 30.0

LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_ALONE_EXT = dict(AASD=dict(H=0.6439, Acc=53.46), Carta2026=dict(H=0.6470, Acc=59.60))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def save_pkl(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as fh:
        pickle.dump(obj, fh, protocol=pickle.HIGHEST_PROTOCOL)


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def highpass_1d(x, tau=TAU, hop=HOP):
    return V4.highpass(np.asarray(x, np.float64), tau=float(tau), hop=hop)


def highpass_2d(D, tau=TAU, hop=HOP):
    return V4.highpass(np.asarray(D, np.float64), tau=float(tau), hop=hop)


def ensure_seeds4_lodo():
    """Full-length 4-seed classifier deltas {ds: {key: (T,4)}}."""
    path = os.path.join(CACHE_DIR, "seeds4_lodo.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    log_line("loading 4-seed LODO folds …")
    t0 = time.time()
    evs, lookahead = [], None
    for _seed, exp in EC.SEEDS:
        ev, info = S.load_folds(exp, "causal")
        evs.append(ev)
        for ds in DS_KEYS:
            la = float(info[ds]["lookahead_s"])
            lookahead = la if lookahead is None else lookahead
    logits = {}
    for ds in DS_KEYS:
        logits[ds] = {}
        for k in evs[0][ds].keys():
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            logits[ds][k] = np.stack(cols, 1).astype(np.float64)
    save_pkl(path, (logits, lookahead))
    log_line(f"seeds4 LODO cache wrote ({time.time() - t0:.0f}s) lookahead={lookahead}")
    return logits, lookahead


def align_seeds_to_sw2(seeds4_ds, sw2_ds, off=OFF):
    """Slice (T,4) seed matrix to SW2 length with DEC_OFFSET; return (n,4) or None."""
    out = {}
    for k, f in sw2_ds.items():
        if k not in seeds4_ds:
            continue
        D = np.asarray(seeds4_ds[k], np.float64)
        n = len(f)
        if D.ndim != 2 or D.shape[1] != 4:
            raise ValueError(f"expected (T,4) seeds for {k}, got {D.shape}")
        if len(D) >= off + n:
            out[k] = D[off:off + n].copy()
        else:
            sl = D[off:off + n] if len(D) > off else D[:n]
            pad = np.zeros((n, 4), np.float64)
            pad[: len(sl)] = sl
            out[k] = pad
    return out


def stack5(seeds_al, sw2_ds):
    """{key: (n,5)} = [s0,s1,s2,s3,sw2]."""
    out = {}
    for k, f in sw2_ds.items():
        if k not in seeds_al:
            continue
        S4 = seeds_al[k]
        n = min(len(S4), len(f))
        out[k] = np.column_stack([S4[:n], np.asarray(f, np.float64)[:n]])
    return out


def agg_stream(X5_ds, kind, w=None):
    """Reduce {key:(n,5)} → {key:(n,)}."""
    out = {}
    for k, X in X5_ds.items():
        X = np.asarray(X, np.float64)
        s4 = X[:, :4]
        sw = X[:, 4]
        mean4 = s4.mean(1)
        mean5 = X.mean(1)
        if kind == "mean4":
            out[k] = mean4
        elif kind == "mean5":
            out[k] = mean5
        elif kind == "hp30_mean4":
            out[k] = highpass_1d(mean4)
        elif kind == "hp30_mean5":
            out[k] = highpass_1d(mean5)
        elif kind == "mean_hp5":
            out[k] = highpass_2d(X).mean(1)
        elif kind == "mean_hp4":
            out[k] = highpass_2d(s4).mean(1)
        elif kind == "soft_mean4_sw2":
            ww = float(w)
            out[k] = (1.0 - ww) * mean4 + ww * sw
        else:
            raise ValueError(kind)
    return out


def apply_rule_all(seeds4, sw2_all, kind, w=None):
    out = {}
    for ds in sw2_all:
        if ds not in seeds4:
            continue
        al = align_seeds_to_sw2(seeds4[ds], sw2_all[ds])
        X5 = stack5(al, sw2_all[ds])
        out[ds] = agg_stream(X5, kind, w=w)
    return out


def candidate_rules():
    rules = [
        ("mean4", None),
        ("mean5", None),
        ("hp30_mean4", None),
        ("hp30_mean5", None),
        ("mean_hp4", None),
        ("mean_hp5", None),
    ]
    for w in (0.0, 0.25, 0.5, 0.75, 1.0):
        rules.append(("soft_mean4_sw2", w))
    return rules


def rule_key(kind, w):
    if kind == "soft_mean4_sw2":
        return f"soft_mean4_sw2_w={w:g}"
    return kind


def eligible(name):
    """Baselines mean4 / hp30_mean4 / mean_hp4 are reported but not preferred as 'new' winners
    unless they beat novel 5-channel / soft-mean4 rules on Acc then H — still selectable so
    honesty can keep them. Exclude nothing except we never include soft_hp30↔sw2.
    """
    return True


def ev_from_stream(stream, meta):
    ev = {}
    for ds in DS_KEYS:
        ev[ds] = {}
        for k, d in stream[ds].items():
            att, te = meta[ds][k]
            d = np.asarray(d, np.float64)
            n = len(d)
            ev[ds][k] = dict(
                r0=d, r1=np.zeros(n), att=np.asarray(att)[:n],
                t_end=np.asarray(te, np.float64)[:n], pre_switched=True,
            )
    return ev


def score_lodo_held(stream_all, meta, held):
    ev = ev_from_stream(stream_all, meta)
    V4._install_scorer_state(ev, LOOKAHEAD)
    return N.official(held, ev, P_STAY, SCALE)


def mean_lodo_for_rule(Z, meta, seeds4, kind, w):
    hs, acs, per = [], [], {}
    for held in DS_KEYS:
        sw2 = SW2.fuse_sw2_held(Z, held, a=A_SW2)
        stream = apply_rule_all(seeds4, sw2, kind, w=w)
        r = score_lodo_held(stream, meta, held)
        hs.append(r["H"])
        acs.append(r["Acc"])
        per[held] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r.get("L"))
    return float(np.mean(hs)), float(np.mean(acs)), per


def emissions_signed(stream_all, meta, datasets):
    parts = []
    for ds in datasets:
        for k, d in stream_all[ds].items():
            att = meta[ds][k][0]
            n = min(len(d), len(att))
            parts.append(np.asarray(d[:n], np.float64) * np.where(np.asarray(att)[:n] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def corpus_stream_mean_held(Z, meta, seeds4, kind, w):
    """Average held-specific streams over 5 LODO weight fits (same soft OOF as sw2_hp30_blend)."""
    acc = {ds: {} for ds in DS_KEYS}
    counts = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        sw2 = SW2.fuse_sw2_held(Z, held, a=A_SW2)
        stream = apply_rule_all(seeds4, sw2, kind, w=w)
        for ds in DS_KEYS:
            for k, d in stream[ds].items():
                if k not in acc[ds]:
                    acc[ds][k] = np.zeros_like(d, np.float64)
                    counts[ds][k] = 0
                n = min(len(acc[ds][k]), len(d))
                acc[ds][k][:n] += d[:n]
                counts[ds][k] += 1
    return {ds: {k: acc[ds][k] / max(counts[ds][k], 1) for k in acc[ds]} for ds in DS_KEYS}


def ensure_seeds4_external():
    path = os.path.join(CACHE_DIR, "seeds4_external.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    log_line("loading 4-seed external (V4) …")
    train_logits, train_att = V4.load_train()
    test_logits, test_meta = V4.load_test()
    save_pkl(path, (train_logits, train_att, test_logits, test_meta))
    return train_logits, train_att, test_logits, test_meta


def apply_rule_ext(seeds4_ds, sw2_ev, kind, w=None):
    """External: sw2_ev is {key: evidence dict}; seeds4_ds is {key:(T,4)}."""
    sw2_1d = {k: np.asarray(e["r0"], np.float64) for k, e in sw2_ev.items()}
    al = align_seeds_to_sw2(seeds4_ds, sw2_1d)
    X5 = stack5(al, sw2_1d)
    stream = agg_stream(X5, kind, w=w)
    out = {}
    for k, d in stream.items():
        e = sw2_ev[k]
        n = len(d)
        out[k] = dict(
            r0=d, r1=np.zeros(n), att=e["att"][:n], t_end=e["t_end"][:n],
            att128=e["att128"], n128=e["n128"], subject=e["subject"],
        )
    return out


def pack_ext_from_stream(stream, meta_ref):
    """Pack 1d stream with meta from SW2-aligned trials."""
    return stream  # already evidence dicts from apply_rule_ext


def _json_default(o):
    if isinstance(o, (np.floating, np.integer)):
        return float(o)
    if isinstance(o, np.ndarray):
        return o.tolist()
    raise TypeError(type(o))


def write_md(summary):
    sel = summary["selected"]
    ext = summary["external"]
    lines = []
    lines.append("# results_sw2_5th_seed\n")
    lines.append(
        "Stack **published SW2 fuse** (`fuse_general` a=0.5) as a **5th logit channel** beside "
        "4 direct-classifier seeds, then equal-mean / HP τ=30 / soft mean4↔SW2. "
        "**Not** soft SW2↔already-formed `linear_hp30` (`sw2-hp30-blend`).\n"
    )
    lines.append(
        f"HMM preset **{P_STAY} / {SCALE}**. OFF={OFF}. Code: `sw2_5th_seed.py`. "
        f"JSON: `results/research/sw2_5th_seed.json`.\n"
    )
    lines.append("## Selection (mean LODO Acc then H; never AASD/Carta)\n")
    lines.append(
        f"**SELECTED `{sel['key']}`** — mean LODO H **{sel['mean_LODO_H']:.4f}** "
        f"Acc **{sel['mean_LODO_Acc']:.2f}**\n"
    )
    lines.append("| Held-out | H | Acc % |")
    lines.append("|---|---:|---:|")
    for ds in DS_KEYS:
        p = sel["per_ds"][ds]
        lines.append(f"| {ds} | {p['H']:.4f} | {p['Acc']:.2f} |")
    lines.append(
        f"| **mean** | **{sel['mean_LODO_H']:.4f}** | **{sel['mean_LODO_Acc']:.2f}** |\n"
    )
    lines.append("### Rule grid (reference)\n")
    lines.append("| Rule | mean H | mean Acc |")
    lines.append("|---|---:|---:|")
    for r in summary["lodo_grid"]:
        mark = " **←**" if r["key"] == sel["key"] else ""
        lines.append(
            f"| `{r['key']}`{mark} | {r['mean_LODO_H']:.4f} | {r['mean_LODO_Acc']:.2f} |"
        )
    lines.append("\n## External (preset)\n")
    lines.append(
        "| Set | selected H/Acc | vs linear_hp30 | vs SW2-alone | Carta baseline |"
    )
    lines.append("|---|---|---|---|---|")
    for ds in ("AASD", "Carta2026"):
        s = ext[ds]["selected"]
        b = LINEAR_HP30_EXT[ds]
        w = SW2_ALONE_EXT[ds]
        lines.append(
            f"| {ds} | {s['H']:.4f} / **{s['Acc']:.2f}** | "
            f"{b['H']:.4f}/{b['Acc']:.2f} | {w['H']:.4f}/{w['Acc']:.2f} | "
            f"{LINEAR_HP30_EXT['Carta2026']['Acc']:.2f} |"
        )
    # also dump ref tags if present
    if "sw2_only" in ext.get("AASD", {}):
        lines.append("\n### Reference tags on same scorer\n")
        lines.append("| Set | selected | sw2_only | mean5 | hp30_mean5 |")
        lines.append("|---|---|---|---|---|")
        for ds in ("AASD", "Carta2026"):
            tags = ext[ds]
            def fmt(t):
                if t not in tags:
                    return "—"
                x = tags[t]
                return f"{x['H']:.4f}/{x['Acc']:.2f}"
            lines.append(
                f"| {ds} | {fmt('selected')} | {fmt('sw2_only')} | "
                f"{fmt('mean5')} | {fmt('hp30_mean5')} |"
            )
    aasd = ext["AASD"]["selected"]
    carta = ext["Carta2026"]["selected"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    d_sw = aasd["Acc"] - SW2_ALONE_EXT["AASD"]["Acc"]
    lines.append("\n## Targets\n")
    lines.append("| Target | Result |")
    lines.append("|---|---|")
    lines.append(
        f"| AASD Acc vs linear_hp30 53.12 | Acc **{aasd['Acc']:.2f}** (Δ {d_hp:+.2f}); "
        f"H {aasd['H']:.4f} |"
    )
    lines.append(
        f"| AASD Acc vs SW2-alone 53.46 | Acc **{aasd['Acc']:.2f}** (Δ {d_sw:+.2f}) |"
    )
    lines.append(
        f"| Carta vs linear_hp30 60.54 | Acc **{carta['Acc']:.2f}**; H {carta['H']:.4f} |"
    )
    lines.append(
        f"| LODO mean | H {sel['mean_LODO_H']:.4f} Acc {sel['mean_LODO_Acc']:.2f} |\n"
    )
    lines.append("## Verdict\n")
    if aasd["Acc"] > 60:
        lines.append("Selected 5th-seed rule clears AASD Acc>60.\n")
    elif aasd["Acc"] > max(LINEAR_HP30_EXT["AASD"]["Acc"], SW2_ALONE_EXT["AASD"]["Acc"]) + 0.15:
        lines.append(
            "Selected rule improves AASD Acc vs both linear_hp30 and SW2-alone, still short of 60.\n"
        )
    else:
        lines.append(
            "Honest LODO Acc/H selector on 5-channel SW2+seeds does not beat SW2-alone / "
            "linear_hp30 on AASD Acc; stacking SW2 as a 5th raw seed is not an Acc>60 path.\n"
        )
    lines.append("## NEXT IDEA\n")
    lines.append(summary.get("next_idea", "") + "\n")
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(
        f"sw2-5th-seed start  p_stay={P_STAY} scale={SCALE} a_sw2={A_SW2} OFF={OFF} LA={LOOKAHEAD}"
    )
    log_line("DISTINCT from sw2-hp30-blend: SW2 is raw 5th channel before aggregation")

    try:
        Z, meta = SW2.ensure_sw2_lodo()
        seeds4, _la = ensure_seeds4_lodo()
    except FileNotFoundError as e:
        log_line(f"ABORT: {e}")
        summary = dict(status="abandoned", reason=str(e))
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2)
        return

    # sanity AVGC
    sw2_avgc = SW2.fuse_sw2_held(Z, "AVGC", a=A_SW2)
    for kind, w in (("mean5", None), ("mean4", None), ("soft_mean4_sw2", 1.0)):
        stream = apply_rule_all(seeds4, sw2_avgc, kind, w=w)
        r = score_lodo_held(stream, meta, "AVGC")
        log_line(
            f"sanity AVGC {rule_key(kind, w):24s} H {r['H']:.4f} Acc {r['Acc']:.2f}"
        )

    rules = candidate_rules()
    log_line(f"scoring {len(rules)} rules on mean LODO Acc then H")
    rows = []
    best = None
    for kind, w in rules:
        t0 = time.time()
        mean_h, mean_a, per = mean_lodo_for_rule(Z, meta, seeds4, kind, w)
        key = rule_key(kind, w)
        row = dict(
            kind=kind, w=w, key=key,
            mean_LODO_H=mean_h, mean_LODO_Acc=mean_a, per_ds=per,
        )
        rows.append(row)
        log_line(
            f"  LODO {key:28s} H {mean_h:.4f} Acc {mean_a:.2f} ({time.time() - t0:.0f}s)"
        )
        if not eligible(key):
            continue
        sel = (mean_a, mean_h)
        if best is None or sel > best[0]:
            best = (sel, row)

    selected = best[1]
    log_line(
        f"SELECTED {selected['key']} mean LODO H {selected['mean_LODO_H']:.4f} "
        f"Acc {selected['mean_LODO_Acc']:.2f}"
    )

    # External
    try:
        EV_sw2, emis_sw2 = SW2.ensure_sw2_external()
        train_logits, train_att, test_logits, test_meta = ensure_seeds4_external()
    except FileNotFoundError as e:
        log_line(f"ABORT external: {e}")
        summary = dict(
            status="partial", selected=selected, lodo_grid=rows, reason=str(e),
        )
        with open(OUT, "w") as fh:
            json.dump(summary, fh, indent=2, default=_json_default)
        write_md(dict(
            selected=selected, lodo_grid=rows,
            external={ds: {"selected": dict(H=0, Acc=0)} for ds in ("AASD", "Carta2026")},
            next_idea="Restore SW2 external pickles; retry scoring only.",
        ))
        return

    kind, w = selected["kind"], selected["w"]
    corp = corpus_stream_mean_held(Z, meta, seeds4, kind, w)
    mu_c, sd_c = emissions_signed(corp, meta, list(DS_KEYS))
    log_line(f"corpus selected emis mu={mu_c:.4f} sd={sd_c:.4f}")

    # reference streams for tags
    ref_specs = [
        ("selected", kind, w, mu_c, sd_c),
        ("sw2_only", "soft_mean4_sw2", 1.0, None, None),
        ("mean5", "mean5", None, None, None),
        ("hp30_mean5", "hp30_mean5", None, None, None),
    ]

    ext_rows = {}
    for ds in ("AASD", "Carta2026"):
        sw2_ev = EV_sw2[ds][A_SW2]
        seeds_ds = test_logits[ds]
        for tag, knd, ww, mu_fix, sd_fix in ref_specs:
            blended = apply_rule_ext(seeds_ds, sw2_ev, knd, w=ww)
            if mu_fix is None:
                # fit emis on corpus with same rule
                corp_t = corpus_stream_mean_held(Z, meta, seeds4, knd, ww)
                mu_u, sd_u = emissions_signed(corp_t, meta, list(DS_KEYS))
            else:
                mu_u, sd_u = mu_fix, sd_fix
            # for pure SW2, prefer FE emis if available
            if tag == "sw2_only":
                mu_u, sd_u = float(emis_sw2[A_SW2][0]), float(emis_sw2[A_SW2][1])
                blended = {
                    k: dict(
                        r0=np.asarray(e["r0"], np.float64),
                        r1=np.zeros(len(e["r0"])),
                        att=e["att"], t_end=e["t_end"],
                        att128=e["att128"], n128=e["n128"], subject=e["subject"],
                    )
                    for k, e in sw2_ev.items()
                }
            r = E.score(blended, mu_u, sd_u * SCALE, P_STAY, LOOKAHEAD)
            pl = E.plain(blended)
            ext_rows.setdefault(ds, {})[tag] = dict(
                H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r["L"],
                acc1=pl["acc1"], nSwitches=r["nSwitches"], nUndetected=r["nUndetected"],
            )
            base = LINEAR_HP30_EXT[ds]["Acc"]
            log_line(
                f"EXT {ds} {tag:12s} H {r['H']:.4f} Acc {r['Acc']:.2f} "
                f"(Δ vs hp30 {r['Acc'] - base:+.2f})"
            )

    next_idea = (
        "Complement newseed-hp30 (5th *classifier* seed → linear_hp30) or a front-end Acc "
        "path; do not re-grid SW2↔hp30 soft/hard or SW2-as-5th-seed aggregates."
    )
    summary = dict(
        status="done",
        method="sw2-5th-seed",
        selected=selected,
        p_stay=P_STAY,
        scale=SCALE,
        a_sw2=A_SW2,
        OFF=OFF,
        lodo_grid=rows,
        external=ext_rows,
        baselines=dict(
            linear_hp30_ext=LINEAR_HP30_EXT,
            sw2_alone_ext=SW2_ALONE_EXT,
        ),
        next_idea=next_idea,
        elapsed_s=time.time() - t_all,
    )
    with open(OUT, "w") as fh:
        json.dump(summary, fh, indent=2, default=_json_default)
    write_md(summary)
    log_line(f"done in {time.time() - t_all:.0f}s → {OUT}")


if __name__ == "__main__":
    main()
