"""decoder-sw3: denser synthetic attention switches for the published vlaai_ws reconstruction decoder.

Retrain / fine-tune from SparrKULee pretrain (same recipe as README_BEST) with:
  seg_switch=0.75, seg_switch2=0.6, seg_post=2.5  (↑p_switch + multi-switch; denser than clf sw2 0.5/0.5)
Then rebuild SW2-style fuse (new decoder seeds ± published clf-sw2) and score AASD/Carta @0.994/0.8
vs Acc leader 53.53 / Carta linear_hp30 60.54.

Does NOT train on AASD/Carta labels. Does NOT soft-blend old SW2↔hp30/swmix evidence only —
produces NEW decoder fold + external evidence.

usage (under gpu.lock):
  GLIMPSE_THR=norm PY=.../python
  $PY -u decoder_sw3.py            # LODO 4 seeds + all5 external + fuse + score
  SEEDS=0 $PY -u decoder_sw3.py    # single seed (faster smoke / fallback)
  FALLBACK_BAND=1 $PY -u decoder_sw3.py   # if denser retrain abandoned: one band-limited env seed
"""
from __future__ import annotations

import json
import os
import pickle
import subprocess
import sys
import time

import numpy as np

import decoders
import ensemble_clf_nested as N
import eval_external as E
import fuse_external as FE
import fuse_general as FG
import score as S
from common import DS_KEYS
from common_exp import CFG, EXP, fold_file, resolve_model
from fuse_evidence import LA, OFF
from run_experiment import init_net, load_external

PY = os.environ.get(
    "PY",
    "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python",
)
PREP = "eaP_n5_t60"
TAG_BASE = "_aug_cluster_onset_dom_xFUG_ptSPK_sw3"
EXT_TAG = "_sw3P"
SEEDS = [int(x) for x in os.environ.get("SEEDS", "0,1,2,3").split(",") if x.strip() != ""]
# denser than classifier sw2 (0.5 / 0.5) and than prior seq_p_switch=0.5 single-cut
SEG_SWITCH = float(os.environ.get("SEG_SWITCH", "0.75"))
SEG_SWITCH2 = float(os.environ.get("SEG_SWITCH2", "0.60"))
SEG_POST = float(os.environ.get("SEG_POST", "2.5"))
SEG_MIN_GAP_S = float(os.environ.get("SEG_MIN_GAP_S", "2.0"))
A_FUSE = 0.5
P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = float(LA)
FALLBACK_BAND = os.environ.get("FALLBACK_BAND", "0") == "1"

CLFS = [
    "direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2",
    "direct_clf__eaP_n5_t60_w10b3_ptSPK_s1_aug-sw2",
    "direct_clf__eaP_n5_t60_w10b3_ptSPK_s2_aug-sw2",
    "direct_clf__eaP_n5_t60_w10b3_ptSPK_s3_aug-sw2",
]
EXT_CLFS = [f"{c}:_w10b3sw2P:{i}" for i, c in enumerate(CLFS)]

OLD_DECS = [
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s1",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s2",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s3",
]

LOG = os.path.join(EXP, "experiments", "decoder_sw3_console.log")
OUT_JSON = os.path.join(EXP, "results", "research", "decoder_sw3.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_decoder_sw3.md")
CACHE_DIR = os.path.join(EXP, "models", "decoder_sw3", "cache")

LEADER_AASD_ACC = 53.53
HP30_CARTA_ACC = 60.54
HP30_AASD = dict(H=0.6422, Acc=53.12)


def log_line(msg):
    print(msg, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(msg + "\n")


def exp_name(seed, band=False):
    tag = TAG_BASE + ("_bandenv" if band else "")
    return f"vlaai_ws__{PREP}{tag}" if seed == 0 else f"vlaai_ws__{PREP}{tag}_s{seed}"


def pretrain_path(seed):
    return resolve_model(f"models/pretrain_spk_vlaai_ws_s{seed}.pt")


def apply_sw3_D(band=False):
    """Denser segment-switch contrastive on top of published cluster aug + env/onset/dom."""
    decoders.D.update(CFG["dnn"])
    decoders.D["augment"] = {**CFG["dnn_augment"], **CFG["aug_variants"]["cluster"]}
    if band:
        # Fallback Acc path: reconstruct band-limited env (causal 2–4 Hz) + onset; NOT delta/theta speech-bank repeat
        decoders.D["targets"] = ["env", "onset"]
        decoders.D["band_env_hz_lo"] = 2.0
        decoders.D["band_env_hz_hi"] = 4.0
    else:
        decoders.D["targets"] = ["env", "onset", "dom"]
        decoders.D.pop("band_env_hz_lo", None)
        decoders.D.pop("band_env_hz_hi", None)
    decoders.D["seg_switch"] = SEG_SWITCH
    decoders.D["seg_switch2"] = SEG_SWITCH2
    decoders.D["seg_post"] = SEG_POST
    decoders.D["seg_min_gap_s"] = SEG_MIN_GAP_S
    decoders.D["loss"] = "contrastive"


def run_lodo_seed(seed, band=False):
    """LODO via run_experiment.py CLI so fold caching / scoring match the published stack."""
    exp = exp_name(seed, band=band)
    missing = [h for h in DS_KEYS if not os.path.exists(fold_file(exp, h))]
    if not missing:
        log_line(f"LODO cache hit {exp}")
        return exp
    tag = TAG_BASE + ("_bandenv" if band else "")
    if seed != 0:
        tag = f"{tag}_s{seed}"
    cmd = [
        PY, "-u", os.path.join(EXP, "run_experiment.py"),
        "--decoder", "vlaai_ws", "--prep", PREP,
        "--aug-variant", "cluster",
        "--extra-train", "Fuglsang2020",
        "--targets", "env,onset" if band else "env,onset,dom",
        "--init-from", pretrain_path(seed),
        "--init-targets", "env,onset",
        "--evidence-weights", "lda",
        "--train-seed", str(seed),
        "--tag", tag,
        "--set",
        f"seg_switch={SEG_SWITCH}",
        f"seg_switch2={SEG_SWITCH2}",
        f"seg_post={SEG_POST}",
        f"seg_min_gap_s={SEG_MIN_GAP_S}",
    ]
    if band:
        # force env target through a causal 2–4 Hz band in _targets via D; set before import path
        cmd += ["--set", "band_env_hz_lo=2", "band_env_hz_hi=4"]
    log_line(f"LODO train {exp}: {' '.join(cmd)}")
    env = os.environ.copy()
    env["GLIMPSE_THR"] = "norm"
    env["PYTHONDONTWRITEBYTECODE"] = "1"
    env["PYTHONUNBUFFERED"] = "1"
    r = subprocess.run(cmd, cwd=EXP, env=env)
    if r.returncode != 0:
        raise RuntimeError(f"run_experiment failed rc={r.returncode} for {exp}")
    return exp


def train_all5_and_external(seed, ref_exp, band=False):
    """Train on all 5 + Fuglsang with denser switches; dump AASD/Carta evidence under EXT_TAG."""
    out = os.path.join(EXP, "results", "research")
    os.makedirs(out, exist_ok=True)
    tag = EXT_TAG + ("_bandenv" if band else "")
    ev_paths = {
        ds: os.path.join(out, f"{('aasd_c1' if ds == 'AASD' else f'ext_c1_{ds}')}{tag}_evidence_s{seed}.pkl")
        for ds in ("AASD", "Carta2026")
    }
    if all(os.path.exists(p) for p in ev_paths.values()):
        log_line(f"external cache hit seed={seed} tag={tag}")
        return tag

    apply_sw3_D(band=band)
    decoders.D["train_seed"] = seed
    E.CONF.update(
        decoder="vlaai_ws", prep=PREP,
        targets=list(decoders.D["targets"]),
        ref=ref_exp,
    )
    log_line(f"all5+ext train seed={seed} seg_switch={SEG_SWITCH}/{SEG_SWITCH2} post={SEG_POST} band={band}")
    # Mirror eval_external.train but keep denser D flags (train() overwrites augment/targets only)
    rng = np.random.default_rng(CFG["seed"] + 99)
    from common_exp import load_trial, rows_of
    train_, val = [], []
    for ds in DS_KEYS:
        rows = rows_of(ds)
        vs = rng.choice(sorted(rows.subject.unique()))
        for _, r in rows.iterrows():
            tr = load_trial(PREP, r["key"])
            (val if r["subject"] == vs else train_).append(tr)
    train_ += load_external(["Fuglsang2020"], PREP, log_line)
    # re-assert denser switches after any side effects
    apply_sw3_D(band=band)
    decoders.D["train_seed"] = seed
    dec = decoders.make("vlaai_ws", 58, log_line)
    init_net(dec, pretrain_path(seed), log_line)
    t0 = time.time()
    dec.fit(train_, val, [(0, 0)] * len(train_))
    log_line(f"  all5 fit done in {time.time() - t0:.0f}s")
    for ds, path in ev_paths.items():
        if os.path.exists(path):
            continue
        ev = E.evidence(dec, ds)
        with open(path, "wb") as fh:
            pickle.dump(ev, fh, protocol=pickle.HIGHEST_PROTOCOL)
        log_line(f"  wrote {path}")
    del dec
    import torch
    torch.cuda.empty_cache()
    return tag


def fuse_lodo_score(decs):
    """SW2-style fuse_general a=0.5 with new decoders + published clf-sw2; score @0.994/0.8."""
    log_line(f"fuse LODO decs={decs} clfs={CLFS} a={A_FUSE}")
    Z, meta = FG.build(decs, CLFS)
    per, hs, acs = {}, [], []
    for held in DS_KEYS:
        zd, zc = Z[held]
        stream = {
            d: {
                k: (1 - A_FUSE) * zd[d][k] + A_FUSE * zc[d][k]
                for k in zd[d]
            }
            for d in DS_KEYS
        }
        ev = {}
        for d in DS_KEYS:
            ev[d] = {}
            for k, delta in stream[d].items():
                att, te = meta[d][k]
                n = len(delta)
                ev[d][k] = dict(
                    r0=np.asarray(delta, np.float64),
                    r1=np.zeros(n),
                    att=np.asarray(att)[:n],
                    t_end=np.asarray(te, np.float64)[:n],
                    pre_switched=True,
                )
        # install + official HMM with scale
        import ensemble_clf_transformer_v4 as V4
        V4._install_scorer_state(ev, LOOKAHEAD)
        r = N.official(held, ev, P_STAY, SCALE)
        per[held] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r.get("L"))
        hs.append(r["H"])
        acs.append(r["Acc"])
        log_line(f"  held={held} H={r['H']:.4f} Acc={r['Acc']:.2f}")
    return dict(mean_H=float(np.mean(hs)), mean_Acc=float(np.mean(acs)), per=per), Z, meta


def fuse_external_score(decs, dec_tag):
    """Rebuild external fuse with NEW decoder evidence + published clf-sw2; score @0.994/0.8."""
    # fuse_external expects [[corpus_exp, seed], ...] where seed indexes evidence_s{seed}.pkl
    decs_fe = [[decs[i], SEEDS[i]] for i in range(len(decs))]
    clfs_fe = [x.split(":") for x in EXT_CLFS]
    log_line(f"fuse external decs={decs_fe} tag={dec_tag}")
    # verify pickles
    res = FE.RES
    missing = []
    for d, s in decs_fe:
        for ds_tag in (f"aasd_c1{dec_tag}", f"ext_c1_Carta2026{dec_tag}"):
            p = os.path.join(res, f"{ds_tag}_evidence_s{s}.pkl")
            if not os.path.exists(p):
                missing.append(p)
    for c in clfs_fe:
        for ds in ("AASD", "Carta2026"):
            p = os.path.join(res, f"ext_clf_{ds}{c[1]}_evidence_s{c[2]}.pkl")
            if not os.path.exists(p):
                missing.append(p)
    if missing:
        raise FileNotFoundError("external evidence missing:\n" + "\n".join(missing[:16]))
    EV, emis = FE.build_external(decs_fe, clfs_fe, [A_FUSE], ["AASD", "Carta2026"], dec_tag)
    mu, sd = emis[A_FUSE]
    out = {}
    for ds in ("AASD", "Carta2026"):
        # also score decoder-only and clf-only references at same preset
        r = E.score(EV[ds][A_FUSE], mu, sd * SCALE, P_STAY, LOOKAHEAD)
        pl = E.plain(EV[ds][A_FUSE])
        out[ds] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r.get("L"), acc1=pl["acc1"],
                       emis_mu=mu, emis_sd=sd)
        log_line(f"  {ds} H={r['H']:.4f} Acc={r['Acc']:.2f} acc1={pl['acc1']:.3f}")
    return out, emis


def write_results(payload):
    os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
    with open(OUT_JSON, "w") as fh:
        json.dump(payload, fh, indent=2, default=float)
    aasd = payload["external"]["AASD"]
    carta = payload["external"]["Carta2026"]
    lodo = payload["lodo"]
    band = payload.get("band_fallback", False)
    lines = [
        "# results_decoder_sw3",
        "",
        ("**Band-limited env fallback** (one seed) — denser-switch retrain abandoned/infeasible."
         if band else
         "Retrain published **vlaai_ws** reconstruction decoder with **denser synthetic attention switches** "
         f"(seg_switch={SEG_SWITCH}, seg_switch2={SEG_SWITCH2}, seg_post={SEG_POST}); "
         "rebuild SW2-style fuse (new decoder ± published clf-sw2 a=0.5). "
         "**Not** SW2↔hp30/swmix soft blend of old evidence."),
        "",
        f"HMM preset **{P_STAY} / {SCALE}**. Code: `decoder_sw3.py`. JSON: `results/research/decoder_sw3.json`.",
        "",
        "## LODO (new decoder × published clf-sw2)",
        "",
        f"Mean H **{lodo['mean_H']:.4f}** Acc **{lodo['mean_Acc']:.2f}**",
        "",
        "| Held-out | H | Acc % |",
        "|---|---:|---:|",
    ]
    for h in DS_KEYS:
        p = lodo["per"][h]
        lines.append(f"| {h} | {p['H']:.4f} | {p['Acc']:.2f} |")
    lines += [
        f"| **mean** | **{lodo['mean_H']:.4f}** | **{lodo['mean_Acc']:.2f}** |",
        "",
        "## External (preset)",
        "",
        "| Set | H / Acc | vs Acc leader 53.53 / Carta hp30 60.54 | linear_hp30 |",
        "|---|---|---|---|",
        f"| AASD | {aasd['H']:.4f} / **{aasd['Acc']:.2f}** | ΔAcc {aasd['Acc'] - LEADER_AASD_ACC:+.2f} | "
        f"{HP30_AASD['H']:.4f}/{HP30_AASD['Acc']:.2f} |",
        f"| Carta2026 | {carta['H']:.4f} / **{carta['Acc']:.2f}** | ΔAcc {carta['Acc'] - HP30_CARTA_ACC:+.2f} | "
        f"0.6520/{HP30_CARTA_ACC} |",
        "",
        "## Targets",
        "",
        "| Target | Result |",
        "|---|---|",
        f"| AASD Acc vs leader {LEADER_AASD_ACC} | Acc **{aasd['Acc']:.2f}** "
        f"(Δ{aasd['Acc'] - LEADER_AASD_ACC:+.2f}); H {aasd['H']:.4f} |",
        f"| Carta vs linear_hp30 {HP30_CARTA_ACC} | Acc **{carta['Acc']:.2f}**; H {carta['H']:.4f} |",
        f"| LODO mean | H {lodo['mean_H']:.4f} Acc {lodo['mean_Acc']:.2f} |",
        f"| AASD Acc>60? | {'YES' if aasd['Acc'] > 60 else 'NO'} |",
        "",
        "## Verdict",
        "",
        payload.get("verdict", ""),
        "",
        "## NEXT IDEA",
        "",
        payload.get("next_idea", ""),
        "",
    ]
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))
    log_line(f"wrote {RESULTS_MD}")


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    band = FALLBACK_BAND
    log_line(
        f"=== decoder-sw3 start seeds={SEEDS} seg={SEG_SWITCH}/{SEG_SWITCH2} "
        f"post={SEG_POST} band={band} ==="
    )

    decs = []
    for seed in SEEDS:
        exp = run_lodo_seed(seed, band=band)
        decs.append(exp)
        train_all5_and_external(seed, exp, band=band)

    dec_tag = EXT_TAG + ("_bandenv" if band else "")
    lodo, _Z, _meta = fuse_lodo_score(decs)
    external, _emis = fuse_external_score(decs, dec_tag)

    aasd_acc = external["AASD"]["Acc"]
    verdict = (
        f"Denser-switch decoder fuse AASD Acc {aasd_acc:.2f} vs leader {LEADER_AASD_ACC} "
        f"(Δ{aasd_acc - LEADER_AASD_ACC:+.2f}). "
        + ("Acc≪60; denser synth switches on reconstruction decoder did not lift real-switch Acc."
           if aasd_acc < 60 else "Acc>60 MET.")
    )
    next_idea = (
        "Do not re-grid denser seg_switch on this vlaai_ws recipe. Prefer a qualitatively new Acc "
        "frame path (e.g. band-limited TRF Path A, or a non-vlaai reconstruction target family) — "
        "not SW2↔hp30/swmix evidence blends."
        if not band else
        "Band-limited env decoder seed scored; if Acc still mid-50s, abandon decoder-family Acc push "
        "on vlaai_ws and try a non-reconstruction Acc front-end that clears chance on AASD acc1."
    )
    payload = dict(
        method="decoder-sw3",
        band_fallback=band,
        seeds=SEEDS,
        seg_switch=SEG_SWITCH,
        seg_switch2=SEG_SWITCH2,
        seg_post=SEG_POST,
        decs=decs,
        clfs=CLFS,
        a=A_FUSE,
        p_stay=P_STAY,
        scale=SCALE,
        lodo=lodo,
        external=external,
        leader_aasd_acc=LEADER_AASD_ACC,
        verdict=verdict,
        next_idea=next_idea,
    )
    write_results(payload)
    log_line(verdict)
    log_line("=== decoder-sw3 done ===")
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except Exception as e:
        log_line(f"FATAL: {type(e).__name__}: {e}")
        # Honest abandon path: if denser retrain blew up mid-way and band not yet tried, hint fallback
        if not FALLBACK_BAND:
            log_line("HINT: re-run with FALLBACK_BAND=1 SEEDS=0 for band-limited envelope target seed.")
        raise
