"""Fine-tune stem_ms96 from SWA with published SW2 fuse evidence as extra ms channels.

Concatenates README_BEST SW2 decoder+clf fuse (a=0.5; same stream as sw2_hp30_blend.py,
AASD Acc 53.46) onto the 24-d multi-scale logit features after DEC_OFFSET alignment:
  24 → 25  (SW2 only)  or  24 → 26  (SW2 + causal τ=30 HP of SW2). Default: 26.

Init from models/stem_ms96_swa/{held-*,all5}.pt:
  expand dw / proj / skip input dims; copy shared weights; zero-init new columns
  so the network starts ≡ SWA on the original 24-d subspace.

Recipe: AdamW, EMA 0.995, causal mask, abs pos, ~1000 LODO steps (same as SWA/IRM).
Score LODO 5 + AASD + Carta at HMM 0.994/0.8; also report hp30 on new evidence.
Never uses AASD/Carta labels for training or selection.

If SW2 fold/external alignment fails → abandon with reason (no invented scores).

usage:
  STEPS=1000 BATCH=512 EXTRA_HP30=1 \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u stem_ms96_sw2_ms.py
"""
from __future__ import annotations

import copy
import json
import math
import os
import pickle
import time

import numpy as np
import torch
import torch.nn.functional as F

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import fuse_general as FG
import fuse_external as FE
import sw2_hp30_blend as SW2
from common import DS_KEYS
from fuse_evidence import OFF

EXP = os.path.dirname(os.path.abspath(__file__))
SWA_DIR = os.path.join(EXP, "models", "stem_ms96_swa")
CKPT_DIR = os.path.join(EXP, "models", "stem_ms96_sw2_ms")
CACHE_DIR = os.path.join(EXP, "models", "stem_ms96_sw2_ms", "cache")
LOG = os.path.join(EXP, "experiments", "stem_ms96_sw2_ms.log")
OUT = os.path.join(EXP, "results", "research", "stem_ms96_sw2_ms.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_stem_sw2_ms.md")
EVID = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research"
TEST_NAMES = ("AASD", "Carta2026")

STEPS = int(os.environ.get("STEPS", "1000"))
BATCH = int(os.environ.get("BATCH", "512"))
EXTRA_HP30 = os.environ.get("EXTRA_HP30", "1") == "1"
EMA_DECAY = 0.995
LR = 8e-4
P_STAY = 0.994
SCALE = 0.8
MS_DIM = 24
CFG = ("stem_ms96_sw2_ms", "ms_sw2", 96, 64, 2, True, LR)
WINDOW = 96
A_SW2 = SW2.A_SW2
LINEAR_HP30_AASD = 53.12
SW2_AASD = 53.46


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def align_sw2_to_ms_len(sw2_1d, t_ms, off=OFF):
    """Pad SW2 (decoder timeline) up to classifier/ms length T with left zeros of size OFF."""
    s = np.asarray(sw2_1d, np.float64).reshape(-1)
    out = np.zeros(t_ms, np.float64)
    n = min(len(s), max(t_ms - off, 0))
    if n > 0:
        out[off: off + n] = s[:n]
    return out.astype(np.float32)


def concat_ms_sw2(ms_x, sw2_1d, with_hp30=EXTRA_HP30):
    """ms_x (T, 24) + aligned SW2 (+ optional hp30 of aligned SW2) → (T, 25|26)."""
    t = len(ms_x)
    aligned = align_sw2_to_ms_len(sw2_1d, t)
    cols = [ms_x.astype(np.float32), aligned[:, None]]
    if with_hp30:
        hp = V4.highpass(aligned[:, None]).reshape(-1).astype(np.float32)
        cols.append(hp[:, None])
    return np.concatenate(cols, axis=1)


def build_feat_held(ms_all, sw2_held, with_hp30=EXTRA_HP30):
    """Build expanded features for all DS_KEYS using one held-specific SW2 fuse map."""
    out = {}
    missing = []
    len_mismatch = []
    for ds in DS_KEYS:
        out[ds] = {}
        if ds not in sw2_held:
            missing.append(f"{ds}: (no sw2 ds)")
            continue
        for key, X in ms_all[ds].items():
            if key not in sw2_held[ds]:
                missing.append(f"{ds}/{key}")
                continue
            sw = np.asarray(sw2_held[ds][key], np.float64).reshape(-1)
            # After OFF pad, usable SW2 length should cover most of the trial.
            expect = max(len(X) - OFF, 0)
            if abs(len(sw) - expect) > 2 and abs(len(sw) - len(X)) > 2:
                len_mismatch.append(
                    f"{ds}/{key}: ms={len(X)} sw2={len(sw)} expect≈{expect}+pad")
            out[ds][key] = concat_ms_sw2(X, sw, with_hp30=with_hp30)
    if missing:
        raise RuntimeError(
            "SW2↔ms key alignment failed (missing trials):\n" + "\n".join(missing[:20]))
    if len_mismatch:
        # Soft warning only if a few; hard fail if many
        log_line(f"  WARN length mismatches ({len(len_mismatch)}): {len_mismatch[0]}")
        if len(len_mismatch) > 50:
            raise RuntimeError(
                "SW2↔ms length alignment failed on many trials:\n"
                + "\n".join(len_mismatch[:12]))
    return out


def build_feat_all5_mean(ms_all, Z, with_hp30=EXTRA_HP30):
    """Mean of the five held-specific SW2 streams (soft all-train SW2 for all5 fold)."""
    streams = [SW2.fuse_sw2_held(Z, held) for held in DS_KEYS]
    sw2_mean = {}
    for ds in DS_KEYS:
        keys = set.intersection(*(set(s[ds].keys()) for s in streams))
        sw2_mean[ds] = {}
        for k in keys:
            arrs = [np.asarray(s[ds][k], np.float64).reshape(-1) for s in streams]
            n = min(len(a) for a in arrs)
            sw2_mean[ds][k] = np.mean([a[:n] for a in arrs], axis=0)
    return build_feat_held(ms_all, sw2_mean, with_hp30=with_hp30)


def expand_swa_to_cin(blob, c_new, device):
    """Load SWA stem; expand input dims; zero-init new columns → start ≡ SWA."""
    name, family, window, d_model, nlayers, stem, _lr = blob["cfg"]
    if name != "stem_ms96" or not stem or int(window) != WINDOW:
        raise RuntimeError(f"unexpected stem cfg {blob['cfg']}")
    c_old = int(np.asarray(blob["mu"]).shape[0])
    if c_old != MS_DIM:
        raise RuntimeError(f"expected SWA c_in={MS_DIM}, got {c_old}")
    if c_new <= c_old:
        raise RuntimeError(f"c_new={c_new} must be > {c_old}")

    net = V.LogitTransformer(c_new, WINDOW, d_model=d_model, nlayers=nlayers, stem=True)
    good_mask = net.mask.detach().clone()
    old = blob["state"]
    new_sd = net.state_dict()

    for k, v in old.items():
        if k == "mask":
            continue
        if k not in new_sd:
            raise RuntimeError(f"unexpected key in SWA state: {k}")
        tv = new_sd[k]
        v = v.detach().cpu()
        if tuple(v.shape) == tuple(tv.shape):
            new_sd[k] = v
            continue
        if k == "proj.weight":  # (d_model, c)
            t = tv.detach().cpu().clone()
            t.zero_()
            t[:, :c_old] = v
            new_sd[k] = t
        elif k == "skip.weight":  # (1, c)
            t = tv.detach().cpu().clone()
            t.zero_()
            t[:, :c_old] = v
            new_sd[k] = t
        elif k == "dw.weight":  # (c, 1, k)
            t = tv.detach().cpu().clone()
            t.zero_()
            t[:c_old] = v
            new_sd[k] = t
        elif k == "dw.bias":
            t = tv.detach().cpu().clone()
            t.zero_()
            t[:c_old] = v
            new_sd[k] = t
        else:
            raise RuntimeError(f"shape mismatch on {k}: old {tuple(v.shape)} new {tuple(tv.shape)}")

    net.load_state_dict(new_sd)
    net.mask.copy_(good_mask)
    if torch.isnan(net.mask).any():
        raise RuntimeError("causal mask has nan after expand")
    net = net.to(device)
    return net, c_old


def set_lr(opt, step, total, base):
    warm = min(40, max(total // 10, 1))
    if step < warm:
        lr = base * (step + 1) / warm
    else:
        p = (step - warm) / max(total - warm, 1)
        lr = 1e-5 + 0.5 * (base - 1e-5) * (1.0 + math.cos(math.pi * p))
    for group in opt.param_groups:
        group["lr"] = lr
    return lr


def weighted_bce(logit, y, m, pos_weight):
    sw = V.switch_weights(y, m)
    bce = F.binary_cross_entropy_with_logits(logit, y, pos_weight=pos_weight, reduction="none")
    denom = (sw * m).sum().clamp_min(1.0)
    loss = (bce * sw * m).sum() / denom
    dv = (logit[:, 1:] - logit[:, :-1]).abs()
    m2 = m[:, 1:] * m[:, :-1]
    loss = loss + 0.05 * (dv * m2).sum() / m2.sum().clamp_min(1.0)
    return loss


def train_fold(model, groups, y_all, mu, sd, device, steps, batch, tag):
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
    rng = np.random.default_rng(V.TRAIN_SEED)
    ema = copy.deepcopy(model).eval()
    model.train()
    t0 = time.time()
    last = None
    for step in range(steps):
        set_lr(opt, step, steps, LR)
        xb, yb, mb = V.sample_batch(rng, groups, WINDOW, batch, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        m = torch.from_numpy(mb).to(device, non_blocking=True)
        x = V.augment(x)
        opt.zero_grad(set_to_none=True)
        logit = model(x)
        loss = weighted_bce(logit, y, m, pos_weight)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss at step {step} for {tag}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        V.ema_update(model, ema, EMA_DECAY)
        last = float(loss.item())
        if step == 0 or (step + 1) % 200 == 0 or step + 1 == steps:
            log_line(f"    {tag} step {step + 1}/{steps} loss {last:.4f}")
    ema.eval()
    acc = V.train_frame_acc(ema, groups, WINDOW, mu, sd, device)
    log_line(f"  trained {tag} loss {last:.4f} frame-acc {acc:.3f} ({time.time() - t0:.0f}s)")
    return ema, dict(loss=last, frame_acc=acc)


def fit_with_oom(swa_path, feat, train_ds, device, steps, batch, tag, halved=False):
    blob = torch.load(swa_path, map_location="cpu", weights_only=False)
    groups, y_all = V.trial_groups(feat, train_ds)
    c_in = groups[0]["trials"][0][0].shape[1]
    mu, sd = V.norm_stats(feat, train_ds)
    model, c_old = expand_swa_to_cin(blob, c_in, device)
    log_line(f"  expanded SWA {c_old}→{c_in} from {swa_path}")
    try:
        ema, stats = train_fold(model, groups, y_all, mu, sd, device, steps, batch, tag)
        return ema, mu, sd, batch, stats, c_in
    except torch.cuda.OutOfMemoryError:
        if halved:
            raise
        log_line(f"  OOM on {tag} at batch {batch}; halving once")
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
        return fit_with_oom(
            swa_path, feat, train_ds, device, steps, batch // 2, tag, halved=True)


def hp30_stream(stream):
    out = {}
    for key, d in stream.items():
        out[key] = V4.highpass(np.asarray(d, np.float64)[:, None]).reshape(-1).astype(np.float64)
    return out


def score_lodo(model, feat, mu, sd, held, device):
    streams = V.infer_all(model, feat, mu, sd, WINDOW, device)
    ev = N.to_ev(streams)
    off = N.official(held, ev, P_STAY, SCALE)
    hp = {ds: hp30_stream(streams[ds]) for ds in streams}
    ev_hp = N.to_ev(hp)
    off_hp = N.official(held, ev_hp, P_STAY, SCALE)
    log_line(f"LODO {held} p {P_STAY} sc {SCALE}  H {off['H']:.4f} Acc {off['Acc']:.2f} "
             f"L {off['L']:.3f} SDT {off['SDT']:.2f}")
    log_line(f"LODO_hp30 {held} p {P_STAY} sc {SCALE}  H {off_hp['H']:.4f} Acc {off_hp['Acc']:.2f}")
    return dict(sw2_ms=off, hp30=off_hp), streams


def score_external(model, feat_ext, meta, mu_in, sd_in, corpus_streams, corpus_att, device):
    parts = []
    for ds, table in corpus_streams.items():
        labels = corpus_att[ds]
        for key, d in table.items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    mu_e = float(np.concatenate(parts).mean())
    sd_e = float(np.concatenate(parts).std())
    log_line(f"  external emissions mu {mu_e:.4f} sd {sd_e:.4f} scale {SCALE}")

    hp_corpus = {ds: hp30_stream(corpus_streams[ds]) for ds in corpus_streams}
    parts_hp = []
    for ds, table in hp_corpus.items():
        labels = corpus_att[ds]
        for key, d in table.items():
            parts_hp.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    mu_hp = float(np.concatenate(parts_hp).mean())
    sd_hp = float(np.concatenate(parts_hp).std())
    log_line(f"  external hp30 emissions mu {mu_hp:.4f} sd {sd_hp:.4f} scale {SCALE}")

    rows = {}
    for ds in TEST_NAMES:
        stream = V.infer_all(model, {ds: feat_ext[ds]}, mu_in, sd_in, WINDOW, device)[ds]
        ev = {}
        for key, d in stream.items():
            meta_k = meta[ds][key]
            d = np.asarray(d, np.float64)
            ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=meta_k["att"], t_end=meta_k["t_end"],
                           att128=meta_k["att128"], n128=meta_k["n128"], subject=meta_k["subject"])
        r = E.score(ev, mu_e, sd_e * SCALE, P_STAY, 0.75)
        stream_hp = hp30_stream(stream)
        ev_hp = {}
        for key, d in stream_hp.items():
            meta_k = meta[ds][key]
            d = np.asarray(d, np.float64)
            ev_hp[key] = dict(r0=d, r1=np.zeros(len(d)), att=meta_k["att"], t_end=meta_k["t_end"],
                              att128=meta_k["att128"], n128=meta_k["n128"], subject=meta_k["subject"])
        r_hp = E.score(ev_hp, mu_hp, sd_hp * SCALE, P_STAY, 0.75)
        rows[ds] = dict(sw2_ms=r, hp30=r_hp)
        log_line(f"EXT {ds} p {P_STAY} sc {SCALE}  H {r['H']:.4f} Acc {r['Acc']:.2f} "
                 f"L {r['L']:.3f} SDT {r['SDT']:.2f}")
        log_line(f"EXT_hp30 {ds} p {P_STAY} sc {SCALE}  H {r_hp['H']:.4f} Acc {r_hp['Acc']:.2f}")
    return rows


def load_external_logits_meta():
    E.CONF.update(prep="eaP_n5_t60", targets=["env", "onset", "dom"])
    logits, meta = {}, {}
    for ds in TEST_NAMES:
        evs = []
        for i in range(4):
            path = os.path.join(EVID, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]), t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]), n128=int(e0["n128"]), subject=int(e0["subject"]))
        log_line(f"  external {ds}: {len(logits[ds])} trials")
    return logits, meta


def build_ext_feat(ms_ext, EV_sw2, with_hp30=EXTRA_HP30):
    """ms_ext from 4-seed clf; SW2 from fuse_external a=0.5 evidence dicts."""
    out = {}
    missing = []
    for ds in TEST_NAMES:
        out[ds] = {}
        sw2_ev = EV_sw2[ds][A_SW2]
        for key, X in ms_ext[ds].items():
            if key not in sw2_ev:
                missing.append(f"{ds}/{key}")
                continue
            sw = np.asarray(sw2_ev[key]["r0"], np.float64).reshape(-1)
            out[ds][key] = concat_ms_sw2(X, sw, with_hp30=with_hp30)
    if missing:
        raise RuntimeError(
            "External SW2↔ms key alignment failed:\n" + "\n".join(missing[:20]))
    return out


def write_md(summary):
    lodo = summary["lodo"]
    mean = summary["lodo_mean"]
    ext = summary["external"]
    lines = [
        "# results_stem_sw2_ms\n",
        "Fine-tune `stem_ms96` from **SWA** with published **SW2 fuse** evidence "
        f"(a={A_SW2}, DEC_OFFSET={OFF}) concatenated onto 24-d ms features "
        f"→ **{summary['c_in']}-d**. Zero-init new input columns. EMA {EMA_DECAY}. "
        f"HMM **{P_STAY}/{SCALE}**.\n",
        f"Code: `stem_ms96_sw2_ms.py`. Ckpts: `models/stem_ms96_sw2_ms/`. "
        f"JSON: `results/research/stem_ms96_sw2_ms.json`.\n",
        "## LODO (preset)\n",
        "| Held-out | H | Acc % | hp30 H | hp30 Acc |",
        "|---|---:|---:|---:|---:|",
    ]
    for ds in DS_KEYS:
        s = lodo[ds]["sw2_ms"]
        h = lodo[ds]["hp30"]
        lines.append(
            f"| {ds} | {s['H']:.4f} | {s['Acc']:.2f} | {h['H']:.4f} | {h['Acc']:.2f} |")
    lines.append(f"| **mean** | **{mean['H']:.4f}** | **{mean['Acc']:.2f}** | | |\n")
    lines.append("## External (preset)\n")
    lines.append("| Set | sw2_ms H/Acc | hp30 H/Acc | vs linear_hp30 53.12 / SW2 53.46 |")
    lines.append("|---|---|---|---|")
    for ds in TEST_NAMES:
        s = ext[ds]["sw2_ms"]
        h = ext[ds]["hp30"]
        if ds == "AASD":
            note = (f"Acc {s['Acc']:.2f} (Δ {s['Acc']-LINEAR_HP30_AASD:+.2f} vs hp30; "
                    f"Δ {s['Acc']-SW2_AASD:+.2f} vs SW2)")
        else:
            note = f"Acc {s['Acc']:.2f}"
        lines.append(
            f"| {ds} | {s['H']:.4f}/**{s['Acc']:.2f}** | {h['H']:.4f}/{h['Acc']:.2f} | {note} |")
    aasd = ext["AASD"]["sw2_ms"]
    carta = ext["Carta2026"]["sw2_ms"]
    avgc = lodo["AVGC"]["sw2_ms"]
    uhd = lodo["UHD"]["sw2_ms"]
    lines.append("\n## Targets\n")
    lines.append("| Target | Result |")
    lines.append("|---|---|")
    lines.append(f"| LODO mean | **{mean['H']:.4f}/{mean['Acc']:.2f}** |")
    lines.append(f"| AVGC ≥0.85/78 | {avgc['H']:.4f}/{avgc['Acc']:.2f} |")
    lines.append(f"| UHD ≥0.85 | {uhd['H']:.4f} |")
    lines.append(
        f"| AASD Acc>60 | **{aasd['Acc']:.2f}** (vs 53.12 / SW2 53.46); H {aasd['H']:.4f} |")
    lines.append(f"| Carta | {carta['H']:.4f}/{carta['Acc']:.2f} |\n")
    lines.append("## Verdict\n")
    moved = []
    if avgc["H"] >= 0.85 and avgc["Acc"] >= 78:
        moved.append("AVGC target met")
    if aasd["Acc"] > 60:
        moved.append("AASD Acc>60 met")
    if not moved:
        lines.append(
            "No open target moved. SW2-as-ms-channel fine-tune does not import SW2's "
            "external Acc into the stem stream.\n")
    else:
        lines.append("; ".join(moved) + "\n")
    lines.append("## NEXT IDEA\n")
    lines.append(
        "Leave for claimed **xcorr-matched-clf** (new backbone). Do not re-grid SW2↔stem "
        "channel weights or reopen fuse↔hp30 / SW2 bank expands.\n"
    )
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))


def write_md_abandon(reason):
    text = f"""# results_stem_sw2_ms

**Status: abandoned**

Could not align published SW2 fuse evidence onto 24-d ms features.

Reason:
```
{reason}
```

## NEXT IDEA

Proceed to claimed **xcorr-matched-clf** (do not steal if another agent holds it). Do not invent SW2 scores or rebuild the decoder LODO stack for this concat.
"""
    with open(RESULTS_MD, "w") as fh:
        fh.write(text)


def main():
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    c_extra = 2 if EXTRA_HP30 else 1
    c_in_expect = MS_DIM + c_extra
    log_line(
        f"stem-sw2-ms start steps={STEPS} batch={BATCH} EXTRA_HP30={EXTRA_HP30} "
        f"c_in={c_in_expect} OFF={OFF} a_sw2={A_SW2}")

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    log_line(f"device {device}")
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    V.assert_causal(device)
    N.load_all()
    ms_all = V.build_features()["ms"]
    # sanity: 24-d
    sample_key = next(iter(ms_all[DS_KEYS[0]].values()))
    if sample_key.shape[1] != MS_DIM:
        raise RuntimeError(f"ms dim {sample_key.shape[1]} != {MS_DIM}")

    try:
        Z, meta = SW2.ensure_sw2_lodo()
        # quick sanity: fuse one held and align one trial
        sw2_avgc = SW2.fuse_sw2_held(Z, "AVGC")
        feat_smoke = build_feat_held(ms_all, sw2_avgc)
        c0 = next(iter(feat_smoke["AVGC"].values())).shape[1]
        log_line(f"SW2 LODO aligned OK; expanded dim={c0} (expect {c_in_expect})")
        if c0 != c_in_expect:
            raise RuntimeError(f"expanded dim {c0} != expect {c_in_expect}")
    except Exception as e:
        log_line(f"ABORT SW2 align: {e}")
        write_md_abandon(str(e))
        with open(OUT, "w") as fh:
            json.dump(dict(status="abandoned", reason=str(e)), fh, indent=2)
        return

    per = {}
    for held in DS_KEYS:
        path = os.path.join(SWA_DIR, f"held-{held}.pt")
        if not os.path.isfile(path):
            raise FileNotFoundError(path)
        train_ds = [d for d in DS_KEYS if d != held]
        sw2 = SW2.fuse_sw2_held(Z, held)
        feat = build_feat_held(ms_all, sw2)
        log_line(f"fold held-out={held} init {path}")
        ema, mu, sd, used, stats, c_in = fit_with_oom(
            path, feat, train_ds, device, STEPS, BATCH, f"held-{held}")
        torch.save(
            {"state": ema.state_dict(), "mu": mu, "sd": sd, "swa": path,
             "train": train_ds, "steps": STEPS, "batch": used, "stats": stats,
             "cfg": CFG, "c_in": c_in, "extra_hp30": EXTRA_HP30, "OFF": OFF},
            os.path.join(CKPT_DIR, f"held-{held}.pt"))
        scores, _ = score_lodo(ema, feat, mu, sd, held, device)
        per[held] = dict(scores=scores, stats=stats, batch=used, c_in=c_in)
        del ema
        if device.type == "cuda":
            torch.cuda.empty_cache()

    log_line(f"all-5 fine-tune for AASD/Carta, init {SWA_DIR}/all5.pt")
    feat5 = build_feat_all5_mean(ms_all, Z)
    all_path = os.path.join(SWA_DIR, "all5.pt")
    ema, mu, sd, used, stats, c_in = fit_with_oom(
        all_path, feat5, list(DS_KEYS), device, STEPS, BATCH, "all5")
    torch.save(
        {"state": ema.state_dict(), "mu": mu, "sd": sd, "swa": all_path,
         "train": list(DS_KEYS), "steps": STEPS, "batch": used, "stats": stats,
         "cfg": CFG, "c_in": c_in, "extra_hp30": EXTRA_HP30, "OFF": OFF},
        os.path.join(CKPT_DIR, "all5.pt"))
    corpus_streams = V.infer_all(ema, feat5, mu, sd, WINDOW, device)

    try:
        EV_sw2, _emis = SW2.ensure_sw2_external()
        ext_logits, ext_meta = load_external_logits_meta()
        ms_ext = V4.build_features_from(ext_logits)["ms"]
        feat_ext = build_ext_feat(ms_ext, EV_sw2)
        # drop trials missing from feat_ext meta alignment
        for ds in TEST_NAMES:
            keep = set(feat_ext[ds])
            ext_meta[ds] = {k: v for k, v in ext_meta[ds].items() if k in keep}
        ext = score_external(
            ema, feat_ext, ext_meta, mu, sd, corpus_streams, N.G["att"], device)
    except Exception as e:
        log_line(f"ABORT external: {e}")
        write_md_abandon(str(e))
        mean_h = float(np.mean([per[d]["scores"]["sw2_ms"]["H"] for d in DS_KEYS]))
        mean_acc = float(np.mean([per[d]["scores"]["sw2_ms"]["Acc"] for d in DS_KEYS]))
        with open(OUT, "w") as fh:
            json.dump(dict(
                status="partial_lodo", reason=str(e),
                lodo={d: per[d]["scores"] for d in DS_KEYS},
                lodo_mean=dict(H=mean_h, Acc=mean_acc),
            ), fh, indent=2, default=float)
        return

    mean_h = float(np.mean([per[d]["scores"]["sw2_ms"]["H"] for d in DS_KEYS]))
    mean_acc = float(np.mean([per[d]["scores"]["sw2_ms"]["Acc"] for d in DS_KEYS]))
    log_line(f"LODO mean H {mean_h:.4f} Acc {mean_acc:.2f}")
    aasd_acc = ext["AASD"]["sw2_ms"]["Acc"]
    log_line(
        f"AASD Acc {aasd_acc:.2f} vs linear_hp30 {LINEAR_HP30_AASD:.2f} "
        f"(Δ{aasd_acc-LINEAR_HP30_AASD:+.2f}) / SW2 {SW2_AASD:.2f} "
        f"(Δ{aasd_acc-SW2_AASD:+.2f})")

    out = dict(
        status="done",
        method="stem-sw2-ms",
        p_stay=P_STAY, scale=SCALE, steps=STEPS, batch=BATCH,
        extra_hp30=EXTRA_HP30, c_in=c_in, OFF=OFF, a_sw2=A_SW2,
        lodo={d: per[d]["scores"] for d in DS_KEYS},
        lodo_stats={d: per[d]["stats"] for d in DS_KEYS},
        lodo_mean=dict(H=mean_h, Acc=mean_acc),
        external=ext,
        all5_stats=stats,
        baselines=dict(linear_hp30_aasd=LINEAR_HP30_AASD, sw2_aasd=SW2_AASD),
        elapsed_s=time.time() - t_all,
        next_idea=(
            "Proceed to claimed xcorr-matched-clf (new backbone). "
            "Do not re-grid SW2 stem-channel concat / fuse↔hp30 / SW2 bank expands."
        ),
    )
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=2, default=float)
    write_md(out)
    log_line(f"wrote {OUT} and {RESULTS_MD} in {time.time()-t_all:.0f}s")


if __name__ == "__main__":
    main()
