"""Confidence-gated residual on frozen IRM↔synth fuse w=0.5 (stays on fuse stream).

Distinct from conf-gate-xstream (that gated fuse→hp30σ). Here a tiny causal residual
(affine / 1-layer MLP / 1-layer GRU, zero-init output) is fit on LODO **train** folds
only and applied **only when** past-only EWMA |fuse| is below a threshold; else raw fuse.

Select thr/α (and resid family) by mean LODO Acc then H @ HMM 0.994/0.8.
Never AASD/Carta labels for selection. Score LODO 5 + AASD + Carta.

Prefer CPU: reuse MoE IRM stream caches when present; infer synth once and cache
IRM↔synth fuse under models/fuse_conf_resid/cache/. Residual fit is CPU torch.

usage:
  CUDA_VISIBLE_DEVICES= \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u \\
    fuse_conf_resid.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as Fnn
from scipy.signal import lfilter

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
import stem_ms96_irm_synth_fuse as SF
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "fuse_conf_resid", "cache")
MOE_CACHE = M.CACHE_DIR
IRM_DIR = SF.IRM_DIR
SYNTH_DIR = SF.SYNTH_DIR
LOG = os.path.join(EXP, "experiments", "fuse_conf_resid.log")
OUT = os.path.join(EXP, "results", "research", "fuse_conf_resid.json")
CKPT = os.path.join(EXP, "models", "fuse_conf_resid")

P_STAY = SF.P_STAY
SCALE = SF.SCALE
TEST_NAMES = SF.TEST_NAMES
WINDOW = SF.WINDOW
W_FUSE = 0.5
HOP = 0.25
TAU_GATE = 4.0
EPS = 1e-6

FUSE_TIP = dict(  # IRM↔synth w=0.5 primary
    mean=dict(H=0.8790, Acc=83.11),
    AVGC=dict(H=0.8192, Acc=75.52),
    AASD=dict(H=0.6187, Acc=50.92),
    Carta2026=dict(H=0.5497, Acc=49.09),
)
LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_SWMIX_TIP = dict(AASD=dict(Acc=53.53), Carta2026=dict(Acc=59.71))

# (name, kind, thresh, alpha, selectable) — kind None = raw fuse
# kind: affine | mlp | gru
VARIANT_SPECS = [
    ("fuse_w05", None, None, None, False),
    ("affine_a0.25_t0.5", "affine", 0.5, 0.25, True),
    ("affine_a0.5_t0.5", "affine", 0.5, 0.5, True),
    ("affine_a1.0_t0.5", "affine", 0.5, 1.0, True),
    ("affine_a0.5_t1.0", "affine", 1.0, 0.5, True),
    ("affine_a1.0_t1.0", "affine", 1.0, 1.0, True),
    ("affine_a0.5_t1.5", "affine", 1.5, 0.5, True),
    ("mlp_a0.5_t0.5", "mlp", 0.5, 0.5, True),
    ("mlp_a1.0_t0.5", "mlp", 0.5, 1.0, True),
    ("mlp_a0.5_t1.0", "mlp", 1.0, 0.5, True),
    ("mlp_a1.0_t1.0", "mlp", 1.0, 1.0, True),
    # Compact GRU grid (CPU-costly); claim allows affine OR mlp OR gru
    ("gru_a0.5_t0.5", "gru", 0.5, 0.5, True),
    ("gru_a1.0_t1.0", "gru", 1.0, 1.0, True),
]

# Always-on residual refs (not selectable) — diagnose gate necessity
# (skip always_gru: too slow on CPU for marginal diagnostic value)
ALWAYS_SPECS = [
    ("always_affine_a1", "affine", None, 1.0, False),
    ("always_mlp_a1", "mlp", None, 1.0, False),
]

TRAIN_STEPS = int(os.environ.get("RESID_STEPS", "400"))
GRU_STEPS = int(os.environ.get("GRU_STEPS", "50"))  # CPU: ~380s@400 → ~50s@50
TRAIN_LR = 3e-3
MAX_TRAIN_FRAMES = 250_000
HID = 8
DEVICE = torch.device("cpu")  # caches hit; never hold GPU / never flock
RESID_DEVICE = torch.device("cpu")


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def save_pkl(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as fh:
        pickle.dump(obj, fh, protocol=pickle.HIGHEST_PROTOCOL)


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def ewma_1d(x, tau_s=TAU_GATE, hop=HOP):
    x = np.asarray(x, np.float64)
    alpha = 1.0 - np.exp(-hop / float(tau_s))
    return lfilter([alpha], [1.0, -(1.0 - alpha)], x)


def ewma_abs_past(f, tau_s=TAU_GATE):
    """Past-only EWMA of |fuse| (gate at t uses history through t-1)."""
    feat = ewma_1d(np.abs(np.asarray(f, np.float64)), tau_s=tau_s)
    past = np.empty_like(feat)
    past[0] = 0.0
    if len(feat) > 1:
        past[1:] = feat[:-1]
    return past


def _infer_one(ckpt_path, feat, device, tag):
    t0 = time.time()
    log_line(f"  infer {tag} {ckpt_path} on {device}")
    net, mu, sd = F.load_ckpt(ckpt_path, device)
    st = V.infer_all(net, feat, mu, sd, WINDOW, device)
    del net
    log_line(f"  infer {tag} done ({time.time() - t0:.0f}s)")
    return st


def _load_or_infer_stream(cache_path, ckpt_path, feat, tag, prefer_moe=None):
    if os.path.isfile(cache_path):
        log_line(f"cache hit {cache_path}")
        return load_pkl(cache_path)
    if prefer_moe and os.path.isfile(prefer_moe):
        log_line(f"reuse MoE {prefer_moe} → {cache_path}")
        st = load_pkl(prefer_moe)
        save_pkl(cache_path, st)
        return st
    st = _infer_one(ckpt_path, feat, DEVICE, tag)
    save_pkl(cache_path, st)
    return st


def ensure_held_fuse(held, feat_all):
    path = os.path.join(CACHE_DIR, f"irm_synth_w05__held-{held}.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_c = os.path.join(CACHE_DIR, f"_tmp_irm__held-{held}.pkl")
    syn_c = os.path.join(CACHE_DIR, f"_tmp_synth__held-{held}.pkl")
    moe_irm = os.path.join(MOE_CACHE, f"_tmp_irm__held-{held}.pkl")
    irm = _load_or_infer_stream(
        irm_c, os.path.join(IRM_DIR, f"held-{held}.pt"), feat_all,
        f"held-{held}-IRM", prefer_moe=moe_irm,
    )
    syn = _load_or_infer_stream(
        syn_c, os.path.join(SYNTH_DIR, f"held-{held}.pt"), feat_all,
        f"held-{held}-synth", prefer_moe=None,
    )
    blended = SF.blend_streams(irm, syn, W_FUSE)
    save_pkl(path, blended)
    log_line(f"cache write {path}")
    return blended


def ensure_all5_corpus(feat_all):
    path = os.path.join(CACHE_DIR, "irm_synth_w05__all5_corpus.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_c = os.path.join(CACHE_DIR, "_tmp_irm__all5_corpus.pkl")
    syn_c = os.path.join(CACHE_DIR, "_tmp_synth__all5_corpus.pkl")
    moe_irm = os.path.join(MOE_CACHE, "_tmp_irm__all5_corpus.pkl")
    irm = _load_or_infer_stream(
        irm_c, os.path.join(IRM_DIR, "all5.pt"), feat_all,
        "all5-corpus-IRM", prefer_moe=moe_irm,
    )
    syn = _load_or_infer_stream(
        syn_c, os.path.join(SYNTH_DIR, "all5.pt"), feat_all,
        "all5-corpus-synth", prefer_moe=None,
    )
    blended = SF.blend_streams(irm, syn, W_FUSE)
    save_pkl(path, blended)
    log_line(f"cache write {path}")
    return blended


def ensure_all5_external(ext_feat):
    path = os.path.join(CACHE_DIR, "irm_synth_w05__all5_external.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_c = os.path.join(CACHE_DIR, "_tmp_irm__all5_external.pkl")
    syn_c = os.path.join(CACHE_DIR, "_tmp_synth__all5_external.pkl")
    moe_irm = os.path.join(MOE_CACHE, "_tmp_irm__all5_external.pkl")
    irm = _load_or_infer_stream(
        irm_c, os.path.join(IRM_DIR, "all5.pt"), ext_feat,
        "all5-ext-IRM", prefer_moe=moe_irm,
    )
    syn = _load_or_infer_stream(
        syn_c, os.path.join(SYNTH_DIR, "all5.pt"), ext_feat,
        "all5-ext-synth", prefer_moe=None,
    )
    out = {ds: SF.blend_one(irm[ds], syn[ds], W_FUSE) for ds in irm}
    save_pkl(path, out)
    log_line(f"cache write {path}")
    return out


class AffineResid(nn.Module):
    """resid = a*f + b; zero-init → identity fuse+α·resid."""

    def __init__(self):
        super().__init__()
        self.a = nn.Parameter(torch.zeros(1))
        self.b = nn.Parameter(torch.zeros(1))

    def forward(self, f, ewma_abs):
        return self.a * f + self.b


class MLPResid(nn.Module):
    """1-hidden MLP on [f, ewma_|f|, 1]; last layer zero-init."""

    def __init__(self, hid=HID):
        super().__init__()
        self.fc1 = nn.Linear(3, hid)
        self.fc2 = nn.Linear(hid, 1)
        nn.init.zeros_(self.fc2.weight)
        nn.init.zeros_(self.fc2.bias)

    def forward(self, f, ewma_abs):
        x = torch.stack([f, ewma_abs, torch.ones_like(f)], -1)
        return self.fc2(Fnn.relu(self.fc1(x))).squeeze(-1)


class GRUResid(nn.Module):
    """1-layer GRU on fuse; zero-init readout."""

    def __init__(self, hid=HID):
        super().__init__()
        self.gru = nn.GRU(1, hid, batch_first=True)
        self.out = nn.Linear(hid, 1)
        nn.init.zeros_(self.out.weight)
        nn.init.zeros_(self.out.bias)

    def forward_seq(self, f):
        # f: (T,)
        x = f.view(1, -1, 1)
        h, _ = self.gru(x)
        return self.out(h).view(-1)


def make_resid(kind):
    if kind == "affine":
        return AffineResid()
    if kind == "mlp":
        return MLPResid()
    if kind == "gru":
        return GRUResid()
    raise ValueError(kind)


def collect_train_frames(streams, att, datasets, thr_fit, max_frames, seed=0):
    """Frames with past-only EWMA |fuse| < thr_fit (or all if thr_fit is None)."""
    rng = np.random.default_rng(seed)
    fs, es, ys = [], [], []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            if key not in labels:
                continue
            f = np.asarray(d, np.float64).reshape(-1)
            lab = np.asarray(labels[key]).reshape(-1)
            n = min(len(f), len(lab))
            f, lab = f[:n], lab[:n]
            e = ewma_abs_past(f)
            if thr_fit is not None:
                m = e < float(thr_fit)
                if not np.any(m):
                    continue
                f, e, lab = f[m], e[m], lab[m]
            fs.append(f)
            es.append(e)
            ys.append(lab.astype(np.float64))
    if not fs:
        return (np.zeros(0), np.zeros(0), np.zeros(0))
    f = np.concatenate(fs)
    e = np.concatenate(es)
    y = np.concatenate(ys)
    if len(f) > max_frames:
        idx = rng.choice(len(f), max_frames, replace=False)
        f, e, y = f[idx], e[idx], y[idx]
    return f, e, y


def fit_resid(kind, streams, att, datasets, thr_fit, steps=TRAIN_STEPS, seed=0):
    """Fit residual on train folds; thr_fit gates which frames enter the loss."""
    net = make_resid(kind).to(RESID_DEVICE)
    rng = np.random.default_rng(seed + 7)
    opt = torch.optim.Adam(net.parameters(), lr=TRAIN_LR)
    last = None
    n_used = 0
    net.train()
    if kind == "gru":
        steps = min(steps, GRU_STEPS)
        for _step in range(steps):
            loss = _fit_gru_step(net, opt, streams, att, datasets, thr_fit, rng,
                                 n_trials=4)
            if loss is not None:
                last = loss
                n_used += 1
        net.eval()
        return net, dict(n=n_used, loss=last, steps=steps, thr_fit=thr_fit)

    f, e, y = collect_train_frames(streams, att, datasets, thr_fit, MAX_TRAIN_FRAMES, seed)
    if len(f) < 100:
        log_line(f"  warn fit {kind} thr={thr_fit}: only {len(f)} frames — keep zero-init")
        net.eval()
        return net, dict(n=int(len(f)), loss=None, steps=0, thr_fit=thr_fit)
    ft = torch.from_numpy(f.astype(np.float32))
    et = torch.from_numpy(e.astype(np.float32))
    yt = torch.from_numpy(y.astype(np.float32))
    # att==0 → class 1 (same polarity as N.to_ev / official)
    target = (yt == 0).float()
    bs = min(8192, len(f))
    for _step in range(steps):
        idx = rng.choice(len(f), bs, replace=False)
        fb, eb, tb = ft[idx], et[idx], target[idx]
        resid = net(fb, eb)
        logits = fb + resid
        loss = Fnn.binary_cross_entropy_with_logits(logits, tb)
        opt.zero_grad()
        loss.backward()
        opt.step()
        last = float(loss.detach())
    net.eval()
    return net, dict(n=int(len(f)), loss=last, steps=steps, thr_fit=thr_fit)


def _fit_gru_step(net, opt, streams, att, datasets, thr_fit, rng, n_trials=8):
    keys = []
    for ds in datasets:
        for key in streams[ds]:
            if key in att[ds]:
                keys.append((ds, key))
    if not keys:
        return None
    losses = []
    opt.zero_grad()
    for _ in range(n_trials):
        ds, key = keys[int(rng.integers(0, len(keys)))]
        f = np.asarray(streams[ds][key], np.float64).reshape(-1)
        lab = np.asarray(att[ds][key]).reshape(-1)
        n = min(len(f), len(lab))
        f, lab = f[:n], lab[:n]
        e = ewma_abs_past(f)
        ft = torch.from_numpy(f.astype(np.float32))
        resid = net.forward_seq(ft)
        logits = ft + resid
        target = torch.from_numpy((lab == 0).astype(np.float32))
        if thr_fit is not None:
            m = torch.from_numpy((e < float(thr_fit)).astype(np.float32))
            if float(m.sum()) < 1:
                continue
            loss = (Fnn.binary_cross_entropy_with_logits(logits, target, reduction="none") * m).sum() / m.sum()
        else:
            loss = Fnn.binary_cross_entropy_with_logits(logits, target)
        loss.backward()
        losses.append(float(loss.detach()))
    if losses:
        opt.step()
        return float(np.mean(losses))
    return None


@torch.no_grad()
def resid_stream(net, kind, fuse_st):
    out = {}
    for key, d in fuse_st.items():
        f = np.asarray(d, np.float64).reshape(-1)
        ft = torch.from_numpy(f.astype(np.float32))
        if kind == "gru":
            r = net.forward_seq(ft).cpu().numpy().astype(np.float64)
        else:
            e = ewma_abs_past(f)
            et = torch.from_numpy(e.astype(np.float32))
            r = net(ft, et).cpu().numpy().astype(np.float64)
        out[key] = r
    return out


def apply_gated(fuse_st, resid_st, thresh, alpha):
    """If thresh is None → always fuse+α·resid; else hard gate on past EWMA |fuse|."""
    out = {}
    for key, raw in fuse_st.items():
        if key not in resid_st:
            continue
        f = np.asarray(raw, np.float64).reshape(-1)
        r = np.asarray(resid_st[key], np.float64).reshape(-1)
        n = min(len(f), len(r))
        f, r = f[:n], r[:n]
        blended = f + float(alpha) * r
        if thresh is None:
            out[key] = blended
        else:
            feat = ewma_abs_past(f)
            out[key] = np.where(feat < float(thresh), blended, f).astype(np.float64)
    return out


def apply_all(fuse_all, resid_all, thresh, alpha):
    return {ds: apply_gated(fuse_all[ds], resid_all[ds], thresh, alpha)
            for ds in fuse_all if ds in resid_all}


def frac_gated(fuse_st, thresh):
    if thresh is None:
        return 1.0
    ws, tot = 0.0, 0
    for table in fuse_st.values():
        for raw in table.values():
            f = np.asarray(raw, np.float64).reshape(-1)
            if not len(f):
                continue
            feat = ewma_abs_past(f)
            ws += float(np.sum(feat < float(thresh)))
            tot += len(f)
    return float(ws / tot) if tot else 0.0


def pick_variant(lodo_means, candidates):
    best_name, best_key = None, None
    for name in candidates:
        m = lodo_means[name]
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def fit_bundle(kinds, streams, att, datasets, thr_fits, seed=0):
    """Fit one model per (kind, thr_fit) used by variants; thr_fit=None for always-on."""
    models = {}
    meta = {}
    needed = set()
    for kind in kinds:
        for thr in thr_fits:
            needed.add((kind, thr))
    for kind, thr in sorted(needed, key=lambda x: (x[0], -1 if x[1] is None else x[1])):
        key = (kind, thr)
        t0 = time.time()
        net, info = fit_resid(kind, streams, att, datasets, thr, seed=seed)
        models[key] = net
        meta[f"{kind}|t{thr}"] = {**info, "sec": time.time() - t0}
        log_line(f"  fit {kind} thr_fit={thr} n={info['n']} loss={info.get('loss')} "
                 f"({time.time()-t0:.1f}s)")
    return models, meta


def resid_all_from(models, kind, thr_fit, fuse_all):
    net = models[(kind, thr_fit)]
    return {ds: resid_stream(net, kind, fuse_all[ds]) for ds in fuse_all}


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"fuse-conf-resid CPU  p_stay {P_STAY} scale {SCALE}  gate_tau {TAU_GATE}s past-only")
    log_line("honesty: residual on IRM↔synth fuse w=0.5 only; NO hp30 mix; train folds only; "
             "select by mean LODO Acc then H")
    log_line("distinct from conf-gate-xstream (fuse→hp30σ)")

    torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "8")))
    N.load_all()
    att = N.G["att"]
    feat_all = V.build_features()["ms"]

    log_line("=== cache IRM↔synth fuse w=0.5 (CPU infer if needed) ===")
    fuse_by_held = {}
    for held in DS_KEYS:
        fuse_by_held[held] = ensure_held_fuse(held, feat_all)

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity raw IRM↔synth fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} "
             f"(expect ~0.8192/75.52)")

    all_specs = VARIANT_SPECS + ALWAYS_SPECS
    kinds = sorted({k for _, k, *_ in all_specs if k})
    # Fit resid on frames with ewma < thr when gated; always-on fits thr_fit=None
    thr_fits_for_kind = {k: set() for k in kinds}
    for name, kind, thr, alpha, sel in all_specs:
        if kind is None:
            continue
        thr_fits_for_kind[kind].add(thr)  # None for always-on

    results = dict(
        method=("confidence-gated residual on frozen IRM↔synth fuse w=0.5; "
                "apply resid only when past-only EWMA |fuse| low; else raw fuse"),
        p_stay=P_STAY, scale=SCALE, tau_gate=TAU_GATE, w_fuse=W_FUSE,
        honesty=("Train residual on LODO train folds only (labels). "
                 "Past-only EWMA |fuse| gate. No AASD/Carta for selection. "
                 "Distinct from conf-gate-xstream fuse→hp30σ."),
        baselines=dict(irm_synth_fuse_w05=FUSE_TIP, linear_hp30_ext=LINEAR_HP30_EXT,
                       sw2_swmix_tip=SW2_SWMIX_TIP),
        variants={},
    )

    log_line("=== LODO (fit resid per held on train folds; select gated) ===")
    lodo_means = {}
    # Precompute resid streams per held per (kind, thr_fit)
    resid_cache = {}  # held -> (kind, thr_fit) -> resid_all

    for held in DS_KEYS:
        fuse = fuse_by_held[held]
        train_ds = [d for d in DS_KEYS if d != held]
        # Fit each kind only on its own thr set (avoid GRU×unused thr=1.5/None)
        models, meta = {}, {}
        for kind in kinds:
            m_k, meta_k = fit_bundle(
                [kind], fuse, att, train_ds, thr_fits_for_kind[kind],
                seed=hash(held + kind) % 10_000,
            )
            models.update(m_k)
            meta.update(meta_k)
        results.setdefault("fit_meta", {})[held] = meta
        resid_cache[held] = {}
        for kind in kinds:
            for thr in thr_fits_for_kind[kind]:
                resid_cache[held][(kind, thr)] = resid_all_from(models, kind, thr, fuse)
        log_line(f"held={held} resid streams ready")

    for name, kind, thr, alpha, selectable in all_specs:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            fuse = fuse_by_held[held]
            if kind is None:
                gated = fuse
                frac = 0.0
            else:
                thr_fit = thr  # train on same thr as gate; always-on uses None
                resid_all = resid_cache[held][(kind, thr_fit)]
                gated = apply_all(fuse, resid_all, thr, alpha)
                frac = frac_gated(fuse, thr)
            off = M.score_lodo(gated, held)
            per[held] = dict(H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"),
                             L=off.get("L"), acc1=off.get("acc1"), frac_gated=frac)
            log_line(f"LODO {name:22s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"frac {frac:.3f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:22s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            kind=kind, thresh=thr, alpha=alpha, selectable=selectable,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    gated_names = [n for n, k, *_, sel in VARIANT_SPECS if sel]
    selected = pick_variant(lodo_means, gated_names)
    ranked = sorted(gated_names, key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
                    reverse=True)
    log_line(f"SELECTED (LODO Acc then H, gated only): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    log_line(f"ranked: {ranked}")
    results["selection"] = dict(
        criterion="mean LODO Acc then H (gated variants only; never AASD/Carta)",
        ranked=ranked, selected=selected,
        excluded_from_select=["fuse_w05"] + [n for n, *_ in ALWAYS_SPECS],
    )
    results["selected"] = selected

    log_line("=== external (all5 corpus fit + external apply) ===")
    corp = ensure_all5_corpus(feat_all)
    ext_logits, ext_meta = SF.load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    ext_fuse = ensure_all5_external(ext_feat)

    models_ext, meta_ext = {}, {}
    for kind in kinds:
        m_k, meta_k = fit_bundle(
            [kind], corp, att, list(V4.CORPUS), thr_fits_for_kind[kind], seed=42 + hash(kind) % 100,
        )
        models_ext.update(m_k)
        meta_ext.update(meta_k)
    results["fit_meta_external"] = meta_ext
    resid_corp = {}
    resid_ext = {}
    for kind in kinds:
        for thr in thr_fits_for_kind[kind]:
            resid_corp[(kind, thr)] = resid_all_from(models_ext, kind, thr, corp)
            net = models_ext[(kind, thr)]
            resid_ext[(kind, thr)] = {
                ds: resid_stream(net, kind, ext_fuse[ds]) for ds in ext_fuse
            }

    for name, kind, thr, alpha, _sel in all_specs:
        if kind is None:
            gated_train = corp
            gated_test = ext_fuse
            frac = 0.0
        else:
            thr_fit = thr
            gated_train = apply_all(corp, resid_corp[(kind, thr_fit)], thr, alpha)
            gated_test = {
                ds: apply_gated(ext_fuse[ds], resid_ext[(kind, thr_fit)][ds], thr, alpha)
                for ds in ext_fuse
            }
            frac = frac_gated(ext_fuse, thr)
        mu_e, sd_e = SF.emissions_from(gated_train, att, datasets=list(V4.CORPUS))
        ext = {}
        for ds in TEST_NAMES:
            r = SF.score_external(gated_test[ds], ext_meta[ds], mu_e, sd_e)
            ext[ds] = r
            d_fuse = r["Acc"] - FUSE_TIP[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            log_line(f"EXT {name:22s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs fuse {d_fuse:+.2f}  vs hp30 {d_hp:+.2f}  frac {frac:.3f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)
        results["variants"][name]["frac_gated_ext"] = frac

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    avgc = prim["lodo"]["AVGC"]
    d_tip = dict(H=avgc["H"] - FUSE_TIP["AVGC"]["H"], Acc=avgc["Acc"] - FUSE_TIP["AVGC"]["Acc"])
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=avgc,
        AVGC_delta_vs_fuse_tip=d_tip,
        AASD_Acc_delta_vs_fuse=aasd["Acc"] - FUSE_TIP["AASD"]["Acc"],
        AASD_Acc_delta_vs_linear_hp30=aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"],
        AASD_Acc_delta_vs_sw2_swmix=aasd["Acc"] - SW2_SWMIX_TIP["AASD"]["Acc"],
        frac_gated_ext=prim.get("frac_gated_ext"),
        targets=dict(
            AVGC_ge_085_78=bool(avgc["H"] >= 0.85 and avgc["Acc"] >= 78),
            AASD_Acc_gt_60=bool(aasd["Acc"] > 60),
        ),
    )
    log_line(f"HEADLINE {selected} AVGC H {avgc['H']:.4f} Acc {avgc['Acc']:.2f} "
             f"(Δtip {d_tip['H']:+.4f}/{d_tip['Acc']:+.2f})  "
             f"AASD Acc {aasd['Acc']:.2f}  Carta Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "summary.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       headline=results["headline"]), fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
