"""Causal input-conditioned MoE gate: IRM↔SWA fuse w=0.5 ↔ linear_hp30.

GPU stem forward (do NOT set CUDA_VISIBLE_DEVICES=). Hold gpu.lock for infer.
No new network training. Cache under models/causal_moe_fuse/cache/.

Streams per trial (aligned by key):
  fuse  = 0.5 * IRM + 0.5 * SWA   (frozen stem_ms96 checkpoints)
  hp30  = causal tau-30 high-pass of mean of 4 seed classifier logits

Gate g_t ∈ [0,1] from causal input stats only (no labels at test).
Evidence = g * fuse + (1-g) * hp30.

Gate family (pre-specified; pick by mean inner Acc then H, never AASD/Carta):
  favor hp30 when |fuse| is small and/or |fuse-hp30| disagreement is high.
Optional tiny logistic gate fit on LODO non-held-out folds only.

HMM preset p_stay 0.994 scale 0.8.

usage:
  /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u causal_moe_fuse.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np
import torch
from scipy.signal import lfilter
from scipy.special import expit

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "causal_moe_fuse", "cache")
LOG = os.path.join(EXP, "experiments", "causal_moe_fuse.log")
OUT = os.path.join(EXP, "results", "research", "causal_moe_fuse.json")
IRM_DIR = F.IRM_DIR
SWA_DIR = F.SWA_DIR
WINDOW = F.WINDOW
P_STAY = F.P_STAY
SCALE = F.SCALE
TEST_NAMES = F.TEST_NAMES
W_FUSE = 0.5

# baselines from prior write-ups (for reporting)
FUSE_LODO = dict(
    AVGC=dict(H=0.8100, Acc=75.14), DTU=dict(H=0.9007, Acc=89.24),
    KUL=dict(H=0.8749, Acc=82.20), UHD=dict(H=0.8580, Acc=80.10),
    USTC=dict(H=0.9188, Acc=86.25),
)
FUSE_MEAN = dict(H=0.8725, Acc=82.58)
LINEAR_HP30_LODO = dict(H=0.7899, Acc=69.21)  # mean from ALREADY_TRIED
LINEAR_HP30_EXT = F.LINEAR_HP30_BASELINE
FUSE_EXT = dict(AASD=dict(H=0.6179, Acc=50.94), Carta2026=dict(H=0.5461, Acc=48.38))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def save_pkl(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as fh:
        pickle.dump(obj, fh, protocol=pickle.HIGHEST_PROTOCOL)


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def ewma_1d(x, tau_s=4.0, hop=0.25):
    """Causal EWMA (one-pole) of a 1-d series."""
    x = np.asarray(x, np.float64)
    alpha = 1.0 - np.exp(-hop / tau_s)
    return lfilter([alpha], [1.0, -(1.0 - alpha)], x)


def gate_series(fuse, hp30, kind, thresh, sharp, ewma_tau=4.0):
    """Return g in [0,1]; high g → trust fuse."""
    f = np.asarray(fuse, np.float64)
    h = np.asarray(hp30, np.float64)
    n = min(len(f), len(h))
    f, h = f[:n], h[:n]
    if kind == "const_fuse":
        return np.ones(n, np.float64)
    if kind == "const_hp30":
        return np.zeros(n, np.float64)
    if kind == "abs_fuse":
        # favor hp30 when |fuse| small
        feat = np.abs(f)
        return expit(sharp * (feat - thresh))
    if kind == "ewma_abs_fuse":
        feat = ewma_1d(np.abs(f), tau_s=ewma_tau)
        return expit(sharp * (feat - thresh))
    if kind == "disagree":
        # favor hp30 when |fuse-hp30| large
        feat = np.abs(f - h)
        return expit(sharp * (thresh - feat))
    if kind == "ewma_disagree":
        feat = ewma_1d(np.abs(f - h), tau_s=ewma_tau)
        return expit(sharp * (thresh - feat))
    if kind == "abs_and_disagree":
        # both: high |fuse| and low disagreement → fuse
        a = expit(sharp * (np.abs(f) - thresh))
        d = expit(sharp * (thresh - np.abs(f - h)))
        return a * d
    if kind == "hard_abs":
        return (np.abs(f) >= thresh).astype(np.float64)
    raise ValueError(kind)


def apply_gate_stream(fuse_st, hp_st, kind, thresh, sharp, ewma_tau=4.0):
    out = {}
    for key in fuse_st:
        if key not in hp_st:
            continue
        f = np.asarray(fuse_st[key], np.float64)
        h = np.asarray(hp_st[key], np.float64)
        n = min(len(f), len(h))
        g = gate_series(f[:n], h[:n], kind, thresh, sharp, ewma_tau)
        out[key] = (g * f[:n] + (1.0 - g) * h[:n]).astype(np.float64)
    return out


def apply_gate_all(fuse_all, hp_all, kind, thresh, sharp, ewma_tau=4.0):
    return {ds: apply_gate_stream(fuse_all[ds], hp_all[ds], kind, thresh, sharp, ewma_tau)
            for ds in fuse_all if ds in hp_all}


def logistic_gate_stream(fuse_st, hp_st, coef, intercept, ewma_tau=4.0):
    """g = sigmoid(w·x + b); x = [|f|, |h|, |f-h|, ewma|f|, ewma|f-h|]."""
    out = {}
    w = np.asarray(coef, np.float64)
    for key in fuse_st:
        if key not in hp_st:
            continue
        f = np.asarray(fuse_st[key], np.float64)
        h = np.asarray(hp_st[key], np.float64)
        n = min(len(f), len(h))
        f, h = f[:n], h[:n]
        af, ah, ad = np.abs(f), np.abs(h), np.abs(f - h)
        X = np.stack([af, ah, ad, ewma_1d(af, ewma_tau), ewma_1d(ad, ewma_tau)], 1)
        g = expit(X @ w + intercept)
        out[key] = (g * f + (1.0 - g) * h).astype(np.float64)
    return out


def fit_logistic_gate(fuse_all, hp_all, att, datasets, max_frames=300_000, seed=0):
    """Fit logistic: P(prefer fuse) from signed-correctness on train folds only."""
    rng = np.random.default_rng(seed)
    xs, ys = [], []
    for ds in datasets:
        labels = att[ds]
        for key in fuse_all[ds]:
            if key not in hp_all[ds] or key not in labels:
                continue
            f = np.asarray(fuse_all[ds][key], np.float64)
            h = np.asarray(hp_all[ds][key], np.float64)
            lab = np.asarray(labels[key])
            n = min(len(f), len(h), len(lab))
            f, h, lab = f[:n], h[:n], lab[:n]
            sgn = np.where(lab == 0, 1.0, -1.0)
            # prefer fuse when signed fuse is more positive than signed hp30
            y = (f * sgn > h * sgn).astype(np.float64)
            af, ah, ad = np.abs(f), np.abs(h), np.abs(f - h)
            X = np.stack([af, ah, ad, ewma_1d(af), ewma_1d(ad)], 1)
            xs.append(X)
            ys.append(y)
    X = np.concatenate(xs, 0)
    y = np.concatenate(ys, 0)
    if len(X) > max_frames:
        idx = rng.choice(len(X), max_frames, replace=False)
        X, y = X[idx], y[idx]
    # standardize features on train only
    mu = X.mean(0)
    sd = X.std(0).clip(1e-6)
    Xz = (X - mu) / sd
    # L2 logistic via IRLS-lite (few Newton steps)
    w = np.zeros(Xz.shape[1], np.float64)
    b = 0.0
    lam = 1e-2
    for _ in range(25):
        z = Xz @ w + b
        p = expit(z)
        # gradient / Hessian diagonal approx
        r = p - y
        gw = Xz.T @ r / len(y) + lam * w
        gb = r.mean()
        W = p * (1.0 - p)
        # damping
        Hdiag = (Xz * Xz * W[:, None]).mean(0) + lam
        w -= 0.5 * gw / Hdiag.clip(1e-8)
        b -= 0.5 * gb / (W.mean() + 1e-8)
    # return weights in original feature space: w_raw = w / sd, b_raw = b - (w/sd)·mu
    coef = w / sd
    intercept = float(b - np.dot(coef, mu))
    return dict(coef=coef.tolist(), intercept=intercept, mu=mu.tolist(), sd=sd.tolist(), n=int(len(y)))


def candidate_rules():
    """Compact pre-specified gate grid (favor hp30 when |fuse| small / disagreement high)."""
    rules = [
        ("const_fuse", 0.0, 1.0),
        ("const_hp30", 0.0, 1.0),
        # |fuse| soft: trust fuse when magnitude large
        ("abs_fuse", 0.5, 4.0),
        ("abs_fuse", 1.0, 4.0),
        ("abs_fuse", 1.5, 4.0),
        ("abs_fuse", 1.0, 8.0),
        ("ewma_abs_fuse", 0.5, 4.0),
        ("ewma_abs_fuse", 1.0, 4.0),
        ("ewma_abs_fuse", 1.5, 4.0),
        # disagreement: trust fuse when streams agree
        ("disagree", 0.5, 4.0),
        ("disagree", 1.0, 4.0),
        ("disagree", 1.5, 4.0),
        ("ewma_disagree", 0.5, 4.0),
        ("ewma_disagree", 1.0, 4.0),
        ("ewma_disagree", 1.5, 4.0),
        # hard threshold on |fuse|
        ("hard_abs", 0.5, 1.0),
        ("hard_abs", 1.0, 1.0),
        ("hard_abs", 1.5, 1.0),
        # both conditions
        ("abs_and_disagree", 0.75, 4.0),
        ("abs_and_disagree", 1.0, 4.0),
        ("abs_and_disagree", 1.25, 8.0),
    ]
    return rules


def rule_key(kind, thr, sharp):
    return f"{kind}|t{thr:g}|s{sharp:g}"


def _infer_one(ckpt_path, feat, device, tag):
    t0 = time.time()
    log_line(f"  infer-one {tag} {ckpt_path}")
    net, mu, sd = F.load_ckpt(ckpt_path, device)
    st = V.infer_all(net, feat, mu, sd, WINDOW, device)
    del net
    log_line(f"  infer-one {tag} done ({time.time() - t0:.0f}s)")
    return st


def ensure_fuse_cache(held, feat_all, device):
    path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_path = os.path.join(IRM_DIR, f"held-{held}.pt")
    swa_path = os.path.join(SWA_DIR, f"held-{held}.pt")
    irm_cache = os.path.join(CACHE_DIR, f"_tmp_irm__held-{held}.pkl")
    swa_cache = os.path.join(CACHE_DIR, f"_tmp_swa__held-{held}.pkl")
    if not os.path.isfile(irm_cache):
        irm_st = _infer_one(irm_path, feat_all, device, f"held-{held}-IRM")
        save_pkl(irm_cache, irm_st)
    else:
        log_line(f"cache hit {irm_cache}")
        irm_st = load_pkl(irm_cache)
    if not os.path.isfile(swa_cache):
        swa_st = _infer_one(swa_path, feat_all, device, f"held-{held}-SWA")
        save_pkl(swa_cache, swa_st)
    else:
        log_line(f"cache hit {swa_cache}")
        swa_st = load_pkl(swa_cache)
    blended = F.blend_streams(irm_st, swa_st, W_FUSE)
    save_pkl(path, blended)
    log_line(f"cache write {path}")
    return blended


def ensure_all5_corpus_fuse(feat_all, device):
    path = os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_path = os.path.join(IRM_DIR, "all5.pt")
    swa_path = os.path.join(SWA_DIR, "all5.pt")
    irm_cache = os.path.join(CACHE_DIR, "_tmp_irm__all5_corpus.pkl")
    swa_cache = os.path.join(CACHE_DIR, "_tmp_swa__all5_corpus.pkl")
    if not os.path.isfile(irm_cache):
        irm_st = _infer_one(irm_path, feat_all, device, "all5-corpus-IRM")
        save_pkl(irm_cache, irm_st)
    else:
        irm_st = load_pkl(irm_cache)
    if not os.path.isfile(swa_cache):
        swa_st = _infer_one(swa_path, feat_all, device, "all5-corpus-SWA")
        save_pkl(swa_cache, swa_st)
    else:
        swa_st = load_pkl(swa_cache)
    blended = F.blend_streams(irm_st, swa_st, W_FUSE)
    save_pkl(path, blended)
    log_line(f"cache write {path}")
    return blended


def ensure_all5_ext_fuse(ext_feat, device):
    path = os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl")
    if os.path.isfile(path):
        log_line(f"cache hit {path}")
        return load_pkl(path)
    irm_path = os.path.join(IRM_DIR, "all5.pt")
    swa_path = os.path.join(SWA_DIR, "all5.pt")
    irm_cache = os.path.join(CACHE_DIR, "_tmp_irm__all5_external.pkl")
    swa_cache = os.path.join(CACHE_DIR, "_tmp_swa__all5_external.pkl")
    if not os.path.isfile(irm_cache):
        irm_st = _infer_one(irm_path, ext_feat, device, "all5-ext-IRM")
        save_pkl(irm_cache, irm_st)
    else:
        irm_st = load_pkl(irm_cache)
    if not os.path.isfile(swa_cache):
        swa_st = _infer_one(swa_path, ext_feat, device, "all5-ext-SWA")
        save_pkl(swa_cache, swa_st)
    else:
        swa_st = load_pkl(swa_cache)
    out = {ds: F.blend_one(irm_st[ds], swa_st[ds], W_FUSE) for ds in irm_st}
    save_pkl(path, out)
    log_line(f"cache write {path}")
    return out


def load_lodo_hp30():
    """Per-dataset LODO linear_hp30 streams (r0 only). Uses N.G from load_all (fuse schedule)."""
    mean_ev, hp_ev, lookahead = V4._mean_and_hp_from_folds()
    hp = {ds: {k: e["r0"] for k, e in hp_ev[ds].items()} for ds in DS_KEYS}
    mean = {ds: {k: e["r0"] for k, e in mean_ev[ds].items()} for ds in DS_KEYS}
    return hp, mean, lookahead


def load_ext_hp30():
    """Corpus + external linear_hp30 from 4-seed logits (v4 external recipe)."""
    train_logits, train_att = V4.load_train()
    test_logits, test_meta = V4.load_test()
    base = {}
    base.update(train_logits)
    base.update(test_logits)
    streams = V4.linear_streams(base, "hp30")
    corp = {ds: streams[ds] for ds in V4.CORPUS}
    ext = {ds: streams[ds] for ds in TEST_NAMES}
    return corp, ext, train_att, test_meta


def score_lodo(streams, held):
    return N.official(held, N.to_ev(streams), P_STAY, SCALE)


def fast_score(streams, ds, train_ds):
    """Fast inner scorer (N.evaluate); emissions from train_ds only."""
    mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
    return N.evaluate(ds, streams[ds], mu, sd * SCALE, P_STAY)


def inner_mean_for_rule(fuse_by_held, hp30, kind, thr, sharp):
    """Mean Acc/H over all (held, inner) cells; never uses held-out labels as selector."""
    hs, acs = [], []
    for held in DS_KEYS:
        fuse = fuse_by_held[held]
        gated = apply_gate_all(fuse, hp30, kind, thr, sharp)
        for inner in DS_KEYS:
            if inner == held:
                continue
            train = [d for d in DS_KEYS if d not in (held, inner)]
            r = fast_score(gated, inner, train)
            hs.append(r["H"])
            acs.append(r["Acc"])
    return float(np.mean(hs)), float(np.mean(acs)), len(hs)


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    # Prefer CUDA for stem forward. Do NOT blank CUDA_VISIBLE_DEVICES.
    if "CUDA_VISIBLE_DEVICES" in os.environ and os.environ["CUDA_VISIBLE_DEVICES"].strip() == "":
        log_line("WARNING: CUDA_VISIBLE_DEVICES is empty; unsetting for GPU stem infer")
        del os.environ["CUDA_VISIBLE_DEVICES"]
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "8")))
    log_line(f"device {device} threads {torch.get_num_threads()} w_fuse={W_FUSE} "
             f"p_stay {P_STAY} scale {SCALE}")

    t_all = time.time()
    N.load_all()
    # load_all already set G; reload hp30 may reinstall — do hp30 after for consistent att
    log_line("loading linear_hp30 LODO folds")
    hp30, mean4, lookahead = load_lodo_hp30()
    log_line(f"lookahead {lookahead}")

    log_line("building ms features for stem infer")
    feat_all = V.build_features()["ms"]

    # Stem infer under GPU; release CUDA after all fuse caches are warm (scoring is CPU).
    fuse_by_held = {}
    for held in DS_KEYS:
        fuse_by_held[held] = ensure_fuse_cache(held, feat_all, device)

    # Preload external/corpus fuse while GPU still held (caller may keep lock for whole job).
    log_line("=== external feature prep (still GPU for stem) ===")
    corp_hp, ext_hp, train_att, test_meta = load_ext_hp30()
    ext_logits, _ = F.load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    corp_fuse = ensure_all5_corpus_fuse(feat_all, device)
    ext_fuse = ensure_all5_ext_fuse(ext_feat, device)

    if device.type == "cuda":
        torch.cuda.empty_cache()
        log_line("stem infer complete; CUDA cache emptied (scoring CPU-only from here)")

    # sanity: fuse alone should reproduce prior AVGC
    san = score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} "
             f"(expect ~0.8100/75.14)")
    san_hp = score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    # --- select gate on mean inner Acc/H ---
    rules = candidate_rules()
    log_line(f"scoring {len(rules)} pre-specified gates on inner cells")
    inner_rows = []
    best = None
    for kind, thr, sharp in rules:
        t0 = time.time()
        mean_h, mean_a, n = inner_mean_for_rule(fuse_by_held, hp30, kind, thr, sharp)
        row = dict(kind=kind, thresh=thr, sharp=sharp, mean_inner_H=mean_h,
                   mean_inner_Acc=mean_a, n=n, key=rule_key(kind, thr, sharp))
        inner_rows.append(row)
        log_line(f"  INNER {row['key']:32s} H {mean_h:.4f} Acc {mean_a:.2f} ({time.time()-t0:.0f}s)")
        key = (mean_a, mean_h)
        if best is None or key > best[0]:
            best = (key, row)

    selected = best[1]
    log_line(f"SELECTED gate {selected['key']} inner H {selected['mean_inner_H']:.4f} "
             f"Acc {selected['mean_inner_Acc']:.2f}")

    # optional logistic (report; select vs pre-spec by inner Acc/H)
    log_fit = {}
    log_inner = []
    for held in DS_KEYS:
        train_ds = [d for d in DS_KEYS if d != held]
        fit = fit_logistic_gate(fuse_by_held[held], hp30, N.G["att"], train_ds, seed=hash(held) % 10_000)
        log_fit[held] = fit
        gated = {ds: logistic_gate_stream(fuse_by_held[held][ds], hp30[ds],
                                          fit["coef"], fit["intercept"])
                 for ds in DS_KEYS}
        for inner in train_ds:
            train = [d for d in DS_KEYS if d not in (held, inner)]
            r = fast_score(gated, inner, train)
            log_inner.append(dict(held=held, inner=inner, H=r["H"], Acc=r["Acc"]))
    log_mean_h = float(np.mean([r["H"] for r in log_inner]))
    log_mean_a = float(np.mean([r["Acc"] for r in log_inner]))
    log_line(f"LOGISTIC inner mean H {log_mean_h:.4f} Acc {log_mean_a:.2f}")

    use_logistic = (log_mean_a, log_mean_h) > (selected["mean_inner_Acc"], selected["mean_inner_H"])
    if use_logistic:
        log_line("HEADLINE uses logistic gate (beats pre-spec on inner Acc/H)")
        headline_mode = "logistic"
    else:
        log_line("HEADLINE uses pre-specified gate")
        headline_mode = "prespec"
        use_logistic = False

    def gated_for_held(held):
        if headline_mode == "logistic":
            fit = log_fit[held]
            return {ds: logistic_gate_stream(fuse_by_held[held][ds], hp30[ds],
                                             fit["coef"], fit["intercept"])
                    for ds in DS_KEYS}
        return apply_gate_all(fuse_by_held[held], hp30,
                              selected["kind"], selected["thresh"], selected["sharp"])

    # --- LODO held-out scores ---
    lodo = {}
    for held in DS_KEYS:
        gated = gated_for_held(held)
        off = score_lodo(gated, held)
        off_fuse = score_lodo(fuse_by_held[held], held)
        off_hp = score_lodo(hp30, held)
        lodo[held] = dict(moe=off, fuse=off_fuse, hp30=off_hp)
        log_line(f"LODO held={held}  moe H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                 f"fuse {off_fuse['H']:.4f}/{off_fuse['Acc']:.2f}  "
                 f"hp30 {off_hp['H']:.4f}/{off_hp['Acc']:.2f}")

    mean_moe = dict(H=float(np.mean([lodo[d]["moe"]["H"] for d in DS_KEYS])),
                    Acc=float(np.mean([lodo[d]["moe"]["Acc"] for d in DS_KEYS])))
    log_line(f"LODO mean moe H {mean_moe['H']:.4f} Acc {mean_moe['Acc']:.2f}  "
             f"(fuse {FUSE_MEAN['H']:.4f}/{FUSE_MEAN['Acc']:.2f})")

    # --- External scoring (streams already cached) ---
    log_line("=== external ===")

    # gate params: if logistic, refit on all 5 corpus; else use selected pre-spec
    if headline_mode == "logistic":
        fit5 = fit_logistic_gate(corp_fuse, corp_hp, train_att, list(V4.CORPUS), seed=42)
        corp_gated = {ds: logistic_gate_stream(corp_fuse[ds], corp_hp[ds],
                                               fit5["coef"], fit5["intercept"])
                      for ds in V4.CORPUS}
        ext_gated = {ds: logistic_gate_stream(ext_fuse[ds], ext_hp[ds],
                                              fit5["coef"], fit5["intercept"])
                     for ds in TEST_NAMES}
        ext_gate_info = dict(mode="logistic", fit=fit5)
    else:
        corp_gated = apply_gate_all(corp_fuse, corp_hp,
                                    selected["kind"], selected["thresh"], selected["sharp"])
        ext_gated = apply_gate_all(ext_fuse, ext_hp,
                                   selected["kind"], selected["thresh"], selected["sharp"])
        ext_gate_info = dict(mode="prespec", **{k: selected[k] for k in
                                                ("kind", "thresh", "sharp", "key")})

    mu_e, sd_e = F.emissions_from(corp_gated, train_att, datasets=list(V4.CORPUS))
    # also score pure baselines with matching emissions for fairness check
    mu_f, sd_f = F.emissions_from(corp_fuse, train_att, datasets=list(V4.CORPUS))
    mu_h, sd_h = F.emissions_from(corp_hp, train_att, datasets=list(V4.CORPUS))

    external = {}
    for ds in TEST_NAMES:
        r_moe = F.score_external(ext_gated[ds], test_meta[ds], mu_e, sd_e)
        r_fuse = F.score_external(ext_fuse[ds], test_meta[ds], mu_f, sd_f)
        r_hp = F.score_external(ext_hp[ds], test_meta[ds], mu_h, sd_h)
        external[ds] = dict(moe=r_moe, fuse=r_fuse, hp30=r_hp)
        log_line(f"EXT {ds}  moe H {r_moe['H']:.4f} Acc {r_moe['Acc']:.2f}  "
                 f"fuse {r_fuse['H']:.4f}/{r_fuse['Acc']:.2f}  "
                 f"hp30 {r_hp['H']:.4f}/{r_hp['Acc']:.2f}  "
                 f"(published linear_hp30 {LINEAR_HP30_EXT[ds]['H']:.4f}/"
                 f"{LINEAR_HP30_EXT[ds]['Acc']:.2f})")

    out = dict(
        p_stay=P_STAY, scale=SCALE, w_fuse=W_FUSE,
        headline_mode=headline_mode,
        selected_prespec=selected,
        logistic_inner=dict(mean_H=log_mean_h, mean_Acc=log_mean_a),
        logistic_fits={k: {kk: vv for kk, vv in v.items() if kk != "mu"}
                       for k, v in log_fit.items()},
        inner_rows=inner_rows,
        lodo=lodo, lodo_mean=mean_moe,
        external=external, ext_gate=ext_gate_info,
        baselines=dict(fuse_lodo=FUSE_LODO, fuse_mean=FUSE_MEAN,
                       linear_hp30_lodo_mean=LINEAR_HP30_LODO,
                       linear_hp30_ext=LINEAR_HP30_EXT, fuse_ext=FUSE_EXT),
        seconds=time.time() - t_all,
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT} in {time.time()-t_all:.0f}s")


if __name__ == "__main__":
    main()
