"""clf-shorthold-ft: short FT of published SW2 vlaai_ws 10s classifiers with
*shorter-hold* switch curriculum (rapid attention flips; NOT higher p_switch).

Init from existing held-*/all5 SW2 checkpoints. Architecture unchanged.

Contrast with:
  clf-switch-dur-ft — longer holds via soft-map long_frac∈(0.5,1) → Acc 53.20
  clf-hardswitch-ft — denser p_switch=0.75 / p_sw2=0.6 → Acc 53.44
  tip-sdur-abs-gate — tip↔sdur |logit| gate → Acc 53.06

Here p_switch=0.5, clf_p_switch2=0.5 (same *rate* as published sw2), but
in-window τ / gaps are drawn from the *short half* of the LODO-train-only
hold-length prior (lower 50% of official synth segment lengths), mapped to
short side lengths → rapid flips inside the 10 s window.

≠ decoder-sw3 denser recon, ≠ frozen-trunk GRU, ≠ tip Acc PP, ≠ multitask.

Short FT: 4 epochs × 250 steps. Seeds s0–s3. Score LODO5 + equal-mean fuse
+ SW2-style fuse a=0.5 with published reconstruction decoders + AASD/Carta
@ HMM 0.994/0.8 vs tip Acc 53.53.

usage (under gpu.lock):
  $PY -u clf_shorthold_ft_lodo.py
  SEEDS=0 $PY -u clf_shorthold_ft_lodo.py   # single-seed smoke
"""
from __future__ import annotations

import json
import os
import pickle
import time
from types import SimpleNamespace

import numpy as np
import torch
import torch.nn.functional as F

import decoders
import direct_classifier as DC
import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import fuse_external as FE
import fuse_general as FG
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, fold_file, load_trial, resolve_model, rows_of
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times
from run_experiment import load_external, log_to

P_SWITCH = float(os.environ.get("P_SWITCH", "0.5"))
P_SWITCH2 = float(os.environ.get("P_SWITCH2", "0.5"))
MAX_EPOCHS = int(os.environ.get("MAX_EPOCHS", "4"))
MIN_EPOCHS = int(os.environ.get("MIN_EPOCHS", "4"))
BATCH = int(os.environ.get("BATCH", "64"))
FT_LR = float(os.environ.get("FT_LR", "2.5e-4"))
STEPS = int(os.environ.get("STEPS_PER_EPOCH", "250"))
SEEDS = [int(x) for x in os.environ.get("SEEDS", "0,1,2,3").split(",") if x.strip() != ""]
PREP = "eaP_n5_t60"
P_STAY, SCALE = 0.994, 0.8
A_FUSE = 0.5
LOOKAHEAD = 0.75
LEADER_AASD = 53.53
HP30_AASD, HP30_CARTA = 53.12, 60.54
TAG_BASE = "_w10b3_ptSW2_shold_aug-sw2"
EXT_TAG = "_w10b3_shold_aug-sw2"

LOG_PATH = os.path.join(EXP, "experiments", "clf_shorthold_ft_lodo.log")
OUT_JSON = os.path.join(EXP, "results", "research", "clf_shorthold_ft.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_clf_shorthold_ft.md")
CKPT_ROOT = os.path.join(EXP, "models", "clf_shorthold_ft")
EXT_DIR = os.path.join(EXP, "results", "research")

PUB_DECS = [
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s1",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s2",
    "vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK_s3",
]

# Cache official LODO synth segment lengths per dataset (fit never sees held-out).
_SEGS_CACHE: dict[str, np.ndarray] = {}
# Active short-hold prior: (log_mu, log_sig, median_s, short_median_s)
# Fitted on full LODO-train segs; sampling uses lower half only.
_HOLD_PRIOR: tuple[float, float, float, float] | None = None


def _dataset_segs(ds: str) -> np.ndarray:
    if ds in _SEGS_CACHE:
        return _SEGS_CACHE[ds]
    segs = []
    srng = np.random.default_rng(SEED + DS_KEYS.index(ds))
    for _, r in rows_of(ds).iterrows():
        dur = int(r["n"]) / 128.0
        times = draw_switch_times(srng, dur)
        srng.integers(0, 2)  # consume start-state coin like score path
        edges = [0.0] + list(times) + [float(dur)]
        segs.extend(np.diff(edges))
    _SEGS_CACHE[ds] = np.asarray(segs, dtype=np.float64)
    return _SEGS_CACHE[ds]


def fit_hold_prior(exclude_held: str | None = None) -> tuple[float, float, float, float]:
    """Lognormal (μ,σ) + medians; short_median = median of lower-half segs (LODO train)."""
    parts = [_dataset_segs(ds) for ds in DS_KEYS if ds != exclude_held]
    segs = np.concatenate(parts)
    log_s = np.log(np.clip(segs, 1e-3, None))
    mu, sig = float(log_s.mean()), float(log_s.std())
    med = float(np.median(segs))
    short = segs[segs <= med]
    short_med = float(np.median(short)) if len(short) else med * 0.5
    return mu, sig, med, short_med


def sample_short_h(rng: np.random.Generator) -> float:
    """Sample hold length from the *short half* of the LODO lognormal prior.

    Inverse-CDF on Uniform(0, 0.5) → lower 50% of LogNormal(μ,σ). Distinct from
    switch-dur which samples the full prior then maps to long_frac∈(0.5,1).
    """
    assert _HOLD_PRIOR is not None
    mu, sig, _med, _sm = _HOLD_PRIOR
    # Φ^{-1}(u) for u∈(0,0.5): use erfinv for no-scipy dependency
    u = float(rng.random()) * 0.5
    u = min(max(u, 1e-6), 0.5 - 1e-6)
    # norm.ppf(u) = √2 * erfinv(2u - 1)
    z = float(np.sqrt(2.0) * _erfinv(2.0 * u - 1.0))
    return float(np.exp(mu + sig * z))


def _erfinv(x: float) -> float:
    """Approximate inverse erf (Abramowitz/Stegun style; fine for sampling)."""
    # Winitzki approximation
    a = 0.147
    sgn = 1.0 if x >= 0 else -1.0
    ln = np.log(1.0 - x * x)
    t1 = 2.0 / (np.pi * a) + 0.5 * ln
    t2 = 1.0 / a * ln
    return float(sgn * np.sqrt(np.sqrt(t1 * t1 - t2) - t1))


def sample_tau(rng: np.random.Generator, W: int) -> int:
    """Place switch for a SHORT hold on a random side (rapid flip curriculum).

    Sample h from LODO short-half prior (seconds), soft-map to short-side fraction
    0.5 * (1 - exp(-h / short_median)) ∈ (0, 0.5), clip to [FS/2, W/2].
    Inverse of switch-dur's long_frac∈(0.5,1) geometry.
    """
    assert _HOLD_PRIOR is not None
    _mu, _sig, _med, short_med = _HOLD_PRIOR
    lo, hi_half = FS // 2, W // 2
    h = sample_short_h(rng)
    short_frac = 0.5 * (1.0 - np.exp(-h / max(short_med, 1e-3)))
    short_n = int(np.clip(round(short_frac * W), lo, hi_half))
    if rng.random() < 0.5:
        return int(short_n)
    return int(W - short_n)


def sample_short_gap(rng: np.random.Generator, rem: int) -> int:
    """Second-switch gap from short-half prior (prefer rapid residual flips)."""
    assert _HOLD_PRIOR is not None
    _mu, _sig, _med, short_med = _HOLD_PRIOR
    h = sample_short_h(rng)
    short_frac = 0.5 * (1.0 - np.exp(-h / max(short_med, 1e-3)))
    min_gap = 2 * FS
    # map into remaining room; keep gap SHORT (not long residual)
    gap = int(np.clip(round(short_frac * rem), min_gap, rem))
    return gap


def windows_shold(trials, n, rng, aug, p_sw, zero_eeg):
    """Like DC.windows but tau/gaps ~ LODO-train short-hold prior (rate unchanged)."""
    W = DC.W
    lens = np.array([len(t["eeg"]) - W for t in trials])
    p = np.maximum(lens, 0).astype(float)
    ti = rng.choice(len(trials), n, p=p / p.sum())
    windows_shold.last_ti = ti
    X = np.empty((n, W, 58), np.float32)
    S_ = np.empty((n, W) + trials[0]["_S"].shape[1:], np.float32)
    y = np.empty(n, np.float32)
    A = (D.get("augment") or {}) if aug is not None else {}
    lag = int(A.get("clf_lag_jitter", 0))
    p_fd = A.get("clf_p_feat_drop", 0.0)
    p_sw2 = A.get("clf_p_switch2", 0.0)
    for j, i in enumerate(ti):
        s = rng.integers(0, lens[i] + 1)
        tr = trials[i]
        s2 = int(np.clip(s + rng.integers(-lag, lag + 1), 0, lens[i])) if lag else s
        X[j], S_[j] = tr["eeg"][s:s + W], tr["_S"][s2:s2 + W]
        col = int(tr["att"][s + DC.LAB])
        if rng.random() < p_sw:
            tau = sample_tau(rng, W)
            S_[j, tau:] = S_[j, tau:, ::-1]
            col = 1 - col if tau <= DC.LAB else col
            # second switch: short gap from short-half prior; keep rate p_sw2
            if p_sw2 and rng.random() < p_sw2 and W - FS // 2 - tau > 2 * FS:
                rem = W - FS // 2 - tau
                gap = sample_short_gap(rng, rem)
                tau2 = int(tau + gap)
                if tau2 < W - FS // 2:
                    S_[j, tau2:] = S_[j, tau2:, ::-1]
                    col = 1 - col if tau2 <= DC.LAB else col
        if p_fd and rng.random() < p_fd:
            S_[j, :, :, rng.integers(S_.shape[-1])] = 0.0
        y[j] = 1.0 if col == 0 else 0.0
    if aug is not None:
        X = aug._augment(X, rng)
    if zero_eeg:
        X[:] = 0.0
    return torch.from_numpy(X), torch.from_numpy(S_), torch.from_numpy(y)


def exp_name(seed: int) -> str:
    if seed == 0:
        return f"direct_clf__{PREP}{TAG_BASE}"
    return f"direct_clf__{PREP}_w10b3_ptSW2_shold_s{seed}_aug-sw2"


def tag_for(seed: int) -> str:
    if seed == 0:
        return TAG_BASE
    return f"_w10b3_ptSW2_shold_s{seed}_aug-sw2"


def pub_fold_ckpt(seed: int, held: str) -> str:
    if seed == 0:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts__{held}.pt"
    else:
        name = f"fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s{seed}_aug-sw2_wts__{held}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    if not os.path.exists(path):
        raise FileNotFoundError(f"published SW2 fold ckpt missing: {name}")
    return path


def pub_all5_ckpt(seed: int) -> str:
    name = f"all5_direct_clf_w10b3sw2P_s{seed}.pt"
    path = os.path.join(DATA_EXP, "models", name)
    if not os.path.exists(path):
        path = resolve_model(os.path.join("models", name))
    if not os.path.exists(path):
        raise FileNotFoundError(f"published SW2 all5 ckpt missing: {name}")
    return path


def configure(seed: int):
    DC.W = int(10 * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS
    D["targets"] = ["env", "onset", "dom"]
    D["augment"] = {
        **CFG["dnn_augment"],
        **CFG["aug_variants"]["cluster"],
        "clf_p_switch2": P_SWITCH2,
    }
    D["train_seed"] = int(seed)
    D["model_blocks"] = 3
    D["max_epochs"] = MAX_EPOCHS
    D["steps_per_epoch"] = STEPS
    D["batch_size"] = BATCH
    D["patience"] = 6
    D["lr"] = FT_LR
    D["val_windows"] = int(os.environ.get("VAL_WINDOWS", "512"))
    decoders.apply_overrides(["model_blocks=3", f"lr={FT_LR}"], [f"clf_p_switch2={P_SWITCH2}"])


def namespace(seed: int, init_from: str):
    return SimpleNamespace(
        backbone="vlaai_ws",
        p_switch=P_SWITCH,
        zero_eeg=False,
        min_epochs=MIN_EPOCHS,
        init_from=init_from,
        prep=PREP,
        tag=tag_for(seed),
        trial_switch="",
    )


def fit_net(train, val, log, a):
    global _HOLD_PRIOR
    Fs = train[0]["_S"].shape[-1]
    seed = D.get("train_seed", CFG["seed"])
    torch.manual_seed(seed)
    net = DC.DirectNet(58, Fs, a.backbone).to(DEV)
    init_from = resolve_model(a.init_from)
    sd = torch.load(init_from, map_location=DEV, weights_only=False)
    missing, unexpected = net.load_state_dict(sd, strict=True)
    log(f"  FT-init from {init_from} (missing {len(missing)} unexpected {len(unexpected)})")
    mu, sig, med, short_med = _HOLD_PRIOR
    n_param = sum(p.numel() for p in net.parameters())
    log(f"  vlaai_ws DirectNet params {n_param}  p_sw={a.p_switch} p_sw2={P_SWITCH2} "
        f"shorthold logN(μ={mu:.3f},σ={sig:.3f}) med={med:.1f}s short_med={short_med:.1f}s  "
        f"lr={D['lr']} batch={D['batch_size']} epochs={D['max_epochs']}×{D['steps_per_epoch']}")
    aug = decoders.DNNDec.__new__(decoders.DNNDec)
    aug.W, aug.fut, aug.hop, aug.C = DC.W, DC.FUT, DC.HOP, 58
    rs = np.random.default_rng(seed)
    # val: no switch aug (same as published recipe)
    Xv, Sv, yv = DC.windows(val, D["val_windows"], rs, None, a.p_switch, a.zero_eeg)
    opt = torch.optim.AdamW(net.parameters(), lr=D["lr"], weight_decay=D["weight_decay"])
    sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
        opt, mode="min", factor=0.5, patience=D["lr_patience"])
    best, bad, state = np.inf, 0, None
    for ep in range(D["max_epochs"]):
        t1, tl = time.time(), []
        net.train()
        for _ in range(D["steps_per_epoch"]):
            X, S_, y = [z.to(DEV) for z in windows_shold(
                train, D["batch_size"], rs, aug, a.p_switch, a.zero_eeg)]
            loss = F.binary_cross_entropy_with_logits(net(X, S_), y)
            opt.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0)
            opt.step()
            tl.append(float(loss.detach()))
        net.eval()
        with torch.no_grad():
            lv = torch.cat([
                net(Xv[i:i + 256].to(DEV), Sv[i:i + 256].to(DEV)).cpu()
                for i in range(0, len(Xv), 256)])
        vl = float(F.binary_cross_entropy_with_logits(lv, yv))
        va = float(((lv > 0).float() == yv).float().mean())
        if ep >= a.min_epochs:
            sch.step(vl)
        log(f"    epoch {ep:02d} train {np.mean(tl):.4f} val BCE {vl:.4f} acc {va:.3f} "
            f"({time.time() - t1:.0f}s)")
        if vl < best - 1e-4:
            best, bad, state = vl, 0, {k: v.detach().clone() for k, v in net.state_dict().items()}
        elif ep >= a.min_epochs:
            bad += 1
            if bad >= D["patience"]:
                break
    if state is None:
        state = {k: v.detach().clone() for k, v in net.state_dict().items()}
    net.load_state_dict(state)
    net.eval()
    return net, best


def run_fold(held, exp, log, a, seed):
    global _HOLD_PRIOR
    _HOLD_PRIOR = fit_hold_prior(exclude_held=held)
    log(f"  shorthold_prior (train\\{held}): logN(μ={_HOLD_PRIOR[0]:.3f},σ={_HOLD_PRIOR[1]:.3f}) "
        f"med={_HOLD_PRIOR[2]:.1f}s short_med={_HOLD_PRIOR[3]:.1f}s")
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held) + 17 * seed)
    train, val = [], []
    for ds in DS_KEYS:
        if ds == held:
            continue
        rows = rows_of(ds)
        vs = rng.choice(sorted(rows.subject.unique()))
        for _, r in rows.iterrows():
            (val if r["subject"] == vs else train).append(load_trial(a.prep, r["key"]))
    train += load_external(["Fuglsang2020"], a.prep, log)
    for tr in train + val:
        tr["_S"] = _targets(tr)
    log(f"  fold held-out={held}: {len(train)} train / {len(val)} val")
    a.init_from = pub_fold_ckpt(seed, held)
    t0 = time.time()
    net, best = fit_net(train, val, log, a)
    t_fit = time.time() - t0
    del train, val
    os.makedirs(os.path.join(CKPT_ROOT, f"s{seed}"), exist_ok=True)
    ckpt = os.path.join(CKPT_ROOT, f"s{seed}", f"held-{held}.pt")
    torch.save(net.state_dict(), ckpt)
    log(f"  saved {ckpt}")
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev, ctl = {}, {k: {} for k in ("normal", "zero_eeg", "shift_eeg", "no_frontal")}
    for _, r in rows_of(held).iterrows():
        tr = load_trial(a.prep, r["key"])
        Sfeat, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128)
        srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur
            cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(
            T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], Sfeat[:, ::-1], Sfeat)
        lg, starts = DC.logits(net, x, Ssw)
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        n10 = len(lg) // 40
        ev[r["key"]] = dict(
            r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            r0_10=lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            r1_10=-lg[:n10 * 40].reshape(n10, 40).mean(1) / 2,
            att10=(truth[:n10 * 40].reshape(n10, 40).mean(1) > 0.5).astype(np.int8),
            pre_switched=True)
        xf = x.copy()
        xf[:, DC.FRONTAL] = 0.0
        for name, xc in (("normal", x), ("zero_eeg", np.zeros_like(x)),
                         ("shift_eeg", np.roll(x, len(x) // 2, 0)), ("no_frontal", xf)):
            l1, s1 = DC.logits(net, xc, Sfeat, step=4)
            ctl[name].setdefault(r["subj_uid"], []).append(
                (l1 > 0) == (att[np.maximum(s1 - DC.HOP, 0)] == 0))
    controls = {
        k: dict(pooled=float(np.mean(np.concatenate([np.concatenate(v) for v in d.values()]))),
                subject_median=float(np.median([np.mean(np.concatenate(v)) for v in d.values()])))
        for k, d in ctl.items()}
    info = dict(decoder="direct_clf", prep=a.prep, held_out=held, fit_s=t_fit,
                total_s=time.time() - t0, lookahead_s=(DC.FUT + DC.HOP) / FS,
                best_val=best, channels=58, p_switch=P_SWITCH, clf_p_switch2=P_SWITCH2,
                hold_prior=dict(log_mu=_HOLD_PRIOR[0], log_sig=_HOLD_PRIOR[1],
                                median_s=_HOLD_PRIOR[2], short_median_s=_HOLD_PRIOR[3],
                                mode="short_half_icdf", exclude_held=held),
                ft_from="published_sw2", controls=controls)
    from common_exp import save_pickle
    save_pickle(dict(evidence={"causal": ev}, info=info), fold_file(exp, held))
    log(f"  fold held-out={held} done in {info['total_s']:.0f}s best_val {best:.4f} "
        + "  ".join(f"{k} {v['pooled']:.3f}" for k, v in controls.items()))
    del net
    if DEV.type == "cuda":
        torch.cuda.empty_cache()


def train_all5_external(seed, log, a):
    global _HOLD_PRIOR
    all5_pt = os.path.join(CKPT_ROOT, f"s{seed}", "all5.pt")
    ev_paths = {
        ds: os.path.join(EXT_DIR, f"ext_clf_{ds}{EXT_TAG}_evidence_s{seed}.pkl")
        for ds in ("AASD", "Carta2026")
    }
    os.makedirs(os.path.dirname(all5_pt), exist_ok=True)
    os.makedirs(EXT_DIR, exist_ok=True)
    if all(os.path.exists(p) for p in ev_paths.values()) and os.path.exists(all5_pt):
        log(f"  all5+ext cache hit seed={seed}")
        return all5_pt, ev_paths

    _HOLD_PRIOR = fit_hold_prior(exclude_held=None)
    log(f"  shorthold_prior (all5): logN(μ={_HOLD_PRIOR[0]:.3f},σ={_HOLD_PRIOR[1]:.3f}) "
        f"med={_HOLD_PRIOR[2]:.1f}s short_med={_HOLD_PRIOR[3]:.1f}s")

    if os.path.exists(all5_pt):
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        net.load_state_dict(torch.load(all5_pt, map_location=DEV, weights_only=False))
        net.eval()
        log(f"  loaded {all5_pt}")
    else:
        rng = np.random.default_rng(CFG["seed"] + 99 + 13 * seed)
        train, val = [], []
        for ds in DS_KEYS:
            rows = rows_of(ds)
            vs = rng.choice(sorted(rows.subject.unique()))
            for _, r in rows.iterrows():
                (val if r["subject"] == vs else train).append(load_trial(a.prep, r["key"]))
        train += load_external(["Fuglsang2020"], a.prep, log)
        for tr in train + val:
            tr["_S"] = _targets(tr)
        a.init_from = pub_all5_ckpt(seed)
        log(f"  all-5 FT train {len(train)} / val {len(val)} from {a.init_from}")
        net, best = fit_net(train, val, log, a)
        torch.save(net.state_dict(), all5_pt)
        log(f"  saved {all5_pt} best val {best:.4f}")
        del train, val

    for ds, path in ev_paths.items():
        if os.path.exists(path):
            continue
        EV = {}
        trials = E.ext_trials(ds)
        log(f"  external {ds}: {len(trials)} trials")
        for i, tr in enumerate(trials):
            lg, starts = DC.logits(net, tr["eeg"], _targets(tr))
            truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
            EV[tr["key"]] = dict(
                r0=lg / 2, r1=-lg / 2, att=truth.astype(np.int8),
                t_end=(starts / FS + tr["t0"]).astype(np.float32),
                att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
            if (i + 1) % 40 == 0 or i + 1 == len(trials):
                log(f"    {ds} {i + 1}/{len(trials)}")
        with open(path, "wb") as fh:
            pickle.dump(EV, fh)
        log(f"  wrote {path}")
    del net
    if DEV.type == "cuda":
        torch.cuda.empty_cache()
    return all5_pt, ev_paths


def score_lodo_one(exp, log):
    la = (DC.FUT + DC.HOP) / FS
    ev, _ = S.load_folds(exp, "causal")
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                            for d in DS_KEYS if d != ds for e in ev[d].values()])
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, la, emis=(mu, sd * SCALE), p_stay=P_STAY)
        per[ds] = r
        log(f"  LODO {exp} {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f}")
    mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log(f"  LODO MEAN {exp}  H {mH:.4f} Acc {mA:.2f}")
    return per, mH, mA, ev


def score_ext_one(ev_lodo, ev_paths, log, name="clf"):
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                        for ds in DS_KEYS for e in ev_lodo[ds].values()])
    mu, sd = float(A.mean()), float(A.std())
    log(f"  {name} external emis mu {mu:.4f} sd {sd:.4f}")
    ext = {}
    for ds, path in ev_paths.items():
        with open(path, "rb") as fh:
            EV = pickle.load(fh)
        r = E.score(EV, mu, sd * SCALE, P_STAY, LOOKAHEAD)
        plain = E.plain(EV)
        ext[ds] = dict(**r, acc1=plain["acc1"])
        log(f"  EXT {name} {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f} acc1 {plain['acc1']:.3f}")
    return ext, mu, sd


def load_lodo_nd(exps, log):
    evs, lookahead = [], None
    for exp in exps:
        ev, info = S.load_folds(exp, "causal")
        evs.append(ev)
        log(f"  stack LODO {exp}")
        for ds in DS_KEYS:
            la = float(info[ds]["lookahead_s"])
            lookahead = la if lookahead is None else lookahead
    out = {}
    for ds in DS_KEYS:
        out[ds] = {}
        for k in evs[0][ds]:
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            out[ds][k] = np.stack(cols, 1).astype(np.float32)
    return out, lookahead, evs


def load_ext_nd(n_seeds, log):
    logits, meta = {}, {}
    for ds in ("AASD", "Carta2026"):
        evs = []
        for i in range(n_seeds):
            path = os.path.join(EXT_DIR, f"ext_clf_{ds}{EXT_TAG}_evidence_s{SEEDS[i]}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
            log(f"  stack EXT {ds} seed{SEEDS[i]}")
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]),
                t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]),
                n128=int(e0["n128"]),
                subject=int(e0["subject"]),
            )
    return logits, meta


def mean_stream(nd):
    return {ds: {k: np.asarray(D, np.float64).mean(1) for k, D in table.items()}
            for ds, table in nd.items()}


def to_ev(stream, ref_ev):
    out = {}
    for ds in DS_KEYS:
        out[ds] = {}
        for k, d in stream[ds].items():
            e0 = ref_ev[ds][k]
            d = np.asarray(d, np.float64)
            out[ds][k] = dict(
                r0=d, r1=np.zeros_like(d), att=e0["att"], t_end=e0["t_end"],
                pre_switched=True)
    return out


def score_lodo_stream(name, stream, ref_ev, log):
    ev = to_ev(stream, ref_ev)
    V4._install_scorer_state(ev, LOOKAHEAD)
    per = {}
    for held in DS_KEYS:
        r = N.official(held, ev, P_STAY, SCALE)
        per[held] = r
        log(f"  LODO {name:14s} {held:5s}  H {r['H']:.4f} Acc {r['Acc']:.2f}")
    mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log(f"  LODO_MEAN {name:14s}  H {mH:.4f} Acc {mA:.2f}")
    return per, mH, mA


def score_ext_stream(name, stream, meta, mu, sd, log):
    out = {}
    for ds in ("AASD", "Carta2026"):
        ev = V4.pack_ev(stream[ds], meta[ds])
        plain = E.plain(ev)
        r = E.score(ev, mu, sd * SCALE, P_STAY, LOOKAHEAD)
        out[ds] = dict(**r, acc1=plain["acc1"])
        dacc = r["Acc"] - LEADER_AASD if ds == "AASD" else None
        extra = f"  ΔAcc vs tip {dacc:+.2f}" if dacc is not None else ""
        log(f"  EXT  {name:14s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f} "
            f"acc1 {plain['acc1']:.3f}{extra}")
    return out


def fuse_sw2_lodo(clfs, log):
    log(f"  fuse LODO decs={PUB_DECS} clfs={clfs} a={A_FUSE}")
    Z, meta = FG.build(PUB_DECS, clfs)
    per = {}
    for held in DS_KEYS:
        zd, zc = Z[held]
        stream = {
            d: {k: (1 - A_FUSE) * zd[d][k] + A_FUSE * zc[d][k] for k in zd[d]}
            for d in DS_KEYS
        }
        ev = {}
        for d in DS_KEYS:
            ev[d] = {}
            for k, delta in stream[d].items():
                att, te = meta[d][k]
                n = len(delta)
                ev[d][k] = dict(
                    r0=np.asarray(delta, np.float64),
                    r1=np.zeros(n),
                    att=np.asarray(att)[:n],
                    t_end=np.asarray(te, np.float64)[:n],
                    pre_switched=True,
                )
        V4._install_scorer_state(ev, LOOKAHEAD)
        r = N.official(held, ev, P_STAY, SCALE)
        per[held] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"])
        log(f"  fuse LODO {held}  H {r['H']:.4f} Acc {r['Acc']:.2f}")
    mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log(f"  fuse LODO MEAN  H {mH:.4f} Acc {mA:.2f}")
    return per, mH, mA


def fuse_sw2_external(clfs_fe, log):
    decs_fe = [[PUB_DECS[i], i] for i in range(4)]
    log(f"  fuse EXT decs={decs_fe} clfs={clfs_fe}")
    dec_tag = "_bestP"
    EV, emis = FE.build_external(decs_fe, clfs_fe, [A_FUSE], ["AASD", "Carta2026"], dec_tag)
    mu, sd = emis[A_FUSE]
    out = {}
    for ds in ("AASD", "Carta2026"):
        r = E.score(EV[ds][A_FUSE], mu, sd * SCALE, P_STAY, LOOKAHEAD)
        pl = E.plain(EV[ds][A_FUSE])
        out[ds] = dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], acc1=pl["acc1"])
        log(f"  fuse EXT {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f} acc1 {pl['acc1']:.3f}")
    return out


def write_results(payload):
    os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
    with open(OUT_JSON, "w") as fh:
        json.dump(payload, fh, indent=2, default=float)

    seeds = payload["seeds"]
    mean_fuse = payload.get("mean_fuse")
    sw2_fuse = payload.get("sw2_fuse")
    if sw2_fuse and "external" in sw2_fuse:
        head = sw2_fuse["external"]["AASD"]
        head_name = "sw2_fuse_a0.5"
        carta = sw2_fuse["external"]["Carta2026"]
        lodo_mH, lodo_mA = sw2_fuse["lodo_mean_H"], sw2_fuse["lodo_mean_Acc"]
    elif mean_fuse:
        head = mean_fuse["external"]["AASD"]
        head_name = "mean_seeds"
        carta = mean_fuse["external"]["Carta2026"]
        lodo_mH, lodo_mA = mean_fuse["lodo_mean_H"], mean_fuse["lodo_mean_Acc"]
    else:
        s0 = seeds[str(SEEDS[0])]
        head = s0["external"]["AASD"]
        head_name = f"seed{SEEDS[0]}"
        carta = s0["external"]["Carta2026"]
        lodo_mH, lodo_mA = s0["lodo_mean_H"], s0["lodo_mean_Acc"]

    dacc = head["Acc"] - LEADER_AASD
    lines = [
        "# results_clf_shorthold_ft",
        "",
        "Fine-tune published **SW2** `vlaai_ws` 10 s classifiers from held-*/all5 checkpoints "
        f"with **shorter-hold switch curriculum** (`p_switch={P_SWITCH}`, `clf_p_switch2={P_SWITCH2}` "
        "= same rate as sw2). In-window τ/gaps from LODO-train *short-half* hold prior "
        "(lower 50% ICDF of synth segment lengths) soft-mapped to short-side holds → rapid flips. "
        "Architecture unchanged.",
        "",
        "**Not** clf-switch-dur-ft longer holds, **not** clf-hardswitch-ft denser p_switch, "
        "**not** tip-sdur gate, **not** decoder-sw3 / frozen-trunk GRU / tip Acc PP.",
        "",
        f"Prep `{PREP}`, FT lr={FT_LR}, {MAX_EPOCHS}×{STEPS} steps, seeds={SEEDS}. "
        f"HMM p_stay {P_STAY} scale {SCALE}.",
        "",
        f"Code: `clf_shorthold_ft_lodo.py`. JSON: `{OUT_JSON}`. "
        f"Ckpts: `models/clf_shorthold_ft/s*/`.",
        "",
        "## Headline Acc",
        "",
        f"Selector stream: **{head_name}**",
        "",
        "| Set | H / Acc | vs tip 53.53 / Carta hp30 60.54 |",
        "|---|---|---|",
        f"| AASD | {head['H']:.4f} / **{head['Acc']:.2f}** | ΔAcc {dacc:+.2f} |",
        f"| Carta2026 | {carta['H']:.4f} / **{carta['Acc']:.2f}** | ΔAcc {carta['Acc'] - HP30_CARTA:+.2f} |",
        f"| LODO mean | {lodo_mH:.4f} / {lodo_mA:.2f} | |",
        "",
        f"AASD Acc>60? **{'YES' if head['Acc'] > 60 else 'NO'}**",
        "",
        "## Per-seed LODO + external",
        "",
        "| Seed | LODO H/Acc | AASD H/Acc | Carta H/Acc |",
        "|---|---|---|---|",
    ]
    for s in SEEDS:
        row = seeds[str(s)]
        aasd = row["external"]["AASD"]
        car = row["external"]["Carta2026"]
        lines.append(
            f"| {s} | {row['lodo_mean_H']:.4f}/{row['lodo_mean_Acc']:.2f} | "
            f"{aasd['H']:.4f}/{aasd['Acc']:.2f} | {car['H']:.4f}/{car['Acc']:.2f} |")
    if mean_fuse:
        aasd = mean_fuse["external"]["AASD"]
        car = mean_fuse["external"]["Carta2026"]
        lines += [
            "",
            "## Equal-mean seed fuse",
            "",
            f"LODO mean **{mean_fuse['lodo_mean_H']:.4f}/{mean_fuse['lodo_mean_Acc']:.2f}**. "
            f"AASD **{aasd['H']:.4f}/{aasd['Acc']:.2f}** (Δ{aasd['Acc'] - LEADER_AASD:+.2f}). "
            f"Carta **{car['H']:.4f}/{car['Acc']:.2f}**.",
        ]
    if sw2_fuse:
        aasd = sw2_fuse["external"]["AASD"]
        car = sw2_fuse["external"]["Carta2026"]
        lines += [
            "",
            "## SW2-style fuse (pub decoder × shorthold clf, a=0.5)",
            "",
            f"LODO mean **{sw2_fuse['lodo_mean_H']:.4f}/{sw2_fuse['lodo_mean_Acc']:.2f}**. "
            f"AASD **{aasd['H']:.4f}/{aasd['Acc']:.2f}** (Δ{aasd['Acc'] - LEADER_AASD:+.2f}). "
            f"Carta **{car['H']:.4f}/{car['Acc']:.2f}**.",
        ]
    lines += [
        "",
        "## Targets",
        "",
        "| Target | Result |",
        "|---|---|",
        f"| AASD Acc vs tip 53.53 | Acc **{head['Acc']:.2f}** (Δ{dacc:+.2f}) |",
        f"| AASD Acc>60? | {'YES' if head['Acc'] > 60 else 'NO'} |",
        f"| Carta vs linear_hp30 60.54 | Acc **{carta['Acc']:.2f}** |",
        f"| vs linear_hp30 AASD 53.12 | Δ{head['Acc'] - HP30_AASD:+.2f} |",
        "",
        "## Verdict",
        "",
        (
            f"Shorter-hold switch curriculum FT from published SW2: AASD Acc {head['Acc']:.2f} "
            f"vs tip 53.53 (Δ{dacc:+.2f}) via {head_name}."
        ),
        "",
        "## NEXT IDEA",
        "",
        (
            "If Acc flat/~53: short-hold schedule FT (like longer-hold / denser p_switch) does not "
            "close the real-switch gap — close the switch-schedule family. Prefer a qualitatively "
            "new Acc frame that is not tip PP, Acc soft-w, vlaai identity residuals, switch-schedule "
            "re-grids, or frozen-trunk head-swap — e.g. non-vlaai Acc front-end warmed from published "
            "evidence, or subject-conditional evidence calibration without AASD labels."
            if head["Acc"] <= LEADER_AASD + 0.5 else
            "If Acc moved: soft-blend shorthold FT ↔ Acc tip under LODO Acc selector (one new stream), "
            "or Carta-aware emission scale."
        ),
        "",
    ]
    open(RESULTS_MD, "w").write("\n".join(lines))
    return RESULTS_MD, head_name, head, carta, dacc, lodo_mH, lodo_mA


def mark_claim(status, note=None):
    path = os.path.join(EXP, "multi_agent_opt", "CLAIMS.md")
    rows = open(path).read().splitlines()
    out = []
    for line in rows:
        if "| clf-shorthold-ft |" in line:
            parts = line.split("|")
            if len(parts) >= 6:
                parts[4] = f" {status} "
                if note:
                    parts[5] = f" {note} "
                line = "|".join(parts)
        out.append(line)
    open(path, "w").write("\n".join(out) + ("\n" if rows and not rows[-1].endswith("\n") else ""))


def update_status(done_note: str):
    path = os.path.join(EXP, "multi_agent_opt", "STATUS_GAPS.md")
    text = open(path).read()
    # clear in-flight shorthold line(s)
    for old in (
        "1. **clf-shorthold-ft** — short-hold switch curriculum FT (GPU Acc) if claimed.\n",
        "1. **clf-shorthold-ft** — short-hold switch curriculum FT (GPU Acc).\n",
        "2. **clf-shorthold-ft** — short-hold switch curriculum FT (GPU Acc).\n",
        "2. **clf-shorthold-ft** — FT SW2 with shorter-hold switch curriculum (GPU Acc).\n",
    ):
        if old in text:
            text = text.replace(old, "")
    if "## Recently closed" in text:
        closed = text.split("## Recently closed")[1][:1500]
        if "**clf-shorthold-ft**" not in closed:
            text = text.replace(
                "## Recently closed\n",
                f"## Recently closed\n\n- **clf-shorthold-ft** — {done_note}\n",
            )
    open(path, "w").write(text)


def append_already(note: str):
    path = os.path.join(EXP, "multi_agent_opt", "ALREADY_TRIED.md")
    with open(path, "a") as fh:
        fh.write(
            "\n- **clf shorter-hold switch curriculum FT from published SW2** (`clf-shorthold-ft`; "
            f"p_switch={P_SWITCH}/clf_p_switch2={P_SWITCH2} same rate as sw2; LODO-train short-half "
            f"hold prior ICDF→short-side τ/gaps; held-*/all5 init; {MAX_EPOCHS}×{STEPS}; seeds {SEEDS}; "
            f"≠ switch-dur longer holds / hardswitch denser p_switch / tip-sdur / decoder-sw3 / "
            f"frozen-trunk GRU / tip PP): {note} "
            "`multi_agent_opt/results_clf_shorthold_ft.md`, `clf_shorthold_ft_lodo.py`.\n"
        )


def main():
    log = log_to(LOG_PATH)
    mark_claim("running", "GPU FT shorter-hold switch curriculum under flock; results_clf_shorthold_ft.md")
    log(f"=== clf-shorthold-ft device={DEV} seeds={SEEDS} "
        f"p_sw={P_SWITCH} p_sw2={P_SWITCH2} lr={FT_LR} epochs={MAX_EPOCHS}×{STEPS} ===")

    seed_payload = {}
    exps = []
    for seed in SEEDS:
        configure(seed)
        exp = exp_name(seed)
        exps.append(exp)
        a = namespace(seed, init_from="")
        log(f"--- seed {seed} exp={exp} ---")
        for held in DS_KEYS:
            path = fold_file(exp, held)
            if os.path.exists(path):
                log(f"  fold held-out={held}: cached {path}")
                continue
            run_fold(held, exp, log, a, seed)
        per, mH, mA, ev = score_lodo_one(exp, log)
        _, ev_paths = train_all5_external(seed, log, a)
        ext, mu, sd = score_ext_one(ev, ev_paths, log, name=f"s{seed}")
        seed_payload[str(seed)] = dict(
            exp=exp, lodo=per, lodo_mean_H=mH, lodo_mean_Acc=mA,
            external=ext, emis_mu=mu, emis_sd=sd,
        )

    payload = dict(
        method="clf-shorthold-ft",
        p_switch=P_SWITCH, clf_p_switch2=P_SWITCH2, ft_lr=FT_LR,
        max_epochs=MAX_EPOCHS, steps_per_epoch=STEPS, seeds=SEEDS,
        hold_prior="lodo_train_short_half_icdf_softmap",
    )
    payload["seeds"] = seed_payload

    mean_fuse = None
    if len(SEEDS) > 1:
        log("--- equal-mean seed fuse ---")
        nd, _, evs = load_lodo_nd(exps, log)
        stream = mean_stream(nd)
        per, mH, mA = score_lodo_stream("mean_seeds", stream, evs[0], log)
        ext_nd, meta = load_ext_nd(len(SEEDS), log)
        ext_stream = {ds: {k: np.asarray(D, np.float64).mean(1)
                           for k, D in table.items()} for ds, table in ext_nd.items()}
        A = np.concatenate([
            stream[ds][k] * np.where(evs[0][ds][k]["att"] == 0, 1.0, -1.0)
            for ds in DS_KEYS for k in stream[ds]
        ])
        mu, sd = float(A.mean()), float(A.std())
        ext = score_ext_stream("mean_seeds", ext_stream, meta, mu, sd, log)
        mean_fuse = dict(lodo=per, lodo_mean_H=mH, lodo_mean_Acc=mA, external=ext,
                         emis_mu=mu, emis_sd=sd)
        payload["mean_fuse"] = mean_fuse

    sw2_fuse = None
    if len(SEEDS) >= 4:
        log("--- SW2-style fuse pub-decoder × shorthold clf ---")
        try:
            clfs = [exp_name(s) for s in range(4)]
            per, mH, mA = fuse_sw2_lodo(clfs, log)
            clfs_fe = [[exp_name(s), EXT_TAG, s] for s in range(4)]
            ext = fuse_sw2_external(clfs_fe, log)
            sw2_fuse = dict(lodo=per, lodo_mean_H=mH, lodo_mean_Acc=mA, external=ext)
            payload["sw2_fuse"] = sw2_fuse
        except Exception as e:
            log(f"  SW2 fuse skipped: {e!r}")
            payload["sw2_fuse_error"] = repr(e)

    rpath, head_name, head, carta, dacc, lodo_mH, lodo_mA = write_results(payload)
    note = (
        f"LODO {lodo_mH:.4f}/{lodo_mA:.2f} via {head_name}. "
        f"AASD {head['H']:.4f}/{head['Acc']:.2f} (Δ{dacc:+.2f} vs tip 53.53). "
        f"Carta {carta['H']:.4f}/{carta['Acc']:.2f}. Acc>60={'YES' if head['Acc'] > 60 else 'NO'}. "
        f"`{rpath}`."
    )
    mark_claim("done", note)
    append_already(note)
    update_status(
        f"AASD Acc {head['Acc']:.2f} (Δ{dacc:+.2f} vs tip). "
        f"Acc>60={'YES' if head['Acc'] > 60 else 'NO'}."
    )
    log(f"wrote {OUT_JSON} and {rpath}")
    log("=== done ===")
    print(
        f"\nRESULT AASD Acc={head['Acc']:.2f} vs tip 53.53 (Δ{dacc:+.2f}); "
        f"Acc>60={'YES' if head['Acc'] > 60 else 'NO'}; stream={head_name}",
        flush=True,
    )


if __name__ == "__main__":
    main()
