"""csp-mdm-fuse — Path A joint EEG+speech SPD MDM (+ CSP→env) as Acc fuse stream.

DISTINCT from:
  * FBCSP / Riemann left-right *EEG-only* spatial decode (C1, 49–53%)
  * Decoder Riemann WF package
  * TRF / DSS / SSD / dual-band Path A lag-banks
  * tip Acc PP / SW2↔swmix soft-w re-grids / Mixup-focal stem

Prep stores attended speech in env[:,0] (att≡0). So we do NOT CSP on att0/att1 EEG.
Instead (train-only; never AASD/Carta):
  1. Windowed joint signals Zs=[EEG; env_s] → Ledoit-Wolf SPD (C+1).
  2. Match class = cov(Z0), mismatch = cov(Z1). Fit CSP then MDM on match vs mismatch.
  3. Evidence A (`joint_mdm`): per window LLR = log P(C0=match) − log P(C1=match).
  4. Evidence B (`csp_env`): CSP filters on EEG block → sliding corr vs env0/env1.
  5. Soft/hard fuse selected stream ↔ Acc tip hard_abs_swmix|t1; LODO Acc then H.
  6. Score AASD+Carta @0.994/0.8.

CPU only. Smoke-abort if AVGC Path A acc1 ≪ gate.

usage:
  PROBE_ONLY=1 .../python -u csp_mdm_fuse.py
  .../python -u csp_mdm_fuse.py
"""
from __future__ import annotations

import json
import os
import pickle
import time
from dataclasses import dataclass

import numpy as np
from pyriemann.spatialfilters import CSP
from scipy.linalg import eigh
from scipy.signal import lfilter
from scipy.special import expit


def _covs(X, shrink=0.1):
    """Fast batch shrunk sample cov; X (n, C, T) → (n, C, C) float64."""
    X = np.asarray(X, np.float64)
    Xc = X - X.mean(axis=-1, keepdims=True)
    cov = np.einsum("nct,ndt->ncd", Xc, Xc) / max(X.shape[-1] - 1, 1)
    tr = np.einsum("nii->n", cov) / cov.shape[1]
    eye = np.eye(cov.shape[1], dtype=np.float64)
    return (1.0 - shrink) * cov + shrink * tr[:, None, None] * eye


def _spd_mean(covs, n_iter=12, max_n=800):
    """Affine-invariant Riemannian mean (numpy); subsample if large."""
    covs = np.asarray(covs, np.float64)
    if len(covs) > max_n:
        rng = np.random.default_rng(0)
        covs = covs[rng.choice(len(covs), size=max_n, replace=False)]
    M = np.mean(covs, axis=0)
    for _ in range(n_iter):
        w, V = eigh(M)
        w = np.clip(w, 1e-12, None)
        Mm12 = (V * (1.0 / np.sqrt(w))) @ V.T
        M12 = (V * np.sqrt(w)) @ V.T
        S = Mm12 @ covs @ Mm12  # (n, K, K)
        # batch logm via eigh
        sw, sV = np.linalg.eigh(S)
        sw = np.clip(sw, 1e-12, None)
        logS = np.einsum("nij,nj,nkj->nik", sV, np.log(sw), sV)
        acc = logS.mean(0)
        ew, eV = eigh(acc)
        expT = (eV * np.exp(ew)) @ eV.T
        M = M12 @ expT @ M12
        M = 0.5 * (M + M.T)
    return M


def _rie_dist2(A, B):
    """Squared Riemannian distance between SPD A and B."""
    w, V = eigh(A)
    w = np.clip(w, 1e-12, None)
    Am12 = (V * (1.0 / np.sqrt(w))) @ V.T
    S = Am12 @ B @ Am12
    sw, _ = eigh(S)
    sw = np.clip(sw, 1e-12, None)
    return float(np.sum(np.log(sw) ** 2))


class NumpyMDM:
    """Minimal 2-class Riemannian MDM (numpy-only)."""

    def __init__(self):
        self.means_ = None  # (2, K, K)

    def fit(self, covs, y):
        covs = np.asarray(covs, np.float64)
        y = np.asarray(y, np.int64)
        means = []
        for c in (0, 1):
            means.append(_spd_mean(covs[y == c]))
        self.means_ = np.stack(means, 0)
        return self

    def predict_proba(self, covs):
        covs = np.asarray(covs, np.float64)
        n = len(covs)
        d = np.empty((n, 2), np.float64)
        for i, C in enumerate(covs):
            d[i, 0] = _rie_dist2(C, self.means_[0])
            d[i, 1] = _rie_dist2(C, self.means_[1])
        # softmax(-d) — nearer mean → higher prob
        z = -d
        z -= z.max(1, keepdims=True)
        e = np.exp(z)
        return e / e.sum(1, keepdims=True)

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
import sw2_hp30_blend as SW2
import sw2_swmix_blend as SM
import trf_lagbank_clf as T
from common import DS_KEYS
from common_exp import CFG, EXP, FS, evidence_from_features, fold_file, load_trial, rows_of, save_pickle
from fuse_evidence import LA
from run_experiment import load_external, log_to

# -------------------------------------------------------------------------------------------------
EXP_NAME = "csp_mdm__eaP_n5_t60"
TAG = "_csp_mdm"
PREP = "eaP_n5_t60"
LOG_PATH = os.path.join(EXP, "experiments", "csp_mdm_fuse.log")
OUT_JSON = os.path.join(EXP, "results", "research", "csp_mdm_fuse.json")
CKPT_DIR = os.path.join(EXP, "models", "csp_mdm")
EXT_DIR = os.path.join(EXP, "results", "research")
CACHE_DIR = os.path.join(CKPT_DIR, "cache")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_csp_mdm_fuse.md")
CLAIMS_PATH = os.path.join(EXP, "multi_agent_opt", "CLAIMS.md")
ALREADY_PATH = os.path.join(EXP, "multi_agent_opt", "ALREADY_TRIED.md")
STATUS_PATH = os.path.join(EXP, "multi_agent_opt", "STATUS_GAPS.md")

P_STAY, SCALE = 0.994, 0.8
WIN_S = float(os.environ.get("WIN_S", "2.0"))
HOP_S = float(os.environ.get("HOP_S", "0.25"))
N_FILTER = int(os.environ.get("N_FILTER", "6"))
MAX_TRAIN_WIN = int(os.environ.get("MAX_TRAIN_WIN", "3000"))  # → 2× for match+mismatch
PROBE_ONLY = os.environ.get("PROBE_ONLY", "0") == "1"
ACC1_GATE = float(os.environ.get("ACC1_GATE", "0.53"))
SOFT_ACC1 = float(os.environ.get("SOFT_ACC1", "0.52"))
SWMIX_AASD = 53.53
LINEAR_HP30_AASD = 53.12
SW2_AASD = 53.46
LEADER_KIND, LEADER_THR, LEADER_SHARP = "hard_abs_swmix", 1.0, 1.0
LOOKAHEAD = float(LA)


def log_line(msg):
    print(msg, flush=True)


@dataclass
class CspMdmModel:
    filters: np.ndarray          # (C+1, K) CSP on joint SPD space
    eeg_filters: np.ndarray      # (C, K) EEG rows of filters (for csp_env)
    mdm: object
    n_filter: int
    win: int
    hop: int
    sign: float = 1.0


def _window_starts(T, win, hop):
    return np.arange(0, T - win + 1, hop)


def _joint_epoch(eeg_win, env_win):
    """eeg_win (C,T), env_win (T,) → joint (C+1, T) centered."""
    z = np.concatenate([eeg_win, env_win[None, :]], axis=0)
    z = z - z.mean(1, keepdims=True)
    return z.astype(np.float32)


def collect_match_mismatch(trials, win, hop, max_n=MAX_TRAIN_WIN, rng=None):
    """Build match (EEG+env0) vs mismatch (EEG+env1) joint epochs.

    Returns X (2n, C+1, T), y (2n,) with 0=match, 1=mismatch.
    """
    rng = rng or np.random.default_rng(CFG["seed"])
    Xs, ys = [], []
    ordered = sorted(trials, key=lambda tr: 1 if tr.get("_grp") == "Fuglsang2020" else 0)
    n_tr = len(ordered)
    for ti, tr in enumerate(ordered):
        eeg = np.asarray(tr["eeg"], np.float64)
        env = np.asarray(tr["env"], np.float64)
        T = min(len(eeg), len(env))
        if T < win + 8:
            continue
        starts = _window_starts(T, win, hop)
        if len(starts) == 0:
            continue
        if len(Xs) // 2 >= max_n:
            break
        remain = max(1, n_tr - ti)
        take = min(len(starts), max(4, (max_n - len(Xs) // 2) // remain))
        if take < len(starts):
            starts = rng.choice(starts, size=take, replace=False)
            starts.sort()
        for s0 in starts:
            ew = eeg[s0: s0 + win].T
            Xs.append(_joint_epoch(ew, env[s0: s0 + win, 0]))
            ys.append(0)  # match
            Xs.append(_joint_epoch(ew, env[s0: s0 + win, 1]))
            ys.append(1)  # mismatch
            if len(Xs) // 2 >= max_n:
                break
    X = np.stack(Xs, axis=0)
    y = np.asarray(ys, np.int64)
    return X, y


def fit_csp_mdm(train, log=None):
    win = int(WIN_S * FS)
    hop = int(HOP_S * FS)
    X, y = collect_match_mismatch(train, win, hop)
    if log:
        log(f"    joint CSP/MDM n={len(y)} C+1={X.shape[1]} T={X.shape[2]} "
            f"match={(y==0).sum()} mismatch={(y==1).sum()}")
    if len(y) < 400 or len(np.unique(y)) < 2:
        raise RuntimeError(f"insufficient windows: n={len(y)} classes={np.unique(y)}")
    covs = _covs(X)
    csp = CSP(nfilter=N_FILTER, metric="euclid", log=False)
    csp.fit(covs, y)
    # pyriemann CSP.filters_ is (nfilter, n_channels)
    filters = np.asarray(csp.filters_[:N_FILTER], np.float64).T  # (C+1, K)
    # project joint signals → filtered covs for MDM
    Xf = np.einsum("ck,nct->nkt", filters, X)
    covs_f = _covs(Xf)
    mdm = NumpyMDM().fit(covs_f, y)
    proba = mdm.predict_proba(covs_f)
    # P(match)=P(class0); prefer env0 if P_match(C0) > P_match(C1)
    llr = np.log(proba[:, 0] + 1e-12) - np.log(proba[:, 1] + 1e-12)
    acc_raw = float(((llr > 0) == (y == 0)).mean())
    sign = 1.0 if acc_raw >= 0.5 else -1.0
    eeg_filters = filters[:-1, :].copy()  # drop speech row → (C, K)
    if log:
        log(f"    CSP K={filters.shape[1]} MDM train_acc={acc_raw:.3f} sign={sign}")
    return CspMdmModel(filters=filters, eeg_filters=eeg_filters, mdm=mdm,
                       n_filter=filters.shape[1], win=win, hop=hop, sign=sign)


def _joint_covs_trial(model, eeg, env0, env1):
    win, hop = model.win, model.hop
    T = min(len(eeg), len(env0), len(env1))
    starts = _window_starts(T, win, hop)
    if len(starts) == 0:
        raise RuntimeError("trial too short")
    Z0, Z1 = [], []
    for s0 in starts:
        ew = eeg[s0: s0 + win].T
        Z0.append(_joint_epoch(ew, env0[s0: s0 + win]))
        Z1.append(_joint_epoch(ew, env1[s0: s0 + win]))
    Z0 = np.stack(Z0, 0)
    Z1 = np.stack(Z1, 0)
    F0 = np.einsum("ck,nct->nkt", model.filters, Z0)
    F1 = np.einsum("ck,nct->nkt", model.filters, Z1)
    C0 = _covs(F0)
    C1 = _covs(F1)
    return starts, C0, C1


def mdm_evidence_trial(model: CspMdmModel, tr):
    """Joint MDM LLR: positive → prefer env0 (attended under prep convention)."""
    eeg = np.asarray(tr["eeg"], np.float64)
    env = np.asarray(tr["env"], np.float64)
    att = np.asarray(tr["att"], np.int8)
    T = min(len(eeg), len(env), len(att))
    starts, C0, C1 = _joint_covs_trial(model, eeg[:T], env[:T, 0], env[:T, 1])
    p0 = model.mdm.predict_proba(C0)[:, 0]  # P(match | cov with env0)
    p1 = model.mdm.predict_proba(C1)[:, 0]  # P(match | cov with env1)
    llr = model.sign * (np.log(p0 + 1e-12) - np.log(p1 + 1e-12))
    r0 = (0.5 * llr).astype(np.float32)
    r1 = (-0.5 * llr).astype(np.float32)
    # att stays 0 under prep; keep for scorer
    att_w = att[np.clip(starts + model.win // 2, 0, T - 1)].astype(np.int8)
    t_end = ((starts + model.win) / FS + tr["t0"]).astype(np.float32)
    return dict(r0=r0, r1=r1, att=att_w, t_end=t_end)


def csp_env_evidence_trial(model: CspMdmModel, tr):
    """EEG-block CSP filters → sliding corr vs env0/env1."""
    eeg = np.asarray(tr["eeg"], np.float64)
    env = np.asarray(tr["env"], np.float64)
    T = min(len(eeg), len(env), len(tr["att"]))
    z = eeg[:T] @ model.eeg_filters
    A = z.astype(np.float32)
    B0 = np.broadcast_to(env[:T, 0:1], A.shape).copy()
    B1 = np.broadcast_to(env[:T, 1:2], A.shape).copy()
    return evidence_from_features(A, B0, B1, tr["att"][:T], tr["t0"])


def swap_probe_mdm(model, tr):
    e = mdm_evidence_trial(model, tr)
    d = float((e["r0"] - e["r1"]).mean())
    tr_s = dict(tr)
    tr_s["env"] = tr["env"][:, ::-1].copy()
    e_s = mdm_evidence_trial(model, tr_s)
    d_s = float((e_s["r0"] - e_s["r1"]).mean())
    tr0 = dict(tr)
    tr0["eeg"] = np.zeros_like(tr["eeg"])
    e0 = mdm_evidence_trial(model, tr0)
    d0 = float((e0["r0"] - e0["r1"]).mean())
    return dict(d=d, d_swap=d_s, d0=d0,
                swap_delta=float(abs(d - d_s)),
                align_vs_zero=float(abs(d) - abs(d0)), kind="joint_mdm")


def swap_probe_env(model, tr):
    e = csp_env_evidence_trial(model, tr)
    d = float((e["r0"] - e["r1"]).mean())
    tr_s = dict(tr)
    tr_s["env"] = tr["env"][:, ::-1].copy()
    e_s = csp_env_evidence_trial(model, tr_s)
    d_s = float((e_s["r0"] - e_s["r1"]).mean())
    tr0 = dict(tr)
    tr0["eeg"] = np.zeros_like(tr["eeg"])
    e0 = csp_env_evidence_trial(model, tr0)
    d0 = float((e0["r0"] - e0["r1"]).mean())
    return dict(d=d, d_swap=d_s, d0=d0,
                swap_delta=float(abs(d - d_s)),
                align_vs_zero=float(abs(d) - abs(d0)), kind="csp_env")


def gather_train(held, prep, log, extra_fug=True):
    train = []
    for ds in DS_KEYS:
        if held is not None and ds == held:
            continue
        for _, r in rows_of(ds).iterrows():
            train.append(load_trial(prep, r["key"]))
    if extra_fug:
        try:
            train += load_external(["Fuglsang2020"], prep, log)
        except Exception as ex:
            log(f"  Fuglsang skip: {ex}")
    return train


def signed_acc1(ev_dict):
    return T.signed_acc1(ev_dict)


def run_fold_stream(held, stream, model, exp, log):
    rows = rows_of(held)
    ev = {}
    for _, r in rows.iterrows():
        tr = load_trial(PREP, r["key"])
        if stream in ("mdm", "joint_mdm"):
            e = mdm_evidence_trial(model, tr)
        else:
            e = csp_env_evidence_trial(model, tr)
        ev[r["key"]] = e
    acc1, snr = signed_acc1(ev)
    info = dict(decoder=f"csp_mdm_{stream}", prep=PREP, held_out=held,
                lookahead_s=float(LA), stream=stream, win_s=WIN_S, hop_s=HOP_S,
                n_filter=model.n_filter, acc1_pooled=acc1, snr1_pooled=snr)
    save_pickle(dict(evidence={"causal": ev}, info=info), fold_file(exp, held))
    return acc1, snr, ev


def avgc_probe(log):
    log("=== AVGC CSP+MDM Path A probe ===")
    t0 = time.time()
    train = gather_train("AVGC", PREP, log)
    log(f"  train trials={len(train)}")
    model = fit_csp_mdm(train, log=log)
    os.makedirs(CKPT_DIR, exist_ok=True)
    # save filters
    np.savez(os.path.join(CKPT_DIR, "held-AVGC_filters.npz"),
             filters=model.filters, sign=model.sign, win=model.win, hop=model.hop)
    with open(os.path.join(CKPT_DIR, "held-AVGC_mdm.pkl"), "wb") as fh:
        pickle.dump(model.mdm, fh)
    tr0 = load_trial(PREP, rows_of("AVGC").iloc[0]["key"])
    sp_m = swap_probe_mdm(model, tr0)
    sp_e = swap_probe_env(model, tr0)
    log(f"  swap MDM {sp_m}")
    log(f"  swap ENV {sp_e}")
    results = {}
    for stream, sp in (("joint_mdm", sp_m), ("csp_env", sp_e)):
        exp = EXP_NAME + f"__probe_{stream}"
        acc1, snr, _ = run_fold_stream("AVGC", stream, model, exp, log)
        ok = acc1 >= ACC1_GATE and sp["swap_delta"] >= 0.01
        soft = (not ok) and acc1 >= SOFT_ACC1 and sp["swap_delta"] >= 0.01
        tag = "PASS" if ok else ("SOFT" if soft else "FAIL")
        log(f"  PROBE[{stream}] acc1={acc1:.4f} snr={snr:.3f} swapΔ={sp['swap_delta']:.4f} → {tag}")
        results[stream] = dict(acc1=acc1, snr=snr, ok=ok or soft, hard_ok=ok, swap=sp)
    log(f"  probe done in {time.time()-t0:.0f}s")
    # pick best passing stream by acc1
    cands = [(k, v) for k, v in results.items() if v["ok"]]
    if not cands:
        return False, None, model, results
    cands.sort(key=lambda kv: kv[1]["acc1"], reverse=True)
    best = cands[0][0]
    log(f"  SELECT stream={best} acc1={results[best]['acc1']:.4f}")
    return True, best, model, results


def score_lodo(exp, log, name="sum"):
    la = float(LA)
    with open(fold_file(exp, "AVGC"), "rb") as fh:
        la = float(pickle.load(fh)["info"].get("lookahead_s", LA))
    ev, _ = S.load_folds(exp, "causal")
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                            for d in DS_KEYS if d != ds for e in ev[d].values()])
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, la, emis=(mu, sd * SCALE), p_stay=P_STAY)
        per[ds] = r
        log(f"  PRESET LODO[{name}] {ds}  H {r['H']:.4f}  Acc {r['Acc']:.2f}")
    mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log(f"  PRESET LODO[{name}] MEAN  H {mH:.4f}  Acc {mA:.2f}")
    return per, mH, mA, ev, la


def ewma_1d(x, tau_s=4.0, hop=0.25):
    x = np.asarray(x, np.float64)
    alpha = 1.0 - np.exp(-hop / tau_s)
    return lfilter([alpha], [1.0, -(1.0 - alpha)], x)


def gate_series(leader, csp, kind, thresh, sharp, ewma_tau=4.0):
    f = np.asarray(leader, np.float64)
    h = np.asarray(csp, np.float64)
    n = min(len(f), len(h))
    f, h = f[:n], h[:n]
    if kind == "soft_w":
        return np.full(n, float(thresh), np.float64)
    if kind == "const_leader":
        return np.ones(n, np.float64)
    if kind == "const_csp":
        return np.zeros(n, np.float64)
    if kind == "abs_leader":
        return expit(sharp * (np.abs(f) - thresh))
    if kind == "hard_abs_leader":
        return (np.abs(f) >= thresh).astype(np.float64)
    if kind == "hard_abs_csp":
        return (np.abs(h) < thresh).astype(np.float64)
    if kind == "ewma_abs_leader":
        return expit(sharp * (ewma_1d(np.abs(f), ewma_tau) - thresh))
    if kind == "abs_csp":
        return expit(sharp * (thresh - np.abs(h)))
    if kind == "disagree":
        return expit(sharp * (thresh - np.abs(f - h)))
    raise ValueError(kind)


def candidate_rules():
    rules = []
    for w in (0.0, 0.25, 0.5, 0.75, 1.0):
        rules.append(("soft_w", w, 1.0))
    rules += [
        ("const_leader", 0.0, 1.0),
        ("const_csp", 0.0, 1.0),
        ("abs_leader", 0.5, 4.0),
        ("abs_leader", 1.0, 4.0),
        ("abs_leader", 1.5, 4.0),
        ("ewma_abs_leader", 0.5, 4.0),
        ("ewma_abs_leader", 1.0, 4.0),
        ("hard_abs_leader", 0.5, 1.0),
        ("hard_abs_leader", 1.0, 1.0),
        ("hard_abs_leader", 1.5, 1.0),
        ("hard_abs_csp", 0.05, 1.0),
        ("hard_abs_csp", 0.1, 1.0),
        ("hard_abs_csp", 0.2, 1.0),
        ("abs_csp", 0.05, 40.0),
        ("abs_csp", 0.1, 40.0),
        ("disagree", 0.5, 4.0),
        ("disagree", 1.0, 4.0),
    ]
    return rules


def rule_key(kind, thr, sharp):
    if kind == "soft_w":
        return f"soft_w={thr:g}"
    return f"{kind}|t{thr:g}|s{sharp:g}"


def _match_tend(ta, tb, tol=1e-6):
    ia = ib = 0
    idx_a, idx_b = [], []
    ta = np.asarray(ta, np.float64)
    tb = np.asarray(tb, np.float64)
    while ia < len(ta) and ib < len(tb):
        da = ta[ia] - tb[ib]
        if abs(da) <= tol:
            idx_a.append(ia); idx_b.append(ib); ia += 1; ib += 1
        elif da < 0:
            ia += 1
        else:
            ib += 1
    return np.asarray(idx_a, np.int64), np.asarray(idx_b, np.int64)


def ensure_leader_lodo():
    path = os.path.join(CACHE_DIR, "leader_lodo.pkl")
    if os.path.isfile(path):
        with open(path, "rb") as fh:
            return pickle.load(fh)
    os.makedirs(CACHE_DIR, exist_ok=True)
    # reuse sw2_swmix_blend caches
    SM.ensure_caches = getattr(SM, "ensure_caches", None)
    # build via SM helpers
    from sw2_swmix_blend import ensure_sw2_lodo, ensure_swmix_lodo, apply_rule_lodo
    # Actually look for existing functions
    raise NotImplementedError


def build_leader_from_sm():
    """Return LODO evidence dict ds→key→{r0,r1,att,t_end} for Acc tip."""
    # Prefer JSON result + rebuild via SM module APIs if present
    cache = os.path.join(CACHE_DIR, "leader_lodo_ev.pkl")
    if os.path.isfile(cache):
        with open(cache, "rb") as fh:
            return pickle.load(fh)
    os.makedirs(CACHE_DIR, exist_ok=True)
    # Use SM's ensure + blend at tip rule
    if hasattr(SM, "ensure_lodo_streams"):
        sw2, swmix = SM.ensure_lodo_streams()
    else:
        # load from SM cache dir
        p = os.path.join(SM.CACHE_DIR, "lodo_streams.pkl")
        if not os.path.isfile(p):
            # run blend internals lightly
            import sw2_hp30_blend as SW2
            sw2_ev = {}
            # published SW2 fuse evidence via SW2 helpers
            if hasattr(SW2, "ensure_sw2_lodo"):
                sw2_ev = SW2.ensure_sw2_lodo()
            else:
                raise RuntimeError("need SW2 LODO cache")
        with open(p, "rb") as fh:
            blob = pickle.load(fh)
        sw2, swmix = blob["sw2"], blob["swmix"]
    # Apply tip rule hard_abs_swmix|t1
    tip = {}
    for ds in DS_KEYS:
        tip[ds] = {}
        for key in sw2[ds]:
            a = sw2[ds][key]
            b = swmix[ds][key]
            ia, ib = _match_tend(a["t_end"], b["t_end"])
            if len(ia) < 8:
                continue
            fa = (a["r0"] - a["r1"]).astype(np.float64)[ia]
            fb = (b["r0"] - b["r1"]).astype(np.float64)[ib]
            # hard_abs_swmix: use swmix when |swmix|>=1 else sw2
            g = (np.abs(fb) >= LEADER_THR).astype(np.float64)  # 1→swmix
            signed = (1 - g) * fa + g * fb
            tip[ds][key] = dict(
                r0=(0.5 * signed).astype(np.float32),
                r1=(-0.5 * signed).astype(np.float32),
                att=a["att"][ia].astype(np.int8),
                t_end=a["t_end"][ia].astype(np.float32),
            )
    with open(cache, "wb") as fh:
        pickle.dump(tip, fh)
    return tip


def main():
    log = log_to(LOG_PATH) if callable(log_to) else log_line
    # log_to returns a function in this codebase
    try:
        log = log_to(LOG_PATH)
    except TypeError:
        log = log_line
    log("=" * 72)
    log("csp-mdm-fuse START")
    ok, stream, model, probe = avgc_probe(log)
    if not ok:
        log("ABORT_CHANCE: AVGC Path A below gate for both MDM and CSP→env")
        # mark abandoned
        _write_abandon(probe)
        return 2
    if PROBE_ONLY:
        log("PROBE_ONLY=1 stop after probe")
        return 0

    # full LODO for selected stream
    exp = EXP_NAME + f"_{stream}"
    log(f"=== full LODO stream={stream} ===")
    models = {"AVGC": model}
    for held in DS_KEYS:
        if held == "AVGC":
            run_fold_stream(held, stream, model, exp, log)
            continue
        t0 = time.time()
        train = gather_train(held, PREP, log)
        m = fit_csp_mdm(train, log=log)
        models[held] = m
        np.savez(os.path.join(CKPT_DIR, f"held-{held}_filters.npz"),
                 filters=m.filters, sign=m.sign, win=m.win, hop=m.hop)
        with open(os.path.join(CKPT_DIR, f"held-{held}_mdm.pkl"), "wb") as fh:
            pickle.dump(m.mdm, fh)
        acc1, snr, _ = run_fold_stream(held, stream, m, exp, log)
        log(f"  fold {held} acc1={acc1:.4f} snr={snr:.3f} in {time.time()-t0:.0f}s")

    per, mH, mA, ev_csp, la = score_lodo(exp, log, name=stream)

    # --- fuse with Acc tip ---
    log("=== fuse CSP stream ↔ Acc tip hard_abs_swmix|t1 ===")
    tip = None
    try:
        tip = _load_tip_lodo(log)
    except Exception as ex:
        log(f"  tip load failed: {ex}; scoring CSP alone only")

    fuse_sel = None
    fuse_ext = {}
    if tip is not None:
        fuse_sel, fuse_table = _select_fuse(tip, ev_csp, la, log)
        fuse_ext = _score_external_fuse(models, stream, tip, fuse_sel, ev_csp, la, log)
    else:
        fuse_table = {}

    # external CSP alone
    log("=== external CSP alone ===")
    # all5 model
    train_all = gather_train(None, PREP, log)
    m_all = fit_csp_mdm(train_all, log=log)
    ext_alone = _score_external_alone(m_all, stream, ev_csp, la, log)

    out = dict(
        stream=stream, probe= {k: {kk: vv for kk, vv in v.items() if kk != "swap"} | {"swap": v["swap"]}
                               for k, v in probe.items()},
        lodo_csp=dict(per={d: dict(H=per[d]["H"], Acc=per[d]["Acc"]) for d in DS_KEYS},
                      mean_H=mH, mean_Acc=mA),
        fuse_sel=fuse_sel, fuse_table=fuse_table,
        ext_csp=ext_alone, fuse_ext=fuse_ext,
    )
    # jsonify-safe
    def _safe(o):
        if isinstance(o, dict):
            return {str(k): _safe(v) for k, v in o.items()}
        if isinstance(o, (np.floating, float)):
            return float(o)
        if isinstance(o, (np.integer, int)):
            return int(o)
        if isinstance(o, np.ndarray):
            return o.tolist()
        if isinstance(o, (list, tuple)):
            return [_safe(x) for x in o]
        return o
    os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
    with open(OUT_JSON, "w") as fh:
        json.dump(_safe(out), fh, indent=2)
    _write_results(out, stream)
    log("csp-mdm-fuse DONE")
    return 0


def _load_tip_lodo(log):
    """Build tip LODO evidence from SW2↔swmix caches."""
    cache = os.path.join(CACHE_DIR, "leader_lodo_ev.pkl")
    if os.path.isfile(cache):
        with open(cache, "rb") as fh:
            return pickle.load(fh)
    os.makedirs(CACHE_DIR, exist_ok=True)
    # Find stream caches used by sw2_swmix_blend
    cand = [
        os.path.join(SM.CACHE_DIR, "lodo_aligned.pkl"),
        os.path.join(SM.CACHE_DIR, "lodo_streams.pkl"),
        os.path.join(SM.CACHE_DIR, "sw2_lodo.pkl"),
    ]
    for p in cand:
        log(f"  check cache {p}: {os.path.isfile(p)}")
    # Import builders from SM
    import importlib
    sm = importlib.import_module("sw2_swmix_blend")
    # Try common patterns
    for fn_name in ("ensure_streams_lodo", "load_lodo_streams", "build_lodo"):
        if hasattr(sm, fn_name):
            log(f"  using SM.{fn_name}")
            break
    # Fall back: reconstruct from fold pickles of published SW2 + swmix
    import sw2_hp30_blend as SW2b
    sw2_ev = None
    if hasattr(SW2b, "load_sw2_lodo_evidence"):
        sw2_ev = SW2b.load_sw2_lodo_evidence()
    elif hasattr(SW2b, "ensure_sw2_evidence"):
        sw2_ev = SW2b.ensure_sw2_evidence()
    # Direct: score.py load_folds for SW2 fuse experiment name
    # Published SW2 LODO folds live under decoder_lodo or eeg-thao folds
    sw2_name = getattr(SW2b, "SW2_EXP", None) or "fuse_sw2_a0.5"
    # Use SM JSON to find how they cached
    with open(os.path.join(EXP, "results", "research", "sw2_swmix_blend.json")) as fh:
        meta = json.load(fh)
    log(f"  sw2_swmix_blend.json keys={list(meta.keys())[:12]}")
    # Rebuild tip by calling SM gate on cached arrays if present
    sw2_path = os.path.join(SM.CACHE_DIR, "sw2_signed_lodo.pkl")
    swmix_path = os.path.join(SM.CACHE_DIR, "swmix_signed_lodo.pkl")
    if os.path.isfile(sw2_path) and os.path.isfile(swmix_path):
        with open(sw2_path, "rb") as fh:
            sw2 = pickle.load(fh)
        with open(swmix_path, "rb") as fh:
            swmix = pickle.load(fh)
    else:
        # scan cache dir
        files = os.listdir(SM.CACHE_DIR) if os.path.isdir(SM.CACHE_DIR) else []
        log(f"  SM.CACHE_DIR files={files[:30]}")
        # try loading whatever looks right
        sw2 = swmix = None
        for f in files:
            if "sw2" in f.lower() and f.endswith(".pkl"):
                with open(os.path.join(SM.CACHE_DIR, f), "rb") as fh:
                    obj = pickle.load(fh)
                log(f"    loaded {f} type={type(obj)} keys={list(obj)[:8] if isinstance(obj, dict) else None}")
        raise RuntimeError("cannot locate SW2/swmix LODO caches for tip")
    tip = {}
    for ds in DS_KEYS:
        tip[ds] = {}
        for key in sw2[ds]:
            if key not in swmix[ds]:
                continue
            a, b = sw2[ds][key], swmix[ds][key]
            ia, ib = _match_tend(a["t_end"], b["t_end"])
            if len(ia) < 8:
                continue
            fa = (a["r0"] - a["r1"]).astype(np.float64)[ia]
            fb = (b["r0"] - b["r1"]).astype(np.float64)[ib]
            g = (np.abs(fb) >= LEADER_THR).astype(np.float64)
            signed = (1 - g) * fa + g * fb
            tip[ds][key] = dict(
                r0=(0.5 * signed).astype(np.float32),
                r1=(-0.5 * signed).astype(np.float32),
                att=np.asarray(a["att"])[ia].astype(np.int8),
                t_end=np.asarray(a["t_end"])[ia].astype(np.float32),
            )
    with open(cache, "wb") as fh:
        pickle.dump(tip, fh)
    return tip


def _select_fuse(tip, csp_ev, la, log):
    table = []
    best = None
    for kind, thr, sharp in candidate_rules():
        fused = {}
        for ds in DS_KEYS:
            fused[ds] = {}
            for key, e_tip in tip[ds].items():
                if key not in csp_ev.get(ds, {}):
                    continue
                e_c = csp_ev[ds][key]
                ia, ib = _match_tend(e_tip["t_end"], e_c["t_end"])
                if len(ia) < 8:
                    continue
                ft = (e_tip["r0"] - e_tip["r1"]).astype(np.float64)[ia]
                fc = (e_c["r0"] - e_c["r1"]).astype(np.float64)[ib]
                g = gate_series(ft, fc, kind, thr, sharp)
                signed = g * ft + (1 - g) * fc
                fused[ds][key] = dict(
                    r0=(0.5 * signed).astype(np.float32),
                    r1=(-0.5 * signed).astype(np.float32),
                    att=np.asarray(e_tip["att"])[ia].astype(np.int8),
                    t_end=np.asarray(e_tip["t_end"])[ia].astype(np.float32),
                )
        # score
        per = {}
        ok = True
        for ds in DS_KEYS:
            if not fused[ds]:
                ok = False
                break
            A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                                for d in DS_KEYS if d != ds for e in fused[d].values()])
            mu, sd = float(A.mean()), float(A.std() + 1e-12)
            per[ds] = S.score_dataset(ds, fused, la, emis=(mu, sd * SCALE), p_stay=P_STAY)
        if not ok:
            continue
        mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        key = rule_key(kind, thr, sharp)
        row = dict(rule=key, kind=kind, thr=thr, sharp=sharp, mean_H=mH, mean_Acc=mA,
                   per={d: dict(H=per[d]["H"], Acc=per[d]["Acc"]) for d in DS_KEYS})
        table.append(row)
        log(f"  fuse {key}: H {mH:.4f} Acc {mA:.2f}")
        if best is None or (mA, mH) > (best["mean_Acc"], best["mean_H"]):
            best = row
            best["_fused"] = fused
    log(f"  SELECT fuse={best['rule']} Acc={best['mean_Acc']:.2f} H={best['mean_H']:.4f}")
    # drop heavy
    sel = {k: v for k, v in best.items() if k != "_fused"}
    sel["_fused"] = best["_fused"]
    return sel, table


def _score_external_alone(model, stream, ev_train, la, log):
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                        for ds in DS_KEYS for e in ev_train[ds].values()])
    mu, sd = float(A.mean()), float(A.std())
    E.CONF.update(prep=PREP, targets=["env"])
    os.makedirs(EXT_DIR, exist_ok=True)
    out = {}
    for ds in ("AASD", "Carta2026"):
        path = os.path.join(EXT_DIR, f"ext_csp_mdm_{ds}{TAG}_{stream}_evidence.pkl")
        if os.path.exists(path):
            with open(path, "rb") as fh:
                EV = pickle.load(fh)
            log(f"  loaded {path}")
        else:
            EV = {}
            trials = E.ext_trials(ds)
            log(f"  external {ds}: {len(trials)} trials")
            for i, tr in enumerate(trials):
                e = mdm_evidence_trial(model, tr) if stream == "mdm" else csp_env_evidence_trial(model, tr)
                e.update(att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
                EV[tr["key"]] = e
                if (i + 1) % 40 == 0 or i + 1 == len(trials):
                    log(f"    {ds} {i + 1}/{len(trials)}")
            with open(path, "wb") as fh:
                pickle.dump(EV, fh)
        r = E.score(EV, mu, sd * SCALE, P_STAY, 0.75)
        out[ds] = dict(H=float(r["H"]), Acc=float(r["Acc"]))
        log(f"  EXT[{stream}] {ds} H {r['H']:.4f} Acc {r['Acc']:.2f}")
    return out


def _score_external_fuse(models, stream, tip_lodo, fuse_sel, ev_csp, la, log):
    # Build all5 tip from SM external + CSP all5
    # For simplicity: use fold-AVGC model for external CSP (same as other Path A scripts
    # that refit all5 — we already have that in caller via m_all separately)
    # Here tip external from SM tip reconstruction
    kind, thr, sharp = fuse_sel["kind"], fuse_sel["thr"], fuse_sel["sharp"]
    # emissions from fused train
    fused_train = fuse_sel["_fused"]
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                        for ds in DS_KEYS for e in fused_train[ds].values()])
    mu, sd = float(A.mean()), float(A.std())
    # need external tip + external csp — defer: score using tip_ext caches if available
    out = {}
    try:
        tip_ext = _load_tip_ext(log)
        m_all = models.get("ALL") or models.get("AVGC")
        for ds in ("AASD", "Carta2026"):
            path = os.path.join(EXT_DIR, f"ext_csp_mdm_{ds}{TAG}_{stream}_evidence.pkl")
            with open(path, "rb") as fh:
                EV_c = pickle.load(fh)
            EV_t = tip_ext[ds]
            fused = {}
            for key, e_t in EV_t.items():
                if key not in EV_c:
                    continue
                e_c = EV_c[key]
                ia, ib = _match_tend(e_t["t_end"], e_c["t_end"])
                if len(ia) < 8:
                    continue
                ft = (e_t["r0"] - e_t["r1"]).astype(np.float64)[ia]
                fc = (e_c["r0"] - e_c["r1"]).astype(np.float64)[ib]
                g = gate_series(ft, fc, kind, thr, sharp)
                signed = g * ft + (1 - g) * fc
                ne = dict(e_t)
                ne["r0"] = (0.5 * signed).astype(np.float32)
                ne["r1"] = (-0.5 * signed).astype(np.float32)
                ne["att"] = np.asarray(e_t["att"])[ia].astype(np.int8) if "att" in e_t else e_t.get("att")
                ne["t_end"] = np.asarray(e_t["t_end"])[ia].astype(np.float32)
                fused[key] = ne
            r = E.score(fused, mu, sd * SCALE, P_STAY, 0.75)
            out[ds] = dict(H=float(r["H"]), Acc=float(r["Acc"]))
            log(f"  EXT-FUSE[{fuse_sel['rule']}] {ds} H {r['H']:.4f} Acc {r['Acc']:.2f}")
    except Exception as ex:
        log(f"  external fuse skip: {ex}")
    return out


def _load_tip_ext(log):
    cache = os.path.join(CACHE_DIR, "leader_ext_ev.pkl")
    if os.path.isfile(cache):
        with open(cache, "rb") as fh:
            return pickle.load(fh)
    # Build from SM external caches similar to tip LODO
    files = os.listdir(SM.CACHE_DIR) if os.path.isdir(SM.CACHE_DIR) else []
    log(f"  tip_ext SM files={files}")
    raise RuntimeError("tip_ext not cached yet")


def _write_abandon(probe):
    md = f"""# results_csp_mdm_fuse

CSP→Riemann MDM / CSP→env Path A Acc fuse stream.

## Status: ABANDONED (chance)

AVGC probe failed both streams (acc1 below gate ~{ACC1_GATE}).

```
{json.dumps({k: {kk: (float(vv) if isinstance(vv, (float, np.floating)) else vv)
                 for kk, vv in v.items() if kk != 'swap'} for k, v in probe.items()}, indent=2)}
```

## NEXT IDEA

Do not rerun identical CSP/MDM window SPD Path A. Prefer DANN/GRL or VICReg subject-invariant pretrain on stem when GPU free; or SincNet Acc front-end with chance-abort.
"""
    with open(RESULTS_MD, "w") as fh:
        fh.write(md)
    # update claims
    text = open(CLAIMS_PATH).read()
    lines = text.splitlines()
    out = []
    for ln in lines:
        if "csp-mdm-fuse" in ln and "| running |" in ln:
            ln = ln.replace("| running |", "| abandoned |").rstrip()
            if "chance" not in ln.lower():
                ln += " ABORT_CHANCE AVGC probe. results_csp_mdm_fuse.md"
        out.append(ln)
    open(CLAIMS_PATH, "w").write("\n".join(out) + "\n")
    # ALREADY_TRIED
    with open(ALREADY_PATH, "a") as fh:
        fh.write(
            "\n- **csp-mdm-fuse** (CSP→MDM / CSP→env Path A; fuse↔Acc tip): abandoned AVGC chance-probe "
            f"(acc1 below gate). Do not rerun identical CSP/MDM SPD windows. "
            "`multi_agent_opt/results_csp_mdm_fuse.md`.\n"
        )


def _write_results(out, stream):
    lodo = out["lodo_csp"]
    ext = out.get("ext_csp", {})
    fuse = out.get("fuse_sel")
    fext = out.get("fuse_ext", {})
    lines = [
        "# results_csp_mdm_fuse",
        "",
        f"CSP→Riemann MDM / CSP→env Path A (`stream={stream}`) as Acc fuse stream vs tip "
        "`hard_abs_swmix|t1`. HMM **0.994/0.8**. CPU.",
        "",
        f"Code: `csp_mdm_fuse.py`. JSON: `{OUT_JSON}`.",
        "",
        "## Path A LODO (CSP stream alone)",
        "",
        f"**MEAN H {lodo['mean_H']:.4f} Acc {lodo['mean_Acc']:.2f}**",
        "",
        "| Held-out | H | Acc % |",
        "|---|---:|---:|",
    ]
    for d in DS_KEYS:
        p = lodo["per"][d]
        lines.append(f"| {d} | {p['H']:.4f} | {p['Acc']:.2f} |")
    lines += ["", "## External CSP alone", ""]
    for ds, r in ext.items():
        lines.append(f"- {ds}: H {r['H']:.4f} Acc **{r['Acc']:.2f}** "
                     f"(Δtip {r['Acc']-SWMIX_AASD:+.2f}; Δhp30 {r['Acc']-LINEAR_HP30_AASD:+.2f})")
    if fuse:
        lines += [
            "",
            f"## Fuse select `{fuse['rule']}`",
            "",
            f"LODO MEAN H {fuse['mean_H']:.4f} Acc {fuse['mean_Acc']:.2f}",
            "",
        ]
        for ds, r in fext.items():
            lines.append(f"- fuse {ds}: H {r['H']:.4f} Acc **{r['Acc']:.2f}**")
    lines += [
        "",
        "## Targets",
        "",
        f"| Target | Result |",
        f"|---|---|",
        f"| AASD Acc>60? | {'YES' if max([ext.get('AASD',{}).get('Acc',0)]+[fext.get('AASD',{}).get('Acc',0)])>60 else 'NO'} |",
        f"| tip moved? | NO (Acc stream; LODO tip unchanged) |",
        "",
        "## NEXT IDEA",
        "",
        "If CSP/MDM Acc flat mid-50s: close classical SPD fuse-stream family. Prefer DANN/GRL or "
        "VICReg/Barlow subject-invariant pretrain→LODO FT when GPU free — not Mixup/focal/tip soft-attn.",
        "",
    ]
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))
    # claims → done
    text = open(CLAIMS_PATH).read()
    lines2 = []
    for ln in text.splitlines():
        if "csp-mdm-fuse" in ln and "| running |" in ln:
            aasd = fext.get("AASD", ext.get("AASD", {}))
            ln = ln.replace("| running |", "| done |")
            ln += (f" stream={stream} LODO {lodo['mean_H']:.4f}/{lodo['mean_Acc']:.2f}; "
                   f"AASD {aasd.get('H',0):.4f}/{aasd.get('Acc',0):.2f}. results_csp_mdm_fuse.md")
        lines2.append(ln)
    open(CLAIMS_PATH, "w").write("\n".join(lines2) + "\n")
    with open(ALREADY_PATH, "a") as fh:
        aasd = fext.get("AASD", ext.get("AASD", {}))
        fh.write(
            f"\n- **csp-mdm-fuse** (`csp_mdm_fuse.py`; CSP→MDM/CSP→env Path A + fuse↔Acc tip; "
            f"stream={stream}): LODO {lodo['mean_H']:.4f}/{lodo['mean_Acc']:.2f}; "
            f"AASD {aasd.get('Acc', float('nan')):.2f}. ≠ FBCSP spatial-alone / TRF-DSS Path A. "
            f"`multi_agent_opt/results_csp_mdm_fuse.md`.\n"
        )
    # status
    st = open(STATUS_PATH).read()
    if "csp-mdm-fuse" not in st:
        st = st.replace(
            "## Recently closed",
            "## Recently closed\n\n"
            f"- **csp-mdm-fuse** — stream={stream} LODO {lodo['mean_H']:.4f}/{lodo['mean_Acc']:.2f}; "
            f"AASD Acc {aasd.get('Acc', float('nan')):.2f}. Classical SPD fuse closed if Acc≪60.\n",
            1,
        )
        open(STATUS_PATH, "w").write(st)


if __name__ == "__main__":
    raise SystemExit(main())
