"""dss-trf-clf — Path A: train-only DSS/SSD spatial filters → frozen lag/corr HMM evidence.

Distinct from:
  * TRF Path A / band-trf / dual-band-trf (ridge EEG→env lag filters)
  * Decoder CCA/FBCCA/Riemann package (C1)
  * Spatial CCA lag-bank that stayed ~chance in trf_lagbank
  * xcorr CNN

Pipeline:
  1. Fit on LODO train only:
       DSS — envelope-informed bias Σ_τ R_xy(τ)R_xy(τ)^T vs C_xx (gen. eigen).
       SSD — maximize 1–8 Hz power vs flanking bands (no env labels in fit).
  2. Freeze Wx (C, K). Project EEG → K components.
  3. Framewise lag-bank attended−unattended correlation scores (multi-comp → sum
     or train-only LDA on per-comp streams).
  4. Path A: frozen scores → HMM @ 0.994/0.8. Path B tiny head + chance abort.

Prep eaP_n5_t60. CPU preferred (no GPU lock).

usage:
  PROBE_ONLY=1 .../python dss_trf_clf.py
  .../python dss_trf_clf.py
"""
from __future__ import annotations

import json
import os
import pickle
import re
import time
from dataclasses import dataclass

import numpy as np
from scipy.signal import butter, sosfilt

import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
import trf_lagbank_clf as T
from common import DS_KEYS
from common_exp import (
    CFG, EXP, FS, evidence_from_features, fold_file, load_trial, rows_of, save_pickle,
)
from run_experiment import load_external, log_to

# -------------------------------------------------------------------------------------------------
EXP_NAME = "dss_trf__eaP_n5_t60"
TAG = "_dss_trf"
PREP = "eaP_n5_t60"
LOG_PATH = os.path.join(EXP, "experiments", "dss_trf_clf.log")
OUT_JSON = os.path.join(EXP, "results", f"{EXP_NAME}.json")
CKPT_DIR = os.path.join(EXP, "models", "dss_trf")
EXT_DIR = os.path.join(EXP, "results", "research")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_dss_trf.md")
CLAIMS_PATH = os.path.join(EXP, "multi_agent_opt", "CLAIMS.md")
ALREADY_PATH = os.path.join(EXP, "multi_agent_opt", "ALREADY_TRIED.md")
STATUS_PATH = os.path.join(EXP, "multi_agent_opt", "STATUS_GAPS.md")

P_STAY, SCALE = 0.994, 0.8
MAX_LAG_MS = int(os.environ.get("MAX_LAG_MS", "400"))
N_COMP = int(os.environ.get("N_COMP", "4"))
RIDGE_REL = float(os.environ.get("RIDGE_REL", "1e-3"))
PATH_MODE = os.environ.get("PATH_MODE", "both")  # dss | ssd | both
FUSE = os.environ.get("FUSE", "both")  # sum | lda | both  (select LODO Acc then H)
PROBE_ONLY = os.environ.get("PROBE_ONLY", "0") == "1"
ACC1_GATE = float(os.environ.get("ACC1_GATE", "0.53"))
CHANCE_LOSS = float(os.environ.get("CHANCE_LOSS", "0.685"))
CHANCE_STEPS = int(os.environ.get("CHANCE_STEPS", "50"))
SWMIX_AASD = 53.53
LINEAR_HP30_AASD = 53.12
SW2_AASD = 53.46
TRF_AASD = 50.91
DUAL_AASD = 51.09

# SSD bands (Hz) — speech envelope band vs flanks
SSD_SIG = (float(os.environ.get("SSD_LO", "1.0")), float(os.environ.get("SSD_HI", "8.0")))
SSD_FLANK_LO = (0.5, 1.0)
SSD_FLANK_HI = (8.0, 12.0)
SSD_ORDER = 4

_SOS = {}


def _sos(lo, hi, btype="bandpass"):
    key = (lo, hi, btype, SSD_ORDER)
    if key not in _SOS:
        if btype == "bandpass":
            _SOS[key] = butter(SSD_ORDER, [lo, hi], btype="bandpass", fs=float(FS), output="sos")
        else:
            raise ValueError(btype)
    return _SOS[key]


def bandpass(x, lo, hi):
    x = np.asarray(x, np.float64)
    return sosfilt(_sos(lo, hi), x, axis=0)


@dataclass
class SpatialFilters:
    kind: str                 # "dss" | "ssd"
    max_lag: int
    W: np.ndarray             # (C, K) spatial
    Wt: np.ndarray | None = None  # (n_lags, K) temporal TRF on projected comps
    evals: np.ndarray | None = None
    ridge: float = RIDGE_REL
    n_comp: int = N_COMP


def _att_env(tr):
    a = tr["att"].astype(int)
    idx = np.arange(len(tr["env"]))
    return tr["env"][idx, a].astype(np.float64), tr["env"][idx, 1 - a].astype(np.float64)


def fit_dss(train, max_lag, n_comp=N_COMP, ridge_rel=RIDGE_REL, log=None, max_samples=2_000_000):
    """Envelope-informed DSS: C_bias = Σ_τ R_xy(τ) R_xy(τ)^T ; solve C_bias w = λ C_xx w.

    Distinct from CCA (joint EEG×lagged-env eigen) and from TRF (ridge W_τ).
    Bias uses attended envelope only (train labels). Never held-out/ext labels.
    """
    C = train[0]["eeg"].shape[1]
    n_lags = max_lag + 1
    Cxx = np.zeros((C, C), np.float64)
    Rxy = np.zeros((n_lags, C), np.float64)  # E[eeg(t+τ) env(t)]
    n = 0
    mid = max_lag // 2
    ordered = sorted(train, key=lambda tr: 1 if tr.get("_grp") == "Fuglsang2020" else 0)
    for tr in ordered:
        if n >= max_samples:
            break
        X = np.asarray(tr["eeg"], np.float64)
        y, _ = _att_env(tr)
        T = min(len(X), len(y))
        if T <= max_lag + 8:
            continue
        n_valid = T - max_lag
        take = min(n_valid, max_samples - n)
        sel = (np.linspace(0, n_valid - 1, take, dtype=int) if take < n_valid
               else np.arange(n_valid))
        Xc = X - X.mean(0, keepdims=True)
        yc = y - y.mean()
        for τ in range(n_lags):
            Rxy[τ] += Xc[sel + τ].T @ yc[sel]
        Cxx += Xc[sel + mid].T @ Xc[sel + mid]
        n += take
    if n < C:
        raise RuntimeError(f"DSS underdetermined: n={n} C={C}")
    Rxy /= n
    Cxx /= n
    C_bias = np.zeros((C, C), np.float64)
    for τ in range(n_lags):
        v = Rxy[τ][:, None]
        C_bias += v @ v.T
    lam = ridge_rel * (np.trace(Cxx) / C)
    Cxx_r = Cxx + lam * np.eye(C)
    M = np.linalg.solve(Cxx_r, C_bias)
    evals, evecs = np.linalg.eigh(M)
    order = np.argsort(evals)[::-1]
    K = min(n_comp, C)
    Wx = evecs[:, order[:K]].copy()
    ev = evals[order[:K]].copy()
    for k in range(K):
        nrm = np.sqrt(max(Wx[:, k] @ Cxx @ Wx[:, k], 1e-18))
        Wx[:, k] /= nrm
        if float(Rxy[mid] @ Wx[:, k]) < 0:
            Wx[:, k] *= -1
    if log:
        log(f"    DSS fit n={n} C={C} K={K} topλ={float(ev[0]):.4e} "
            f"lookahead={max_lag/FS:.3f}s")
    return SpatialFilters(kind="dss", max_lag=max_lag, W=Wx.astype(np.float64),
                          evals=ev.astype(np.float64), ridge=ridge_rel, n_comp=K)


def fit_comp_trf(train, Wx, max_lag, ridge_rel=RIDGE_REL, log=None, max_samples=2_000_000):
    """Ridge TRF on DSS/SSD-projected components: env(t) ≈ Σ_τ z(t+τ)·w_τ, z=eeg@Wx.

    Keeps spatial filters frozen; learns only K-dim temporal bank (≠ full-channel TRF).
    """
    K = Wx.shape[1]
    n_lags = max_lag + 1
    p = n_lags * K
    XtX = np.zeros((p, p), np.float64)
    Xty = np.zeros(p, np.float64)
    n = 0
    chunk = 4096
    ordered = sorted(train, key=lambda tr: 1 if tr.get("_grp") == "Fuglsang2020" else 0)
    for tr in ordered:
        if n >= max_samples:
            break
        X = np.asarray(tr["eeg"], np.float64)
        y, _ = _att_env(tr)
        T = min(len(X), len(y))
        if T <= max_lag + 8:
            continue
        z = X @ Wx  # (T, K)
        n_valid = T - max_lag
        take = min(n_valid, max_samples - n)
        idx = (np.linspace(0, n_valid - 1, take, dtype=int) if take < n_valid
               else np.arange(n_valid))
        for s0 in range(0, len(idx), chunk):
            ii = idx[s0:s0 + chunk]
            m = len(ii)
            Xt = np.empty((m, p), np.float64)
            for τ in range(n_lags):
                Xt[:, τ * K:(τ + 1) * K] = z[ii + τ]
            yt = y[ii]
            XtX += Xt.T @ Xt
            Xty += Xt.T @ yt
            n += m
            if n >= max_samples:
                break
    if n < p:
        raise RuntimeError(f"comp-TRF underdetermined: n={n} p={p}")
    XtX /= n
    Xty /= n
    lam = ridge_rel * (np.trace(XtX) / p)
    XtX.flat[:: p + 1] += lam
    w = np.linalg.solve(XtX, Xty)
    Wt = w.reshape(n_lags, K)
    if log:
        log(f"    comp-TRF fit n={n} p={p} ||Wt||={np.linalg.norm(Wt):.4f}")
    return Wt.astype(np.float64)


def fit_ssd(train, max_lag, n_comp=N_COMP, ridge_rel=RIDGE_REL, log=None, max_samples=2_000_000):
    """SSD: maximize speech-band (1–8 Hz) power vs flanking bands. No envelope in fit.

    Spatial filters still scored vs env0/env1 at inference (lag/corr Path A).
    """
    C = train[0]["eeg"].shape[1]
    Csig = np.zeros((C, C), np.float64)
    Cnoi = np.zeros((C, C), np.float64)
    n = 0
    ordered = sorted(train, key=lambda tr: 1 if tr.get("_grp") == "Fuglsang2020" else 0)
    for tr in ordered:
        if n >= max_samples:
            break
        X = np.asarray(tr["eeg"], np.float64)
        T = len(X)
        if T < int(2 * FS):
            continue
        take = min(T, max_samples - n)
        if take < T:
            X = X[:take]
        X = X - X.mean(0, keepdims=True)
        Xs = bandpass(X, *SSD_SIG)
        Xl = bandpass(X, *SSD_FLANK_LO)
        Xh = bandpass(X, *SSD_FLANK_HI)
        # drop filter transient
        drop = int(0.5 * FS)
        if len(Xs) <= drop + 8:
            continue
        Xs, Xl, Xh = Xs[drop:], Xl[drop:], Xh[drop:]
        Csig += Xs.T @ Xs
        Cnoi += Xl.T @ Xl + Xh.T @ Xh
        n += len(Xs)
    if n < C:
        raise RuntimeError(f"SSD underdetermined: n={n} C={C}")
    Csig /= n
    Cnoi /= n
    lam = ridge_rel * (np.trace(Cnoi) / C)
    Cnoi_r = Cnoi + lam * np.eye(C)
    M = np.linalg.solve(Cnoi_r, Csig)
    evals, evecs = np.linalg.eigh(M)
    order = np.argsort(evals)[::-1]
    K = min(n_comp, C)
    Wx = evecs[:, order[:K]].copy()
    ev = evals[order[:K]].copy()
    for k in range(K):
        nrm = np.sqrt(max(Wx[:, k] @ Csig @ Wx[:, k], 1e-18))
        Wx[:, k] /= nrm
    if log:
        log(f"    SSD fit n={n} C={C} K={K} topλ={float(ev[0]):.4f} "
            f"sig={SSD_SIG} flanks={SSD_FLANK_LO}+{SSD_FLANK_HI}")
    return SpatialFilters(kind="ssd", max_lag=max_lag, W=Wx.astype(np.float64),
                          evals=ev.astype(np.float64), ridge=ridge_rel, n_comp=K)


def lagbank_scores(filt: SpatialFilters, eeg, env0, env1):
    """DSS/SSD spatial → optional low-D TRF reconstruction bank (TRF-shaped evidence)."""
    eeg = np.asarray(eeg, np.float64)
    env0 = np.asarray(env0, np.float64)
    env1 = np.asarray(env1, np.float64)
    T = min(len(eeg), len(env0), len(env1))
    eeg, env0, env1 = eeg[:T], env0[:T], env1[:T]
    L = filt.max_lag
    Tv = T - L
    z = eeg @ filt.W  # (T, K)
    K = z.shape[1]
    n_lags = L + 1
    if filt.Wt is not None:
        # ŷ_τ(t) = z(t+τ) · Wt[τ]; A = [Σŷ, ŷ_0..ŷ_L] like TRF Path A but on K comps
        Yhat = np.zeros((Tv, n_lags), np.float64)
        for τ in range(n_lags):
            Yhat[:, τ] = (z[τ: τ + Tv] * filt.Wt[τ][None, :]).sum(1)
        ysum = Yhat.sum(1, keepdims=True)
        A = np.concatenate([ysum, Yhat], axis=1)
    else:
        # fallback: raw lagged components
        A = np.empty((Tv, K * n_lags), np.float64)
        for τ in range(n_lags):
            A[:, τ * K:(τ + 1) * K] = z[τ: τ + Tv]
    e0, e1 = env0[:Tv], env1[:Tv]
    B0 = np.broadcast_to(e0[:, None], A.shape).copy()
    B1 = np.broadcast_to(e1[:, None], A.shape).copy()
    return A.astype(np.float32), B0.astype(np.float32), B1.astype(np.float32), L / FS


def evidence_trial(filt, tr):
    A, B0, B1, la = lagbank_scores(filt, tr["eeg"], tr["env"][:, 0], tr["env"][:, 1])
    att = tr["att"][: len(A)]
    return evidence_from_features(A, B0, B1, att, tr["t0"]), la


def evidence_trial_percomp(filt, tr):
    """Per-lag 1-D streams from DSS-TRF reconstruction packed as r0f/r1f for LDA fuse."""
    A, B0, B1, la = lagbank_scores(filt, tr["eeg"], tr["env"][:, 0], tr["env"][:, 1])
    # If TRF-shaped: cols = [sum, lag0..lagL] — use each lag (+sum) as a fuse dim
    from common_exp import sliding_corr
    W, Hh = int(CFG["evidence"]["win_s"] * FS), int(CFG["evidence"]["hop_s"] * FS)
    e0 = B0[:, 0]
    e1 = B1[:, 0]
    r0_cols, r1_cols = [], []
    # cap fuse dims
    cols = list(range(A.shape[1]))
    if len(cols) > 8:
        cols = [0] + list(np.linspace(1, A.shape[1] - 1, 7, dtype=int))
    for j in cols:
        r0, st = sliding_corr(A[:, j], e0, W, Hh)
        r1, _ = sliding_corr(A[:, j], e1, W, Hh)
        r0_cols.append(r0)
        r1_cols.append(r1)
    r0f = np.stack(r0_cols, axis=1).astype(np.float32)
    r1f = np.stack(r1_cols, axis=1).astype(np.float32)
    c = np.concatenate([[0], np.cumsum(tr["att"][: len(A)].astype(np.float64))])
    att_w = ((c[st + W] - c[st]) / W > 0.5).astype(np.int8)
    t_end = ((st + W) / FS + tr["t0"]).astype(np.float32)
    out = dict(
        r0=r0f.mean(1).astype(np.float32),
        r1=r1f.mean(1).astype(np.float32),
        att=att_w, t_end=t_end,
        r0f=r0f.astype(np.float16), r1f=r1f.astype(np.float16),
    )
    return out, la


def signed_acc1(ev_dict):
    return T.signed_acc1(ev_dict)


def swap_probe(filt, tr):
    """Speaker-order swap must flip evidence; zero-EEG must collapse |d|."""
    A, B0, B1, _ = lagbank_scores(filt, tr["eeg"], tr["env"][:, 0], tr["env"][:, 1])
    a = A.mean(1)
    r0 = np.corrcoef(a, B0.mean(1))[0, 1]
    r1 = np.corrcoef(a, B1.mean(1))[0, 1]
    d = r0 - r1
    r0s = np.corrcoef(a, B1.mean(1))[0, 1]
    r1s = np.corrcoef(a, B0.mean(1))[0, 1]
    d_swap = r0s - r1s
    A0, B00, B10, _ = lagbank_scores(filt, np.zeros_like(tr["eeg"]), tr["env"][:, 0], tr["env"][:, 1])
    a0 = A0.mean(1)
    if a0.std() < 1e-12:
        d0 = 0.0
    else:
        d0 = np.corrcoef(a0, B00.mean(1))[0, 1] - np.corrcoef(a0, B10.mean(1))[0, 1]
    return dict(d=float(d), d_swap=float(d_swap), d0=float(d0),
                swap_delta=float(abs(d - d_swap)),
                align_vs_zero=float(abs(d) - abs(d0)))


def gather_train(held, prep, log, extra_fug=False):
    train = []
    for ds in DS_KEYS:
        if held is not None and ds == held:
            continue
        for _, r in rows_of(ds).iterrows():
            train.append(load_trial(prep, r["key"]))
    if extra_fug:
        try:
            train += load_external(["Fuglsang2020"], prep, log)
        except Exception as ex:
            log(f"  Fuglsang skip: {ex}")
    return train


def fit_filters(train, kind, max_lag, log):
    if kind == "dss":
        spat = fit_dss(train, max_lag, log=log)
    elif kind == "ssd":
        spat = fit_ssd(train, max_lag, log=log)
    else:
        raise ValueError(kind)
    Wt = fit_comp_trf(train, spat.W, max_lag, log=log)
    spat.Wt = Wt
    return spat


def apply_w_ev(ev, w):
    w = np.asarray(w, np.float64)
    w = w / (np.abs(w).sum() + 1e-12)
    out = {}
    for d in ev:
        out[d] = {}
        for k, e in ev[d].items():
            if "r0f" not in e:
                out[d][k] = e
                continue
            r0f = e["r0f"].astype(np.float64)
            r1f = e["r1f"].astype(np.float64)
            ne = dict(e)
            ne["r0"] = (r0f @ w).astype(np.float32)
            ne["r1"] = (r1f @ w).astype(np.float32)
            out[d][k] = ne
    return out


def path_b_chance_probe(filt, train_trials, log):
    """BCE on frozen DSS-TRF window features; abort if mean loss ≥ CHANCE_LOSS."""
    import torch
    import torch.nn.functional as F

    W = int(CFG["evidence"]["win_s"] * FS)
    feats, labels = [], []
    rng = np.random.default_rng(CFG["seed"])
    for tr in train_trials:
        A, B0, B1, _ = lagbank_scores(filt, tr["eeg"], tr["env"][:, 0], tr["env"][:, 1])
        T = len(A)
        att = tr["att"][:T].astype(int)
        starts = np.arange(0, T - W + 1, W)
        if len(starts) == 0:
            continue
        for s0 in starts:
            sl = slice(s0, s0 + W)
            a = A[sl]
            e0 = B0[sl, 0]
            e1 = B1[sl, 0]

            def _r(u, v):
                u, v = u - u.mean(), v - v.mean()
                return float((u @ v) / (np.linalg.norm(u) * np.linalg.norm(v) + 1e-12))

            am = a.mean(1)
            diff = _r(am, e0) - _r(am, e1)
            cols = np.linspace(0, a.shape[1] - 1, num=min(8, a.shape[1]), dtype=int)
            extra = [_r(a[:, j], e0) - _r(a[:, j], e1) for j in cols]
            x = np.array([diff] + extra, np.float32)
            y = 1.0 if (att[s0:s0 + W] == 0).mean() > 0.5 else 0.0
            feats.append(x)
            labels.append(y)
        if len(feats) > 4000:
            break
    X = np.stack(feats)
    y = np.asarray(labels, np.float32)
    mu, sd = X.mean(0), X.std(0) + 1e-6
    X = (X - mu) / sd
    Xt = torch.from_numpy(X)
    yt = torch.from_numpy(y)
    head = torch.nn.Linear(X.shape[1], 1)
    opt = torch.optim.Adam(head.parameters(), lr=1e-2)
    losses = []
    for _ in range(CHANCE_STEPS):
        idx = rng.choice(len(X), size=min(256, len(X)), replace=False)
        loss = F.binary_cross_entropy_with_logits(head(Xt[idx]).squeeze(-1), yt[idx])
        opt.zero_grad()
        loss.backward()
        opt.step()
        losses.append(float(loss.detach()))
    mean_loss = float(np.mean(losses))
    log(f"  Path B chance-probe mean_loss@{CHANCE_STEPS}={mean_loss:.4f} "
        f"(abort if >={CHANCE_LOSS})")
    return mean_loss, mean_loss < CHANCE_LOSS


def run_fold(held, kind, max_lag, exp, log, use_percomp=True):
    t0 = time.time()
    train = gather_train(held, PREP, log)
    log(f"  fold held-out={held} kind={kind}: {len(train)} train trials, "
        f"L={max_lag} ({max_lag/FS*1000:.0f} ms)")
    filt = fit_filters(train, kind, max_lag, log)
    os.makedirs(CKPT_DIR, exist_ok=True)
    ckpt = os.path.join(CKPT_DIR, f"held-{held}_{kind}.npz")
    np.savez(ckpt, kind=filt.kind, max_lag=filt.max_lag, W=filt.W,
             Wt=filt.Wt if filt.Wt is not None else np.array([]),
             ridge=filt.ridge, n_comp=filt.n_comp,
             evals=filt.evals if filt.evals is not None else np.array([]))
    del train
    rows = rows_of(held)
    tr0 = load_trial(PREP, rows.iloc[0]["key"])
    sp = swap_probe(filt, tr0)
    log(f"  swap-probe {sp}")
    ev = {}
    la = max_lag / FS
    for _, r in rows.iterrows():
        tr = load_trial(PREP, r["key"])
        if use_percomp:
            e, la = evidence_trial_percomp(filt, tr)
        else:
            e, la = evidence_trial(filt, tr)
        ev[r["key"]] = e
    acc1, snr = signed_acc1(ev)
    info = dict(decoder=f"dss_trf_{kind}", prep=PREP, held_out=held,
                fit_s=time.time() - t0, total_s=time.time() - t0,
                lookahead_s=la, channels=58, max_lag=max_lag, kind=kind,
                swap=sp, acc1_pooled=acc1, snr1_pooled=snr, n_comp=filt.n_comp)
    save_pickle(dict(evidence={"causal": ev}, info=info), fold_file(exp, held))
    log(f"  fold {held} done in {info['total_s']:.0f}s acc1={acc1:.3f} snr={snr:.3f}")
    return filt, acc1, snr, sp


def avgc_probe(kind, max_lag, log):
    log(f"=== AVGC Path A probe kind={kind} max_lag_ms={max_lag/FS*1000:.0f} ===")
    filt, acc1, snr, sp = run_fold("AVGC", kind, max_lag, EXP_NAME + f"__probe_{kind}", log)
    ok = acc1 >= ACC1_GATE and sp["swap_delta"] >= 0.01
    # soft: swap-clear and acc1>=0.52
    soft = (not ok) and acc1 >= 0.52 and sp["swap_delta"] >= 0.01
    log(f"  PROBE RESULT acc1={acc1:.4f} snr={snr:.3f} swapΔ={sp['swap_delta']:.4f} "
        f"gate>={ACC1_GATE} → {'PASS' if ok else ('SOFT' if soft else 'FAIL')}")
    return ok or soft, acc1, snr, sp, filt, ok


def score_lodo(exp, log, w=None, fuse_name="sum"):
    la = 0.75
    with open(fold_file(exp, "AVGC"), "rb") as fh:
        la = float(pickle.load(fh)["info"].get("lookahead_s", 0.75))
    ev, _ = S.load_folds(exp, "causal")
    if w is not None:
        ev = apply_w_ev(ev, w)
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                            for d in DS_KEYS if d != ds for e in ev[d].values()])
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, la, emis=(mu, sd * SCALE), p_stay=P_STAY)
        per[ds] = r
        log(f"  PRESET LODO[{fuse_name}] {ds}  H {r['H']:.4f}  Acc {r['Acc']:.2f}")
    mH = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mA = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log(f"  PRESET LODO[{fuse_name}] MEAN  H {mH:.4f}  Acc {mA:.2f}")
    return per, mH, mA, ev, la


def score_hp30(ev, la, log):
    hp_ev = {}
    for ds in DS_KEYS:
        hp_ev[ds] = {}
        for key, e in ev[ds].items():
            signed = (e["r0"] - e["r1"]).astype(np.float64)
            hp = V4.highpass(signed[:, None], tau=30.0, hop=0.25)[:, 0]
            hp_ev[ds][key] = dict(e)
            hp_ev[ds][key]["r0"] = (hp / 2).astype(np.float32)
            hp_ev[ds][key]["r1"] = (-hp / 2).astype(np.float32)
    hp_per = {}
    for ds in DS_KEYS:
        A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                            for d in DS_KEYS if d != ds for e in hp_ev[d].values()])
        mu, sd = float(A.mean()), float(A.std())
        hp_per[ds] = S.score_dataset(ds, hp_ev, la, emis=(mu, sd * SCALE), p_stay=P_STAY)
    hp_mH = float(np.mean([hp_per[d]["H"] for d in DS_KEYS]))
    hp_mA = float(np.mean([hp_per[d]["Acc"] for d in DS_KEYS]))
    log(f"  HP30 LODO MEAN  H {hp_mH:.4f}  Acc {hp_mA:.2f}")
    return hp_per, hp_mH, hp_mA, hp_ev


def select_fuse(exp, log):
    """sum (equal mean over comps) vs train-only LDA; pick by mean LODO Acc then H."""
    ev_raw, _ = S.load_folds(exp, "causal")
    # check if multi-comp
    sample = next(iter(next(iter(ev_raw.values())).values()))
    has_f = "r0f" in sample and sample["r0f"].ndim == 2 and sample["r0f"].shape[1] > 1
    cands = []
    # sum
    per, mH, mA, ev_sum, la = score_lodo(exp, log, w=None, fuse_name="sum")
    cands.append(("sum", None, per, mH, mA, ev_sum, la))
    if has_f and FUSE in ("lda", "both"):
        # LODO-mean LDA: average fold-wise lda weights from other-4
        ws = []
        for held in DS_KEYS:
            try:
                w = S.lda_weights(ev_raw, held)
                ws.append(np.asarray(w, np.float64))
            except Exception as ex:
                log(f"  lda_weights({held}) skip: {ex}")
        if ws:
            w_mean = np.mean(np.stack(ws), axis=0)
            per_l, mH_l, mA_l, ev_l, la = score_lodo(exp, log, w=w_mean, fuse_name="lda")
            cands.append(("lda", w_mean, per_l, mH_l, mA_l, ev_l, la))
    # select Acc then H
    cands.sort(key=lambda c: (c[4], c[3]), reverse=True)
    best = cands[0]
    log(f"  SELECT fuse={best[0]} Acc={best[4]:.2f} H={best[3]:.4f}")
    return best, cands


def score_external(filt, ev_train, la, log, w=None, kind="dss"):
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                        for ds in DS_KEYS for e in ev_train[ds].values()])
    mu, sd = float(A.mean()), float(A.std())
    log(f"  external emissions mu {mu:.4f} sd {sd:.4f}")
    hp_parts = []
    for ds in DS_KEYS:
        for e in ev_train[ds].values():
            signed = (e["r0"] - e["r1"]).astype(np.float64)
            hp = V4.highpass(signed[:, None], tau=30.0, hop=0.25)[:, 0]
            hp_parts.append(hp * np.where(e["att"] == 0, 1.0, -1.0))
    mu_hp = float(np.concatenate(hp_parts).mean())
    sd_hp = float(np.concatenate(hp_parts).std())

    E.CONF.update(prep=PREP, targets=["env"])
    os.makedirs(EXT_DIR, exist_ok=True)
    ext = {}
    for ds in ("AASD", "Carta2026"):
        path = os.path.join(EXT_DIR, f"ext_dss_trf_{ds}{TAG}_{kind}_evidence.pkl")
        if os.path.exists(path):
            with open(path, "rb") as fh:
                EV = pickle.load(fh)
            log(f"  loaded {path}")
        else:
            EV = {}
            trials = E.ext_trials(ds)
            log(f"  external {ds}: {len(trials)} trials")
            for i, tr in enumerate(trials):
                e, _ = evidence_trial_percomp(filt, tr)
                e.update(att128=tr["att128"], n128=tr["n128"], subject=tr["subject"])
                EV[tr["key"]] = e
                if (i + 1) % 40 == 0 or i + 1 == len(trials):
                    log(f"    {ds} {i + 1}/{len(trials)}")
            with open(path, "wb") as fh:
                pickle.dump(EV, fh)
        if w is not None:
            EV_use = {}
            for k, e in EV.items():
                r0f = e["r0f"].astype(np.float64)
                r1f = e["r1f"].astype(np.float64)
                ww = np.asarray(w, np.float64)
                ww = ww / (np.abs(ww).sum() + 1e-12)
                ne = dict(e)
                ne["r0"] = (r0f @ ww).astype(np.float32)
                ne["r1"] = (r1f @ ww).astype(np.float32)
                EV_use[k] = ne
        else:
            EV_use = EV
        r = E.score(EV_use, mu, sd * SCALE, P_STAY, 0.75)
        EV_hp = {}
        for key, e in EV_use.items():
            signed = (e["r0"] - e["r1"]).astype(np.float64)
            hp = V4.highpass(signed[:, None], tau=30.0, hop=0.25)[:, 0]
            EV_hp[key] = dict(e)
            EV_hp[key]["r0"] = (hp / 2).astype(np.float32)
            EV_hp[key]["r1"] = (-hp / 2).astype(np.float32)
        r_hp = E.score(EV_hp, mu_hp, sd_hp * SCALE, P_STAY, 0.75)
        plain = E.plain(EV_use)
        ext[ds] = dict(raw=r, hp30=r_hp, plain=plain)
        log(f"  PRESET EXT {ds}  H {r['H']:.4f} Acc {r['Acc']:.2f}  plain_acc1 {plain['acc1']:.3f}")
        log(f"  PRESET EXT_hp30 {ds}  H {r_hp['H']:.4f} Acc {r_hp['Acc']:.2f}")
    return ext, mu, sd


def write_results(kind, fuse, per, ext, mH, mA, hp_per, hp_mH, hp_mA, probe, cands):
    lines = [
        "# results_dss_trf",
        "",
        f"**Method: dss-trf-clf / Path A ({kind}, fuse=`{fuse}`)** — train-only "
        f"{'envelope-informed DSS (Σ_τ R_xy R_xyᵀ vs C_xx)' if kind=='dss' else 'SSD 1–8 Hz vs flanks'} "
        f"spatial filters → frozen K-dim ridge TRF recon lag-bank → HMM.",
        "",
        "Distinct from full-channel TRF Path A / band-trf / dual-band-trf, decoder CCA/FBCCA package, "
        "and spatial-CCA lag-bank (~chance).",
        f"Prep `eaP_n5_t60`, max_lag={MAX_LAG_MS} ms @ {FS} Hz, n_comp={N_COMP}, "
        f"ridge_rel={RIDGE_REL}. HMM p_stay {P_STAY} scale {SCALE}. Also causal hp30.",
        "",
        f"Code: `dss_trf_clf.py`. Checkpoints: `models/dss_trf/`. JSON: `{OUT_JSON}`.",
        "",
        "## Path / probe / fuse",
        "",
        f"- Chosen: **Path A / {kind}** fuse=`{fuse}`",
        f"- AVGC probe acc1={probe.get('acc1')} snr={probe.get('snr')} "
        f"swapΔ={probe.get('swap_delta')} → "
        f"{'PASS' if probe.get('hard_pass') else 'SOFT-PASS' if probe.get('ok') else 'FAIL'}",
        "",
        "## Fuse grid (LODO Acc then H)",
        "",
        "| Fuse | mean H | mean Acc % |",
        "|---|---:|---:|",
    ]
    for name, _w, _per, h, a, _ev, _la in cands:
        mark = " ←" if name == fuse else ""
        lines.append(f"| {name}{mark} | {h:.4f} | {a:.2f} |")
    lines += [
        "",
        "## LODO (preset, selected fuse)",
        "",
        "| Held-out | H | Acc % | hp30 H/Acc |",
        "|---|---:|---:|---|",
    ]
    for ds in DS_KEYS:
        r, h = per[ds], hp_per[ds]
        lines.append(f"| {ds} | {r['H']:.4f} | {r['Acc']:.2f} | {h['H']:.4f}/{h['Acc']:.2f} |")
    lines += [
        f"| **mean** | **{mH:.4f}** | **{mA:.2f}** | **{hp_mH:.4f}/{hp_mA:.2f}** |",
        "",
        "## External (preset)",
        "",
        "| Set | H | Acc % | hp30 H/Acc | vs SW2↔swmix 53.53 / hp30 53.12 |",
        "|---|---:|---:|---|---|",
        f"| AASD | {ext['AASD']['raw']['H']:.4f} | {ext['AASD']['raw']['Acc']:.2f} | "
        f"{ext['AASD']['hp30']['H']:.4f}/{ext['AASD']['hp30']['Acc']:.2f} | "
        f"Δ {ext['AASD']['raw']['Acc']-SWMIX_AASD:+.2f} / "
        f"{ext['AASD']['raw']['Acc']-LINEAR_HP30_AASD:+.2f} |",
        f"| Carta2026 | {ext['Carta2026']['raw']['H']:.4f} | {ext['Carta2026']['raw']['Acc']:.2f} | "
        f"{ext['Carta2026']['hp30']['H']:.4f}/{ext['Carta2026']['hp30']['Acc']:.2f} | — |",
        "",
        "## Targets",
        "",
        "| Target | Result |",
        "|---|---|",
        f"| Probe AVGC acc1≥0.53 | "
        f"{'PASS' if probe.get('hard_pass') else 'SOFT/FAIL'} ({probe.get('acc1')}) |",
        f"| LODO mean H≥0.85 Acc≥78 | "
        f"{'met' if mH >= 0.85 and mA >= 78 else 'miss'} ({mH:.4f}/{mA:.2f}) |",
        f"| AASD Acc>60 vs 53.53/53.12 | "
        f"{'met' if ext['AASD']['raw']['Acc'] > 60 else 'miss'} "
        f"({ext['AASD']['raw']['Acc']:.2f}; Δlead {ext['AASD']['raw']['Acc']-SWMIX_AASD:+.2f}; "
        f"Δhp {ext['AASD']['raw']['Acc']-LINEAR_HP30_AASD:+.2f}) |",
        "",
        "## NEXT IDEA",
        "",
        "DSS/SSD Path A Acc stayed in the TRF-family mid-50s band if flat. Prefer a "
        "qualitatively new Acc front-end (bipolar montage if probe clears) — do not "
        "re-grid TRF Path A / band / dual-band / identical DSS lag-bank.",
        "",
    ]
    open(RESULTS_MD, "w").write("\n".join(lines) + "\n")
    return RESULTS_MD


def write_abandon(reason, probe=None):
    lines = [
        "# results_dss_trf",
        "",
        "**Method: dss-trf-clf**. Path **abandoned**.",
        "",
        f"Reason: {reason}",
        "",
    ]
    if probe:
        lines.append(f"Probe: {json.dumps(probe, default=float)}")
        lines.append("")
    lines += [
        "NEXT IDEA: bipolar-vlaai Acc front-end if still open; do not reopen TRF Path A "
        "variants / decoder CCA package / xcorr CNN.",
        "",
    ]
    open(RESULTS_MD, "w").write("\n".join(lines) + "\n")


def update_claims(status, note):
    text = open(CLAIMS_PATH).read()
    # match the dss-trf-clf row status
    pat = re.compile(
        r"(\| 2026-10-29 16:51 \+07 \| dss-trf-clf \|[^|]+\| )(claimed|running|done|abandoned)( \| )([^|]*)(\|)"
    )
    m = pat.search(text)
    if m:
        new = m.group(1) + status + m.group(3) + note + m.group(5)
        text = text[: m.start()] + new + text[m.end():]
        open(CLAIMS_PATH, "w").write(text)
        return
    open(CLAIMS_PATH, "a").write(
        f"\n| 2026-10-29 +07 | dss-trf-clf | DSS/SSD Path A | {status} | {note} |\n")


def append_already(note_block):
    text = open(ALREADY_PATH).read()
    if "dss-trf-clf" in text and "dss_trf_clf.py" in text:
        return
    open(ALREADY_PATH, "a").write("\n" + note_block.rstrip() + "\n")


def update_status(done_line, aasd_acc=None):
    text = open(STATUS_PATH).read()
    # remove dss from in flight
    text = re.sub(r"\n2\. \*\*dss-trf-clf\*\*[^\n]*", "", text)
    text = re.sub(r"\n1\. \*\*bipolar-vlaai\*\*", "\n1. **bipolar-vlaai**", text)
    # add to recently closed
    if "## Recently closed" in text and "dss-trf-clf" not in text.split("## Recently closed")[1][:800]:
        text = text.replace(
            "## Recently closed\n",
            f"## Recently closed\n\n- **dss-trf-clf** — {done_line}\n",
            1,
        )
    # bump updated stamp lightly
    text = re.sub(
        r"Updated: [^\n]+",
        "Updated: 2026-10-29 (dss-trf-clf closed).",
        text,
        count=1,
    )
    open(STATUS_PATH, "w").write(text)


def main():
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(OUT_JSON), exist_ok=True)
    log = log_to(LOG_PATH)
    max_lag = int(round(MAX_LAG_MS / 1000.0 * FS))
    log(f"dss-trf-clf start PATH_MODE={PATH_MODE} max_lag={max_lag} ({MAX_LAG_MS} ms) "
        f"n_comp={N_COMP} ridge={RIDGE_REL} probe_only={PROBE_ONLY}")
    update_claims("running", "CPU Path A DSS/SSD probe→score (no GPU).")

    kinds = {"dss": ["dss"], "ssd": ["ssd"], "both": ["dss", "ssd"]}[PATH_MODE]
    probes = {}
    best = None
    for kind in kinds:
        ok, acc1, snr, sp, filt, hard = avgc_probe(kind, max_lag, log)
        probes[kind] = dict(ok=ok, hard_pass=hard, acc1=acc1, snr=snr, **sp)
        if ok and (best is None or acc1 > best[1]):
            best = (kind, acc1, snr, sp, hard)
        if not ok:
            train = gather_train("AVGC", PREP, log)
            mean_loss, b_ok = path_b_chance_probe(filt, train, log)
            probes[kind]["path_b_loss"] = mean_loss
            probes[kind]["path_b_ok"] = b_ok
            del train
            if not b_ok:
                log(f"  Path B also chance for {kind}; skip")

    if best is None:
        write_abandon("AVGC Path A below chance gate and Path B chance-stuck", probes)
        update_claims("abandoned", f"probe fail { {k: round(v.get('acc1',0),4) for k,v in probes.items()} }")
        log("ABANDONED")
        return 3

    kind = best[0]
    if PROBE_ONLY:
        log(f"PROBE_ONLY done best={kind} {best[1]:.4f}")
        update_claims("running", f"probe PASS {kind} acc1={best[1]:.4f}; full score pending")
        json.dump(dict(probe=probes, best=kind), open(OUT_JSON, "w"), indent=1, default=float)
        return 0

    exp = f"{EXP_NAME}_{kind}"
    log(f"=== Full LODO kind={kind} exp={exp} ===")
    for held in DS_KEYS:
        run_fold(held, kind, max_lag, exp, log)

    (fuse, w, per, mH, mA, ev, la), cands = select_fuse(exp, log)
    hp_per, hp_mH, hp_mA, _ = score_hp30(ev, la, log)

    # all5 filters for external
    train = gather_train(None, PREP, log)
    filt = fit_filters(train, kind, max_lag, log)
    np.savez(os.path.join(CKPT_DIR, f"all5_{kind}.npz"),
             kind=filt.kind, max_lag=filt.max_lag, W=filt.W,
             Wt=filt.Wt if filt.Wt is not None else np.array([]),
             ridge=filt.ridge, n_comp=filt.n_comp,
             evals=filt.evals if filt.evals is not None else np.array([]))
    del train

    ext, mu, sd = score_external(filt, ev, la, log, w=w, kind=kind)
    probe = probes[kind]
    write_results(kind, fuse, per, ext, mH, mA, hp_per, hp_mH, hp_mA, probe, cands)
    out = dict(
        method="dss-trf-clf", path="A", kind=kind, fuse=fuse, prep=PREP,
        max_lag_ms=MAX_LAG_MS, n_comp=N_COMP, ridge_rel=RIDGE_REL,
        p_stay=P_STAY, scale=SCALE, lookahead_s=la, probe=probes,
        lodo=per, lodo_mean=dict(H=mH, Acc=mA),
        lodo_hp30={d: hp_per[d] for d in DS_KEYS},
        lodo_hp30_mean=dict(H=hp_mH, Acc=hp_mA),
        external=ext, emis=dict(mu=mu, sd=sd),
        fuse_w=(w.tolist() if w is not None else None),
        baselines=dict(swmix_aasd=SWMIX_AASD, linear_hp30_aasd=LINEAR_HP30_AASD,
                       trf_aasd=TRF_AASD, dual_aasd=DUAL_AASD),
        exp=exp,
    )
    json.dump(out, open(OUT_JSON, "w"), indent=1, default=float)
    aasd = ext["AASD"]["raw"]["Acc"]
    carta = ext["Carta2026"]["raw"]["Acc"]
    note = (f"Path A {kind}/{fuse}: probe acc1={probe['acc1']:.4f}; "
            f"LODO {mH:.4f}/{mA:.2f}; AASD {aasd:.2f}; Carta {carta:.2f}. "
            f"`results_dss_trf.md`.")
    update_claims("done", note)
    append_already(
        f"- **dss-trf-clf Path A** (`dss_trf_clf.py`; train-only "
        f"{'envelope-informed DSS' if kind=='dss' else 'SSD 1–8 vs flanks'} "
        f"spatial filters → frozen multi-comp lag/corr scores; fuse=`{fuse}`; "
        f"≠ TRF Path A/band/dual-band, ≠ decoder CCA package): AVGC probe "
        f"acc1=**{probe['acc1']:.4f}** swapΔ=**{probe['swap_delta']:.4f}**. "
        f"LODO mean **{mH:.4f}/{mA:.2f}**. AASD **{ext['AASD']['raw']['H']:.4f}/{aasd:.2f}** "
        f"(Δ{aasd-SWMIX_AASD:+.2f} vs swmix 53.53; Δ{aasd-LINEAR_HP30_AASD:+.2f} vs hp30 53.12); "
        f"Carta **{ext['Carta2026']['raw']['H']:.4f}/{carta:.2f}**. "
        f"Do not rerun identical DSS/SSD lag-bank Path A. "
        f"`multi_agent_opt/results_dss_trf.md`, `{OUT_JSON}`."
    )
    update_status(
        f"{kind}/{fuse}; AASD **{aasd:.2f}** (Δ{aasd-SWMIX_AASD:+.2f} vs 53.53). "
        f"Do not re-grid identical DSS/SSD Path A.",
        aasd_acc=aasd,
    )
    log(f"DONE {note}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
