"""Cross-dataset prototype-memory emission shift on Acc tip streams (CPU).

DISTINCT from:
  - domain-emis-acc-tip: tip↔hp30 emission-LL gate → collapsed to const_tip
  - subj-adapt-emis-tip: past-only EWMA of μ±,σ within trial/subject

Method:
  1. Build a memory of prototypes from LODO-train (never AASD/Carta labels):
     per-(dataset, subject) clusters of unlabeled logit moments
     (mean|z|, std, rms, switch-rate proxy, abs-skew) + labeled signed
     emission (μ,σ) for that subject.
  2. Optionally k-means compress subject prototypes → K cluster centers
     (emissions = mean of members).
  3. At inference: match test trial/session unlabeled moments to nearest
     prototype(s) via hard-NN / soft-kNN / Sinkhorn; set HMM emissions to
     soft-weighted prototype (μ,σ). Evidence stream frozen.
  4. Log argmax prototype ids → diversity (must be non-constant).
  5. Select by mean LODO Acc then H @0.994/0.8; score AASD + Carta.

Streams: tip=hard_abs_swmix|t1; SW2 fuse a=0.5. Also report linear_hp30 Carta.

usage:
  cd /root/disk_3tb/eeg-thao && \\
  /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u proto_mem_emis.py
"""
from __future__ import annotations

import json
import os
import re
import time
from collections import Counter

import numpy as np

import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
import sw2_hp30_blend as SW2
import sw2_swmix_blend as SM
from common import DS_KEYS, FS as FS128, SEED
from e1_switch import expand, synth_trial
from fuse_evidence import LA, OFF
from scorer import aggregate, fast_score_subject

try:
    import numba
except ImportError:
    numba = None

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "proto_mem_emis.log")
OUT = os.path.join(EXP, "results", "research", "proto_mem_emis.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_proto_mem_emis.md")
CKPT = os.path.join(EXP, "models", "proto_mem_emis")

P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = float(LA)
A_SW2 = SW2.A_SW2
HOP = 0.25
EPS = 1e-6
MU_CLIP = (0.25, 4.0)
SD_CLIP = (0.25, 4.0)

TIP_KIND, TIP_THR, TIP_SHARP = "hard_abs_swmix", 1.0, 1.0

LINEAR_HP30_EXT = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
SW2_ALONE_EXT = dict(AASD=dict(H=0.6439, Acc=53.46), Carta2026=dict(H=0.6470, Acc=59.60))
SW2_SWMIX_LEADER = dict(AASD=dict(H=0.6424, Acc=53.53), Carta2026=dict(H=0.6460, Acc=59.71))

# Feature dim for matching (unlabeled)
FEAT_NAMES = ("mean_abs", "std", "rms", "switch_proxy", "abs_skew")

# Variants: (name, stream, level, match, k, temp, persist_subj)
# level: subj | ds | km8 | km16
VARIANT_SPECS = []
for stream in ("tip", "sw2"):
    VARIANT_SPECS.append((f"{stream}_raw", stream, "subj", "none", 0, 1.0, True))
    # hard NN on subject prototypes
    VARIANT_SPECS.append((f"{stream}_hard_subj", stream, "subj", "hard", 1, 1.0, False))
    VARIANT_SPECS.append((f"{stream}_hard_subj_persist", stream, "subj", "hard", 1, 1.0, True))
    # soft kNN
    for k, temp in ((3, 1.0), (5, 1.0), (5, 4.0), (8, 1.0)):
        VARIANT_SPECS.append(
            (f"{stream}_soft_k{k}_T{int(temp)}", stream, "subj", "soft", k, temp, False)
        )
    VARIANT_SPECS.append((f"{stream}_soft_k5_T1_subj", stream, "subj", "soft", 5, 1.0, True))
    # dataset-level prototypes (5 train DS means — strong diversity across DS)
    VARIANT_SPECS.append((f"{stream}_soft_ds_k3_T1", stream, "ds", "soft", 3, 1.0, False))
    VARIANT_SPECS.append((f"{stream}_hard_ds", stream, "ds", "hard", 1, 1.0, False))
    # k-means compressed memory
    VARIANT_SPECS.append((f"{stream}_soft_km8_k3_T1", stream, "km8", "soft", 3, 1.0, False))
    VARIANT_SPECS.append((f"{stream}_soft_km16_k5_T1", stream, "km16", "soft", 5, 1.0, False))
    # Sinkhorn over subject prototypes (trial→proto assignment)
    VARIANT_SPECS.append((f"{stream}_sink_k8_T1", stream, "subj", "sinkhorn", 8, 1.0, False))


def _njit(fn):
    return numba.njit(cache=False)(fn) if numba is not None else fn


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def subject_of(key, meta_entry=None):
    if isinstance(meta_entry, dict) and "subject" in meta_entry:
        return int(meta_entry["subject"])
    m = re.search(r"subject[-_]?(\d+)", str(key), re.I)
    if m:
        return int(m.group(1))
    m = re.search(r"_S(\d+)_", str(key), re.I)
    if m:
        return int(m.group(1))
    return hash(str(key)) % 10_000_007


def tip_stream(sw2, swmix_aligned):
    return SM.apply_gate_all(sw2, swmix_aligned, TIP_KIND, TIP_THR, TIP_SHARP)


def build_raw_held(Z, swmix_full):
    sw2_held, tip_held = {}, {}
    for held in DS_KEYS:
        sw2 = SW2.fuse_sw2_held(Z, held, a=A_SW2)
        sm = SM.align_swmix_to_sw2(swmix_full, sw2)
        sw2_held[held] = sw2
        tip_held[held] = tip_stream(sw2, sm)
    return sw2_held, tip_held


def train_signed_emis(streams, meta, datasets):
    parts = []
    for ds in datasets:
        if ds not in streams:
            continue
        for k, d in streams[ds].items():
            if ds not in meta or k not in meta[ds]:
                continue
            att = meta[ds][k][0]
            n = min(len(d), len(att))
            parts.append(
                np.asarray(d[:n], np.float64)
                * np.where(np.asarray(att)[:n] == 0, 1.0, -1.0)
            )
    if not parts:
        return 1.0, 1.0
    A = np.concatenate(parts)
    return float(A.mean()), float(max(A.std(), EPS))


def unlabeled_feats(z):
    """5-D unlabeled moment vector from a decision stream (no labels)."""
    z = np.asarray(z, np.float64).reshape(-1)
    if z.size == 0:
        return np.zeros(len(FEAT_NAMES), np.float64)
    az = np.abs(z)
    mean_abs = float(az.mean())
    std = float(z.std() + EPS)
    rms = float(np.sqrt(np.mean(z * z) + EPS))
    # switch-rate proxy: zero-crossings of causal EWMA(|z|)-centered z
    if z.size >= 3:
        alpha = 1.0 - np.exp(-HOP / 2.0)
        ema = np.empty_like(z)
        ema[0] = z[0]
        for t in range(1, len(z)):
            ema[t] = (1.0 - alpha) * ema[t - 1] + alpha * z[t]
        sgn = np.sign(ema)
        sgn[sgn == 0] = 1.0
        switch_proxy = float(np.mean(sgn[1:] != sgn[:-1]))
    else:
        switch_proxy = 0.0
    # abs-skew of |z|
    c = az - mean_abs
    m3 = float(np.mean(c ** 3))
    abs_skew = m3 / (std ** 3 + EPS)
    return np.asarray([mean_abs, std, rms, switch_proxy, abs_skew], np.float64)


def signed_emis_of(z, att):
    z = np.asarray(z, np.float64).reshape(-1)
    att = np.asarray(att).reshape(-1)
    n = min(len(z), len(att))
    if n == 0:
        return 1.0, 1.0
    s = z[:n] * np.where(att[:n] == 0, 1.0, -1.0)
    return float(s.mean()), float(max(s.std(), EPS))


def build_subject_prototypes(streams, meta, datasets):
    """One prototype per (ds, subject) from train datasets only."""
    buckets = {}  # (ds, sid) -> list of (z, att)
    for ds in datasets:
        if ds not in streams:
            continue
        for k, d in streams[ds].items():
            if ds not in meta or k not in meta[ds]:
                continue
            att = meta[ds][k][0]
            sid = subject_of(k)
            buckets.setdefault((ds, sid), []).append((np.asarray(d, np.float64), np.asarray(att)))
    feats, mus, sds, ids = [], [], [], []
    for (ds, sid), trials in sorted(buckets.items(), key=lambda x: (x[0][0], x[0][1])):
        z_all = np.concatenate([t[0] for t in trials])
        att_all = np.concatenate([t[1][: len(t[0])] for t in trials])
        f = unlabeled_feats(z_all)
        mu, sd = signed_emis_of(z_all, att_all)
        feats.append(f)
        mus.append(mu)
        sds.append(sd)
        ids.append(f"{ds}:S{sid}")
    if not feats:
        return (
            np.zeros((1, len(FEAT_NAMES))),
            np.asarray([1.0]),
            np.asarray([1.0]),
            ["empty"],
        )
    return (
        np.stack(feats, 0),
        np.asarray(mus, np.float64),
        np.asarray(sds, np.float64),
        ids,
    )


def build_dataset_prototypes(streams, meta, datasets):
    feats, mus, sds, ids = [], [], [], []
    for ds in datasets:
        if ds not in streams:
            continue
        zs, atts = [], []
        for k, d in streams[ds].items():
            if k not in meta[ds]:
                continue
            zs.append(np.asarray(d, np.float64))
            atts.append(np.asarray(meta[ds][k][0])[: len(d)])
        if not zs:
            continue
        z_all = np.concatenate(zs)
        att_all = np.concatenate(atts)
        feats.append(unlabeled_feats(z_all))
        mu, sd = signed_emis_of(z_all, att_all)
        mus.append(mu)
        sds.append(sd)
        ids.append(f"DS:{ds}")
    if not feats:
        return (
            np.zeros((1, len(FEAT_NAMES))),
            np.asarray([1.0]),
            np.asarray([1.0]),
            ["empty"],
        )
    return (
        np.stack(feats, 0),
        np.asarray(mus, np.float64),
        np.asarray(sds, np.float64),
        ids,
    )


def kmeans_compress(feats, mus, sds, ids, K, seed=0):
    """Simple k-means on feats; emissions = mean of assigned members."""
    X = np.asarray(feats, np.float64)
    n = X.shape[0]
    K = int(min(K, n))
    if K <= 0:
        K = 1
    rng = np.random.default_rng(seed)
    # init: random subset
    centers = X[rng.choice(n, size=K, replace=False)].copy()
    for _ in range(25):
        d2 = ((X[:, None, :] - centers[None, :, :]) ** 2).sum(-1)
        lab = d2.argmin(1)
        new_c = centers.copy()
        for k in range(K):
            m = lab == k
            if m.any():
                new_c[k] = X[m].mean(0)
        if np.allclose(new_c, centers):
            centers = new_c
            break
        centers = new_c
    d2 = ((X[:, None, :] - centers[None, :, :]) ** 2).sum(-1)
    lab = d2.argmin(1)
    c_mu, c_sd, c_ids = [], [], []
    for k in range(K):
        m = lab == k
        if not m.any():
            # orphan: use nearest original
            j = int(d2[:, k].argmin())
            c_mu.append(float(mus[j]))
            c_sd.append(float(sds[j]))
            c_ids.append(f"km{K}:{k}~{ids[j]}")
            continue
        c_mu.append(float(np.mean(mus[m])))
        c_sd.append(float(np.mean(sds[m])))
        c_ids.append(f"km{K}:{k}(n={int(m.sum())})")
    return centers, np.asarray(c_mu), np.asarray(c_sd), c_ids


def normalize_feats(train_feats, query_feats=None):
    mu = train_feats.mean(0)
    sd = train_feats.std(0) + EPS
    tn = (train_feats - mu) / sd
    if query_feats is None:
        return tn, mu, sd
    qn = (query_feats - mu) / sd
    return tn, qn, mu, sd


def soft_knn_weights(q, P, k, temp):
    """q: (D,), P: (N,D) → weights (N,) summing to 1; only top-k non-zero."""
    d2 = ((P - q[None, :]) ** 2).sum(1)
    k = int(min(max(k, 1), len(d2)))
    idx = np.argpartition(d2, k - 1)[:k]
    logits = -d2[idx] / max(float(temp), 1e-6)
    logits = logits - logits.max()
    w = np.exp(logits)
    w = w / (w.sum() + EPS)
    out = np.zeros_like(d2)
    out[idx] = w
    return out


def hard_nn_weights(q, P):
    d2 = ((P - q[None, :]) ** 2).sum(1)
    j = int(d2.argmin())
    out = np.zeros_like(d2)
    out[j] = 1.0
    return out


def sinkhorn_weights(Q, P, k, temp, n_iter=20):
    """Batch Sinkhorn: Q (M,D) queries, P (N,D) prototypes → W (M,N).

    Restrict each query to its top-k prototypes (sparse cost) then scale.
    """
    M, N = Q.shape[0], P.shape[0]
    d2 = ((Q[:, None, :] - P[None, :, :]) ** 2).sum(-1)  # (M,N)
    k = int(min(max(k, 1), N))
    # mask non-topk to large cost
    cost = np.full_like(d2, 1e6)
    for i in range(M):
        idx = np.argpartition(d2[i], k - 1)[:k]
        cost[i, idx] = d2[i, idx]
    Kmat = np.exp(-cost / max(float(temp), 1e-6))
    # Sinkhorn scaling toward uniform marginals
    u = np.ones(M) / M
    v = np.ones(N) / N
    a = np.ones(M)
    b = np.ones(N)
    for _ in range(n_iter):
        Ka = Kmat @ b
        a = u / (Ka + EPS)
        Kb = Kmat.T @ a
        b = v / (Kb + EPS)
    W = (a[:, None] * Kmat) * b[None, :]
    # row-normalize for emission blend
    W = W / (W.sum(1, keepdims=True) + EPS)
    return W


def blend_emis(weights, mus, sds, mu_global, sd_global):
    w = np.asarray(weights, np.float64)
    mu = float(np.dot(w, mus))
    sd = float(np.dot(w, sds))
    # clip relative to global train
    mu = float(np.clip(mu, MU_CLIP[0] * mu_global, MU_CLIP[1] * mu_global))
    sd = float(np.clip(sd, SD_CLIP[0] * sd_global, SD_CLIP[1] * sd_global))
    return mu, sd


@_njit
def integ_fixed_nb(d, p_stay, mu, sd):
    n = d.shape[0]
    out = np.empty(n, np.int8)
    lp0 = np.log(0.5)
    lp1 = np.log(0.5)
    ls = np.log(p_stay)
    lsw = np.log(1.0 - p_stay)
    inv = 1.0 / max(sd, 1e-6)
    for t in range(n):
        if t > 0:
            aa = np.logaddexp(lp0 + ls, lp1 + lsw)
            bb = np.logaddexp(lp1 + ls, lp0 + lsw)
            lp0, lp1 = aa, bb
        z0 = (d[t] - mu) * inv
        z1 = (d[t] + mu) * inv
        lp0 = lp0 - 0.5 * z0 * z0
        lp1 = lp1 - 0.5 * z1 * z1
        nrm = np.logaddexp(lp0, lp1)
        lp0 = lp0 - nrm
        lp1 = lp1 - nrm
        out[t] = 1 if np.exp(lp1) > 0.5 else 0
    return out


def integ_fixed(d, p_stay, mu, sd):
    d = np.ascontiguousarray(np.asarray(d, np.float64).reshape(-1))
    if len(d) == 0:
        return np.zeros(0, np.int8)
    return integ_fixed_nb(d, float(p_stay), float(mu), float(sd))


def avg_corpus_stream(streams_by_held):
    acc = {ds: {} for ds in DS_KEYS}
    counts = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        stream = streams_by_held[held]
        for ds in DS_KEYS:
            for k, d in stream[ds].items():
                d = np.asarray(d, np.float64)
                if k not in acc[ds]:
                    acc[ds][k] = np.zeros_like(d, np.float64)
                    counts[ds][k] = 0
                n = min(len(acc[ds][k]), len(d))
                acc[ds][k][:n] += d[:n]
                counts[ds][k] += 1
    out = {}
    for ds in DS_KEYS:
        out[ds] = {k: acc[ds][k] / max(counts[ds][k], 1) for k in acc[ds]}
    return out


def make_memory(streams, meta, train_ds, level, seed=0):
    if level == "ds":
        feats, mus, sds, ids = build_dataset_prototypes(streams, meta, train_ds)
    else:
        feats, mus, sds, ids = build_subject_prototypes(streams, meta, train_ds)
        if level.startswith("km"):
            K = int(level[2:])
            feats, mus, sds, ids = kmeans_compress(feats, mus, sds, ids, K, seed=seed)
    return feats, mus, sds, ids


def match_emis_for_queries(queries, mem_feats, mem_mus, mem_sds, match, k, temp, mu_g, sd_g):
    """queries: (M,D) normalized; mem_feats already normalized.
    Returns list of (mu, sd, argmax_id) length M.
    """
    M = queries.shape[0]
    if match == "none" or mem_feats.shape[0] == 0:
        return [(mu_g, sd_g, -1)] * M
    if match == "sinkhorn":
        W = sinkhorn_weights(queries, mem_feats, k, temp)
        out = []
        for i in range(M):
            mu, sd = blend_emis(W[i], mem_mus, mem_sds, mu_g, sd_g)
            out.append((mu, sd, int(W[i].argmax())))
        return out
    out = []
    for i in range(M):
        if match == "hard":
            w = hard_nn_weights(queries[i], mem_feats)
        else:
            w = soft_knn_weights(queries[i], mem_feats, k, temp)
        mu, sd = blend_emis(w, mem_mus, mem_sds, mu_g, sd_g)
        out.append((mu, sd, int(w.argmax())))
    return out


def score_lodo_held_proto(
    stream_all, meta, held, mu_g, sd_g, mem_feats, mem_mus, mem_sds, mem_ids,
    feat_mu, feat_sd, match, k, temp, persist_subject,
):
    ev = SW2.ev_from_stream(stream_all, meta)
    V4._install_scorer_state(ev, LOOKAHEAD)
    rng = np.random.default_rng(SEED + DS_KEYS.index(held))
    per = {}
    rows = list(S.rows_of(held).iterrows())
    by_sub = {}
    for _, r in rows:
        sid = r["subj_uid"]
        by_sub.setdefault(sid, []).append(r)

    proto_picks = []
    mem_n = (mem_feats - feat_mu) / feat_sd

    for sid in sorted(by_sub.keys(), key=lambda x: str(x)):
        rlist = sorted(by_sub[sid], key=lambda r: str(r["key"]))
        # collect query feats
        qs_raw = []
        trials_data = []
        for r in rlist:
            T = int(r["n"])
            e = ev[held][r["key"]]
            lab = np.load(os.path.join(S.EDA_CACHE, "cnd", f"{r['key']}.npz"))
            att = lab["att_col"][:T].astype(int)
            d = (e["r0"] - e["r1"]).astype(float)
            t_end = e["t_end"].astype(float)
            st, d_syn, _ = synth_trial(r["key"], T, d, t_end, rng)
            if e.get("pre_switched"):
                d_syn = d
            guess = int(rng.integers(0, 2))
            qs_raw.append(unlabeled_feats(d_syn))
            trials_data.append((d_syn, t_end, T, guess, st, att))

        Q = (np.stack(qs_raw, 0) - feat_mu) / feat_sd
        if match == "none":
            pairs = [(mu_g, sd_g, -1)] * len(trials_data)
        elif persist_subject:
            # match once on mean subject query
            q_mean = Q.mean(0, keepdims=True)
            one = match_emis_for_queries(
                q_mean, mem_n, mem_mus, mem_sds, match, k, temp, mu_g, sd_g
            )[0]
            pairs = [one] * len(trials_data)
        else:
            pairs = match_emis_for_queries(
                Q, mem_n, mem_mus, mem_sds, match, k, temp, mu_g, sd_g
            )

        for (d_syn, t_end, T, guess, st, att), (mu_e, sd_e, pid) in zip(trials_data, pairs):
            proto_picks.append(int(pid))
            dec = integ_fixed(d_syn, P_STAY, mu_e, sd_e)
            labs = expand(dec, t_end, LOOKAHEAD, T, guess)
            per.setdefault(sid, []).append((labs, np.where(st, 1 - att, att)))

    res = []
    for _subj, lst in per.items():
        pred = np.concatenate([a for a, _ in lst])
        gt = np.concatenate([b for _, b in lst])
        trial = np.concatenate([np.full(len(a), i) for i, (a, _) in enumerate(lst)])
        res.append(fast_score_subject(pred, gt, trial, fs=FS128))
    a = aggregate(res)
    diversity = _diversity_stats(proto_picks, mem_ids)
    return dict(
        Acc=a["Acc"], SDT=a["SDT"], L=a["L"], H=a["H"],
        nSwitches=a["nSwitches"], nUndetected=a["nUndetected"],
        diversity=diversity,
    )


def score_ext_proto(
    ev, mu_g, sd_g, mem_feats, mem_mus, mem_sds, mem_ids,
    feat_mu, feat_sd, match, k, temp, persist_subject,
):
    rng = np.random.default_rng(SEED)
    by_sub = {}
    for key in sorted(ev):
        e = ev[key]
        sid = e["subject"]
        by_sub.setdefault(sid, []).append(key)

    per = {}
    proto_picks = []
    mem_n = (mem_feats - feat_mu) / feat_sd

    for sid in sorted(by_sub.keys(), key=lambda x: str(x)):
        keys = by_sub[sid]
        qs_raw = []
        trials_data = []
        for key in keys:
            e = ev[key]
            d = (e["r0"] - e["r1"]).astype(float)
            qs_raw.append(unlabeled_feats(d))
            trials_data.append((key, d, e))

        Q = (np.stack(qs_raw, 0) - feat_mu) / feat_sd
        if match == "none":
            pairs = [(mu_g, sd_g, -1)] * len(trials_data)
        elif persist_subject:
            q_mean = Q.mean(0, keepdims=True)
            one = match_emis_for_queries(
                q_mean, mem_n, mem_mus, mem_sds, match, k, temp, mu_g, sd_g
            )[0]
            pairs = [one] * len(trials_data)
        else:
            pairs = match_emis_for_queries(
                Q, mem_n, mem_mus, mem_sds, match, k, temp, mu_g, sd_g
            )

        for (key, d, e), (mu_e, sd_e, pid) in zip(trials_data, pairs):
            proto_picks.append(int(pid))
            dec = integ_fixed(d, P_STAY, mu_e, sd_e)
            labs = expand(
                dec, e["t_end"].astype(float), LOOKAHEAD, e["n128"], int(rng.integers(0, 2))
            )
            per.setdefault(sid, []).append((labs, e["att128"].astype(float)))

    res = []
    for _s, lst in per.items():
        pred = np.concatenate([a for a, _ in lst])
        gt = np.concatenate([b for _, b in lst])
        trial = np.concatenate([np.full(len(a), i) for i, (a, _) in enumerate(lst)])
        res.append(fast_score_subject(pred, gt, trial, fs=FS128))
    a = aggregate(res)
    diversity = _diversity_stats(proto_picks, mem_ids)
    return dict(
        Acc=a["Acc"], SDT=a["SDT"], L=a["L"], H=a["H"],
        nSwitches=a["nSwitches"], nUndetected=a["nUndetected"],
        diversity=diversity,
    )


def _diversity_stats(picks, mem_ids):
    picks = [int(p) for p in picks if int(p) >= 0]
    if not picks:
        return dict(
            n_trials=0, n_unique=0, entropy=0.0, is_const=True,
            top_proto=None, top_frac=1.0, hist={},
        )
    c = Counter(picks)
    total = sum(c.values())
    probs = np.asarray([v / total for v in c.values()], np.float64)
    ent = float(-(probs * np.log(probs + EPS)).sum())
    top_i, top_n = c.most_common(1)[0]
    top_name = mem_ids[top_i] if 0 <= top_i < len(mem_ids) else str(top_i)
    hist = {mem_ids[i] if 0 <= i < len(mem_ids) else str(i): int(n) for i, n in c.items()}
    return dict(
        n_trials=total,
        n_unique=len(c),
        entropy=ent,
        is_const=bool(len(c) <= 1),
        top_proto=top_name,
        top_frac=float(top_n / total),
        hist=hist,
    )


def blend_ext_tip(sw2_ev, sm_aligned):
    return SM.blend_ext_ev(sw2_ev, sm_aligned, TIP_KIND, TIP_THR, TIP_SHARP)


def pick_variant(lodo_means):
    best_name, best_key = None, None
    for name, m in lodo_means.items():
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def _json_default(o):
    if isinstance(o, (np.floating, np.integer)):
        return float(o)
    if isinstance(o, np.ndarray):
        return o.tolist()
    raise TypeError(type(o))


def write_md(results):
    sel = results["selected"]
    prim = results["variants"][sel]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    tip_acc = SW2_SWMIX_LEADER["AASD"]["Acc"]
    hp_acc = LINEAR_HP30_EXT["AASD"]["Acc"]
    lines = []
    lines.append("# results_proto_mem_emis\n")
    lines.append(
        "**Claim:** `proto-mem-emis` — cross-dataset **prototype-memory emission shift** "
        "on Acc tip streams (CPU):\n"
        "1. LODO-train per-subject / per-dataset / k-means clusters of unlabeled "
        "logit moments + labeled signed emission (μ,σ)\n"
        "2. At test: soft-kNN / hard-NN / Sinkhorn match of session/trial moments → "
        "blend prototype HMM emissions (evidence frozen)\n"
        "3. Diversity of selected prototypes logged (must be ≠ const)\n\n"
        "**DISTINCT** from `domain-emis-acc-tip` (tip↔hp30 LL gate → const_tip) and "
        "`subj-adapt-emis-tip` (EWMA μ/σ within trial).\n"
    )
    lines.append(
        f"HMM preset **0.994 / 0.8**. Code: `proto_mem_emis.py`. "
        f"JSON: `results/research/proto_mem_emis.json`.\n"
    )
    lines.append("## Selection (mean LODO Acc then H; never AASD/Carta)\n")
    lines.append(
        f"**SELECTED `{sel}`** — mean LODO H **{prim['lodo_mean']['H']:.4f}** "
        f"Acc **{prim['lodo_mean']['Acc']:.2f}**\n"
    )
    lines.append("| Held-out | H | Acc % |")
    lines.append("|---|---:|---:|")
    for ds in DS_KEYS:
        r = prim["lodo"][ds]
        lines.append(f"| {ds} | {r['H']:.4f} | {r['Acc']:.2f} |")
    lines.append(
        f"| **mean** | **{prim['lodo_mean']['H']:.4f}** | "
        f"**{prim['lodo_mean']['Acc']:.2f}** |\n"
    )

    lines.append("### Variant table (LODO Acc desc; selected first)\n")
    lines.append(
        "| Variant | LODO H / Acc | AASD H / Acc | ΔAcc vs 53.53 | "
        "Carta H / Acc | n_unique proto |"
    )
    lines.append("|---------|--------------|--------------|---------------|---------------|----------------|")
    ranked = results["selection"]["ranked"]
    for name in ranked:
        v = results["variants"][name]
        ext_a = v["external"]["AASD"]
        ext_c = v["external"]["Carta2026"]
        d_tip = ext_a["Acc"] - tip_acc
        mark = " **←**" if name == sel else ""
        div = ext_a.get("diversity", {})
        nu = div.get("n_unique", "?")
        lines.append(
            f"| `{name}`{mark} | {v['lodo_mean']['H']:.4f} / {v['lodo_mean']['Acc']:.2f} | "
            f"{ext_a['H']:.4f} / {ext_a['Acc']:.2f} | {d_tip:+.2f} | "
            f"{ext_c['H']:.4f} / {ext_c['Acc']:.2f} | {nu} |"
        )

    lines.append("\n## Prototype diversity (selected, external)\n")
    for ds in ("AASD", "Carta2026"):
        div = prim["external"][ds].get("diversity", {})
        lines.append(
            f"- **{ds}**: n_unique={div.get('n_unique')}, entropy={div.get('entropy', 0):.3f}, "
            f"is_const={div.get('is_const')}, top=`{div.get('top_proto')}` "
            f"({100*div.get('top_frac', 1):.1f}%)"
        )
    lines.append("")

    lines.append("## Targets\n")
    lines.append("| Target | Result |")
    lines.append("|---|---|")
    d_tip = aasd["Acc"] - tip_acc
    d_hp = aasd["Acc"] - hp_acc
    lines.append(
        f"| AASD Acc vs tip {tip_acc:.2f} / hp30 {hp_acc:.2f} | "
        f"Acc **{aasd['Acc']:.2f}** (Δtip {d_tip:+.2f}; Δhp {d_hp:+.2f}); H {aasd['H']:.4f} |"
    )
    lines.append(f"| AASD Acc>60? | {'YES' if aasd['Acc'] > 60 else 'NO'} |")
    lines.append(
        f"| Carta vs linear_hp30 {LINEAR_HP30_EXT['Carta2026']['Acc']:.2f} | "
        f"Acc **{carta['Acc']:.2f}** (Δ {carta['Acc']-LINEAR_HP30_EXT['Carta2026']['Acc']:+.2f}); "
        f"H {carta['H']:.4f} |"
    )
    lines.append(
        f"| linear_hp30 Carta (portfolio) | **{LINEAR_HP30_EXT['Carta2026']['Acc']:.2f}** |"
    )
    lines.append(
        f"| LODO mean | H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f} |\n"
    )

    lines.append("## Verdict\n")
    lines.append(results.get("verdict", "") + "\n")
    lines.append("## Do not repeat\n")
    lines.append(results.get("do_not_repeat", "") + "\n")
    lines.append("## NEXT IDEA\n")
    lines.append(results.get("next_idea", "") + "\n")
    with open(RESULTS_MD, "w") as fh:
        fh.write("\n".join(lines))


def main():
    os.makedirs(CKPT, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(
        f"proto-mem-emis CPU  p_stay={P_STAY} scale={SCALE} a_sw2={A_SW2} "
        f"tip={TIP_KIND}|t{TIP_THR:g} n_var={len(VARIANT_SPECS)}"
    )
    log_line(
        "honesty: train-only prototypes; unlabeled moment match; emission shift only; "
        "no AASD/Carta labels; ≠ domain-emis tip↔hp30; ≠ subj-adapt EWMA"
    )

    _ = integ_fixed(np.zeros(8), P_STAY, 1.0, 1.0)

    try:
        Z, meta = SM.ensure_sw2_lodo()
        swmix_full, la_sm = SM.ensure_swmix_lodo()
        log_line(f"lookahead LA={LOOKAHEAD} swmix_fold_la={la_sm} OFF={OFF}")
    except FileNotFoundError as e:
        log_line(f"ABORT: {e}")
        with open(OUT, "w") as fh:
            json.dump(dict(status="abandoned", reason=str(e)), fh, indent=2)
        with open(RESULTS_MD, "w") as fh:
            fh.write(f"# results_proto_mem_emis\n\n**Status: abandoned**\n\n{e}\n")
        return

    log_line("building raw SW2 fuse + tip (hard_abs_swmix|t1) per held …")
    sw2_held, tip_held = build_raw_held(Z, swmix_full)
    raw_by_stream = dict(sw2=sw2_held, tip=tip_held)

    mu_s, sd_s = train_signed_emis(tip_held["AVGC"], meta, [d for d in DS_KEYS if d != "AVGC"])
    # sanity with raw path
    train_ds = [d for d in DS_KEYS if d != "AVGC"]
    feats, mus, sds, ids = make_memory(tip_held["AVGC"], meta, train_ds, "subj")
    feat_mu, feat_sd = feats.mean(0), feats.std(0) + EPS
    r_tip = score_lodo_held_proto(
        tip_held["AVGC"], meta, "AVGC", mu_s, sd_s * SCALE,
        feats, mus, sds, ids, feat_mu, feat_sd, "none", 0, 1.0, True,
    )
    log_line(
        f"sanity AVGC tip_raw H {r_tip['H']:.4f} Acc {r_tip['Acc']:.2f} "
        f"(expect ~0.7159 / ~67.94); n_proto={len(ids)}"
    )

    results = dict(
        status="running",
        method="proto-mem-emis",
        p_stay=P_STAY,
        scale=SCALE,
        a_sw2=A_SW2,
        tip=dict(kind=TIP_KIND, thresh=TIP_THR, sharp=TIP_SHARP),
        feat_names=list(FEAT_NAMES),
        honesty=(
            "Cross-dataset prototype memory from LODO-train subjects/datasets/k-means; "
            "unlabeled soft-kNN/Sinkhorn match → HMM emission (μ,σ) shift. Evidence frozen. "
            "≠ domain-emis tip↔hp30, ≠ subj-adapt EWMA, ≠ online-moment logit affine."
        ),
        baselines=dict(
            linear_hp30_ext=LINEAR_HP30_EXT,
            sw2_alone_ext=SW2_ALONE_EXT,
            sw2_swmix_leader=SW2_SWMIX_LEADER,
        ),
        variants={},
    )
    lodo_means = {}

    log_line(f"=== LODO ({len(VARIANT_SPECS)} variants; selection) ===")
    for name, stream, level, match, k, temp, persist in VARIANT_SPECS:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            raw = raw_by_stream[stream][held]
            train_ds = [d for d in DS_KEYS if d != held]
            mu_t, sd_t = train_signed_emis(raw, meta, train_ds)
            sd_dec = sd_t * SCALE
            feats, mus, sds, ids = make_memory(
                raw, meta, train_ds, level, seed=SEED + DS_KEYS.index(held)
            )
            feat_mu = feats.mean(0)
            feat_sd = feats.std(0) + EPS
            off = score_lodo_held_proto(
                raw, meta, held, mu_t, sd_dec,
                feats, mus, sds, ids, feat_mu, feat_sd,
                match, k, temp, persist,
            )
            per[held] = dict(
                H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"), L=off.get("L"),
                train_mu=mu_t, train_sd=sd_t, sd_decode=sd_dec,
                n_proto=len(ids),
                diversity=off["diversity"],
            )
            log_line(
                f"LODO {name:32s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                f"uniq={off['diversity']['n_unique']} const={off['diversity']['is_const']}"
            )
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        # mean diversity uniqueness across held
        mean_u = float(np.mean([per[d]["diversity"]["n_unique"] for d in DS_KEYS]))
        log_line(
            f"LODO_MEAN {name:32s}  H {mean_h:.4f} Acc {mean_a:.2f}  "
            f"mean_uniq={mean_u:.1f}  ({time.time()-t0:.0f}s)"
        )
        lodo_means[name] = dict(H=mean_h, Acc=mean_a, mean_uniq=mean_u)
        results["variants"][name] = dict(
            stream=stream, level=level, match=match, k=k, temp=temp,
            persist_subject=persist,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a, mean_uniq=mean_u),
        )

    selected = pick_variant(lodo_means)
    ranked = sorted(
        lodo_means.keys(),
        key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
        reverse=True,
    )
    log_line(
        f"SELECTED (LODO Acc then H): {selected}  "
        f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}"
    )
    log_line(f"top5: {ranked[:5]}")
    results["selected"] = selected
    results["selection"] = dict(
        criterion="mean LODO Acc then H (never AASD/Carta)",
        ranked=ranked,
        selected=selected,
    )

    try:
        EV_sw2, emis_sw2 = SM.ensure_sw2_external()
        sm_ext = SM.ensure_swmix_external()
    except FileNotFoundError as e:
        log_line(f"ABORT external: {e}")
        results["status"] = "partial"
        results["reason"] = str(e)
        with open(OUT, "w") as fh:
            json.dump(results, fh, indent=2, default=_json_default)
        write_md(results)
        return

    corp_raw = {
        "sw2": avg_corpus_stream(sw2_held),
        "tip": avg_corpus_stream(tip_held),
    }
    mu_corp, sd_corp = {}, {}
    mem_corp = {}
    for stream in ("sw2", "tip"):
        mu_corp[stream], sd_corp[stream] = train_signed_emis(
            corp_raw[stream], meta, list(DS_KEYS)
        )
        log_line(
            f"corpus {stream} signed emis mu {mu_corp[stream]:.6f} sd {sd_corp[stream]:.6f}"
        )
        # prebuild memories at each level for external
        for level in ("subj", "ds", "km8", "km16"):
            feats, mus, sds, ids = make_memory(
                corp_raw[stream], meta, list(DS_KEYS), level, seed=SEED
            )
            mem_corp[(stream, level)] = dict(
                feats=feats, mus=mus, sds=sds, ids=ids,
                feat_mu=feats.mean(0), feat_sd=feats.std(0) + EPS,
            )
            log_line(f"corpus mem {stream}/{level}: n_proto={len(ids)}")

    results["train_emis_corpus"] = {
        s: dict(mu=mu_corp[s], sd=sd_corp[s]) for s in ("sw2", "tip")
    }

    tip_ext_ev = {}
    for ds in ("AASD", "Carta2026"):
        sw2_ev = EV_sw2[ds][A_SW2]
        sm_al = SM.align_ext_swmix_to_sw2(sm_ext[ds], sw2_ev)
        tip_ext_ev[ds] = blend_ext_tip(sw2_ev, sm_al)

    log_line("=== external (all variants; headline = selected) ===")
    for name, stream, level, match, k, temp, persist in VARIANT_SPECS:
        mu_e, sd_e = mu_corp[stream], sd_corp[stream]
        if match == "none" and stream == "sw2":
            mu_e, sd_e = float(emis_sw2[A_SW2][0]), float(emis_sw2[A_SW2][1])
        sd_dec = sd_e * SCALE
        mem = mem_corp[(stream, level if match != "none" else "subj")]
        ext = {}
        for ds in ("AASD", "Carta2026"):
            base_ev = EV_sw2[ds][A_SW2] if stream == "sw2" else tip_ext_ev[ds]
            if match == "none":
                r = E.score(base_ev, mu_e, sd_dec, P_STAY, LOOKAHEAD)
                # diversity N/A for raw — mark as const emission (no proto select)
                r = dict(r)
                r["diversity"] = dict(
                    n_trials=0, n_unique=0, entropy=0.0, is_const=True,
                    top_proto="global_train", top_frac=1.0, hist={},
                )
            else:
                r = score_ext_proto(
                    base_ev, mu_e, sd_dec,
                    mem["feats"], mem["mus"], mem["sds"], mem["ids"],
                    mem["feat_mu"], mem["feat_sd"],
                    match, k, temp, persist,
                )
            pl = E.plain(base_ev)
            ext[ds] = dict(
                H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r["L"],
                acc1=pl["acc1"], nSwitches=r.get("nSwitches"), nUndetected=r.get("nUndetected"),
                diversity=r.get("diversity"),
            )
            d_tip = r["Acc"] - SW2_SWMIX_LEADER[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            div = r.get("diversity", {})
            log_line(
                f"EXT {name:32s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                f"Δtip {d_tip:+.2f}  Δhp30 {d_hp:+.2f}  "
                f"uniq={div.get('n_unique')} const={div.get('is_const')}"
            )
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e, sd_decode=sd_dec)
        results["variants"][name]["n_proto_corpus"] = len(mem["ids"])

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_tip = aasd["Acc"] - SW2_SWMIX_LEADER["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    div_a = aasd.get("diversity", {})
    nonconst = not div_a.get("is_const", True) if prim["match"] != "none" else False

    # Prefer a non-const variant for headline diversity documentation if selected is raw
    # but selection is honest LODO Acc — keep selected; note diversity of best non-const.
    best_nonconst = None
    for name in ranked:
        v = results["variants"][name]
        if v["match"] == "none":
            continue
        da = v["external"]["AASD"].get("diversity", {})
        if not da.get("is_const", True) and da.get("n_unique", 0) >= 2:
            best_nonconst = name
            break

    if aasd["Acc"] > 60:
        verdict = (
            f"Prototype-memory emission shift `{selected}` cleared AASD Acc>60 "
            f"({aasd['Acc']:.2f}). Diversity const={div_a.get('is_const')}."
        )
        do_not = "Freeze this proto-mem recipe; avoid re-grid of tip soft-w / domain-emis."
        next_idea = (
            "Stress-test Carta with same memory; avoid banned tip Acc PP / SSL / Path A."
        )
    elif aasd["Acc"] <= SW2_SWMIX_LEADER["AASD"]["Acc"] + 0.05 and aasd["Acc"] < 55:
        verdict = (
            f"Honest LODO selector picks `{selected}` (LODO Acc "
            f"{prim['lodo_mean']['Acc']:.2f}) but AASD Acc **{aasd['Acc']:.2f}** "
            f"(Δtip {d_tip:+.2f}) stays mid-50s — same Acc tip ceiling. "
            f"Prototype diversity on AASD: n_unique={div_a.get('n_unique')}, "
            f"is_const={div_a.get('is_const')} "
            f"(best non-const under LODO rank: `{best_nonconst}`)."
        )
        do_not = (
            "Do not rerun cross-dataset prototype-memory emission shift / soft-kNN / "
            "Sinkhorn emission blending on frozen SW2/tip Acc streams. "
            "Do not reopen domain-emis tip↔hp30 or subj-adapt EWMA μ/σ."
        )
        next_idea = (
            "CLOSE emis-shift family (domain / subj-adapt / proto-mem). Prefer a "
            "**qualitatively new Acc front-end** unused on this montage — e.g. raw-EEG "
            "Conformer-style with different speech coupling already tried → try "
            "**speech cross-attn decoder** ≠ temp-attn-spk, or **noise-aug FT** "
            "(clf-noise-aug-ft orphaned), or Carta-aware train — not another emis/gate PP."
        )
    else:
        verdict = (
            f"`{selected}` AASD Acc {aasd['Acc']:.2f} (Δtip {d_tip:+.2f}); "
            f"diversity n_unique={div_a.get('n_unique')}."
        )
        do_not = "Do not re-grid this proto-mem soft-kNN temperature family on tip/SW2."
        next_idea = (
            "If Acc still <60, close proto-mem-emis; move off emission-shift Acc PP."
        )

    results["verdict"] = verdict
    results["do_not_repeat"] = do_not
    results["next_idea"] = next_idea
    results["diversity_note"] = dict(
        selected_aasd=div_a,
        best_nonconst_under_lodo=best_nonconst,
        nonconst_demonstrated=bool(best_nonconst is not None) or nonconst,
    )

    results["headline"] = dict(
        selected=selected,
        AASD=aasd,
        Carta2026=carta,
        AASD_Acc_delta_vs_tip=d_tip,
        AASD_Acc_delta_vs_linear_hp30=d_hp,
        AASD_Acc_gt_60=bool(aasd["Acc"] > 60),
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=prim["lodo"]["AVGC"],
        linear_hp30_Carta=LINEAR_HP30_EXT["Carta2026"],
        tip_moved=bool(aasd["Acc"] > SW2_SWMIX_LEADER["AASD"]["Acc"]),
    )
    results["status"] = "done"
    results["elapsed_s"] = time.time() - t_all
    log_line(
        f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
        f"(Δtip {d_tip:+.2f}, Δhp30 {d_hp:+.2f}; >60={'YES' if aasd['Acc'] > 60 else 'NO'})  "
        f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}  "
        f"uniq={div_a.get('n_unique')} const={div_a.get('is_const')}"
    )
    log_line(
        f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}  "
        f"AVGC H {prim['lodo']['AVGC']['H']:.4f} Acc {prim['lodo']['AVGC']['Acc']:.2f}"
    )
    log_line(f"diversity nonconst_demonstrated={results['diversity_note']['nonconst_demonstrated']}")

    # serialize mem sizes only (not full arrays)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=2, default=_json_default)
    with open(os.path.join(CKPT, "selection.json"), "w") as fh:
        json.dump(
            dict(
                selected=selected,
                lodo_means=lodo_means,
                train_emis_corpus=results["train_emis_corpus"],
                headline=results["headline"],
                diversity_note=results["diversity_note"],
            ),
            fh,
            indent=2,
            default=_json_default,
        )
    write_md(results)
    log_line(f"wrote {OUT} + {RESULTS_MD} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
