"""Whole-trial unlabeled energy/MMD stream assignment: fuse w=0.5 ↔ linear_hp30.

Distinct from framewise domain-emis-ll (which collapsed to const_fuse):
  1. Per-trial summary stats of each stream's evidence (moments / quantiles).
  2. Train-only reference "ball" per stream (mean/cov + trial cloud).
  3. Distance of that trial's summary to its stream's train ball
     (Mahalanobis / energy / RBF-MMD); hard pick or soft weight.
  4. Whole-trial gate is constant across frames of that trial.

No labels at test. Select metric/features/temp on LODO inner Acc then H.
Score LODO 5 + AASD + Carta at HMM p_stay 0.994 scale 0.8.
Report fraction of trials choosing hp30 on AASD (and LODO).

CPU on models/causal_moe_fuse/cache/ + v4 hp30 loaders.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u trial_mmd_stream.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np
from scipy.special import expit

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "trial_mmd_stream.log")
OUT = os.path.join(EXP, "results", "research", "trial_mmd_stream.json")
CACHE_DIR = M.CACHE_DIR
P_STAY = M.P_STAY
SCALE = M.SCALE
TEST_NAMES = M.TEST_NAMES
EPS = 1e-8
RIDGE = 1e-4

FUSE_MEAN = M.FUSE_MEAN
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT
FUSE_EXT = M.FUSE_EXT

# percentiles for richer summaries
QS = (0.1, 0.25, 0.5, 0.75, 0.9)


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def trial_summary(x, feat_kind):
    """Unlabeled per-trial summary vector of 1-d evidence."""
    x = np.asarray(x, np.float64).reshape(-1)
    if len(x) == 0:
        # fall back; should not happen
        return np.zeros(4 if feat_kind == "mom4" else 14, np.float64)
    mu = float(x.mean())
    sd = float(x.std())
    energy = float(np.mean(x * x))
    abs_mu = abs(mu)
    base = np.array([mu, sd, abs_mu, energy], np.float64)
    if feat_kind == "mom4":
        return base
    ax = np.abs(x)
    q_x = np.quantile(x, QS)
    q_a = np.quantile(ax, QS)
    return np.concatenate([base, q_x, q_a]).astype(np.float64)


def collect_summaries(streams, datasets, feat_kind):
    rows, keys = [], []
    for ds in datasets:
        if ds not in streams:
            continue
        for key, d in streams[ds].items():
            rows.append(trial_summary(d, feat_kind))
            keys.append((ds, key))
    if not rows:
        raise RuntimeError(f"no trials in {datasets}")
    return np.stack(rows, 0), keys


def fit_ball(streams, datasets, feat_kind):
    X, _ = collect_summaries(streams, datasets, feat_kind)
    mu = X.mean(0)
    Xc = X - mu
    n, d = X.shape
    cov = (Xc.T @ Xc) / max(n - 1, 1) + RIDGE * np.eye(d)
    # stable inverse via eigh
    w, V = np.linalg.eigh(cov)
    w = np.maximum(w, RIDGE)
    inv = (V * (1.0 / w)) @ V.T
    # median pairwise distance for RBF bandwidth (subsample)
    rng = np.random.default_rng(0)
    idx = rng.choice(n, size=min(n, 256), replace=False)
    Xs = X[idx]
    # pairwise sq dist
    g = Xs @ Xs.T
    sq = np.maximum(np.diag(g)[:, None] + np.diag(g)[None, :] - 2.0 * g, 0.0)
    tri = sq[np.triu_indices(len(Xs), 1)]
    med = float(np.median(tri)) if len(tri) else 1.0
    bw = max(med, 1e-6)
    # energy constant: 0.5 E||Xi-Xj||
    e_pair = float(np.mean(np.sqrt(tri))) if len(tri) else 0.0
    return dict(mu=mu, cov=cov, inv=inv, X=X, bw=bw, energy_pair=e_pair, n=int(n))


def dist_mahal(s, ball):
    d = s - ball["mu"]
    return float(d @ ball["inv"] @ d)


def dist_euclid(s, ball):
    d = s - ball["mu"]
    return float(np.dot(d, d))


def dist_energy(s, ball):
    """Energy distance of point s to empirical cloud (full, comparable across balls)."""
    X = ball["X"]
    # E||s - Xi||
    diff = X - s[None, :]
    e_sx = float(np.mean(np.linalg.norm(diff, axis=1)))
    return e_sx - 0.5 * float(ball["energy_pair"])


def dist_mmd(s, ball):
    """Biased squared RBF-MMD between {s} and empirical train cloud."""
    X = ball["X"]
    bw = ball["bw"]
    # k(s,s)=1; mean k(s,Xi); mean k(Xi,Xj) via subsample already in bw scale
    # Exact mean k(Xi,Xj) on full cloud can be O(n^2); use subsample of 256
    n = len(X)
    rng = np.random.default_rng(1)
    idx = rng.choice(n, size=min(n, 256), replace=False)
    Xs = X[idx]
    # ||s-Xi||^2
    d2 = np.sum((Xs - s[None, :]) ** 2, axis=1)
    k_sx = float(np.mean(np.exp(-d2 / bw)))
    g = Xs @ Xs.T
    sq = np.maximum(np.diag(g)[:, None] + np.diag(g)[None, :] - 2.0 * g, 0.0)
    k_xx = float(np.mean(np.exp(-sq / bw)))
    # MMD^2 = k(s,s) + k_xx - 2 k_sx  with k(s,s)=1
    return 1.0 + k_xx - 2.0 * k_sx


DIST_FNS = {
    "mahal": dist_mahal,
    "euclid": dist_euclid,
    "energy": dist_energy,
    "mmd": dist_mmd,
}


def gate_weight(d_f, d_h, kind, temp):
    """g∈[0,1]; high g → fuse. kind: hard | soft."""
    if kind == "const_fuse":
        return 1.0
    if kind == "const_hp30":
        return 0.0
    if kind == "hard":
        # smaller distance wins; tie → fuse
        return 1.0 if d_f <= d_h else 0.0
    if kind == "soft":
        # normalized gap so temp is scale-free
        z = (d_h - d_f) / (d_h + d_f + EPS)
        return float(expit(temp * z))
    raise ValueError(kind)


def assign_stream(fuse_st, hp_st, ball_f, ball_h, feat_kind, dist_name, kind, temp):
    """Whole-trial constant gate → blended evidence."""
    dist_fn = DIST_FNS[dist_name]
    out = {}
    picks = {}  # 1=fuse, 0=hp30 (hard); soft stores g
    for key in fuse_st:
        if key not in hp_st:
            continue
        f = np.asarray(fuse_st[key], np.float64)
        h = np.asarray(hp_st[key], np.float64)
        n = min(len(f), len(h))
        f, h = f[:n], h[:n]
        if kind in ("const_fuse", "const_hp30"):
            g = gate_weight(0.0, 0.0, kind, temp)
        else:
            sf = trial_summary(f, feat_kind)
            sh = trial_summary(h, feat_kind)
            d_f = dist_fn(sf, ball_f)
            d_h = dist_fn(sh, ball_h)
            g = gate_weight(d_f, d_h, kind, temp)
        out[key] = (g * f + (1.0 - g) * h).astype(np.float64)
        picks[key] = float(g)
    return out, picks


def assign_all(fuse_all, hp_all, ball_f, ball_h, feat_kind, dist_name, kind, temp):
    out, picks = {}, {}
    for ds in fuse_all:
        if ds not in hp_all:
            continue
        st, pk = assign_stream(
            fuse_all[ds], hp_all[ds], ball_f, ball_h, feat_kind, dist_name, kind, temp)
        out[ds] = st
        picks[ds] = pk
    return out, picks


def candidate_rules():
    """Compact grid + const baselines. Select by inner Acc then H."""
    rules = [
        ("const_fuse", "mom4", "mahal", 1.0),
        ("const_hp30", "mom4", "mahal", 1.0),
    ]
    for feat in ("mom4", "mom_q"):
        for dist in ("mahal", "euclid", "energy", "mmd"):
            rules.append(("hard", feat, dist, 1.0))
            for temp in (1.0, 2.0, 4.0, 8.0):
                rules.append(("soft", feat, dist, temp))
    # dedupe
    seen, out = set(), []
    for r in rules:
        if r not in seen:
            seen.add(r)
            out.append(r)
    return out


def rule_key(kind, feat, dist, temp):
    if kind.startswith("const_"):
        return kind
    if kind == "hard":
        return f"hard|{feat}|{dist}"
    return f"soft|{feat}|{dist}|T{temp:g}"


def fast_score(streams, ds, train_ds):
    mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
    return N.evaluate(ds, streams[ds], mu, sd * SCALE, P_STAY)


def balls_for(fuse, hp, datasets, feat_kind):
    return fit_ball(fuse, datasets, feat_kind), fit_ball(hp, datasets, feat_kind)


def inner_mean_for_rule(fuse_by_held, hp30, kind, feat, dist, temp, ball_cache=None):
    hs, acs = [], []
    if ball_cache is None:
        ball_cache = {}
    for held in DS_KEYS:
        fuse = fuse_by_held[held]
        for inner in DS_KEYS:
            if inner == held:
                continue
            train = [d for d in DS_KEYS if d not in (held, inner)]
            if kind.startswith("const_"):
                ball_f = ball_h = None
            else:
                ck = (held, tuple(train), feat)
                if ck not in ball_cache:
                    ball_cache[ck] = balls_for(fuse, hp30, train, feat)
                ball_f, ball_h = ball_cache[ck]
            gated, _ = assign_all(
                fuse, hp30, ball_f, ball_h, feat, dist, kind, temp)
            r = fast_score(gated, inner, train)
            hs.append(r["H"])
            acs.append(r["Acc"])
    return float(np.mean(hs)), float(np.mean(acs)), len(hs)


def hp30_frac(picks_by_ds, ds=None):
    """Fraction of trials with g < 0.5 (prefer hp30)."""
    vals = []
    if ds is None:
        for table in picks_by_ds.values():
            vals.extend(table.values())
    else:
        vals.extend(picks_by_ds.get(ds, {}).values())
    if not vals:
        return float("nan")
    return float(np.mean([1.0 if g < 0.5 else 0.0 for g in vals]))


def mean_g(picks_by_ds, ds=None):
    vals = []
    if ds is None:
        for table in picks_by_ds.values():
            vals.extend(table.values())
    else:
        vals.extend(picks_by_ds.get(ds, {}).values())
    return float(np.mean(vals)) if vals else float("nan")


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"trial_mmd_stream CPU  p_stay={P_STAY} scale={SCALE}")
    log_line("whole-trial summary→train-ball distance (not framewise emis-LL)")

    N.load_all()
    hp30, _mean4, lookahead = M.load_lodo_hp30()
    log_line(f"lookahead {lookahead}")

    fuse_by_held = {}
    for held in DS_KEYS:
        path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
        if not os.path.isfile(path):
            raise FileNotFoundError(f"missing fuse cache {path}")
        fuse_by_held[held] = M.load_pkl(path)
        log_line(f"cache hit {path}")

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")
    san_hp = M.score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    rules = candidate_rules()
    log_line(f"scoring {len(rules)} trial-MMD rules on inner cells")
    inner_rows = []
    best = None
    ball_cache = {}
    for kind, feat, dist, temp in rules:
        t0 = time.time()
        mean_h, mean_a, n = inner_mean_for_rule(
            fuse_by_held, hp30, kind, feat, dist, temp, ball_cache=ball_cache)
        row = dict(kind=kind, feat=feat, dist=dist, temp=temp,
                   mean_inner_H=mean_h, mean_inner_Acc=mean_a, n=n,
                   key=rule_key(kind, feat, dist, temp))
        inner_rows.append(row)
        log_line(f"  INNER {row['key']:40s} H {mean_h:.4f} Acc {mean_a:.2f} "
                 f"({time.time()-t0:.0f}s)")
        key = (mean_a, mean_h)
        if best is None or key > best[0]:
            best = (key, row)

    selected = best[1]
    log_line(f"SELECTED {selected['key']} inner H {selected['mean_inner_H']:.4f} "
             f"Acc {selected['mean_inner_Acc']:.2f}")

    # LODO held-out
    lodo = {}
    lodo_picks = {}
    for held in DS_KEYS:
        fuse = fuse_by_held[held]
        train = [d for d in DS_KEYS if d != held]
        kind, feat, dist, temp = (
            selected["kind"], selected["feat"], selected["dist"], selected["temp"])
        if kind.startswith("const_"):
            ball_f = ball_h = None
        else:
            ball_f, ball_h = balls_for(fuse, hp30, train, feat)
        gated, picks = assign_all(fuse, hp30, ball_f, ball_h, feat, dist, kind, temp)
        off = M.score_lodo(gated, held)
        off_fuse = M.score_lodo(fuse, held)
        off_hp = M.score_lodo(hp30, held)
        frac = hp30_frac(picks, held)
        lodo[held] = dict(gate=off, fuse=off_fuse, hp30=off_hp,
                          frac_hp30=frac, mean_g=mean_g(picks, held))
        lodo_picks[held] = picks.get(held, {})
        log_line(f"LODO held={held}  gate H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                 f"fuse {off_fuse['H']:.4f}/{off_fuse['Acc']:.2f}  "
                 f"hp30 {off_hp['H']:.4f}/{off_hp['Acc']:.2f}  "
                 f"frac→hp30 {frac:.3f}")

    mean_gate = dict(H=float(np.mean([lodo[d]["gate"]["H"] for d in DS_KEYS])),
                     Acc=float(np.mean([lodo[d]["gate"]["Acc"] for d in DS_KEYS])))
    lodo_frac = float(np.mean([lodo[d]["frac_hp30"] for d in DS_KEYS]))
    log_line(f"LODO mean gate H {mean_gate['H']:.4f} Acc {mean_gate['Acc']:.2f}  "
             f"mean frac→hp30 {lodo_frac:.3f}  "
             f"(fuse {FUSE_MEAN['H']:.4f}/{FUSE_MEAN['Acc']:.2f})")

    # External
    log_line("=== external (cached fuse + 4-seed hp30) ===")
    corp_hp, ext_hp, train_att, test_meta = M.load_ext_hp30()
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))

    kind, feat, dist, temp = (
        selected["kind"], selected["feat"], selected["dist"], selected["temp"])
    if kind.startswith("const_"):
        ball_f = ball_h = None
        corp_gated, _ = assign_all(
            corp_fuse, corp_hp, ball_f, ball_h, feat, dist, kind, temp)
        ext_gated, ext_picks = assign_all(
            ext_fuse, ext_hp, ball_f, ball_h, feat, dist, kind, temp)
    else:
        ball_f, ball_h = balls_for(corp_fuse, corp_hp, list(V4.CORPUS), feat)
        corp_gated, _ = assign_all(
            corp_fuse, corp_hp, ball_f, ball_h, feat, dist, kind, temp)
        ext_gated, ext_picks = assign_all(
            ext_fuse, ext_hp, ball_f, ball_h, feat, dist, kind, temp)

    mu_e, sd_e = F.emissions_from(corp_gated, train_att, datasets=list(V4.CORPUS))
    mu_f, sd_f = F.emissions_from(corp_fuse, train_att, datasets=list(V4.CORPUS))
    mu_h, sd_h = F.emissions_from(corp_hp, train_att, datasets=list(V4.CORPUS))

    external = {}
    for ds in TEST_NAMES:
        r_g = F.score_external(ext_gated[ds], test_meta[ds], mu_e, sd_e)
        r_f = F.score_external(ext_fuse[ds], test_meta[ds], mu_f, sd_f)
        r_h = F.score_external(ext_hp[ds], test_meta[ds], mu_h, sd_h)
        frac = hp30_frac(ext_picks, ds)
        external[ds] = dict(
            gate=r_g, fuse=r_f, hp30=r_h,
            frac_hp30=frac, mean_g=mean_g(ext_picks, ds),
            n_trials=len(ext_picks.get(ds, {})))
        log_line(f"EXT {ds}  gate H {r_g['H']:.4f} Acc {r_g['Acc']:.2f}  "
                 f"fuse {r_f['H']:.4f}/{r_f['Acc']:.2f}  "
                 f"hp30 {r_h['H']:.4f}/{r_h['Acc']:.2f}  "
                 f"frac→hp30 {frac:.3f} (n={external[ds]['n_trials']})  "
                 f"(published linear_hp30 {LINEAR_HP30_EXT[ds]['H']:.4f}/"
                 f"{LINEAR_HP30_EXT[ds]['Acc']:.2f})")

    # top inner non-const for diagnostics
    non_const = [r for r in inner_rows if not r["kind"].startswith("const_")]
    non_const_sorted = sorted(non_const, key=lambda r: (-r["mean_inner_Acc"], -r["mean_inner_H"]))
    top5 = non_const_sorted[:5]

    out = dict(
        p_stay=P_STAY, scale=SCALE,
        selected=selected,
        inner_rows=inner_rows,
        top_non_const=top5,
        lodo=lodo, lodo_mean=mean_gate, lodo_frac_hp30=lodo_frac,
        external=external,
        baselines=dict(fuse_mean=FUSE_MEAN, linear_hp30_ext=LINEAR_HP30_EXT,
                       fuse_ext=FUSE_EXT),
        seconds=time.time() - t_all,
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT} in {time.time()-t_all:.0f}s")


if __name__ == "__main__":
    main()
