"""Causal-online unlabeled EWMA moment match on IRM↔SWA fuse w=0.5.

Same past-only EMA affine / scale-only / whiten as online_moment_hp30, but the
decision stream is the frozen fuse logits from models/causal_moe_fuse/cache/
(fuse_w05__*.pkl). No labels for adaptation or selection. No entropy-min.
Does not touch the 1-D linear_hp30 stream.

Variant selection: mean LODO Acc then H among adapt variants only.
Score LODO 5 + AASD + Carta at HMM p_stay 0.994 scale 0.8.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u online_moment_fuse.py
"""
from __future__ import annotations

import json
import os
import re
import time

import numpy as np
from scipy.signal import lfilter

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = M.CACHE_DIR
LOG = os.path.join(EXP, "experiments", "online_moment_fuse.log")
OUT = os.path.join(EXP, "results", "research", "online_moment_fuse.json")
CKPT = os.path.join(EXP, "models", "online_moment_fuse")

P_STAY = M.P_STAY
SCALE = M.SCALE
TEST_NAMES = M.TEST_NAMES
HOP = 0.25
EPS = 1e-6
TAU_S = 60.0
SCALE_CLIP = (0.5, 2.0)

FUSE_BASELINE = dict(
    AASD=dict(H=0.6179, Acc=50.94),
    Carta2026=dict(H=0.5461, Acc=48.38),
    LODO_mean=dict(H=0.8725, Acc=82.58),
    AVGC=dict(H=0.8100, Acc=75.14),
)
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT

# (name, mode, persist_subject, scale_only); mode None = identity sanity
# Adapt variants only enter selection (same as online-moment-hp30).
VARIANT_SPECS = [
    ("fuse_w05", None, True, False),
    ("trial_match_t60", "affine", False, False),
    ("subj_match_t60", "affine", True, False),
    ("trial_whiten_t60", "whiten", False, False),
    ("trial_scale_t60", "affine", False, True),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def subject_of(key, meta_entry=None):
    if meta_entry is not None and "subject" in meta_entry:
        return int(meta_entry["subject"])
    m = re.search(r"subject[-_]?(\d+)", str(key), re.I)
    if m:
        return int(m.group(1))
    m = re.search(r"_S(\d+)_", str(key), re.I)
    if m:
        return int(m.group(1))
    return hash(str(key)) % 10_000_007


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def train_moments(streams, datasets):
    parts = []
    for ds in datasets:
        for _k, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    x = np.concatenate(parts)
    return float(x.mean()), float(x.std()) + EPS


def _ema_past(x, alpha, init):
    """Past-only EMA at each t (init at t=0); also return final EMA including last x."""
    x = np.asarray(x, np.float64).reshape(-1)
    a = float(alpha)
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x, zi=np.asarray([init * (1.0 - a)], np.float64))
    past = np.empty_like(x)
    past[0] = init
    if len(x) > 1:
        past[1:] = y[:-1]
    return past, float(y[-1]) if len(y) else float(init)


def adapt_1d(z, mu_tgt, sd_tgt, alpha, mode, scale_only=False,
             init_mu=None, init_second=None):
    """Causal EMA moment match. Emit using past EMA, then update with raw z[t]."""
    z = np.asarray(z, np.float64).reshape(-1)
    if mode is None or mode == "none" or len(z) == 0:
        return z.copy(), (mu_tgt if init_mu is None else init_mu), (
            mu_tgt * mu_tgt + sd_tgt * sd_tgt if init_second is None else init_second)
    mu0 = float(mu_tgt if init_mu is None else init_mu)
    sec0 = float((mu_tgt * mu_tgt + sd_tgt * sd_tgt) if init_second is None else init_second)
    mu_past, mu_final = _ema_past(z, alpha, mu0)
    sec_past, sec_final = _ema_past(z * z, alpha, sec0)
    var = np.maximum(sec_past - mu_past * mu_past, EPS)
    sd = np.sqrt(var)
    lo, hi = SCALE_CLIP
    if mode == "affine":
        scale = np.clip(sd_tgt / sd, lo, hi)
        if scale_only:
            out = (z - mu_past) * scale + mu_past
        else:
            out = (z - mu_past) * scale + mu_tgt
    elif mode == "whiten":
        out = (z - mu_past) / sd
    else:
        out = z.copy()
    return out, mu_final, sec_final


def adapt_streams(streams, meta_by_ds, mu_tgt, sd_tgt, alpha, mode,
                  persist_subject=True, scale_only=False):
    """Apply per-subject (or per-trial) causal adapt. Trials sorted by key."""
    if mode is None or mode == "none":
        return {ds: {k: np.asarray(d, np.float64) for k, d in table.items()}
                for ds, table in streams.items()}
    out = {}
    for ds, table in streams.items():
        meta = (meta_by_ds or {}).get(ds) or {}
        by_sub = {}
        for key in table:
            sid = subject_of(key, meta.get(key))
            by_sub.setdefault(sid, []).append(key)
        adapted = {}
        for _sid, keys in by_sub.items():
            keys = sorted(keys)
            state_mu, state_second = None, None
            for key in keys:
                if not persist_subject:
                    state_mu, state_second = None, None
                y, state_mu, state_second = adapt_1d(
                    table[key], mu_tgt, sd_tgt, alpha, mode, scale_only=scale_only,
                    init_mu=state_mu, init_second=state_second,
                )
                adapted[key] = y
        out[ds] = adapted
    return out


def pick_variant(lodo_means, candidates):
    """Max mean Acc, then H among candidates. Never uses AASD/Carta."""
    best_name, best_key = None, None
    for name in candidates:
        m = lodo_means[name]
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def load_held_fuse(held):
    path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
    if not os.path.isfile(path):
        raise FileNotFoundError(f"missing fuse cache {path}")
    return M.load_pkl(path)


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    alpha = alpha_from_tau(TAU_S)
    log_line(f"online-moment-fuse CPU  p_stay {P_STAY} scale {SCALE} tau {TAU_S}s alpha {alpha:.6g}")
    log_line("honesty: unlabeled EMA moments on fuse w=0.5; no entropy-min; no AASD/Carta labels")
    log_line("stream: models/causal_moe_fuse/cache/fuse_w05__*.pkl (NOT linear_hp30)")

    N.load_all()
    att = N.G["att"]

    fuse_by_held = {}
    for held in DS_KEYS:
        fuse_by_held[held] = load_held_fuse(held)
        log_line(f"cache hit fuse_w05__held-{held}.pkl")

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity raw fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")

    results = dict(
        method="causal-online unlabeled moment match on IRM↔SWA fuse w=0.5 (EWMA affine)",
        p_stay=P_STAY, scale=SCALE, tau_s=TAU_S, alpha=alpha,
        scale_clip=list(SCALE_CLIP),
        honesty=("Past-only EWMA mean/var on fuse stream; affine to train μ/σ or whiten. "
                 "No labels, no entropy-min. Distinct from online-moment-hp30 / TTA / CORAL."),
        baselines=dict(fuse=FUSE_BASELINE, linear_hp30_ext=LINEAR_HP30_EXT),
        variants={},
    )

    # --- LODO for all variants ---
    log_line("=== LODO (fixed HMM; selection among adapt variants) ===")
    lodo_means = {}
    for name, mode, persist, scale_only in VARIANT_SPECS:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            streams = fuse_by_held[held]
            train_ds = [d for d in DS_KEYS if d != held]
            mu_t, sd_t = train_moments(streams, train_ds)
            adapted = adapt_streams(
                streams, None, mu_t, sd_t, alpha, mode,
                persist_subject=persist, scale_only=scale_only,
            )
            off = M.score_lodo(adapted, held)
            per[held] = dict(H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"),
                             L=off.get("L"), acc1=off.get("acc1"),
                             train_mu=mu_t, train_sd=sd_t)
            log_line(f"LODO {name:18s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:18s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            mode=mode, tau=TAU_S if mode else None, alpha=alpha if mode else None,
            persist_subject=persist if mode else None,
            scale_only=scale_only if mode else None,
            scope=("subj" if persist else "trial") if mode else None,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    adapt_names = [n for n, m, *_ in VARIANT_SPECS if m is not None]
    selected = pick_variant(lodo_means, adapt_names)
    ranked = sorted(adapt_names, key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
                    reverse=True)
    log_line(f"SELECTED (LODO Acc then H, adapt only): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    log_line(f"ranked: {ranked}")
    results["selection"] = dict(
        criterion="mean LODO Acc then H (adapt variants only; never AASD/Carta)",
        ranked=ranked, selected=selected,
    )
    results["selected"] = selected

    # --- External ---
    log_line("=== external (all5 corpus + external fuse caches) ===")
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))
    _, test_meta = V4.load_test()
    train_att = {ds: att[ds] for ds in V4.CORPUS}
    mu_corp, sd_corp = train_moments(corp_fuse, list(V4.CORPUS))
    log_line(f"corpus fuse moments mu {mu_corp:.6f} sd {sd_corp:.6f}")
    results["train_moments_corpus"] = dict(mu=mu_corp, sd=sd_corp)

    for name, mode, persist, scale_only in VARIANT_SPECS:
        adapted_train = adapt_streams(
            corp_fuse, None, mu_corp, sd_corp, alpha, mode,
            persist_subject=persist, scale_only=scale_only,
        )
        mu_e, sd_e = F.emissions_from(adapted_train, train_att, datasets=list(V4.CORPUS))
        adapted_test = adapt_streams(
            ext_fuse, test_meta, mu_corp, sd_corp, alpha, mode,
            persist_subject=persist, scale_only=scale_only,
        )
        ext = {}
        for ds in TEST_NAMES:
            r = F.score_external(adapted_test[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            d_fuse = r["Acc"] - FUSE_BASELINE[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            log_line(f"EXT {name:18s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs fuse {d_fuse:+.2f}  vs hp30 {d_hp:+.2f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_fuse = aasd["Acc"] - FUSE_BASELINE["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_fuse=d_fuse,
        AASD_Acc_delta_vs_linear_hp30=d_hp,
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=prim["lodo"]["AVGC"],
        baseline_fuse_ext=FUSE_BASELINE,
        baseline_linear_hp30_ext=LINEAR_HP30_EXT,
    )
    log_line(f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δfuse {d_fuse:+.2f}, Δhp30 {d_hp:+.2f})  "
             f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}  "
             f"AVGC H {prim['lodo']['AVGC']['H']:.4f} Acc {prim['lodo']['AVGC']['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "summary.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       train_moments_corpus=results["train_moments_corpus"],
                       headline=results["headline"]), fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
