"""Per-subject causal-online unlabeled moment match on linear_hp30.

CPU-only on cached 4-seed logits (same front-end as linear_hp30 /
ext_coral_logits). No new network train. No entropy-min. No labels for
adaptation or selection.

Within each subject, maintain an EMA of unlabeled decision-stream moments
and apply a causal affine map after emit:

  affine-to-train:  z' = (z - μ_ema) * (σ_train / σ_ema) + μ_train
  online-whiten:    z' = (z - μ_ema) / σ_ema

Distinct from entropy-min TTA (results_tta.md) and global CORAL
(results_ext_coral.md).

Variant selection: LODO mean Acc then H only. Score LODO + AASD + Carta
at fixed HMM p_stay 0.994 scale 0.8.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH python -u online_moment_hp30.py
"""
from __future__ import annotations

import json
import os
import re
import time

import numpy as np
from scipy.signal import lfilter

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "online_moment_hp30.log")
OUT = os.path.join(EXP, "results", "research", "online_moment_hp30.json")
CKPT = os.path.join(EXP, "models", "online_moment_hp30")

CORPUS = ("AVGC", "DTU", "KUL", "UHD", "USTC")
TEST_NAMES = ("AASD", "Carta2026")
P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = 0.75
HOP = 0.25
EPS = 1e-6
SCALE_CLIP = (0.25, 4.0)

LINEAR_HP30_BASELINE = dict(
    AASD=dict(H=0.6422, Acc=53.12),
    Carta2026=dict(H=0.6520, Acc=60.54),
    LODO_mean=dict(H=0.7899, Acc=69.21),
)

# (name, mode, tau_s, persist_subject, scale_only)
# mode: none | affine | whiten
VARIANT_SPECS = [
    ("linear_hp30", "none", None, True, False),
    ("affine_tau5", "affine", 5.0, True, False),
    ("affine_tau15", "affine", 15.0, True, False),
    ("affine_tau30", "affine", 30.0, True, False),
    ("affine_tau60", "affine", 60.0, True, False),
    ("affine_tau30_scale", "affine", 30.0, True, True),
    ("affine_tau30_trial", "affine", 30.0, False, False),
    ("whiten_tau15", "whiten", 15.0, True, False),
    ("whiten_tau30", "whiten", 30.0, True, False),
    ("whiten_tau60", "whiten", 60.0, True, False),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def subject_of(key, meta_entry=None):
    if meta_entry is not None and "subject" in meta_entry:
        return int(meta_entry["subject"])
    m = re.search(r"subject[-_]?(\d+)", str(key), re.I)
    if m:
        return int(m.group(1))
    return hash(str(key)) % 10_000_007


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def train_moments(streams, datasets):
    parts = []
    for ds in datasets:
        for _k, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    x = np.concatenate(parts)
    return float(x.mean()), float(x.std()) + EPS


def _ema_past(x, alpha, init):
    """Past-only EMA at each t (init at t=0); also return final EMA including last x."""
    x = np.asarray(x, np.float64).reshape(-1)
    a = float(alpha)
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x, zi=np.asarray([init * (1.0 - a)], np.float64))
    past = np.empty_like(x)
    past[0] = init
    if len(x) > 1:
        past[1:] = y[:-1]
    return past, float(y[-1]) if len(y) else float(init)


def adapt_1d(z, mu_tgt, sd_tgt, alpha, mode, scale_only=False,
             init_mu=None, init_second=None):
    """Causal EMA moment match. Emit using past EMA, then update with raw z[t]."""
    z = np.asarray(z, np.float64).reshape(-1)
    if mode == "none" or len(z) == 0:
        return z.copy(), (mu_tgt if init_mu is None else init_mu), (
            mu_tgt * mu_tgt + sd_tgt * sd_tgt if init_second is None else init_second)
    mu0 = float(mu_tgt if init_mu is None else init_mu)
    sec0 = float((mu_tgt * mu_tgt + sd_tgt * sd_tgt) if init_second is None else init_second)
    mu_past, mu_final = _ema_past(z, alpha, mu0)
    sec_past, sec_final = _ema_past(z * z, alpha, sec0)
    var = np.maximum(sec_past - mu_past * mu_past, EPS)
    sd = np.sqrt(var)
    lo, hi = SCALE_CLIP
    if mode == "affine":
        scale = np.clip(sd_tgt / sd, lo, hi)
        if scale_only:
            out = (z - mu_past) * scale + mu_past
        else:
            out = (z - mu_past) * scale + mu_tgt
    elif mode == "whiten":
        out = (z - mu_past) / sd
    else:
        out = z.copy()
    return out, mu_final, sec_final


def adapt_streams(streams, meta_by_ds, mu_tgt, sd_tgt, alpha, mode,
                  persist_subject=True, scale_only=False):
    """Apply per-subject (or per-trial) causal adapt. Trials sorted by key."""
    if mode == "none":
        return {ds: {k: np.asarray(d, np.float64) for k, d in table.items()}
                for ds, table in streams.items()}
    out = {}
    for ds, table in streams.items():
        meta = (meta_by_ds or {}).get(ds) or {}
        by_sub = {}
        for key in table:
            sid = subject_of(key, meta.get(key))
            by_sub.setdefault(sid, []).append(key)
        adapted = {}
        for _sid, keys in by_sub.items():
            keys = sorted(keys)
            state_mu, state_second = None, None
            for key in keys:
                if not persist_subject:
                    state_mu, state_second = None, None
                y, state_mu, state_second = adapt_1d(
                    table[key], mu_tgt, sd_tgt, alpha, mode, scale_only=scale_only,
                    init_mu=state_mu, init_second=state_second,
                )
                adapted[key] = y
        out[ds] = adapted
    return out


def mean_hp30(logits_4d):
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {}
        for k, D in table.items():
            X = V4.highpass(np.asarray(D, np.float64))
            out[ds][k] = X.mean(1).astype(np.float64)
    return out


def emissions_from(streams, att, datasets):
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, LOOKAHEAD)


def load_lodo_4d():
    evs = []
    for _seed, exp in EC.SEEDS:
        ev, _info = S_load(exp)
        evs.append(ev)
    logits = {}
    for ds in DS_KEYS:
        logits[ds] = {}
        for k in evs[0][ds].keys():
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            logits[ds][k] = np.stack(cols, 1).astype(np.float64)
    return logits


def S_load(exp):
    import score as S
    return S.load_folds(exp, "causal")


def pick_variant(lodo_means):
    """Max mean Acc, then H. Never uses AASD/Carta."""
    best_name, best_key = None, None
    for name, m in lodo_means.items():
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"online-moment-hp30 CPU-only  p_stay {P_STAY} scale {SCALE}")
    log_line("honesty: unlabeled EMA moments only; no entropy-min; no AASD/Carta labels")

    N.load_all()

    log_line("loading CACHE train + external 4-seed evidence")
    cache_logits, cache_att = V4.load_train()
    src_cache = {ds: cache_logits[ds] for ds in CORPUS}
    train_att = {ds: cache_att[ds] for ds in CORPUS}
    test_logits, test_meta = V4.load_test()

    log_line("loading LODO fold 4-seed logits")
    lodo_4d = load_lodo_4d()

    base_cache = mean_hp30(src_cache)
    base_test = mean_hp30(test_logits)
    base_lodo = mean_hp30(lodo_4d)

    mu_train, sd_train = train_moments(base_cache, list(CORPUS))
    log_line(f"train hp30 moments mu {mu_train:.6f} sd {sd_train:.6f}")

    results = dict(
        method="per-subject causal-online unlabeled moment match on linear_hp30",
        p_stay=P_STAY, scale=SCALE,
        honesty="EMA affine/whiten on unlabeled decision stream; no entropy-min; selection on LODO only.",
        baselines=LINEAR_HP30_BASELINE,
        train_moments=dict(mu=mu_train, sd=sd_train),
        variants={},
    )

    # --- LODO for all variants (selection set) ---
    log_line("=== LODO (fixed HMM; selection) ===")
    lodo_means = {}
    for name, mode, tau, persist, scale_only in VARIANT_SPECS:
        alpha = 0.0 if mode == "none" else alpha_from_tau(tau)
        t0 = time.time()
        adapted = adapt_streams(
            base_lodo, None, mu_train, sd_train, alpha, mode,
            persist_subject=persist, scale_only=scale_only,
        )
        per = {}
        for held in DS_KEYS:
            off = N.official(held, N.to_ev(adapted), P_STAY, SCALE)
            per[held] = off
            log_line(f"LODO {name:22s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:22s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            mode=mode, tau=tau, alpha=alpha, persist_subject=persist,
            scale_only=scale_only, lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    selected = pick_variant(lodo_means)
    log_line(f"SELECTED (LODO Acc then H): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    results["selected"] = selected

    # --- External for all variants (report; primary = selected) ---
    log_line("=== external ===")
    # meta for CORPUS: parse subject from key only
    for name, mode, tau, persist, scale_only in VARIANT_SPECS:
        alpha = 0.0 if mode == "none" else alpha_from_tau(tau)
        adapted_train = adapt_streams(
            base_cache, None, mu_train, sd_train, alpha, mode,
            persist_subject=persist, scale_only=scale_only,
        )
        mu_e, sd_e = emissions_from(adapted_train, train_att, list(CORPUS))
        adapted_test = adapt_streams(
            base_test, test_meta, mu_train, sd_train, alpha, mode,
            persist_subject=persist, scale_only=scale_only,
        )
        ext = {}
        for ds in TEST_NAMES:
            r = score_external(adapted_test[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            base_acc = LINEAR_HP30_BASELINE[ds]["Acc"]
            dacc = r["Acc"] - base_acc
            log_line(f"EXT {name:22s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs linear_hp30 {dacc:+.2f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=prim["lodo_mean"],
    )
    log_line(f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δ {d_aasd:+.2f} vs 53.12)  Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "selection.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       train_moments=results["train_moments"]), fh, indent=1)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
