"""Cross-stream causal scale: match fuse online σ toward train linear_hp30 σ.

Distinct from online-moment-fuse (same-stream fuse→fuse train σ) and from
MoE / emission-LL gates. No labels for adaptation. No entropy-min.

Variants (pre-specified; select by mean LODO Acc then H):
  1. fuse_xσ_t60       — past-only EWMA σ of fuse; scale so σ→train hp30 σ
                         (keep fuse running mean). Per-trial.
  2. fuse_xμσ_t60      — same + bias to train hp30 mean (full affine).
  3. fuse_xσ_subj_t60  — (1) with per-subject EMA carry across trials.
  4–5. align2d_mix_g*  — affine-align BOTH streams online to train hp30 (μ,σ),
                         then g·fuse_al + (1−g)·hp30_al; g∈{0.5, 0.75}
                         chosen by LODO Acc only.

HMM preset p_stay 0.994 scale 0.8. CPU on causal_moe_fuse caches + hp30 loaders.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u cross_stream_scale.py
"""
from __future__ import annotations

import json
import os
import re
import time

import numpy as np
from scipy.signal import lfilter

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = M.CACHE_DIR
LOG = os.path.join(EXP, "experiments", "cross_stream_scale.log")
OUT = os.path.join(EXP, "results", "research", "cross_stream_scale.json")
CKPT = os.path.join(EXP, "models", "cross_stream_scale")

P_STAY = M.P_STAY
SCALE = M.SCALE
TEST_NAMES = M.TEST_NAMES
HOP = 0.25
EPS = 1e-6
TAU_S = 60.0
SCALE_CLIP = (0.5, 2.0)

FUSE_BASELINE = dict(
    AASD=dict(H=0.6179, Acc=50.94),
    Carta2026=dict(H=0.5461, Acc=48.38),
    LODO_mean=dict(H=0.8725, Acc=82.58),
    AVGC=dict(H=0.8100, Acc=75.14),
)
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT
OMF_SELECTED = dict(  # online-moment-fuse trial_scale_t60
    AASD=dict(H=0.6183, Acc=50.97),
    AVGC=dict(H=0.8168, Acc=75.70),
    LODO_mean=dict(H=0.8769, Acc=83.01),
)

# (name, kind, persist_subject, scale_only, mix_g)
# kind: None=raw fuse; "xhp30"=adapt fuse→train hp30 moments;
#       "align2d"=adapt both→train hp30 then soft mix
VARIANT_SPECS = [
    ("fuse_w05", None, True, False, None),
    ("fuse_xσ_t60", "xhp30", False, True, None),
    ("fuse_xμσ_t60", "xhp30", False, False, None),
    ("fuse_xσ_subj_t60", "xhp30", True, True, None),
    ("align2d_mix_g0.50", "align2d", False, False, 0.50),
    ("align2d_mix_g0.75", "align2d", False, False, 0.75),
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def subject_of(key, meta_entry=None):
    if meta_entry is not None and "subject" in meta_entry:
        return int(meta_entry["subject"])
    m = re.search(r"subject[-_]?(\d+)", str(key), re.I)
    if m:
        return int(m.group(1))
    m = re.search(r"_S(\d+)_", str(key), re.I)
    if m:
        return int(m.group(1))
    return hash(str(key)) % 10_000_007


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def train_moments(streams, datasets):
    parts = []
    for ds in datasets:
        for _k, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    x = np.concatenate(parts)
    return float(x.mean()), float(x.std()) + EPS


def _ema_past(x, alpha, init):
    x = np.asarray(x, np.float64).reshape(-1)
    a = float(alpha)
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x, zi=np.asarray([init * (1.0 - a)], np.float64))
    past = np.empty_like(x)
    past[0] = init
    if len(x) > 1:
        past[1:] = y[:-1]
    return past, float(y[-1]) if len(y) else float(init)


def adapt_1d(z, mu_tgt, sd_tgt, alpha, mode="affine", scale_only=False,
             init_mu=None, init_second=None):
    """Causal EMA moment match toward (mu_tgt, sd_tgt). Past-only emit."""
    z = np.asarray(z, np.float64).reshape(-1)
    if mode is None or mode == "none" or len(z) == 0:
        return z.copy(), (mu_tgt if init_mu is None else init_mu), (
            mu_tgt * mu_tgt + sd_tgt * sd_tgt if init_second is None else init_second)
    mu0 = float(mu_tgt if init_mu is None else init_mu)
    sec0 = float((mu_tgt * mu_tgt + sd_tgt * sd_tgt) if init_second is None else init_second)
    mu_past, mu_final = _ema_past(z, alpha, mu0)
    sec_past, sec_final = _ema_past(z * z, alpha, sec0)
    var = np.maximum(sec_past - mu_past * mu_past, EPS)
    sd = np.sqrt(var)
    lo, hi = SCALE_CLIP
    scale = np.clip(sd_tgt / sd, lo, hi)
    if scale_only:
        out = (z - mu_past) * scale + mu_past
    else:
        out = (z - mu_past) * scale + mu_tgt
    return out, mu_final, sec_final


def adapt_streams(streams, meta_by_ds, mu_tgt, sd_tgt, alpha,
                  persist_subject=True, scale_only=False):
    out = {}
    for ds, table in streams.items():
        meta = (meta_by_ds or {}).get(ds) or {}
        by_sub = {}
        for key in table:
            sid = subject_of(key, meta.get(key))
            by_sub.setdefault(sid, []).append(key)
        adapted = {}
        for _sid, keys in by_sub.items():
            keys = sorted(keys)
            state_mu, state_second = None, None
            for key in keys:
                if not persist_subject:
                    state_mu, state_second = None, None
                y, state_mu, state_second = adapt_1d(
                    table[key], mu_tgt, sd_tgt, alpha, mode="affine",
                    scale_only=scale_only,
                    init_mu=state_mu, init_second=state_second,
                )
                adapted[key] = y
        out[ds] = adapted
    return out


def soft_mix(fuse_st, hp_st, g):
    """g·fuse + (1−g)·hp30; truncate to common length."""
    out = {}
    for ds in fuse_st:
        table_f, table_h = fuse_st[ds], hp_st[ds]
        mixed = {}
        for key in table_f:
            if key not in table_h:
                continue
            f = np.asarray(table_f[key], np.float64).reshape(-1)
            h = np.asarray(table_h[key], np.float64).reshape(-1)
            n = min(len(f), len(h))
            mixed[key] = (g * f[:n] + (1.0 - g) * h[:n]).astype(np.float64)
        out[ds] = mixed
    return out


def apply_variant(fuse_st, hp_st, mu_hp, sd_hp, alpha, kind, persist, scale_only, mix_g,
                  meta_by_ds=None):
    if kind is None:
        return {ds: {k: np.asarray(d, np.float64) for k, d in table.items()}
                for ds, table in fuse_st.items()}
    if kind == "xhp30":
        return adapt_streams(
            fuse_st, meta_by_ds, mu_hp, sd_hp, alpha,
            persist_subject=persist, scale_only=scale_only,
        )
    if kind == "align2d":
        fuse_al = adapt_streams(
            fuse_st, meta_by_ds, mu_hp, sd_hp, alpha,
            persist_subject=persist, scale_only=False,
        )
        hp_al = adapt_streams(
            hp_st, meta_by_ds, mu_hp, sd_hp, alpha,
            persist_subject=persist, scale_only=False,
        )
        return soft_mix(fuse_al, hp_al, float(mix_g))
    raise ValueError(f"unknown kind {kind}")


def pick_variant(lodo_means, candidates):
    best_name, best_key = None, None
    for name in candidates:
        m = lodo_means[name]
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def load_held_fuse(held):
    path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
    if not os.path.isfile(path):
        raise FileNotFoundError(f"missing fuse cache {path}")
    return M.load_pkl(path)


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    alpha = alpha_from_tau(TAU_S)
    log_line(f"cross-stream-scale CPU  p_stay {P_STAY} scale {SCALE} tau {TAU_S}s alpha {alpha:.6g}")
    log_line("honesty: fuse online σ → train hp30 σ (or 2-D align + soft mix); no labels; no entropy-min")
    log_line("distinct from online-moment-fuse (same-stream) and MoE/emis-LL")

    N.load_all()
    att = N.G["att"]

    log_line("loading LODO fuse caches + hp30")
    fuse_by_held = {}
    for held in DS_KEYS:
        fuse_by_held[held] = load_held_fuse(held)
        log_line(f"cache hit fuse_w05__held-{held}.pkl")
    hp30, _mean4, _la = M.load_lodo_hp30()

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity raw fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")
    san_hp = M.score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    # train hp30 moments for held=AVGC (sanity print)
    mu0, sd0 = train_moments(hp30, [d for d in DS_KEYS if d != "AVGC"])
    mu_f0, sd_f0 = train_moments(fuse_by_held["AVGC"], [d for d in DS_KEYS if d != "AVGC"])
    log_line(f"ref train moments (held AVGC): fuse μ/σ {mu_f0:.4f}/{sd_f0:.4f}  "
             f"hp30 μ/σ {mu0:.4f}/{sd0:.4f}")

    results = dict(
        method=("cross-stream causal scale: fuse online σ → train linear_hp30 σ; "
                "optional bias; optional 2-D align + LODO soft mix"),
        p_stay=P_STAY, scale=SCALE, tau_s=TAU_S, alpha=alpha,
        scale_clip=list(SCALE_CLIP),
        honesty=("Past-only EWMA on fuse (and hp30 for align2d); target = unlabeled "
                 "train hp30 μ/σ. No labels, no entropy-min. Distinct from same-stream "
                 "online-moment-fuse and MoE/emis-LL."),
        baselines=dict(
            fuse=FUSE_BASELINE,
            linear_hp30_ext=LINEAR_HP30_EXT,
            online_moment_fuse_selected=OMF_SELECTED,
        ),
        variants={},
    )

    log_line("=== LODO (fixed HMM; selection among adapt variants) ===")
    lodo_means = {}
    for name, kind, persist, scale_only, mix_g in VARIANT_SPECS:
        t0 = time.time()
        per = {}
        for held in DS_KEYS:
            fuse = fuse_by_held[held]
            train_ds = [d for d in DS_KEYS if d != held]
            mu_hp, sd_hp = train_moments(hp30, train_ds)
            adapted = apply_variant(
                fuse, hp30, mu_hp, sd_hp, alpha, kind, persist, scale_only, mix_g,
            )
            off = M.score_lodo(adapted, held)
            per[held] = dict(H=off["H"], Acc=off["Acc"], SDT=off.get("SDT"),
                             L=off.get("L"), acc1=off.get("acc1"),
                             train_hp30_mu=mu_hp, train_hp30_sd=sd_hp)
            log_line(f"LODO {name:20s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                     f"(hp30tgt σ={sd_hp:.3f})")
        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:20s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
        lodo_means[name] = dict(H=mean_h, Acc=mean_a)
        results["variants"][name] = dict(
            kind=kind, tau=TAU_S if kind else None, alpha=alpha if kind else None,
            persist_subject=persist if kind else None,
            scale_only=scale_only if kind == "xhp30" else None,
            mix_g=mix_g,
            lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
        )

    adapt_names = [n for n, k, *_ in VARIANT_SPECS if k is not None]
    selected = pick_variant(lodo_means, adapt_names)
    ranked = sorted(adapt_names, key=lambda n: (lodo_means[n]["Acc"], lodo_means[n]["H"]),
                    reverse=True)
    log_line(f"SELECTED (LODO Acc then H, adapt only): {selected}  "
             f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}")
    log_line(f"ranked: {ranked}")
    results["selection"] = dict(
        criterion="mean LODO Acc then H (adapt variants only; never AASD/Carta)",
        ranked=ranked, selected=selected,
    )
    results["selected"] = selected

    log_line("=== external (all5 corpus + external fuse + hp30) ===")
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))
    corp_hp, ext_hp, train_att, test_meta = M.load_ext_hp30()
    mu_corp_hp, sd_corp_hp = train_moments(corp_hp, list(V4.CORPUS))
    mu_corp_f, sd_corp_f = train_moments(corp_fuse, list(V4.CORPUS))
    log_line(f"corpus fuse μ/σ {mu_corp_f:.6f}/{sd_corp_f:.6f}  "
             f"hp30 μ/σ {mu_corp_hp:.6f}/{sd_corp_hp:.6f}")
    results["train_moments_corpus"] = dict(
        fuse=dict(mu=mu_corp_f, sd=sd_corp_f),
        hp30=dict(mu=mu_corp_hp, sd=sd_corp_hp),
    )

    for name, kind, persist, scale_only, mix_g in VARIANT_SPECS:
        adapted_train = apply_variant(
            corp_fuse, corp_hp, mu_corp_hp, sd_corp_hp, alpha,
            kind, persist, scale_only, mix_g,
        )
        mu_e, sd_e = F.emissions_from(adapted_train, train_att, datasets=list(V4.CORPUS))
        adapted_test = apply_variant(
            ext_fuse, ext_hp, mu_corp_hp, sd_corp_hp, alpha,
            kind, persist, scale_only, mix_g, meta_by_ds=test_meta,
        )
        ext = {}
        for ds in TEST_NAMES:
            r = F.score_external(adapted_test[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            d_fuse = r["Acc"] - FUSE_BASELINE[ds]["Acc"]
            d_hp = r["Acc"] - LINEAR_HP30_EXT[ds]["Acc"]
            log_line(f"EXT {name:20s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs fuse {d_fuse:+.2f}  vs hp30 {d_hp:+.2f}")
        results["variants"][name]["external"] = ext
        results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_fuse = aasd["Acc"] - FUSE_BASELINE["AASD"]["Acc"]
    d_hp = aasd["Acc"] - LINEAR_HP30_EXT["AASD"]["Acc"]
    d_omf = aasd["Acc"] - OMF_SELECTED["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_fuse=d_fuse,
        AASD_Acc_delta_vs_linear_hp30=d_hp,
        AASD_Acc_delta_vs_online_moment_fuse=d_omf,
        lodo_mean=prim["lodo_mean"],
        lodo_AVGC=prim["lodo"]["AVGC"],
        baseline_fuse_ext=FUSE_BASELINE,
        baseline_linear_hp30_ext=LINEAR_HP30_EXT,
    )
    log_line(f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δfuse {d_fuse:+.2f}, Δhp30 {d_hp:+.2f}, Δomf {d_omf:+.2f})  "
             f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}  "
             f"AVGC H {prim['lodo']['AVGC']['H']:.4f} Acc {prim['lodo']['AVGC']['Acc']:.2f}")

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "summary.json"), "w") as fh:
        json.dump(dict(selected=selected, lodo_means=lodo_means,
                       train_moments_corpus=results["train_moments_corpus"],
                       headline=results["headline"]), fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
