"""Domain-aware emission log-likelihood gate: fuse w=0.5 ↔ linear_hp30.

Two frozen streams (CPU caches from causal_moe_fuse / v4):
  fuse  = IRM↔SWA blend w=0.5
  hp30  = causal tau-30 high-pass of mean-of-4 seed logits

Method (no labels at test; distinct from stream-stat MoE in causal_moe_fuse):
  1. Fit Gaussian HMM emissions (mu, sd) on TRAIN corpus only, per stream.
  2. Causal frame log-likelihood of observed evidence under that stream's
     train emissions (2-state ±mu mixture, or IID Gaussian on the logit).
  3. Gate g_t = sigmoid(temp · EWMA(LL_fuse − LL_hp30)), or hard pick by
     cumulative LL. Evidence = g·fuse + (1−g)·hp30.
  4. Select temp / EWMA τ / family on LODO train inner Acc then H only.
     Never AASD/Carta labels.

HMM preset p_stay 0.994 scale 0.8.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u domain_emis_ll.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np
from scipy.special import expit, logsumexp
from scipy.signal import lfilter

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "domain_emis_ll.log")
OUT = os.path.join(EXP, "results", "research", "domain_emis_ll.json")
CACHE_DIR = M.CACHE_DIR
P_STAY = M.P_STAY
SCALE = M.SCALE
TEST_NAMES = M.TEST_NAMES
HOP = 0.25

FUSE_MEAN = M.FUSE_MEAN
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT
FUSE_EXT = M.FUSE_EXT


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def ewma_1d(x, tau_s, hop=HOP):
    x = np.asarray(x, np.float64)
    if tau_s <= 0:
        return x.copy()
    alpha = 1.0 - np.exp(-hop / tau_s)
    return lfilter([alpha], [1.0, -(1.0 - alpha)], x)


def fit_signed_emis(streams, att, datasets):
    """HMM emissions from signed train evidence (labels on train only)."""
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            if key not in labels:
                continue
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(max(A.std(), 1e-6))


def fit_iid_emis(streams, datasets):
    """IID Gaussian on raw (unsigned) logits — train moments only, no labels."""
    parts = []
    for ds in datasets:
        for d in streams[ds].values():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    A = np.concatenate(parts)
    return float(A.mean()), float(max(A.std(), 1e-6))


def ll_two_state(d, mu, sd):
    """Marginal loglik under 0.5 N(+mu,sd)+0.5 N(-mu,sd)."""
    d = np.asarray(d, np.float64)
    inv = 1.0 / sd
    c = -0.5 * np.log(2.0 * np.pi * sd * sd)
    a = c - 0.5 * ((d - mu) * inv) ** 2
    b = c - 0.5 * ((d + mu) * inv) ** 2
    return logsumexp(np.stack([a, b], 0), axis=0) - np.log(2.0)


def ll_iid(d, mu, sd):
    d = np.asarray(d, np.float64)
    inv = 1.0 / sd
    c = -0.5 * np.log(2.0 * np.pi * sd * sd)
    return c - 0.5 * ((d - mu) * inv) ** 2


def gate_series(fuse, hp30, emis_f, emis_h, kind, temp, ewma_tau, ll_fn):
    """g∈[0,1]; high g → trust fuse. Uses unlabeled evidence only."""
    f = np.asarray(fuse, np.float64)
    h = np.asarray(hp30, np.float64)
    n = min(len(f), len(h))
    f, h = f[:n], h[:n]
    if kind == "const_fuse":
        return np.ones(n, np.float64)
    if kind == "const_hp30":
        return np.zeros(n, np.float64)
    ll_f = ll_fn(f, *emis_f)
    ll_h = ll_fn(h, *emis_h)
    delta = ll_f - ll_h
    if kind == "hard_cum":
        c = np.cumsum(delta)
        return (c >= 0.0).astype(np.float64)
    if kind == "soft_raw":
        return expit(temp * delta)
    if kind == "soft_ewma":
        return expit(temp * ewma_1d(delta, ewma_tau))
    if kind == "hard_ewma":
        return (ewma_1d(delta, ewma_tau) >= 0.0).astype(np.float64)
    raise ValueError(kind)


def apply_gate_stream(fuse_st, hp_st, emis_f, emis_h, kind, temp, ewma_tau, ll_fn):
    out = {}
    for key in fuse_st:
        if key not in hp_st:
            continue
        f = np.asarray(fuse_st[key], np.float64)
        h = np.asarray(hp_st[key], np.float64)
        n = min(len(f), len(h))
        g = gate_series(f[:n], h[:n], emis_f, emis_h, kind, temp, ewma_tau, ll_fn)
        out[key] = (g * f[:n] + (1.0 - g) * h[:n]).astype(np.float64)
    return out


def apply_gate_all(fuse_all, hp_all, emis_f, emis_h, kind, temp, ewma_tau, ll_fn):
    return {ds: apply_gate_stream(fuse_all[ds], hp_all[ds], emis_f, emis_h,
                                  kind, temp, ewma_tau, ll_fn)
            for ds in fuse_all if ds in hp_all}


def candidate_rules():
    """Compact grid: baselines + soft/hard emission-LL gates."""
    rules = [
        ("const_fuse", "two_state", 1.0, 4.0),
        ("const_hp30", "two_state", 1.0, 4.0),
    ]
    # soft_ewma: temp × τ; hard_cum ignores both; hard_ewma uses τ only; soft_raw uses temp only
    for emis_kind in ("two_state", "iid"):
        rules.append(("hard_cum", emis_kind, 1.0, 4.0))
        for tau in (1.0, 2.0, 4.0, 8.0, 16.0):
            rules.append(("hard_ewma", emis_kind, 1.0, tau))
        for temp in (0.5, 1.0, 2.0, 4.0, 8.0):
            rules.append(("soft_raw", emis_kind, temp, 4.0))
            for tau in (1.0, 2.0, 4.0, 8.0, 16.0):
                rules.append(("soft_ewma", emis_kind, temp, tau))
    # dedupe
    seen, out = set(), []
    for r in rules:
        if r not in seen:
            seen.add(r)
            out.append(r)
    return out


def rule_key(kind, emis_kind, temp, tau):
    return f"{kind}|{emis_kind}|T{temp:g}|τ{tau:g}"


def emis_pair(fuse_all, hp_all, att, datasets, emis_kind):
    if emis_kind == "two_state":
        return (fit_signed_emis(fuse_all, att, datasets),
                fit_signed_emis(hp_all, att, datasets),
                ll_two_state)
    if emis_kind == "iid":
        return (fit_iid_emis(fuse_all, datasets),
                fit_iid_emis(hp_all, datasets),
                ll_iid)
    raise ValueError(emis_kind)


def fast_score(streams, ds, train_ds):
    mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
    return N.evaluate(ds, streams[ds], mu, sd * SCALE, P_STAY)


def inner_mean_for_rule(fuse_by_held, hp30, att, kind, emis_kind, temp, tau):
    """Mean Acc/H over (held, inner) cells; emissions for gate from the 3 legal train sets."""
    hs, acs = [], []
    for held in DS_KEYS:
        fuse = fuse_by_held[held]
        for inner in DS_KEYS:
            if inner == held:
                continue
            train = [d for d in DS_KEYS if d not in (held, inner)]
            emis_f, emis_h, ll_fn = emis_pair(fuse, hp30, att, train, emis_kind)
            gated = apply_gate_all(fuse, hp30, emis_f, emis_h, kind, temp, tau, ll_fn)
            r = fast_score(gated, inner, train)
            hs.append(r["H"])
            acs.append(r["Acc"])
    return float(np.mean(hs)), float(np.mean(acs)), len(hs)


def gated_for_held(fuse, hp30, att, held, kind, emis_kind, temp, tau):
    train = [d for d in DS_KEYS if d != held]
    emis_f, emis_h, ll_fn = emis_pair(fuse, hp30, att, train, emis_kind)
    gated = apply_gate_all(fuse, hp30, emis_f, emis_h, kind, temp, tau, ll_fn)
    return gated, dict(emis_f=emis_f, emis_h=emis_h)


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"domain_emis_ll CPU  p_stay={P_STAY} scale={SCALE}")
    log_line("loading LODO folds + fuse caches (no stem re-infer)")

    N.load_all()
    hp30, _mean4, lookahead = M.load_lodo_hp30()
    log_line(f"lookahead {lookahead}")
    att = N.G["att"]

    fuse_by_held = {}
    for held in DS_KEYS:
        path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
        if not os.path.isfile(path):
            raise FileNotFoundError(f"missing fuse cache {path}; run causal_moe_fuse first")
        fuse_by_held[held] = M.load_pkl(path)
        log_line(f"cache hit {path}")

    # sanity
    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")
    san_hp = M.score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    rules = candidate_rules()
    log_line(f"scoring {len(rules)} emission-LL gates on inner cells")
    inner_rows = []
    best = None
    for kind, emis_kind, temp, tau in rules:
        t0 = time.time()
        mean_h, mean_a, n = inner_mean_for_rule(
            fuse_by_held, hp30, att, kind, emis_kind, temp, tau)
        row = dict(kind=kind, emis_kind=emis_kind, temp=temp, ewma_tau=tau,
                   mean_inner_H=mean_h, mean_inner_Acc=mean_a, n=n,
                   key=rule_key(kind, emis_kind, temp, tau))
        inner_rows.append(row)
        log_line(f"  INNER {row['key']:40s} H {mean_h:.4f} Acc {mean_a:.2f} "
                 f"({time.time()-t0:.0f}s)")
        key = (mean_a, mean_h)
        if best is None or key > best[0]:
            best = (key, row)

    selected = best[1]
    log_line(f"SELECTED {selected['key']} inner H {selected['mean_inner_H']:.4f} "
             f"Acc {selected['mean_inner_Acc']:.2f}")

    # LODO held-out
    lodo = {}
    for held in DS_KEYS:
        gated, emis_info = gated_for_held(
            fuse_by_held[held], hp30, att, held,
            selected["kind"], selected["emis_kind"],
            selected["temp"], selected["ewma_tau"])
        off = M.score_lodo(gated, held)
        off_fuse = M.score_lodo(fuse_by_held[held], held)
        off_hp = M.score_lodo(hp30, held)
        lodo[held] = dict(gate=off, fuse=off_fuse, hp30=off_hp,
                          emis_f=emis_info["emis_f"], emis_h=emis_info["emis_h"])
        log_line(f"LODO held={held}  gate H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                 f"fuse {off_fuse['H']:.4f}/{off_fuse['Acc']:.2f}  "
                 f"hp30 {off_hp['H']:.4f}/{off_hp['Acc']:.2f}")

    mean_gate = dict(H=float(np.mean([lodo[d]["gate"]["H"] for d in DS_KEYS])),
                     Acc=float(np.mean([lodo[d]["gate"]["Acc"] for d in DS_KEYS])))
    log_line(f"LODO mean gate H {mean_gate['H']:.4f} Acc {mean_gate['Acc']:.2f}  "
             f"(fuse {FUSE_MEAN['H']:.4f}/{FUSE_MEAN['Acc']:.2f})")

    # External
    log_line("=== external (cached fuse + 4-seed hp30) ===")
    corp_hp, ext_hp, train_att, test_meta = M.load_ext_hp30()
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))

    emis_f, emis_h, ll_fn = emis_pair(
        corp_fuse, corp_hp, train_att, list(V4.CORPUS), selected["emis_kind"])
    corp_gated = apply_gate_all(
        corp_fuse, corp_hp, emis_f, emis_h,
        selected["kind"], selected["temp"], selected["ewma_tau"], ll_fn)
    ext_gated = apply_gate_all(
        ext_fuse, ext_hp, emis_f, emis_h,
        selected["kind"], selected["temp"], selected["ewma_tau"], ll_fn)

    mu_e, sd_e = F.emissions_from(corp_gated, train_att, datasets=list(V4.CORPUS))
    mu_f, sd_f = F.emissions_from(corp_fuse, train_att, datasets=list(V4.CORPUS))
    mu_h, sd_h = F.emissions_from(corp_hp, train_att, datasets=list(V4.CORPUS))

    external = {}
    for ds in TEST_NAMES:
        r_g = F.score_external(ext_gated[ds], test_meta[ds], mu_e, sd_e)
        r_f = F.score_external(ext_fuse[ds], test_meta[ds], mu_f, sd_f)
        r_h = F.score_external(ext_hp[ds], test_meta[ds], mu_h, sd_h)
        external[ds] = dict(gate=r_g, fuse=r_f, hp30=r_h)
        log_line(f"EXT {ds}  gate H {r_g['H']:.4f} Acc {r_g['Acc']:.2f}  "
                 f"fuse {r_f['H']:.4f}/{r_f['Acc']:.2f}  "
                 f"hp30 {r_h['H']:.4f}/{r_h['Acc']:.2f}  "
                 f"(published linear_hp30 {LINEAR_HP30_EXT[ds]['H']:.4f}/"
                 f"{LINEAR_HP30_EXT[ds]['Acc']:.2f})")

    # mean gate fraction on AASD (diagnostic; no labels used)
    g_frac = {}
    for ds in TEST_NAMES:
        fracs = []
        for key in ext_fuse[ds]:
            if key not in ext_hp[ds]:
                continue
            f = np.asarray(ext_fuse[ds][key], np.float64)
            h = np.asarray(ext_hp[ds][key], np.float64)
            n = min(len(f), len(h))
            g = gate_series(f[:n], h[:n], emis_f, emis_h,
                            selected["kind"], selected["temp"],
                            selected["ewma_tau"], ll_fn)
            fracs.append(float(g.mean()))
        g_frac[ds] = float(np.mean(fracs)) if fracs else float("nan")
        log_line(f"  mean g (fuse weight) on {ds}: {g_frac[ds]:.3f}")

    out = dict(
        p_stay=P_STAY, scale=SCALE,
        selected=selected,
        inner_rows=inner_rows,
        lodo=lodo, lodo_mean=mean_gate,
        external=external,
        ext_emis=dict(emis_f=emis_f, emis_h=emis_h),
        mean_g_frac=g_frac,
        baselines=dict(fuse_mean=FUSE_MEAN, linear_hp30_ext=LINEAR_HP30_EXT,
                       fuse_ext=FUSE_EXT),
        seconds=time.time() - t_all,
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT} in {time.time()-t_all:.0f}s")


if __name__ == "__main__":
    main()
