"""Reselect causal MoE fuse↔hp30 gate by inner Acc at lower p_stay.

Reuses streams/caches from causal_moe_fuse.py (CPU only; no stem re-infer).
Same pre-specified gate grid; selection metric changes only:

  For each gate, mean inner Acc (then H) on LODO train folds at
  p_stay ∈ {0.95, 0.99, 0.994}, emission scale 0.8.

Primary selector: Acc@0.95 (then H@0.95). Also report Acc@0.99 winner.
Final report still scores the selected gate at challenge preset 0.994/0.8
on LODO 5 + AASD + Carta. Never AASD/Carta labels for selection.

Distinct from Acc@0.994-only selector in results_moe_fuse.md.
Distinct from domain_emis_ll (emission-LL gate family).

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    .../.venv/bin/python -u moe_pstay_proxy.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np

import causal_moe_fuse as M
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "moe_pstay_proxy.log")
OUT = os.path.join(EXP, "results", "research", "moe_pstay_proxy.json")
CACHE_DIR = M.CACHE_DIR

# challenge preset (final report only)
P_STAY_FINAL = M.P_STAY  # 0.994
SCALE = M.SCALE          # 0.8
# selection proxies (shorter synthetic dwell → more real-switch-like)
P_STAY_PROXY = (0.95, 0.99, 0.994)
PRIMARY_P = 0.95
ALT_P = 0.99

TEST_NAMES = M.TEST_NAMES
FUSE_MEAN = M.FUSE_MEAN
FUSE_LODO = M.FUSE_LODO
LINEAR_HP30_EXT = M.LINEAR_HP30_EXT
FUSE_EXT = M.FUSE_EXT
LINEAR_HP30_LODO = M.LINEAR_HP30_LODO


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def fast_score_at(streams, ds, train_ds, p_stay, scale=SCALE):
    mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
    return N.evaluate(ds, streams[ds], mu, sd * scale, p_stay)


def inner_multi_pstay(fuse_by_held, hp30, kind, thr, sharp, p_stays=P_STAY_PROXY):
    """Mean Acc/H over all (held, inner) cells at each p_stay; emissions once per cell."""
    by_p = {p: ([], []) for p in p_stays}
    n_cells = 0
    for held in DS_KEYS:
        gated = M.apply_gate_all(fuse_by_held[held], hp30, kind, thr, sharp)
        for inner in DS_KEYS:
            if inner == held:
                continue
            train = [d for d in DS_KEYS if d not in (held, inner)]
            mu, sd = N.emis({d: gated[d] for d in train}, train)
            for p in p_stays:
                r = N.evaluate(inner, gated[inner], mu, sd * SCALE, p)
                by_p[p][0].append(r["H"])
                by_p[p][1].append(r["Acc"])
            n_cells += 1
    out = {}
    for p, (hs, acs) in by_p.items():
        out[p] = dict(mean_inner_H=float(np.mean(hs)),
                      mean_inner_Acc=float(np.mean(acs)),
                      n=n_cells)
    return out


def pick_best(rows, p_key):
    """Max Acc then H at the given p_stay key (stringified float)."""
    best = None
    for row in rows:
        s = row["by_pstay"][p_key]
        key = (s["mean_inner_Acc"], s["mean_inner_H"])
        if best is None or key > best[0]:
            best = (key, row)
    return best[1]


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"moe_pstay_proxy CPU  select@p_stay∈{list(P_STAY_PROXY)} "
             f"scale={SCALE}; final@{P_STAY_FINAL}/{SCALE}")
    log_line("loading LODO folds + fuse caches (no stem re-infer)")

    N.load_all()
    hp30, _mean4, lookahead = M.load_lodo_hp30()
    log_line(f"lookahead {lookahead}")

    fuse_by_held = {}
    for held in DS_KEYS:
        path = os.path.join(CACHE_DIR, f"fuse_w05__held-{held}.pkl")
        if not os.path.isfile(path):
            raise FileNotFoundError(f"missing fuse cache {path}; run causal_moe_fuse first")
        fuse_by_held[held] = M.load_pkl(path)
        log_line(f"cache hit {path}")

    san = M.score_lodo(fuse_by_held["AVGC"], "AVGC")
    log_line(f"sanity fuse AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} (expect ~0.8100/75.14)")
    san_hp = M.score_lodo(hp30, "AVGC")
    log_line(f"sanity hp30 AVGC H {san_hp['H']:.4f} Acc {san_hp['Acc']:.2f}")

    rules = M.candidate_rules()
    log_line(f"scoring {len(rules)} gates × {len(P_STAY_PROXY)} p_stay on inner cells")
    inner_rows = []
    for kind, thr, sharp in rules:
        t0 = time.time()
        by_p = inner_multi_pstay(fuse_by_held, hp30, kind, thr, sharp)
        # JSON keys must be strings
        by_pstay = {f"{p:g}": by_p[p] for p in P_STAY_PROXY}
        row = dict(kind=kind, thresh=thr, sharp=sharp,
                   key=M.rule_key(kind, thr, sharp), by_pstay=by_pstay)
        # flatten primary/alt for convenience
        row["Acc@0.95"] = by_p[0.95]["mean_inner_Acc"]
        row["H@0.95"] = by_p[0.95]["mean_inner_H"]
        row["Acc@0.99"] = by_p[0.99]["mean_inner_Acc"]
        row["H@0.99"] = by_p[0.99]["mean_inner_H"]
        row["Acc@0.994"] = by_p[0.994]["mean_inner_Acc"]
        row["H@0.994"] = by_p[0.994]["mean_inner_H"]
        inner_rows.append(row)
        log_line(
            f"  INNER {row['key']:32s} "
            f"@0.95 Acc {row['Acc@0.95']:.2f} H {row['H@0.95']:.4f}  "
            f"@0.99 Acc {row['Acc@0.99']:.2f} H {row['H@0.99']:.4f}  "
            f"@0.994 Acc {row['Acc@0.994']:.2f} H {row['H@0.994']:.4f}  "
            f"({time.time()-t0:.0f}s)"
        )

    selected_095 = pick_best(inner_rows, "0.95")
    selected_099 = pick_best(inner_rows, "0.99")
    selected_0994 = pick_best(inner_rows, "0.994")  # should match prior const_fuse

    log_line(f"SELECTED@0.95  {selected_095['key']}  "
             f"Acc {selected_095['Acc@0.95']:.2f} H {selected_095['H@0.95']:.4f}")
    log_line(f"SELECTED@0.99  {selected_099['key']}  "
             f"Acc {selected_099['Acc@0.99']:.2f} H {selected_099['H@0.99']:.4f}")
    log_line(f"SELECTED@0.994 {selected_0994['key']}  "
             f"Acc {selected_0994['Acc@0.994']:.2f} H {selected_0994['H@0.994']:.4f}")

    # Headline = Acc@0.95 winner (primary proxy)
    selected = selected_095
    log_line(f"HEADLINE gate = Acc@0.95 winner: {selected['key']}")

    # --- LODO at challenge preset 0.994/0.8 ---
    lodo = {}
    for held in DS_KEYS:
        gated = M.apply_gate_all(
            fuse_by_held[held], hp30,
            selected["kind"], selected["thresh"], selected["sharp"])
        off = M.score_lodo(gated, held)
        off_fuse = M.score_lodo(fuse_by_held[held], held)
        off_hp = M.score_lodo(hp30, held)
        lodo[held] = dict(moe=off, fuse=off_fuse, hp30=off_hp)
        log_line(f"LODO held={held}  moe H {off['H']:.4f} Acc {off['Acc']:.2f}  "
                 f"fuse {off_fuse['H']:.4f}/{off_fuse['Acc']:.2f}  "
                 f"hp30 {off_hp['H']:.4f}/{off_hp['Acc']:.2f}")

    mean_moe = dict(H=float(np.mean([lodo[d]["moe"]["H"] for d in DS_KEYS])),
                    Acc=float(np.mean([lodo[d]["moe"]["Acc"] for d in DS_KEYS])))
    log_line(f"LODO mean moe H {mean_moe['H']:.4f} Acc {mean_moe['Acc']:.2f}  "
             f"(fuse {FUSE_MEAN['H']:.4f}/{FUSE_MEAN['Acc']:.2f})")

    # --- External at preset ---
    log_line("=== external (cached fuse + 4-seed hp30) ===")
    corp_hp, ext_hp, train_att, test_meta = M.load_ext_hp30()
    corp_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_corpus.pkl"))
    ext_fuse = M.load_pkl(os.path.join(CACHE_DIR, "fuse_w05__all5_external.pkl"))

    corp_gated = M.apply_gate_all(
        corp_fuse, corp_hp, selected["kind"], selected["thresh"], selected["sharp"])
    ext_gated = M.apply_gate_all(
        ext_fuse, ext_hp, selected["kind"], selected["thresh"], selected["sharp"])

    mu_e, sd_e = F.emissions_from(corp_gated, train_att, datasets=list(V4.CORPUS))
    mu_f, sd_f = F.emissions_from(corp_fuse, train_att, datasets=list(V4.CORPUS))
    mu_h, sd_h = F.emissions_from(corp_hp, train_att, datasets=list(V4.CORPUS))

    external = {}
    for ds in TEST_NAMES:
        r_moe = F.score_external(ext_gated[ds], test_meta[ds], mu_e, sd_e)
        r_fuse = F.score_external(ext_fuse[ds], test_meta[ds], mu_f, sd_f)
        r_hp = F.score_external(ext_hp[ds], test_meta[ds], mu_h, sd_h)
        external[ds] = dict(moe=r_moe, fuse=r_fuse, hp30=r_hp)
        log_line(f"EXT {ds}  moe H {r_moe['H']:.4f} Acc {r_moe['Acc']:.2f}  "
                 f"fuse {r_fuse['H']:.4f}/{r_fuse['Acc']:.2f}  "
                 f"hp30 {r_hp['H']:.4f}/{r_hp['Acc']:.2f}  "
                 f"(published linear_hp30 {LINEAR_HP30_EXT[ds]['H']:.4f}/"
                 f"{LINEAR_HP30_EXT[ds]['Acc']:.2f})")

    # diagnostic: mean gate weight on external (no labels)
    g_frac = {}
    for ds in TEST_NAMES:
        fracs = []
        for key in ext_fuse[ds]:
            if key not in ext_hp[ds]:
                continue
            f = np.asarray(ext_fuse[ds][key], np.float64)
            h = np.asarray(ext_hp[ds][key], np.float64)
            n = min(len(f), len(h))
            g = M.gate_series(f[:n], h[:n], selected["kind"],
                              selected["thresh"], selected["sharp"])
            fracs.append(float(g.mean()))
        g_frac[ds] = float(np.mean(fracs)) if fracs else float("nan")
        log_line(f"  mean g (fuse weight) on {ds}: {g_frac[ds]:.3f}")

    # If Acc@0.99 picked a different gate, score it too (analysis; not headline)
    alt_ext = None
    alt_lodo_mean = None
    if selected_099["key"] != selected["key"]:
        log_line(f"=== alt Acc@0.99 gate {selected_099['key']} (analysis) ===")
        alt_lodo = {}
        for held in DS_KEYS:
            gated = M.apply_gate_all(
                fuse_by_held[held], hp30,
                selected_099["kind"], selected_099["thresh"], selected_099["sharp"])
            alt_lodo[held] = M.score_lodo(gated, held)
            log_line(f"  ALT LODO held={held} H {alt_lodo[held]['H']:.4f} "
                     f"Acc {alt_lodo[held]['Acc']:.2f}")
        alt_lodo_mean = dict(
            H=float(np.mean([alt_lodo[d]["H"] for d in DS_KEYS])),
            Acc=float(np.mean([alt_lodo[d]["Acc"] for d in DS_KEYS])))
        corp_a = M.apply_gate_all(
            corp_fuse, corp_hp,
            selected_099["kind"], selected_099["thresh"], selected_099["sharp"])
        ext_a = M.apply_gate_all(
            ext_fuse, ext_hp,
            selected_099["kind"], selected_099["thresh"], selected_099["sharp"])
        mu_a, sd_a = F.emissions_from(corp_a, train_att, datasets=list(V4.CORPUS))
        alt_ext = {}
        for ds in TEST_NAMES:
            r = F.score_external(ext_a[ds], test_meta[ds], mu_a, sd_a)
            alt_ext[ds] = r
            log_line(f"  ALT EXT {ds} H {r['H']:.4f} Acc {r['Acc']:.2f}")

    out = dict(
        method="moe_pstay_proxy",
        select_p_stays=list(P_STAY_PROXY),
        primary_select_p=PRIMARY_P,
        alt_select_p=ALT_P,
        final_p_stay=P_STAY_FINAL,
        scale=SCALE,
        selected_prespec=dict(
            kind=selected["kind"], thresh=selected["thresh"], sharp=selected["sharp"],
            key=selected["key"],
            Acc_at_095=selected["Acc@0.95"], H_at_095=selected["H@0.95"],
            Acc_at_099=selected["Acc@0.99"], H_at_099=selected["H@0.99"],
            Acc_at_0994=selected["Acc@0.994"], H_at_0994=selected["H@0.994"],
        ),
        selected_at_095=selected_095["key"],
        selected_at_099=selected_099["key"],
        selected_at_0994=selected_0994["key"],
        inner_rows=inner_rows,
        lodo=lodo, lodo_mean=mean_moe,
        external=external, mean_g_frac=g_frac,
        alt_099=dict(key=selected_099["key"], lodo_mean=alt_lodo_mean, external=alt_ext)
        if selected_099["key"] != selected["key"] else None,
        baselines=dict(
            fuse_lodo=FUSE_LODO, fuse_mean=FUSE_MEAN,
            linear_hp30_lodo_mean=LINEAR_HP30_LODO,
            linear_hp30_ext=LINEAR_HP30_EXT, fuse_ext=FUSE_EXT,
        ),
        seconds=time.time() - t_all,
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT} in {time.time()-t_all:.0f}s")


if __name__ == "__main__":
    main()
