"""Submission fusion for the combinations of stem_ensemble.py (members dec, clf, tr, bm, stem), e.g. ref_t_bm_stem50
(user, 2026-11-11). Same structure as sub_fuse.py (eq3).

calibrate --combo C   constants from the CORPUS only, exactly the outer 'ALL' branch of stem_ensemble.py: LDA target
                      weights and scales of dec / clf (r1_acc_decision_grid.components), signed-std scales of tr (rate-aux
                      Transformer trained on all five), bm (BMMS3 mean of 4 seeds) and stem (stem_ms96, mean of the five
                      fold models), the combination weights and the HMM emissions -> results/submission/calibration_<C>.json.
                      Check: the stored AASD / Carta evidence must reproduce the stem_ensemble.csv row of C at p 0.998.
predict --combo C --set S --out DIR   per test trial, causal, trial by trial; labels and CSV as sub_fuse.write_predictions.
Member weights: see stem_ensemble.COMBOS. Checkpoints: torch.load(weights_only=True); stem_ms96 numpy globals only.
usage: python sub_fuse_multi.py calibrate --combo ref_t_bm_stem50
       python sub_fuse_multi.py predict --combo ref_t_bm_stem50 --set TEST_DLAAD --out <submission>/dlaad
"""
import argparse
import json
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import numpy as np  # noqa: E402
import pandas as pd  # noqa: E402
import torch  # noqa: E402

import stem_ensemble as SE  # noqa: E402
import sub_fuse as SF  # noqa: E402
import tb_members as TB  # noqa: E402
from common import DS_KEYS  # noqa: E402
from common_exp import EXP  # noqa: E402

SEEDS = (0, 1, 2, 3)
P_STAY = 0.998
X16 = SE.X16
DEV = torch.device("cuda" if torch.cuda.is_available() else "cpu")
cal_path = lambda combo: os.path.join(SF.SUB, f"calibration_{combo}.json")


def dec_clf(c, x):
    W, Sc = np.asarray(c["W"]), np.asarray(c["Sc"])
    d = np.mean([x[:, 4 + 3 * i:7 + 3 * i] @ W[i] for i in range(4)], axis=0) / c["sd_dec"]
    k = np.mean([x[:, i] / Sc[i] for i in range(4)], axis=0) / c["sd_clf"]
    return d, k


def stem_at(models, clf_list, k, t_end):
    """stem_ms96 stream (mean of the 5 fold models) from the FULL 4-seed logit sequence, at the times t_end."""
    Dl = np.stack([(c[k]["r0"] - c[k]["r1"]).astype(np.float64) for c in clf_list], 1)
    o, _ = SE.stem_streams(models, list(DS_KEYS), Dl, DEV)
    return SF.bvec(dict(t_end=clf_list[0][k]["t_end"], r0=o, r1=0 * o), t_end)


def members(c, models, x, e, bm_list, clf_list, k):
    d, kk = dec_clf(c, x)
    p = dict(dec=d, clf=kk)
    w = c["weights"]
    if "tr" in w:
        p["tr"] = TB.t_score("ALL", TB.x16(x, **X16)) / c["scale"]["tr"]
    if "bm" in w:
        p["bm"] = np.mean([SF.bvec(b[k], e["t_end"]) for b in bm_list], axis=0) / c["scale"]["bm"]
    if "stem" in w:
        p["stem"] = stem_at(models, clf_list, k, e["t_end"]) / c["scale"]["stem"]
    return sum(v * p[m] for m, v in w.items())


def calibrate(combo):
    import eval_external as E
    import e1_switch
    import nested_schedule_eval as N
    import nested_stack_fusion as F
    import r1_acc_decision_grid as R
    import score as S
    weights = SE.COMBOS[combo]
    assert set(weights) <= {"dec", "clf", "tr", "bm", "stem"}, weights
    R.install_scorer()
    torch.set_num_threads(2)
    clf, info = zip(*[S.load_folds(e, "causal") for e in SE.CLF])
    dec = [S.load_folds(e, "causal")[0] for e in SE.DEC]
    bm = [S.load_folds(e, "causal")[0] for e in SE.BM]
    la = float(info[0][DS_KEYS[0]]["lookahead_s"])
    tab = {ds: F.corpus_table([v[ds] for v in dec], [v[ds] for v in clf], ds) for ds in DS_KEYS}
    trials = [(d, k, t) for d in DS_KEYS for k, t in tab[d].items()]
    x = np.concatenate([t["x"] * np.where(t["y"] == 0, 1., -1.)[:, None] for _, _, t in trials])
    W, Sc = [], []
    for i in SEEDS:                                         # r1_acc_decision_grid.components, written out
        v = x[:, 4 + 3 * i:7 + 3 * i].astype(float)
        w = np.linalg.solve(np.cov(v.T) + np.eye(3) * 1e-8, v.mean(0))
        W.append(w / max((v @ w).std(), 1e-8)); Sc.append(max(x[:, i].std(), 1e-8))
    zd = np.mean([x[:, 4 + 3 * i:7 + 3 * i] @ W[i] for i in SEEDS], axis=0)
    zc = np.mean([x[:, i] / Sc[i] for i in SEEDS], axis=0)
    c = dict(combo=combo, W=[w.tolist() for w in W], Sc=Sc, sd_dec=max(float(zd.std()), 1e-8), sd_clf=max(float(zc.std()), 1e-8),
             weights=weights, p_stay=P_STAY, lookahead_s=la, scale={})
    models = SE.stem_models(DEV) if "stem" in weights else None
    raw = {}
    for d, k, t in trials:
        r = {}
        if "tr" in weights:
            r["tr"] = TB.t_score("ALL", TB.x16(t["x"], **X16))
        if "bm" in weights:
            r["bm"] = TB.bm_mean([b[d][k] for b in bm], t["ev"]["t_end"], t["y"], SEEDS)
        if "stem" in weights:
            r["stem"] = stem_at(models, [cc[d] for cc in clf], k, t["ev"]["t_end"])
        raw[(d, k)] = r
    for m in ("tr", "bm", "stem"):
        if m in weights:
            c["scale"][m] = max(float(np.concatenate([raw[(d, k)][m] * np.where(t["y"] == 0, 1., -1.) for d, k, t in trials]).std()), 1e-8)

    def mix_corpus(d, k, t):
        dd, kk = dec_clf(c, t["x"])
        p = dict(dec=dd, clf=kk, **{m: v / c["scale"][m] for m, v in raw[(d, k)].items()})
        return sum(v * p[m] for m, v in weights.items())
    sg = np.concatenate([mix_corpus(d, k, t) * np.where(t["y"] == 0, 1., -1.) for d, k, t in trials])
    c["mu"], c["sd"] = float(sg.mean()), float(sg.std())
    json.dump(c, open(cal_path(combo), "w"), indent=1)
    print("calibration ->", cal_path(combo), json.dumps({k: v for k, v in c.items() if k not in ("W",)}), flush=True)
    # reproduction check against stem_ensemble.csv (outer ALL, p 0.998)
    ref = pd.read_csv(os.path.join(EXP, "results", "research", "stem_ensemble.csv"))
    ref = ref[(ref.combo == combo) & (ref.p_stay == P_STAY) & (ref.kind == "real")].set_index("ds")
    E.integ_hmm = lambda d, prm: e1_switch.integ_hmm(d, dict(p_stay=P_STAY, mu=prm["mu"], sd=prm["sd"]))
    ok = True
    for ds in ("AASD", "Carta2026"):
        pre = "aasd_c1" if ds == "AASD" else f"ext_c1_{ds}"
        RES = os.path.join(EXP, "results", "research")
        ed = [N.load(os.path.join(RES, f"{pre}_bestP_evidence_s{i}.pkl")) for i in SEEDS]
        ec = [N.load(os.path.join(RES, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")) for i in SEEDS]
        eb = [N.load(os.path.join(RES, f"ext_clf_{ds}_w10bmms3enhP_evidence_s{i}.pkl")) for i in SEEDS]
        ev = {}
        for k in ed[0]:
            xx, e = F.join(ed, ec, k)
            ev[k] = dict(e, r0=members(c, models, xx, e, eb, ec, k), r1=np.zeros(len(e["att"])))
        out = E.score(ev, c["mu"], c["sd"], P_STAY, la)
        want = float(ref.loc[ds, "Acc"])
        print(f"check {ds}: Acc {out['Acc']:.4f} H {out['H']:.4f} (stem_ensemble {combo}: Acc {want:.4f} H {ref.loc[ds, 'H']:.4f})", flush=True)
        ok &= abs(out["Acc"] - want) < 1e-3
    if not ok:
        raise SystemExit("calibration check FAILED")
    print("calibration check passed", flush=True)


def predict(combo, name, out_dir):
    import nested_stack_fusion as F
    from e1_switch import integ_hmm
    c = json.load(open(cal_path(combo)))
    torch.set_num_threads(2)
    models = SE.stem_models(DEV) if "stem" in c["weights"] else None
    ed, ec, eb = SF.load_evidence(name, "dec"), SF.load_evidence(name, "clf"), SF.load_evidence(name, "bmms")
    idx = pd.read_csv(os.path.join(EXP, "..", "external_data", name, "cache", "index.csv"))
    assert set(idx.key) == set(ed[0]) == set(ec[0]) == set(eb[0]), "evidence does not cover every trial"

    def decide(k):
        x, e = F.join(ed, ec, k)
        d = members(c, models, x, e, eb, ec, k)
        return integ_hmm(d, dict(p_stay=c["p_stay"], mu=c["mu"], sd=c["sd"])), e["t_end"], c["lookahead_s"]
    SF.write_predictions(name, out_dir, decide, combo)


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=("calibrate", "predict"))
    ap.add_argument("--combo", required=True, choices=sorted(SE.COMBOS))
    ap.add_argument("--set")
    ap.add_argument("--out")
    a = ap.parse_args()
    calibrate(a.combo) if a.cmd == "calibrate" else predict(a.combo, a.set, a.out)
