"""B1 of REAL_SWITCH_STRATEGY.md: HMM with a dwell-time distribution (semi-Markov) instead of a fixed stay probability.

The 2-state HMM has a geometric dwell time (memoryless, p_stay 0.998 = mean dwell 125 s). Here the dwell time is Weibull,
survival S(a) = exp(-(a dt / lam)^k), with
  m  mean dwell (s)       k < 1 heavy tail (short and long dwells in one prior), k = 1 geometric, k > 1 refractory after a switch
The filter keeps the age a (steps since the last switch) in both classes: P(class, age). The hazard h(a) = 1 - S(a + 1) / S(a) moves
the mass to the other class with age 0. The age at the trial start is drawn from the EQUILIBRIUM distribution pi(a) ~ S(a)
(a trial starts at an arbitrary moment of the process), so the prior does not know where the trial begins: no first-switch-after-30-s
assumption, no position-in-trial prior (organisers' warning). The dwell prior is one parameter pair for all datasets and is NOT tuned
to the synthetic schedule (gaps U(30, 90) s); the choice is cross-fitted (selected on one family, applied to the other).
Everything else is the fixed best pipeline: 4 vlaai_ws decoders + 4 antisymmetric classifiers (sw2), a = 0.5, DEC_OFFSET = 3, lookahead
0.75 s, emissions fitted on the corpus. Replicates (two seeds): ensemble of seeds {0,1} and of {2,3} (and all four = headline).
Real: AASD, Carta2026 (evidence from models trained on the 5 corpus datasets). Synthetic: LODO, five datasets, schedule offset 0.
usage: python hmm_semimarkov.py [--selftest] -> results/research/semimarkov.csv, semimarkov.txt"""
import argparse
import math
import multiprocessing as mp
import os
import sys

import numpy as np
import pandas as pd
from numba import njit

DT = 0.25                                                   # s per HMM step (hop of 8 samples at 32 Hz)
DMAX = 640                                                  # age bins (160 s); the last bin absorbs everything older
MEANS = (5.0, 10.0, 20.0, 40.0, 80.0, 125.0)
SHAPES = (0.6, 0.8, 1.0, 1.5, 2.5)
GEO = (0.998, 0.99, 0.95, 0.9)                              # reference: the usual geometric HMM


@njit(cache=True)
def sm_filter(ll0, ll1, h, pi):
    """P(class 1 | evidence up to t) for every t. State = (class, age index i = steps since the last switch, capped at D - 1)."""
    n, D = len(ll0), len(h)
    a0, a1 = 0.5 * pi.copy(), 0.5 * pi.copy()
    n0, n1 = np.empty(D), np.empty(D)
    out = np.empty(n)
    for t in range(n):
        if t > 0:
            s0, s1 = 0.0, 0.0
            for i in range(D):
                s0 += a0[i] * h[i]
                s1 += a1[i] * h[i]
            n0[0], n1[0] = s1, s0
            for i in range(1, D):
                n0[i] = a0[i - 1] * (1.0 - h[i - 1])
                n1[i] = a1[i - 1] * (1.0 - h[i - 1])
            n0[D - 1] += a0[D - 1] * (1.0 - h[D - 1])
            n1[D - 1] += a1[D - 1] * (1.0 - h[D - 1])
            a0, n0 = n0, a0
            a1, n1 = n1, a1
        m = max(ll0[t], ll1[t])
        e0, e1 = math.exp(ll0[t] - m), math.exp(ll1[t] - m)
        tot, p1 = 0.0, 0.0
        for i in range(D):
            a0[i] *= e0
            a1[i] *= e1
            tot += a0[i] + a1[i]
            p1 += a1[i]
        for i in range(D):
            a0[i] /= tot
            a1[i] /= tot
        out[t] = p1 / tot
    return out


def dwell_prior(mean_s, k):
    """Hazard h[0..D-1] and equilibrium age distribution pi[0..D-1] of a Weibull dwell with the given mean (s) and shape."""
    lam = mean_s / math.gamma(1.0 + 1.0 / k)
    a = np.arange(0, 400 * DMAX + 2) * DT
    S = np.exp(-(a / lam) ** k)                              # S[i] = P(dwell > i steps)
    h = 1.0 - S[1:DMAX + 1] / S[:DMAX]
    pi = S[:400 * DMAX].copy()
    pi[DMAX - 1] = pi[DMAX - 1:].sum()                       # older than the last bin: lumped
    pi = pi[:DMAX] / pi[:DMAX].sum()
    return np.clip(h, 1e-9, 1 - 1e-9), pi


def selftest():
    import common_exp  # noqa: F401  (puts reports/eda_full/scripts on sys.path)
    from e1_switch import hmm_filter
    rng = np.random.default_rng(0)
    d = np.cumsum(rng.normal(size=3000)) * 0.1 + rng.normal(size=3000)
    ll0, ll1 = -0.5 * (d - 0.4) ** 2, -0.5 * (d + 0.4) ** 2
    for p in (0.998, 0.95):
        h = np.full(DMAX, 1 - p)
        pi = (1 - h) ** np.arange(DMAX)
        pi[-1] = pi[-1] / h[0]
        pi /= pi.sum()
        a = sm_filter(ll0, ll1, h, pi)
        b = hmm_filter(ll0, ll1, p)
        print(f"geometric p_stay {p}: max |semi-Markov - HMM| = {np.abs(a - b).max():.2e}")
        assert np.abs(a - b).max() < 1e-6
    h, pi = dwell_prior(125.0, 1.0)
    print("Weibull k=1 mean 125 s: hazard", h[:2], "pi sum", pi.sum())
    # position invariance: with no evidence the age distribution stays at its equilibrium, so P(class) stays 0.5
    for m, k in ((20.0, 0.6), (20.0, 2.5)):
        h, pi = dwell_prior(m, k)
        z = sm_filter(np.zeros(1000), np.zeros(1000), h, pi)
        print(f"mean {m}, k {k}: P(class 1) without evidence, min/max over 1000 steps {z.min():.4f}/{z.max():.4f}")
    print("selftest ok")


# ---------------------------------------------------------------------------------------------------------------------
_TASK = None


def _run(args):
    return _TASK(args)


def main():
    import fuse_external as FX
    import fuse_general as FG
    import eval_external as E
    import score as S
    import e1_switch
    from common import DS_KEYS
    from common_exp import EXP
    from fuse_evidence import LA

    DEC = ["vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK" + t for t in ("", "_s1", "_s2", "_s3")]
    CLF = ["direct_clf__eaP_n5_t60_w10b3_ptSPK" + t + "_aug-sw2" for t in ("", "_s1", "_s2", "_s3")]
    REPS = {"all4": [0, 1, 2, 3], "s01": [0, 1], "s23": [2, 3]}
    CUR = {}

    def integ(d, p):
        if CUR["kind"] == "geo":
            return e1_switch.integ_hmm(d, dict(p_stay=CUR["p"], mu=p["mu"], sd=p["sd"]))
        ll0, ll1 = -0.5 * ((d - p["mu"]) / p["sd"]) ** 2, -0.5 * ((d + p["mu"]) / p["sd"]) ** 2
        return (sm_filter(ll0, ll1, CUR["h"], CUR["pi"]) > 0.5).astype(int)

    S.integ_hmm = integ
    E.integ_hmm = integ

    # ---- evidence of each replicate
    data = {}
    for rep, idx in REPS.items():
        print("building evidence", rep, flush=True)
        z, meta = FG.build([DEC[i] for i in idx], [CLF[i] for i in idx])
        decs, clfs = [[DEC[i], str(i)] for i in idx], [[CLF[i], "_w10b3sw2P", str(i)] for i in idx]
        ev_real, emis = FX.build_external(decs, clfs, [0.5], ["AASD", "Carta2026"], "_bestP")
        data[rep] = (z, meta, ev_real, emis[0.5])

    def task(args):
        rep, ds, kind, par = args
        CUR.clear()
        CUR["kind"] = kind
        if kind == "geo":
            CUR["p"] = par
        else:
            CUR["h"], CUR["pi"] = dwell_prior(*par)
        z, meta, ev_real, emis = data[rep]
        if ds in DS_KEYS:
            zd, zc = z[ds]
            ev = {d: {k: dict(r0=0.5 * zd[d][k] + 0.5 * zc[d][k], r1=np.zeros(len(zd[d][k])), att=meta[d][k][0], t_end=meta[d][k][1],
                              pre_switched=True) for k in zd[d]} for d in DS_KEYS}
            r = S.score_dataset(ds, ev, LA, p_stay=0.998)
        else:
            r = E.score(ev_real[ds][0.5], *emis, 0.998, LA)
        return dict(rep=rep, ds=ds, kind=kind, mean_s=(par if kind == "geo" else par[0]), k=(1.0 if kind == "geo" else par[1]),
                    p_stay=(par if kind == "geo" else np.nan), H=r["H"], Acc=r["Acc"], SDT=r["SDT"])

    points = [("geo", p) for p in GEO] + [("sm", (m, k)) for m in MEANS for k in SHAPES]
    dss = ["AASD", "Carta2026"] + list(DS_KEYS)
    tasks = [(rep, ds, kind, par) for rep in REPS for (kind, par) in points for ds in dss]
    out = os.path.join(EXP, "results", "research", "semimarkov.csv")
    if os.path.exists(out):
        os.remove(out)
    rows = []
    global _TASK
    _TASK = task                                            # forked workers inherit it; a local function cannot be pickled
    ctx = mp.get_context("fork")
    with ctx.Pool(int(os.environ.get("SCORE_JOBS", 3))) as pool:
        for i, r in enumerate(pool.imap_unordered(_run, tasks, chunksize=1)):
            rows.append(r)
            pd.DataFrame([r]).to_csv(out, mode="a", header=not os.path.exists(out), index=False)
            if i % 20 == 0:
                print(f"{i + 1}/{len(tasks)} {r}", flush=True)
    summarize(pd.DataFrame(rows), os.path.join(EXP, "results", "research", "semimarkov.txt"))


def summarize(df, path):
    lines = []

    def say(s=""):
        print(s, flush=True)
        lines.append(s)

    df = df.copy()
    df["pt"] = [f"geo p{p}" if kind == "geo" else f"m{m:g} k{k:g}" for kind, p, m, k in zip(df.kind, df.p_stay, df.mean_s, df.k)]
    syn = ["AVGC", "DTU", "KUL", "UHD", "USTC"]
    df["fam"] = np.where(df.ds.isin(["AASD", "Carta2026"]), df.ds, "synthetic")
    # H per (rep, point, family): synthetic = mean over the five datasets
    g = df.groupby(["rep", "pt", "fam"])[["H", "Acc", "SDT"]].mean().reset_index()
    pv = g.pivot_table(index=["rep", "pt"], columns="fam", values="H")
    pv["real"] = pv[["AASD", "Carta2026"]].mean(1)
    base = pv.xs("geo p0.998", level="pt")
    say("H, 4-seed ensemble (all4); synthetic = mean of the 5 LODO datasets; reference = geometric p_stay 0.998")
    t = pv.xs("all4", level="rep")
    t = t.assign(dAASD=t.AASD - t.loc["geo p0.998", "AASD"], dCarta=t.Carta2026 - t.loc["geo p0.998", "Carta2026"],
                 dSyn=t.synthetic - t.loc["geo p0.998", "synthetic"])
    say(t.round(4).sort_values("real", ascending=False).to_string())
    say("\nSelection (cross-fitted); each choice is scored on both replicates (seeds {0,1}, {2,3}) and on the headline all4:")
    names = list(pv.index.get_level_values("pt").unique())
    ref = "geo p0.998"
    choices = {}
    r4 = pv.xs("all4", level="rep")
    sm_pts = [n for n in names if n.startswith("m")]
    choices["best on AASD (applied to Carta)"] = r4.loc[sm_pts, "AASD"].idxmax()
    choices["best on Carta (applied to AASD)"] = r4.loc[sm_pts, "Carta2026"].idxmax()
    choices["best on real mean (applied to synthetic)"] = r4.loc[sm_pts, "real"].idxmax()
    choices["best on synthetic (applied to real)"] = r4.loc[sm_pts, "synthetic"].idxmax()
    geo_pts = [n for n in sm_pts if n.endswith("k1")]
    choices["best geometric-shape (k 1) on real mean"] = r4.loc[geo_pts, "real"].idxmax()
    rows = []
    for why, pt in choices.items():
        for rep in ("s01", "s23", "all4"):
            for fam in ("AASD", "Carta2026", "synthetic"):
                rows.append(dict(rule=why, point=pt, rep=rep, fam=fam, base=pv.loc[(rep, ref), fam], semi=pv.loc[(rep, pt), fam]))
    c = pd.DataFrame(rows)
    c["delta"] = c.semi - c.base
    say(c.round(4).to_string(index=False))
    say("\nPer dataset (all4), the chosen point 'best on real mean' vs geometric 0.998: H / Acc / SDT")
    pt = choices["best on real mean (applied to synthetic)"]
    q = df[(df.rep == "all4") & (df.pt.isin([ref, pt]))].pivot_table(index="ds", columns="pt", values=["H", "Acc", "SDT"])
    say(q.round(3).to_string())
    open(path, "w").write("\n".join(lines) + "\n")


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--selftest", action="store_true")
    ap.add_argument("--summarize", action="store_true", help="re-print the summary from results/research/semimarkov.csv")
    a = ap.parse_args()
    if a.selftest:
        selftest()
    elif a.summarize:
        from common_exp import EXP
        summarize(pd.read_csv(os.path.join(EXP, "results", "research", "semimarkov.csv")), os.path.join(EXP, "results", "research", "semimarkov.txt"))
    else:
        main()
