"""B3 of REAL_SWITCH_STRATEGY.md: a LEARNED decision rule trained on switches, antisymmetric by construction.

Input: the fused evidence d_t (0.25-s steps; the same series the HMM filters). Features: causal, bias-corrected exponential moving averages of
d at several time constants (0.5 s .. 64 s); e_k(t) = sum_i w_i d_i / sum_i w_i, w_i = (1 - a_k)^(t - i): a weighted mean of the evidence that has
arrived, so there is no zero padding and no start-of-trial marker (the weights are normalised). Read-out
    y_t = sum_k a_k f_k + sum_j v_j tanh(sum_k W_kj f_k)       f_k = e_k / s_k   (s_k a positive scale)
with NO bias anywhere: y(-d) = -y(d) exactly, y > 0 = column 0. Nothing in it sees the position in the trial, the dataset, or the column order.
Target of step j: the true attended column over the 128-Hz samples this decision labels (soft fraction in [s0_j, s0_j+1), s0 = t_end + lookahead - 1 s,
exactly the stamping of e1_switch.expand), so the rule learns the 4-s lag of the evidence on real switches. BCE, weight decay, early stopping on
held-out SUBJECTS of the training data.
Training sets (never the evaluated set):
  V1 real only      target AASD <- Carta ; target Carta <- AASD ; synthetic LODO dataset D <- AASD + Carta
  V2 real + synth   target AASD <- Carta + synthetic of all 5 datasets ; target Carta <- AASD + synthetic ;
                    synthetic D <- AASD + Carta + synthetic of the other four datasets (their own LODO evidence and schedules); groups weighted 1 : 1
Evidence = the fixed best pipeline (4 + 4 models, a = 0.5), replicates: all four seeds, seeds {0,1}, seeds {2,3}; each with two training seeds.
Reference rows: HMM p_stay 0.998 and 0.95. Scoring = the official scorer through the same hooks as hmm_semimarkov.py.
usage: python hmm_learned.py [--selftest | --smoke | --summarize] -> results/research/learned_rule.csv, learned_rule.txt"""
import argparse
import multiprocessing as mp
import os

import numpy as np
import pandas as pd
from scipy.signal import lfilter

DT = 0.25
TAUS_S = (0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0)            # EMA time constants (s)
HID = 16
SEEDS = (0, 1)


def ema_features(x):
    """(n, K) bias-corrected causal EMAs of x; odd in x, scale-free in the position (normalised weights)."""
    x = np.asarray(x, dtype=np.float64)
    n = len(x)
    one = np.ones(n)
    out = np.empty((n, len(TAUS_S)))
    for k, tau in enumerate(TAUS_S):
        a = 1.0 - np.exp(-DT / tau)
        b, aa = [a], [1.0, -(1.0 - a)]
        out[:, k] = lfilter(b, aa, x) / lfilter(b, aa, one)
    return out


class Rule:
    """y = F @ lin + tanh(F @ W) @ v on F = ema_features(d) / scale; no biases (odd)."""

    def __init__(self, lin, W, v, scale):
        self.lin, self.W, self.v, self.scale = lin, W, v, scale

    def logit(self, d):
        F = ema_features(d) / self.scale
        return F @ self.lin + np.tanh(F @ self.W) @ self.v

    def decide(self, d):                                     # 1 = column 1 (as the HMM decision)
        return (self.logit(d) < 0).astype(int)


def fit_rule(trials, seed, epochs=60, patience=8):
    """trials: list of dict(F (n, K) features, t (n,) soft target for column 0, w weight per row, g group id for the subject split)."""
    import torch
    torch.set_num_threads(2)
    rs = np.random.default_rng(seed)
    groups = sorted({tr["g"] for tr in trials})
    val_g = set(rs.choice(len(groups), max(1, len(groups) // 7), replace=False).tolist())
    gi = {g: i for i, g in enumerate(groups)}
    is_val = np.array([gi[tr["g"]] in val_g for tr in trials])
    cat = lambda sel, key: np.concatenate([tr[key] for tr, s in zip(trials, sel) if s])
    F_tr, t_tr, w_tr = (cat(~is_val, k) for k in ("F", "t", "w"))
    F_va, t_va, w_va = (cat(is_val, k) for k in ("F", "t", "w"))
    scale = np.sqrt((F_tr ** 2).mean(0)) + 1e-9
    tt = lambda a: torch.from_numpy(np.ascontiguousarray(a, dtype=np.float32))
    Xtr, ytr, wtr, Xva, yva, wva = tt(F_tr / scale), tt(t_tr), tt(w_tr), tt(F_va / scale), tt(t_va), tt(w_va)
    torch.manual_seed(seed)
    K = F_tr.shape[1]
    lin = torch.nn.Parameter(torch.zeros(K))
    W = torch.nn.Parameter(0.3 * torch.randn(K, HID))
    v = torch.nn.Parameter(0.1 * torch.randn(HID))
    opt = torch.optim.Adam([lin, W, v], lr=3e-3, weight_decay=1e-4)
    f = lambda X: X @ lin + torch.tanh(X @ W) @ v
    bce = torch.nn.functional.binary_cross_entropy_with_logits
    best, bad, state = 1e9, 0, None
    for ep in range(epochs):
        perm = torch.randperm(len(Xtr))
        for i in range(0, len(perm), 8192):
            j = perm[i:i + 8192]
            loss = (bce(f(Xtr[j]), ytr[j], reduction="none") * wtr[j]).sum() / wtr[j].sum()
            opt.zero_grad()
            loss.backward()
            opt.step()
        with torch.no_grad():
            vl = ((bce(f(Xva), yva, reduction="none") * wva).sum() / wva.sum()).item()
        if vl < best - 1e-5:
            best, bad, state = vl, 0, [p.detach().clone() for p in (lin, W, v)]
        else:
            bad += 1
            if bad >= patience:
                break
    return Rule(state[0].numpy().astype(np.float64), state[1].numpy().astype(np.float64), state[2].numpy().astype(np.float64), scale), best, ep + 1


def soft_target(t_end, att128, la):
    """Fraction of column-1 samples among the 128-Hz samples each decision labels (expand's stamping); returns P(column 0) per step."""
    from common import FS as FS128
    T = len(att128)
    s0 = np.clip(np.ceil((np.asarray(t_end, float) + la - 1.0) * FS128).astype(int), 0, T)
    e = np.r_[s0[1:], T]
    cum = np.r_[0.0, np.cumsum(att128.astype(float))]
    ln = np.maximum(e - s0, 0)
    frac = np.where(ln > 0, (cum[e] - cum[s0]) / np.maximum(ln, 1), att128[np.minimum(s0, T - 1)])
    return 1.0 - frac


def selftest():
    rng = np.random.default_rng(0)
    d = rng.normal(size=600)
    lin, W, v, sc = rng.normal(size=8), rng.normal(size=(8, HID)), rng.normal(size=HID), np.ones(8)
    r = Rule(lin, W, v, sc)
    a, b = r.logit(d), r.logit(-d)
    print("antisymmetry max |y(-d) + y(d)| =", np.abs(a + b).max())
    assert np.abs(a + b).max() < 1e-12
    d2 = d.copy()
    d2[300:] = rng.normal(size=300) * 5
    c = r.logit(d2)
    print("causality: outputs before the change identical:", np.array_equal(a[:300], c[:300]), "| changed after:", not np.array_equal(a[300:], c[300:]))
    assert np.array_equal(a[:300], c[:300])
    F = ema_features(d)
    k = 2
    al = 1 - np.exp(-DT / TAUS_S[k])
    t = 100
    w = (1 - al) ** (t - np.arange(t + 1))
    print("EMA vs explicit weighted mean at t=100:", F[t, k], (w * d[:t + 1]).sum() / w.sum())
    assert abs(F[t, k] - (w * d[:t + 1]).sum() / w.sum()) < 1e-10
    print("constant evidence c: features at t = 0 and t = 500 both equal c (no start-of-trial cue):", ema_features(np.full(600, 0.7))[[0, 500]].round(6).tolist())
    z = np.zeros(1000)
    print("zero evidence -> y == 0:", np.abs(r.logit(z)).max())
    n = 3000
    share = np.zeros(3)
    for _ in range(n):
        y = r.logit(rng.normal(size=240))
        share += [np.mean(y[s:e] > 0) for s, e in ((0, 20), (20, 120), (120, 240))]
    print("random odd rule on noise: share y > 0 at 0-5 s, 5-30 s, 30-60 s:", (share / n).round(3))
    print("selftest ok")


# ---------------------------------------------------------------------------------------------------------------------
_TASK = None


def _run(a):
    return _TASK(a)


def main(smoke=False):
    import fuse_external as FX
    import fuse_general as FG
    import eval_external as E
    import score as S
    import e1_switch
    import scorer_kappa_next_inclusive as K
    S.fast_score_subject=K.fast_score_subject;S.aggregate=K.aggregate
    E.fast_score_subject=K.fast_score_subject;E.aggregate=K.aggregate
    from common import DS_KEYS, SEED
    from common_exp import EXP, rows_of
    from fuse_evidence import LA

    DEC = ["vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK" + t for t in ("", "_s1", "_s2", "_s3")]
    CLF = ["direct_clf__eaP_n5_t60_w10b3_ptSPK" + t + "_aug-sw2" for t in ("", "_s1", "_s2", "_s3")]
    REPS = {"all4": [0, 1, 2, 3]} if smoke else {"all4": [0, 1, 2, 3], "s01": [0, 1], "s23": [2, 3]}
    seeds = SEEDS[:1] if smoke else SEEDS
    dss = ["AASD", "DTU"] if smoke else ["AASD", "Carta2026"] + list(DS_KEYS)
    REAL = ("AASD", "Carta2026")
    CUR = {}

    def integ(d, p):
        if CUR["kind"] == "rule":
            return CUR["rule"].decide(d)
        return e1_switch.integ_hmm(d, dict(p_stay=CUR["p"], mu=p["mu"], sd=p["sd"]))

    S.integ_hmm = integ
    E.integ_hmm = integ

    def real_set(ev, ds, tag):
        out = []
        for k, e in ev.items():
            d = e["r0"].astype(float)
            out.append(dict(F=ema_features(d), t=soft_target(e["t_end"], e["att128"], LA), g=f"{ds}:{e['subject']}", w=None, ds=ds))
        return out

    def synth_truth(z, meta, ds):
        """Per trial 128-Hz ground truth of the synthetic schedule of ds (the one score_dataset uses)."""
        rng = np.random.default_rng(SEED + S.SCHED + DS_KEYS.index(ds))
        zd, zc = z[ds][0][ds], z[ds][1][ds]
        truth = {}
        for _, r in rows_of(ds).iterrows():
            T = int(r["n"])
            att = np.load(os.path.join(S.EDA_CACHE, "cnd", f"{r['key']}.npz"))["att_col"][:T].astype(int)
            d = 0.5 * zd[r["key"]] + 0.5 * zc[r["key"]]
            st, _, _ = e1_switch.synth_trial(r["key"], T, d, meta[ds][r["key"]][1].astype(float), rng)
            rng.integers(0, 2)
            truth[r["key"]] = (np.where(st, 1 - att, att).astype(float), r["subj_uid"])
        return truth

    def synth_set(z, meta, fold, ds, truth):
        zd, zc = z[fold]
        out = []
        for k, (gt, subj) in truth.items():
            d = 0.5 * zd[ds][k] + 0.5 * zc[ds][k]
            out.append(dict(F=ema_features(d), t=soft_target(meta[ds][k][1].astype(float), gt, LA), g=f"{ds}:{subj}", w=None, ds=ds))
        return out

    def weight(sets, share):
        """Row weights: every group of rows (a list of trials) gets total weight `share`, uniform inside."""
        for grp, sh in zip(sets, share):
            n = sum(len(tr["t"]) for tr in grp)
            for tr in grp:
                tr["w"] = np.full(len(tr["t"]), sh / n)
        return [tr for grp in sets for tr in grp]

    MODELS = {}
    info = []
    for rep, idx in REPS.items():
        print("building evidence", rep, flush=True)
        z, meta = FG.build([DEC[i] for i in idx], [CLF[i] for i in idx])
        decs, clfs = [[DEC[i], str(i)] for i in idx], [[CLF[i], "_w10b3sw2P", str(i)] for i in idx]
        ev_real, emis = FX.build_external(decs, clfs, [0.5], list(REAL), "_bestP")
        real = {ds: real_set(ev_real[ds][0.5], ds, rep) for ds in REAL}
        truth = {ds: synth_truth(z, meta, ds) for ds in DS_KEYS}
        for ts in seeds:
            def fit(name, sets, share):
                rule, vl, ne = fit_rule(weight([[dict(tr) for tr in g] for g in sets], share), ts, epochs=2 if smoke else 60)
                MODELS[(rep, ts, name)] = rule
                info.append(dict(rep=rep, seed=ts, model=name, val_bce=vl, epochs=ne))
                print(f"  {rep} seed {ts} {name}: val BCE {vl:.4f} after {ne} epochs", flush=True)
            for tgt, oth in (("AASD", "Carta2026"), ("Carta2026", "AASD")):
                fit(f"V1:{tgt}", [real[oth]], [1.0])
                allsyn = [synth_set(z, meta, ds, ds, truth[ds]) for ds in DS_KEYS]         # each dataset's own out-of-fold evidence
                fit(f"V2:{tgt}", [real[oth], [tr for g in allsyn for tr in g]], [0.5, 0.5])
            fit("V1:synth", [real["AASD"] + real["Carta2026"]], [1.0])
            for held in DS_KEYS:
                oth = [synth_set(z, meta, held, ds, truth[ds]) for ds in DS_KEYS if ds != held]
                fit(f"V2:{held}", [real["AASD"] + real["Carta2026"], [tr for g in oth for tr in g]], [0.5, 0.5])
        MODELS[(rep, "data")] = (z, meta, ev_real, emis[0.5])
    pd.DataFrame(info).to_csv(os.path.join(EXP, "results", "research", "learned_rule_kappa_next_inclusive_training.csv"), index=False)

    def task(a):
        rep, ds, kind, par = a
        CUR.clear()
        CUR["kind"] = kind
        z, meta, ev_real, emis = MODELS[(rep, "data")]
        if kind == "hmm":
            CUR["p"], name, ts = par, f"HMM {par}", -1
        else:
            variant, ts = par
            CUR["rule"] = MODELS[(rep, ts, "V1:synth" if (variant == "V1" and ds in DS_KEYS) else f"{variant}:{ds}")]
            name = variant
        if ds in DS_KEYS:
            zd, zc = z[ds]
            ev = {d: {k: dict(r0=0.5 * zd[d][k] + 0.5 * zc[d][k], r1=np.zeros(len(zd[d][k])), att=meta[d][k][0], t_end=meta[d][k][1],
                              pre_switched=True) for k in zd[d]} for d in DS_KEYS}
            r = S.score_dataset(ds, ev, LA, p_stay=0.998)
        else:
            r = E.score(ev_real[ds][0.5], *emis, 0.998, LA)
        return dict(rep=rep, ds=ds, rule=name, seed=ts, H=r["H"], Acc=r["Acc"], SDT=r["SDT"])

    tasks = []
    for rep in REPS:
        for ds in dss:
            for p in (0.998, 0.95):
                tasks.append((rep, ds, "hmm", p))
            for v in ("V1", "V2"):
                for ts in seeds:
                    tasks.append((rep, ds, "rule", (v, ts)))
    out = os.path.join(EXP, "results", "research", "learned_rule_kappa_next_inclusive" + ("_smoke" if smoke else "") + ".csv")
    if os.path.exists(out):
        os.remove(out)
    global _TASK
    _TASK = task
    rows = []
    with mp.get_context("fork").Pool(int(os.environ.get("SCORE_JOBS", 3))) as pool:
        for i, r in enumerate(pool.imap_unordered(_run, tasks, chunksize=1)):
            rows.append(r)
            pd.DataFrame([r]).to_csv(out, mode="a", header=not os.path.exists(out), index=False)
            if i % 10 == 0:
                print(f"{i + 1}/{len(tasks)} {r}", flush=True)
    summarize(pd.DataFrame(rows), os.path.join(EXP, "results", "research", "learned_rule_kappa_next_inclusive" + ("_smoke" if smoke else "") + ".txt"))


def summarize(df, path):
    lines = []

    def say(s=""):
        print(s, flush=True)
        lines.append(s)

    df = df.copy()
    df["fam"] = np.where(df.ds.isin(["AASD", "Carta2026"]), df.ds, "synthetic")
    g = df.groupby(["rep", "rule", "seed", "fam"])[["H", "Acc", "SDT"]].mean().reset_index()
    ref = "HMM 0.998"
    say("H by rule (mean over the two training seeds; synthetic = mean of the 5 LODO datasets); delta vs HMM 0.998 of the same replicate")
    for rep in g.rep.unique():
        t = g[g.rep == rep].groupby(["rule", "fam"]).H.mean().unstack()
        for c in [c for c in t.columns]:
            t["d" + c] = t[c] - t.loc[ref, c]
        say(f"\n[{rep}]\n" + t.round(4).to_string())
    say("\nPer training seed (all4): H of AASD / Carta / synthetic")
    s = g[g.rep == "all4"].pivot_table(index=["rule", "seed"], columns="fam", values="H")
    say(s.round(4).to_string())
    for rule in [r for r in df.rule.unique() if r.startswith("V")]:
        pv = df[(df.rep == "all4") & (df.rule.isin([ref, rule]))].groupby(["ds", "rule"])[["H", "Acc", "SDT"]].mean().unstack()
        say(f"\nPer dataset (all4), {rule} vs HMM 0.998 (H / Acc / SDT)\n" + pv.round(3).to_string())
    open(path, "w").write("\n".join(lines) + "\n")


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--selftest", action="store_true")
    ap.add_argument("--smoke", action="store_true")
    ap.add_argument("--summarize", action="store_true")
    a = ap.parse_args()
    if a.selftest:
        selftest()
    elif a.summarize:
        from common_exp import EXP
        summarize(pd.read_csv(os.path.join(EXP, "results", "research", "learned_rule_kappa_next_inclusive.csv")), os.path.join(EXP, "results", "research", "learned_rule_kappa_next_inclusive.txt"))
    else:
        main(a.smoke)
