"""Steps 1-2 of the improvement plan: better switch integrators on the SAVED LODO evidence (no retraining).

All integrators are causal forward filters (no extra lookahead: labels stamped exactly as in score.py). They run
on the GPU in one batch: one sequence per subject (its trials concatenated in corpus order, HMM state reset at every
trial start), all parameter configurations at once; only the time axis is sequential.

Integrator family (one config = one point of the grid below):
  * base    : 2-state HMM, Gaussian emissions N(+-mu, sd) fitted on the other 4 datasets (= score.py), p_stay
  * temp    : log-likelihood ratio scaled by `temp` (the 4 overlapping 1-s windows per second are not independent)
  * adapt   : step 1, unsupervised causal emission adaptation per subject (decision-directed): mu <- EW mean of
              d * (1 - 2 P(col1)), v <- EW mean of d^2, sd^2 = max(v - mu^2, 0.25 sd0^2); time constant tau_s,
              carried across the subject's trials (a device keeps adapting to its user); off when tau_s = 0
  * cusum   : step 2, state-guided switching (SGAD-like): CUSUM g of the evidence AGAINST the current decision;
              while g > h the HMM uses the fast p_fast instead of p_stay; g resets when the decision flips
Selection is nested: for held-out dataset D the config with the best mean H on the other 4 datasets is used for D.
The base config (p_stay 0.998, temp 1, no adapt, no cusum) must reproduce results/<exp>.json exactly (checked).

usage: python integrators.py EXP [EXP ...]
Writes results/integrators_<exp>.csv (every config x dataset) and results/integrators_<exp>.json (nested summary).
"""
import itertools
import json
import os
import sys
import time

import numpy as np
import pandas as pd
import torch
from joblib import Parallel, delayed

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import score as S  # noqa: E402
from common_exp import EXP, LAB, rows_of  # noqa: E402
from common import CACHE as EDA_CACHE, DS_KEYS, FS as FS128, SEED  # noqa: E402
from e1_switch import BUDGET, synth_trial  # noqa: E402
from scorer import aggregate, fast_score_subject  # noqa: E402

DEV = torch.device("cuda" if torch.cuda.is_available() else "cpu")
HOP_S = 0.25
LOG = __import__("sys").stdout


def log(m):
    line = f"{time.strftime('%F %T')} {m}"
    print(line, flush=True)
    LOG.write(line + "\n")


def grid():
    cf = []
    for p, temp, tau in itertools.product((0.995, 0.997, 0.998, 0.999, 0.9995), (0.25, 0.5, 1.0), (0, 60, 180, 600)):
        cf.append(dict(p_stay=p, temp=temp, tau_s=tau, p_fast=p, h=np.inf))
        for pf, h in itertools.product((0.98, 0.995), (1.0, 2.0, 4.0)):
            if pf < p:
                cf.append(dict(p_stay=p, temp=temp, tau_s=tau, p_fast=pf, h=h))
    return cf


def prepare(exp):
    """Synthetic switches exactly as score.py (same rng order), grouped per subject."""
    ev, _ = S.load_folds(exp, "causal")
    data = {}
    for ds in DS_KEYS:
        A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                            for d in DS_KEYS if d != ds for e in ev[d].values()])
        rng = np.random.default_rng(SEED + DS_KEYS.index(ds))
        subj = {}
        for _, r in rows_of(ds).iterrows():
            T = int(r["n"])
            e = ev[ds][r["key"]]
            att = np.load(os.path.join(EDA_CACHE, "cnd", f"{r['key']}.npz"))["att_col"][:T].astype(int)
            d = (e["r0"] - e["r1"]).astype(float)
            t_end = e["t_end"].astype(float)
            st, d_syn, _ = synth_trial(r["key"], T, d, t_end, rng)
            guess = int(rng.integers(0, 2))
            subj.setdefault(r["subj_uid"], []).append(dict(d=d_syn, t_end=t_end, T=T, gt=np.where(st, 1 - att, att),
                                                           guess=guess))
        data[ds] = dict(mu=float(A.mean()), sd=float(A.std()), subj=subj)
    return data


@torch.no_grad()
def run_gpu(dsd, cf):
    """Batched causal filtering. Returns {subject: [decisions (K, n_trial_steps) per trial]}."""
    subs = list(dsd["subj"])
    seqs = [np.concatenate([t["d"] for t in dsd["subj"][s]]) for s in subs]
    starts = [np.cumsum([0] + [len(t["d"]) for t in dsd["subj"][s]][:-1]) for s in subs]
    Sn, Tm, K = len(subs), max(len(x) for x in seqs), len(cf)
    D = torch.zeros(Sn, Tm, device=DEV, dtype=torch.float64)
    M = torch.zeros(Sn, Tm, device=DEV, dtype=torch.bool)
    B = torch.zeros(Sn, Tm, device=DEV, dtype=torch.bool)
    for i, (x, st) in enumerate(zip(seqs, starts)):
        D[i, :len(x)] = torch.from_numpy(x)
        M[i, :len(x)] = True
        B[i, torch.from_numpy(st.astype(np.int64))] = True
    par = lambda k: torch.tensor([c[k] for c in cf], device=DEV, dtype=torch.float64)[:, None]  # (K, 1)
    ls, lsw = torch.log(par("p_stay")), torch.log1p(-par("p_stay"))
    lf, lfw = torch.log(par("p_fast")), torch.log1p(-par("p_fast"))
    temp, h = par("temp"), par("h")
    tau = par("tau_s")
    a = torch.where(tau > 0, HOP_S / tau.clamp(min=1e-9), torch.zeros_like(tau))  # EW rate per step
    mu0, sd0 = dsd["mu"], dsd["sd"]
    mu = torch.full((K, Sn), mu0, device=DEV, dtype=torch.float64)
    v = torch.full((K, Sn), sd0 ** 2 + mu0 ** 2, device=DEV, dtype=torch.float64)
    lp = torch.full((K, Sn, 2), np.log(0.5), device=DEV, dtype=torch.float64)
    g = torch.zeros(K, Sn, device=DEV, dtype=torch.float64)
    prev = torch.zeros(K, Sn, device=DEV, dtype=torch.bool)
    out = torch.zeros(K, Sn, Tm, device=DEV, dtype=torch.bool)
    for t in range(Tm):
        x, m, b = D[:, t][None], M[:, t][None], B[:, t][None]
        fast = g > h
        s_, w_ = torch.where(fast, lf, ls), torch.where(fast, lfw, lsw)
        a0 = torch.logaddexp(lp[..., 0] + s_, lp[..., 1] + w_)
        a1 = torch.logaddexp(lp[..., 1] + s_, lp[..., 0] + w_)
        a0 = torch.where(b, torch.full_like(a0, np.log(0.5)), a0)          # trial start: uninformed state
        a1 = torch.where(b, torch.full_like(a1, np.log(0.5)), a1)
        var = (v - mu ** 2).clamp(min=0.25 * sd0 ** 2)
        llr = temp * 2.0 * mu * x / var                                    # log N(x; mu) - log N(x; -mu)
        n0, n1 = a0 + 0.5 * llr, a1 - 0.5 * llr
        z = torch.logaddexp(n0, n1)
        n0, n1 = n0 - z, n1 - z
        lp = torch.where(m[..., None], torch.stack([n0, n1], -1), lp)
        p1 = torch.exp(lp[..., 1])
        dec = p1 > 0.5
        out[:, :, t] = dec
        # CUSUM against the current decision, reset at a flip and at trial starts
        against = torch.where(dec, llr, -llr)
        g = torch.where(b | (dec != prev), torch.zeros_like(g), (g + against).clamp(min=0.0))
        g = torch.where(m, g, torch.zeros_like(g))
        prev = torch.where(m, dec, prev)
        # decision-directed emission adaptation (causal: used from the next step on)
        s = x * (1.0 - 2.0 * p1)
        mu = torch.where(m, mu + a * (s - mu), mu)
        v = torch.where(m, v + a * (x ** 2 - v), v)
    out = out.cpu().numpy()
    res = {}
    for i, s in enumerate(subs):
        res[s] = [out[:, i, st:st + len(tr["d"])] for st, tr in zip(starts[i], dsd["subj"][s])]
    return res


def expand(dec, t_end, L, T, guess):
    s0 = np.clip(np.ceil((t_end + L - BUDGET) * FS128).astype(int), 0, T)
    idx = np.searchsorted(s0, np.arange(T), side="right") - 1
    return np.where(idx >= 0, dec[np.maximum(idx, 0)], guess).astype(float)


def score_config(k, trials_dec, subj_trials, L):
    res = []
    for s, trs in subj_trials.items():
        pred = np.concatenate([expand(trials_dec[s][j][k], tr["t_end"], L, tr["T"], tr["guess"]) for j, tr in enumerate(trs)])
        gt = np.concatenate([tr["gt"] for tr in trs])
        trial = np.concatenate([np.full(tr["T"], j) for j, tr in enumerate(trs)])
        res.append(fast_score_subject(pred, gt, trial, fs=FS128))
    a = aggregate(res)
    return dict(Acc=a["Acc"], SDT=a["SDT"], L=a["L"], H=a["H"])


def main(exp):
    js = json.load(open(os.path.join(EXP, "results", f"{exp}.json")))
    L = js["causal"]["info"]["AVGC"]["lookahead_s"]
    cf = grid()
    log(f"=== {exp}: {len(cf)} integrator configs, lookahead {L} s, device {DEV}")
    t0 = time.time()
    data = prepare(exp)
    rows = []
    for ds in DS_KEYS:
        t1 = time.time()
        dec = run_gpu(data[ds], cf)
        t2 = time.time()
        sc = Parallel(n_jobs=int(os.environ.get("SCORE_JOBS", 4)))(delayed(score_config)(k, dec, data[ds]["subj"], L) for k in range(len(cf)))
        for k, r in enumerate(sc):
            rows.append(dict(ds=ds, k=k, **cf[k], **r))
        log(f"  {LAB[ds]}: GPU filter {t2 - t1:.0f}s, scoring {time.time() - t2:.0f}s")
    R = pd.DataFrame(rows)
    R.to_csv(os.path.join(EXP, "results", "research", f"integrators_{exp}.csv"), index=False)
    base = R[(R.p_stay == 0.998) & (R.temp == 1.0) & (R.tau_s == 0) & np.isinf(R.h)].set_index("ds")
    ref = js["causal"]["per_dataset"]
    dev_ = max(abs(base.loc[ds, "H"] - ref[ds]["H"]) for ds in DS_KEYS)
    log(f"  base config reproduces results/{exp}.json: max |dH| = {dev_:.2e}")
    # nested selection: best mean H on the other 4 datasets
    piv = R.pivot_table(index="k", columns="ds", values="H")
    sel = {}
    for ds in DS_KEYS:
        others = [d for d in DS_KEYS if d != ds]
        k = int(piv[others].mean(1).idxmax())
        r = R[(R.ds == ds) & (R.k == k)].iloc[0]
        sel[ds] = dict(k=k, cfg={c: (None if (isinstance(cf[k][c], float) and np.isinf(cf[k][c])) else cf[k][c]) for c in cf[k]},
                       Acc=r.Acc, SDT=r.SDT, L=r.L, H=r.H)
    mean = {m: float(np.mean([sel[d][m] for d in DS_KEYS])) for m in ("Acc", "SDT", "L", "H")}
    bmean = {m: float(base[m].mean()) for m in ("Acc", "SDT", "L", "H")}
    fam = {}
    for name, q in {"temp only": (R.tau_s == 0) & np.isinf(R.h), "adapt (step 1)": (R.tau_s > 0) & np.isinf(R.h),
                    "cusum (step 2)": (R.tau_s == 0) & np.isfinite(R.h), "adapt + cusum": (R.tau_s > 0) & np.isfinite(R.h)}.items():
        pv = R[q].pivot_table(index="k", columns="ds", values="H")
        hs = []
        for ds in DS_KEYS:
            k = pv[[d for d in DS_KEYS if d != ds]].mean(1).idxmax()
            hs.append(pv.loc[k, ds])
        fam[name] = float(np.mean(hs))
    out = dict(experiment=exp, n_configs=len(cf), baseline_mean=bmean, nested_mean=mean, per_dataset=sel,
               nested_H_by_family=fam, base_reproduction_max_dH=dev_)
    json.dump(out, open(os.path.join(EXP, "results", "research", f"integrators_{exp}.json"), "w"), indent=1, default=float)
    log(f"  baseline HMM   : " + "  ".join(f"{m} {bmean[m]:.3f}" for m in bmean))
    log(f"  nested best    : " + "  ".join(f"{m} {mean[m]:.3f}" for m in mean))
    log(f"  nested H by family: " + "  ".join(f"{n}: {h:.3f}" for n, h in fam.items()))
    log(f"=== done {exp} in {time.time() - t0:.0f}s")


if __name__ == "__main__":
    for e in sys.argv[1:]:
        main(e)
