"""Nested no-decoder search on the four direct-classifier logits.

For each held-out dataset D the hyperparameters are chosen only on the other four datasets:
each inner dataset A is scored with LDA weights and HMM emissions fit on the three datasets
that are neither D nor A. The winning setting is then refit on all four and D is scored once
with score.score_dataset. An oracle sweep is stored separately and is not the result.

Families (all causal, past only):
  lda4  the four seed logits (the previous best)
  dyn   logits + 1 s causal slope + 1 s causal mean
Transforms of the logits before LDA: id, hp60 (subtract 60 s causal EMA), z60 (divide by
causal running SD, tau 60 s), sm1 (1 s causal EMA).
HMM knobs inside the official Gaussian emission model: p_stay, and a temperature implemented
as emis_sd * scale (scale < 1 makes the chain switch faster).

usage: python ensemble_clf_nested.py
writes results/research/ensemble_clf_nested.csv and ensemble_clf_nested_inner.csv
"""
import os

import numpy as np
import pandas as pd
from joblib import Parallel, delayed

import score as S
from common import DS_KEYS, SEED
from e1_switch import expand, integ_hmm, synth_trial
from scorer import aggregate, fast_score_subject

import ensemble_clf as E

try:
    import numba
except ImportError:
    numba = None

OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "results", "research", "ensemble_clf_nested.csv")
INNER_OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "results", "research", "ensemble_clf_nested_inner.csv")

FAMILIES = (("lda4", 0.0), ("dyn", 0.1))
XFORMS = ("id", "hp60", "z60", "sm1")
P_STAY = (0.99, 0.995, 0.997, 0.998, 0.999)
SCALES = (0.55, 0.75, 1.0, 1.25)
SMOOTH = (1.0,)

G = {}


def _njit(fn):
    return numba.njit(cache=False)(fn) if numba is not None else fn


@_njit
def ema_nb(x, alpha):
    y = np.empty(x.shape[0])
    if x.shape[0] == 0:
        return y
    b = 1.0 - alpha
    y[0] = alpha * x[0]
    for t in range(1, x.shape[0]):
        y[t] = alpha * x[t] + b * y[t - 1]
    return y


@_njit
def running_div(x, tau_steps):
    n = x.shape[0]
    y = np.empty(n)
    a = 1.0 / tau_steps
    m2 = x[0] * x[0] if n else 1.0
    for t in range(n):
        sd = np.sqrt(m2) if m2 > 1e-12 else 1e-6
        y[t] = x[t] / sd
        m2 = m2 + a * (x[t] * x[t] - m2)
    return y


@_njit
def integ_nb(d, p_stay, mu, sd):
    n = d.shape[0]
    out = np.empty(n, np.int8)
    inv = 1.0 / sd
    lp0 = np.log(0.5)
    lp1 = np.log(0.5)
    ls = np.log(p_stay)
    lsw = np.log(1.0 - p_stay)
    for t in range(n):
        if t > 0:
            a = np.logaddexp(lp0 + ls, lp1 + lsw)
            b = np.logaddexp(lp1 + ls, lp0 + lsw)
            lp0, lp1 = a, b
        z0 = (d[t] - mu) * inv
        z1 = (d[t] + mu) * inv
        lp0 = lp0 - 0.5 * z0 * z0
        lp1 = lp1 - 0.5 * z1 * z1
        nrm = np.logaddexp(lp0, lp1)
        lp0 = lp0 - nrm
        lp1 = lp1 - nrm
        out[t] = 1 if np.exp(lp1) > 0.5 else 0
    return out


def ema_2d(X, alpha):
    Y = np.empty(X.shape, np.float64)
    for j in range(X.shape[1]):
        Y[:, j] = ema_nb(np.ascontiguousarray(X[:, j]), alpha)
    return Y


def xform_array(D, name):
    if name == "id":
        return D
    if name == "hp60":
        alpha = 1.0 - np.exp(-0.25 / 60.0)
        return D - ema_2d(D, alpha)
    if name == "sm1":
        alpha = 1.0 - np.exp(-0.25 / 1.0)
        return ema_2d(D, alpha)
    if name == "z60":
        Y = np.empty(D.shape, np.float64)
        steps = 60.0 / 0.25
        for j in range(D.shape[1]):
            Y[:, j] = running_div(np.ascontiguousarray(D[:, j]), steps)
        return Y
    raise ValueError(name)


def featurize(D, family):
    if family == "lda4":
        return D
    if family != "dyn":
        raise ValueError(family)
    slope = np.zeros_like(D)
    if len(D) > 4:
        slope[4:] = D[4:] - D[:-4]
    cs = np.vstack([np.zeros((1, D.shape[1])), np.cumsum(D, axis=0)])
    idx = np.arange(len(D))
    left = np.maximum(0, idx - 3)
    mean = (cs[idx + 1] - cs[left]) / (idx - left + 1)[:, None]
    return np.concatenate([D, slope, mean], axis=1)


def w_from_stats(parts, ridge):
    n = 0
    s = None
    Gm = None
    for ni, si, Gi in parts:
        n += ni
        s = si.copy() if s is None else s + si
        Gm = Gi.copy() if Gm is None else Gm + Gi
    mean = s / n
    C = (Gm - np.outer(s, s) / n) / (n - 1)
    if ridge:
        C = C + ridge * float(np.trace(C)) / C.shape[0] * np.eye(C.shape[0])
    w = np.linalg.solve(C, mean)
    return w / np.abs(w).sum()


def dataset_stats(ds, xform, family):
    parts_n = 0
    s = None
    Gm = None
    att = G["att"][ds]
    for key, D in G["base"][xform][ds].items():
        F = featurize(D, family)
        sign = np.where(att[key] == 0, 1.0, -1.0)[:, None]
        X = F * sign
        parts_n += len(X)
        si = X.sum(0)
        Gi = X.T @ X
        s = si if s is None else s + si
        Gm = Gi if Gm is None else Gm + Gi
    return parts_n, s, Gm


def apply_streams(xform, family, w, datasets):
    out = {}
    for ds in datasets:
        out[ds] = {}
        for key, D in G["base"][xform][ds].items():
            out[ds][key] = featurize(D, family) @ w
    return out


def smooth_map(dmap, alpha):
    if alpha >= 0.999:
        return dmap
    return {k: ema_nb(np.ascontiguousarray(v), alpha) for k, v in dmap.items()}


def emis(streams, datasets):
    parts = []
    for ds in datasets:
        att = G["att"][ds]
        for key, d in streams[ds].items():
            parts.append(d * np.where(att[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def evaluate(ds, dmap, mu, sd, p_stay):
    per = {}
    L = G["lookahead"]
    for item in G["prep"][ds]:
        dec = integ_nb(np.ascontiguousarray(dmap[item["key"]], np.float64), p_stay, mu, sd)
        labs = expand(dec.astype(np.float64), item["t_end"], L, item["T"], item["guess"])
        per.setdefault(item["subj"], []).append((labs, item["gt"]))
    res = []
    for lst in per.values():
        pred = np.concatenate([a for a, _ in lst])
        gt = np.concatenate([b for _, b in lst])
        trial = np.concatenate([np.full(len(a), i, np.int64) for i, (a, _) in enumerate(lst)])
        res.append(fast_score_subject(pred, gt, trial, fs=128))
    a = aggregate(res)
    per_acc = {}
    att = G["att"][ds]
    for item in G["prep"][ds]:
        d = dmap[item["key"]]
        sgn = np.where(att[item["key"]][::4] == 0, 1.0, -1.0)
        per_acc.setdefault(item["subj"], []).append(d[::4] * sgn)
    acc1 = float(np.median([np.mean(np.concatenate(v) > 0) for v in per_acc.values()]))
    return dict(H=float(a["H"]), Acc=float(a["Acc"]), SDT=float(a["SDT"]), L=float(a["L"]), acc1=acc1)


def block(outer, inner, family, ridge, xform):
    train = [d for d in DS_KEYS if d not in (outer, inner)]
    w = w_from_stats([G["stats"][xform][family][d] for d in train], ridge)
    streams = apply_streams(xform, family, w, train + [inner])
    mu, sd = emis(streams, train)
    rows = []
    dmap = streams[inner]
    for p in P_STAY:
        for scale in SCALES:
            for alpha in SMOOTH:
                sm = smooth_map(dmap, alpha)
                # emissions on the same smoothing, train side only
                sm_train = {d: smooth_map(streams[d], alpha) for d in train}
                mu_s, sd_s = emis(sm_train, train) if alpha < 0.999 else (mu, sd)
                r = evaluate(inner, sm, mu_s, sd_s * scale, p)
                rows.append(dict(outer=outer, inner=inner, family=family, xform=xform, ridge=ridge,
                                 p_stay=p, scale=scale, smooth=alpha, **r))
    print(f"inner {outer}<-{inner} {family} {xform}  H {np.mean([x['H'] for x in rows]):.4f}", flush=True)
    return rows


def streams_refit(outer, family, ridge, xform, smooth):
    train = [d for d in DS_KEYS if d != outer]
    w = w_from_stats([G["stats"][xform][family][d] for d in train], ridge)
    raw = apply_streams(xform, family, w, DS_KEYS)
    out = {d: smooth_map(raw[d], smooth) for d in DS_KEYS}
    return out, w


def to_ev(streams):
    ev = {}
    for ds in DS_KEYS:
        ev[ds] = {}
        att = G["att"][ds]
        tend = G["tend"][ds]
        for item in G["prep"][ds]:
            k = item["key"]
            d = np.asarray(streams[ds][k], np.float64)
            ev[ds][k] = dict(r0=d, r1=np.zeros_like(d), att=att[k], t_end=tend[k], pre_switched=True)
    return ev


def official(ds, ev, p_stay, scale):
    train = [d for d in DS_KEYS if d != ds]
    A = np.concatenate([(e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0) for d in train for e in ev[d].values()])
    mu, sd = float(A.mean()), float(A.std())
    r = S.score_dataset(ds, ev, G["lookahead"], emis=(mu, sd * scale), p_stay=p_stay)
    dmap = {k: e["r0"] - e["r1"] for k, e in ev[ds].items()}
    acc1 = evaluate(ds, dmap, mu, sd * scale, p_stay)["acc1"]
    return dict(H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r["L"], acc1=acc1, emis_mu=mu, emis_sd=sd)


def pick(df):
    """Max mean inner H, then higher Acc, then lower SDT. Uses only rows already restricted to one outer fold."""
    g = df.groupby(["family", "xform", "p_stay", "scale", "smooth"], sort=False)[["H", "Acc", "SDT", "L", "acc1"]].mean()
    g = g.reset_index().sort_values(["H", "Acc", "SDT"], ascending=[False, False, True])
    return g.iloc[0]


def load_all():
    evs = []
    lookahead = None
    for _, exp in E.SEEDS:
        ev, info = S.load_folds(exp, "causal")
        for ds in DS_KEYS:
            la = float(info[ds]["lookahead_s"])
            if lookahead is None:
                lookahead = la
            elif abs(la - lookahead) > 1e-9:
                raise RuntimeError("lookahead mismatch")
        evs.append(ev)
        print(f"loaded {exp}", flush=True)
    base_id = {ds: {} for ds in DS_KEYS}
    att, tend = {ds: {} for ds in DS_KEYS}, {ds: {} for ds in DS_KEYS}
    prep = {ds: [] for ds in DS_KEYS}
    for ds in DS_KEYS:
        rng = np.random.default_rng(SEED + DS_KEYS.index(ds))
        ref = evs[0][ds]
        for _, r in S.rows_of(ds).iterrows():
            k = r["key"]
            cols = [E.delta(ev[ds][k]) for ev in evs]
            D = np.stack(cols, 1).astype(np.float64)
            base_id[ds][k] = D
            att[ds][k] = ref[k]["att"]
            tend[ds][k] = ref[k]["t_end"].astype(np.float64)
            T = int(r["n"])
            st, _, _ = synth_trial(k, T, cols[0], tend[ds][k], rng)
            guess = int(rng.integers(0, 2))
            lab = np.load(os.path.join(S.EDA_CACHE, "cnd", f"{k}.npz"))
            raw_att = lab["att_col"][:T].astype(int)
            gt = np.where(st, 1 - raw_att, raw_att)
            prep[ds].append(dict(key=k, subj=r["subj_uid"], T=T, t_end=tend[ds][k], guess=guess, gt=gt))
    G["lookahead"] = lookahead
    G["att"] = att
    G["tend"] = tend
    G["prep"] = prep
    print("building causal transforms", flush=True)
    G["base"] = {name: {ds: {} for ds in DS_KEYS} for name in XFORMS}
    for ds in DS_KEYS:
        for k, D in base_id[ds].items():
            for name in XFORMS:
                G["base"][name][ds][k] = D if name == "id" else xform_array(D, name)
    print("sufficient statistics", flush=True)
    G["stats"] = {xf: {} for xf in XFORMS}
    for xf in XFORMS:
        for family, _ridge in FAMILIES:
            G["stats"][xf][family] = {ds: dataset_stats(ds, xf, family) for ds in DS_KEYS}
            print(f"  stats {xf} {family}", flush=True)


def check_cov_formula():
    rng = np.random.default_rng(0)
    X = rng.normal(size=(2000, 4))
    C1 = np.cov(X.T)
    n, s, Gm = len(X), X.sum(0), X.T @ X
    C2 = (Gm - np.outer(s, s) / n) / (n - 1)
    if not np.allclose(C1, C2):
        raise RuntimeError("covariance formula does not match np.cov")


def check_fast_matches_official():
    """lda4 / id / p_stay 0.998 / scale 1 must match score.score_dataset on every held-out dataset."""
    rows = []
    for ds in DS_KEYS:
        streams, _w = streams_refit(ds, "lda4", 0.0, "id", 1.0)
        ev = to_ev(streams)
        off = official(ds, ev, 0.998, 1.0)
        mu, sd = emis(streams, [d for d in DS_KEYS if d != ds])
        fast = evaluate(ds, streams[ds], mu, sd, 0.998)
        rows.append((ds, off, fast))
        print(f"check {ds} official H {off['H']:.4f} Acc {off['Acc']:.2f}  fast H {fast['H']:.4f} Acc {fast['Acc']:.2f}", flush=True)
        if abs(off["H"] - fast["H"]) > 1e-6 or abs(off["Acc"] - fast["Acc"]) > 1e-6:
            raise RuntimeError(f"fast scorer diverges on {ds}")
    return rows


def _init(base, att, prep, stats, lookahead):
    G["base"] = base
    G["att"] = att
    G["prep"] = prep
    G["stats"] = stats
    G["lookahead"] = lookahead


def main():
    check_cov_formula()
    load_all()
    # warm numba
    integ_nb(np.zeros(4), 0.998, 0.1, 1.0)
    ema_nb(np.zeros(4), 0.5)
    checks = check_fast_matches_official()
    base_H = float(np.mean([c[1]["H"] for c in checks]))
    base_Acc = float(np.mean([c[1]["Acc"] for c in checks]))
    print(f"reproduced lda4 @ 0.998  H {base_H:.4f}  Acc {base_Acc:.2f}", flush=True)

    tasks = [(outer, inner, fam, ridge, xf)
             for outer in DS_KEYS for inner in DS_KEYS if inner != outer
             for fam, ridge in FAMILIES for xf in XFORMS]
    n_jobs = int(os.environ.get("SCORE_JOBS", "6"))
    parts = Parallel(n_jobs=n_jobs, initializer=_init,
                     initargs=(G["base"], G["att"], G["prep"], G["stats"], G["lookahead"]))(
        delayed(block)(*t) for t in tasks)
    inner = pd.DataFrame([r for part in parts for r in part])
    os.makedirs(os.path.dirname(INNER_OUT), exist_ok=True)
    inner.to_csv(INNER_OUT, index=False)
    print(f"wrote {INNER_OUT}", flush=True)

    chosen = []
    for outer in DS_KEYS:
        best = pick(inner[inner.outer == outer])
        chosen.append(best)
        print(f"select {outer}: {best.family} {best.xform} p {best.p_stay} scale {best.scale} "
              f"smooth {best.smooth}  inner H {best.H:.4f} Acc {best.Acc:.2f}", flush=True)

    # refine p_stay / scale / smooth around the inner winner, still without the held-out labels
    refine_tasks = []
    for outer, best in zip(DS_KEYS, chosen):
        for inner_ds in DS_KEYS:
            if inner_ds == outer:
                continue
            refine_tasks.append((outer, inner_ds, best))

    def refine_block(outer, inner_ds, best):
        family, xform = best.family, best.xform
        ridge = dict(FAMILIES)[family]
        train = [d for d in DS_KEYS if d not in (outer, inner_ds)]
        w = w_from_stats([G["stats"][xform][family][d] for d in train], ridge)
        streams = apply_streams(xform, family, w, train + [inner_ds])
        rows = []
        ps = sorted(set([round(float(best.p_stay) + d, 5) for d in (-0.002, -0.001, 0.0, 0.001)]
                        + [float(best.p_stay)]))
        ps = [p for p in ps if 0.9 < p < 0.9995]
        scales = sorted(set([round(float(best.scale) * m, 4) for m in (0.85, 1.0, 1.15)]))
        for p in ps:
            for scale in scales:
                for alpha in (1.0, 0.5, 0.25):
                    sm_train = {d: smooth_map(streams[d], alpha) for d in train}
                    mu, sd = emis(sm_train, train)
                    sm = smooth_map(streams[inner_ds], alpha)
                    r = evaluate(inner_ds, sm, mu, sd * scale, p)
                    rows.append(dict(outer=outer, inner=inner_ds, family=family, xform=xform, ridge=ridge,
                                     p_stay=p, scale=scale, smooth=alpha, **r))
        return rows

    print("refining around inner winners", flush=True)
    rparts = Parallel(n_jobs=n_jobs, initializer=_init,
                      initargs=(G["base"], G["att"], G["prep"], G["stats"], G["lookahead"]))(
        delayed(refine_block)(*t) for t in refine_tasks)
    refined = pd.DataFrame([r for part in rparts for r in part])
    refined.to_csv(INNER_OUT.replace(".csv", "_refine.csv"), index=False)

    rows = []
    for ds, off, fast in checks:
        rows.append(dict(role="baseline", ds=ds, family="lda4", xform="id", p_stay=0.998, scale=1.0, smooth=1.0,
                         H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"], acc1=off["acc1"],
                         inner_H=np.nan, scorer="official"))

    for outer, coarse in zip(DS_KEYS, chosen):
        best = pick(refined[refined.outer == outer])
        # keep the coarse winner if refine did not improve inner H
        if float(best.H) < float(coarse.H) - 1e-12:
            best = coarse
        family, xform = best.family, best.xform
        ridge = float(dict(FAMILIES)[family])
        streams, w = streams_refit(outer, family, ridge, xform, float(best.smooth))
        ev = to_ev(streams)
        off = official(outer, ev, float(best.p_stay), float(best.scale))
        rec = dict(role="nested", ds=outer, family=family, xform=xform, p_stay=float(best.p_stay),
                   scale=float(best.scale), smooth=float(best.smooth), ridge=ridge,
                   H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"], acc1=off["acc1"],
                   inner_H=float(best.H), scorer="official",
                   **{f"w{i}": float(wi) for i, wi in enumerate(w)})
        rows.append(rec)
        print(f"NESTED {outer} {family} {xform} p {best.p_stay} scale {best.scale} smooth {best.smooth}  "
              f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f}", flush=True)
        # Same refit stream. p_stay and scale below are chosen on the held-out labels: oracle only.
        mu, sd = emis(streams, [d for d in DS_KEYS if d != outer])
        best_o, best_key, best_ps, best_sc = None, None, None, None
        for p in list(P_STAY) + [0.992, 0.996, 0.9995]:
            for scale in list(SCALES) + [0.4, 0.9, 1.5]:
                r = evaluate(outer, streams[outer], mu, sd * scale, p)
                key = (r["H"], r["Acc"], -r["SDT"] if np.isfinite(r["SDT"]) else 0.0)
                if best_o is None or key > best_key:
                    best_o, best_key, best_ps, best_sc = r, key, p, scale
        rows.append(dict(role="oracle_op", ds=outer, family=family, xform=xform, p_stay=best_ps, scale=best_sc,
                         smooth=float(best.smooth), H=best_o["H"], Acc=best_o["Acc"], SDT=best_o["SDT"],
                         L=best_o["L"], acc1=best_o["acc1"], inner_H=np.nan, scorer="fast"))
        print(f"ORACLE {outer} p {best_ps} scale {best_sc}  H {best_o['H']:.4f} Acc {best_o['Acc']:.2f}", flush=True)

    for family, ridge in FAMILIES:
        for outer in DS_KEYS:
            best = pick(inner[(inner.outer == outer) & (inner.family == family)])
            streams, w = streams_refit(outer, family, float(ridge), str(best.xform), float(best.smooth))
            ev = to_ev(streams)
            off = official(outer, ev, float(best.p_stay), float(best.scale))
            rows.append(dict(role=f"nested_{family}", ds=outer, family=family, xform=str(best.xform),
                             p_stay=float(best.p_stay), scale=float(best.scale), smooth=float(best.smooth),
                             ridge=float(ridge), H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"],
                             acc1=off["acc1"], inner_H=float(best.H), scorer="official",
                             **{f"w{i}": float(wi) for i, wi in enumerate(w)}))
            print(f"FAMILY {family} {outer} {best.xform} p {best.p_stay} scale {best.scale}  "
                  f"H {off['H']:.4f} Acc {off['Acc']:.2f}", flush=True)

    df = pd.DataFrame(rows)
    df.to_csv(OUT, index=False)
    print("\nmeans:")
    for role, sub in df.groupby("role"):
        m = sub[["H", "Acc", "SDT", "L", "acc1"]].mean()
        print(f"  {role:12s}  H {m.H:.4f}  Acc {m.Acc:.2f}  SDT {m.SDT:.2f}  L {m.L:.3f}  acc1 {m.acc1:.3f}")
    nest = df[df.role == "nested"]
    print(f"\nPRIMARY nested mean H {nest.H.mean():.4f}  Acc {nest.Acc.mean():.2f}")
    print(f"goal H>0.8 and Acc>73: {bool(nest.H.mean() > 0.8 and nest.Acc.mean() > 73)}")
    print(f"wrote {OUT}")


if __name__ == "__main__":
    main()
