"""Backward (stimulus-reconstruction) ridge decoders from sufficient statistics.

Usage:  c2_decoders.py <cfg> <step> [dry]
  cfg  = <variant>-<montage>-<car|nocar>, e.g. c19-common57-car, z19-common57-car, sup-common57-car,
         cwide-common57-car, c19-common57-nocar, c19-b64ustc58-car, c19-b64uhd62-car
  step = stats   per-subject sufficient statistics (+ within-subject LOTO if cfg is the main one)
         loso    leave-one-subject-out within each dataset
         lodo    leave-one-dataset-out: 'pool' (dataset-equal, subject-equal covariance), 'raw'
                 (sample-weighted pooled covariance) and 'avgdec' (average of per-subject decoders)
         crosshalf  KUL-AV-GC 5-min cross-half test (H6)
Design: EEG lags 0..L (EEG after the stimulus sample; L = 250 or 500 ms), no intercept (data are
running-z-scored), relative ridge (R + lambda * trace(R)/p * I), lambda picked by inner CV on the
training groups only (trials for LOTO, subjects for LOSO, datasets for LODO).
Target = the attended envelope per sample (CND attendedStream -> SuperHuge column).
Outputs: cache/recon/<cfg>/<scheme>/<key>.npz (yhat per lag set) and cache/recon/<cfg>/_env/<key>.npz
(env (T,2), att (T,), az (T,), fs, t0) + tables/lambda_<cfg>_<step>.csv
"""
import os
import sys

import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from threadpoolctl import threadpool_limits

sys.path.insert(0, os.path.dirname(__file__))
from common import CACHE, CFG, DS_KEYS, SEED, TAB, get_logger, load_pp, trial_index  # noqa: E402

LAMBDAS = np.array(CFG["decoder"]["lambdas"], float)
MAIN_CFG = "c19-common57-car"


def parse_cfg(cfg):
    v, m, c = cfg.split("-")
    return v, m, c == "car"


def fs_of(variant):
    return CFG["pipeline"]["fs_out_wide"] if variant.endswith("wide") else CFG["pipeline"]["fs_out"]


def lagsets(fs):
    return {f"{a}-{b}ms": int(round(b / 1000 * fs)) for a, b in CFG["decoder"]["lags_ms"]}


def lagged(x, L):
    """(T, C) -> (T-L, (L+1)*C), lag-major: columns [lag0 ch0..chC-1, lag1 ..., lagL ...]."""
    T, C = x.shape
    n = T - L
    X = np.empty((n, (L + 1) * C), np.float32)
    for k in range(L + 1):
        X[:, k * C:(k + 1) * C] = x[k:k + n]
    return X


def sub_idx(C, L):
    return np.arange((L + 1) * C)


def ridge_path(R, r):
    """Decoders for every lambda: (p, nL)."""
    e, V = np.linalg.eigh(R)
    m = max(e.mean(), 1e-12)
    Vr = V.T @ r
    return V @ (Vr[:, None] / (np.maximum(e, 0)[:, None] + LAMBDAS[None, :] * m))


def corr_from_stats(D, R, r, yy):
    num = D.T @ r
    den = np.sqrt(np.maximum(np.einsum("pl,pq,ql->l", D, R, D), 1e-30) * max(yy, 1e-30))
    return num / den


def pick_lambda(groups, K=3, seed=SEED):
    """groups: list of (R, r, yy) (already normalised as desired). Returns (best index, mean inner corr)."""
    G = len(groups)
    if G < 2:
        return int(np.argmin(np.abs(LAMBDAS - 1e-2))), np.full(len(LAMBDAS), np.nan)
    K = min(K, G)
    perm = np.random.default_rng(seed).permutation(G)
    folds = [perm[i::K] for i in range(K)]
    sc = []
    for f in folds:
        tr = [i for i in range(G) if i not in set(f)]
        Rt = sum(groups[i][0] for i in tr)
        rt = sum(groups[i][1] for i in tr)
        D = ridge_path(Rt, rt)
        Rh = sum(groups[i][0] for i in f)
        rh = sum(groups[i][1] for i in f)
        yh = sum(groups[i][2] for i in f)
        sc.append(corr_from_stats(D, Rh, rh, yh))
    sc = np.mean(sc, 0)
    return int(np.nanargmax(sc)), sc


def fit(groups, K=3):
    li, sc = pick_lambda(groups, K)
    R = sum(g[0] for g in groups)
    r = sum(g[1] for g in groups)
    return ridge_path(R, r)[:, li], li


def trial_data(row, cfg):
    v, m, car = parse_cfg(cfg)
    d = load_pp(row, v, m, do_car=car)
    return d


def trial_stats(X, y):
    X64 = X
    R = (X64.T @ X64).astype(np.float64)
    r = (X64.T @ y.astype(np.float32)).astype(np.float64)
    return R, r, float(y @ y), len(y)


def save_env(cfg, row, d):
    p = os.path.join(CACHE, "recon", cfg, "_env", f"{row['key']}.npz")
    if not os.path.exists(p):
        os.makedirs(os.path.dirname(p), exist_ok=True)
        np.savez(p, env=d["env"].astype(np.float32), att=d["att_col"], az=d["az"].astype(np.float32),
                 fs=d["fs"], t0=d["t0_s"])


def save_recon(cfg, scheme, key, yh):
    p = os.path.join(CACHE, "recon", cfg, scheme, f"{key}.npz")
    os.makedirs(os.path.dirname(p), exist_ok=True)
    np.savez(p, **{k: v.astype(np.float32) for k, v in yh.items()})


# ---------------------------------------------------------------------------
def job_stats(cfg, rows, do_loto):
    """Per-subject sums of per-trial statistics for the largest lag set; LOTO recon if requested."""
    with threadpool_limits(2):
        v, m, car = parse_cfg(cfg)
        fs = fs_of(v)
        LS = lagsets(fs)
        Lmax = max(LS.values())
        uid = rows[0]["subj_uid"]
        out = os.path.join(CACHE, "stats", cfg, f"{uid}.npz")
        if os.path.exists(out) and (not do_loto or all(
                os.path.exists(os.path.join(CACHE, "recon", cfg, "loto", f"{r['key']}.npz")) for r in rows)):
            return uid, "cached", []
        per, eegs = [], []
        halves = []
        for row in rows:
            d = trial_data(row, cfg)
            save_env(cfg, row, d)
            X = lagged(d["eeg"].astype(np.float32), Lmax)
            n = len(X)
            ya = d["env"][np.arange(n), d["att_col"][:n]]
            per.append(trial_stats(X, ya))
            eegs.append((row["key"], d["eeg"].astype(np.float32), d["env"], d["att_col"]))
            if row["ds"] == "AVGC":
                h = int(round((300.0 - d["t0_s"]) * d["fs"]))
                h1 = slice(0, h - Lmax)            # rows whose lag window stays inside half 1
                h2 = slice(h, n)
                e0, e1 = d["env"][:n, 0], d["env"][:n, 1]
                halves.append(dict(
                    R1=(X[h1].T @ X[h1]).astype(np.float64), r0_1=(X[h1].T @ e0[h1].astype(np.float32)).astype(np.float64),
                    yy0_1=float(e0[h1] @ e0[h1]),
                    R2=(X[h2].T @ X[h2]).astype(np.float64), r0_2=(X[h2].T @ e0[h2].astype(np.float32)).astype(np.float64),
                    r1_2=(X[h2].T @ e1[h2].astype(np.float32)).astype(np.float64), yy0_2=float(e0[h2] @ e0[h2]),
                    yy1_2=float(e1[h2] @ e1[h2]), h=h))
            del X
        Rs = sum(p[0] for p in per)
        rs = sum(p[1] for p in per)
        yys = sum(p[2] for p in per)
        ns = sum(p[3] for p in per)
        os.makedirs(os.path.dirname(out), exist_ok=True)
        extra = {}
        if halves:
            for k in ("R1", "r0_1", "R2", "r0_2", "r1_2"):
                extra[f"half_{k}"] = np.array([hh[k] for hh in halves])
            for k in ("yy0_1", "yy0_2", "yy1_2", "h"):
                extra[f"half_{k}"] = np.array([hh[k] for hh in halves])
            extra["half_keys"] = np.array([r["key"] for r in rows])
        np.savez(out, R=Rs, r=rs, yy=yys, n=ns, ntr=len(per), keys=np.array([r["key"] for r in rows]),
                 trial_n=np.array([p[3] for p in per]), **extra)
        lam_rows = []
        if do_loto:
            C = eegs[0][1].shape[1]
            for k, (key, eeg, env, att) in enumerate(eegs):
                yh = {}
                for ln, L in LS.items():
                    idx = sub_idx(C, L)
                    groups = [(p[0][np.ix_(idx, idx)], p[1][idx], p[2]) for j, p in enumerate(per) if j != k]
                    dvec, li = fit(groups)
                    X = lagged(eeg, L)
                    yh[ln] = X @ dvec.astype(np.float32)
                    lam_rows.append(dict(key=key, lag=ln, lam=LAMBDAS[li]))
                save_recon(cfg, "loto", key, yh)
        return uid, "done", lam_rows


def load_subject_stats(cfg, ti=None):
    ti = trial_index() if ti is None else ti
    S = {}
    for uid, g in ti.groupby("subj_uid"):
        z = np.load(os.path.join(CACHE, "stats", cfg, f"{uid}.npz"))
        S[uid] = dict(R=z["R"], r=z["r"], yy=float(z["yy"]), n=int(z["n"]), ds=g.ds.iloc[0])
    return S


def norm_group(s, idx):
    n = s["n"]
    return (s["R"][np.ix_(idx, idx)] / n, s["r"][idx] / n, s["yy"] / n)


def recon_rows(cfg, scheme, rows, decs):
    """decs: {lagname: (L, dvec)}"""
    with threadpool_limits(2):
        for row in rows:
            d = trial_data(row, cfg)
            yh = {ln: lagged(d["eeg"].astype(np.float32), L) @ dv.astype(np.float32) for ln, (L, dv) in decs.items()}
            save_recon(cfg, scheme, row["key"], yh)
    return len(rows)


def run_loso(cfg, ti, S, C, LS, log):
    lam, jobs = [], []
    for ds in DS_KEYS:
        uids = sorted(ti[ti.ds == ds].subj_uid.unique())
        for u in uids:
            decs = {}
            for ln, L in LS.items():
                idx = sub_idx(C, L)
                groups = [norm_group(S[v], idx) for v in uids if v != u]
                dv, li = fit(groups)
                decs[ln] = (L, dv)
                lam.append(dict(ds=ds, test=u, lag=ln, lam=LAMBDAS[li]))
            rows = [r for _, r in ti[ti.subj_uid == u].iterrows()]
            jobs.append((rows, decs))
    Parallel(n_jobs=CFG["n_jobs"])(delayed(recon_rows)(cfg, "loso", rows, decs) for rows, decs in jobs)
    return pd.DataFrame(lam)


def ds_mean(S, uids, idx):
    g = [norm_group(S[u], idx) for u in uids]
    return (sum(x[0] for x in g) / len(g), sum(x[1] for x in g) / len(g), sum(x[2] for x in g) / len(g))


def run_lodo(cfg, ti, S, C, LS, log):
    lam = []
    by_ds = {ds: sorted(ti[ti.ds == ds].subj_uid.unique()) for ds in DS_KEYS}
    for test in DS_KEYS:
        train = [d for d in DS_KEYS if d != test]
        decs = {"pool": {}, "raw": {}, "avgdec": {}}
        for ln, L in LS.items():
            idx = sub_idx(C, L)
            # pool: dataset-equal weighting of subject-equal normalised covariance
            groups = [ds_mean(S, by_ds[d], idx) for d in train]
            dv, li = fit(groups, K=len(groups))
            decs["pool"][ln] = (L, dv)
            lam.append(dict(test=test, agg="pool", lag=ln, lam=LAMBDAS[li]))
            # raw: sample-weighted pooled sums (datasets as inner folds)
            groups = [(sum(S[u]["R"][np.ix_(idx, idx)] for u in by_ds[d]), sum(S[u]["r"][idx] for u in by_ds[d]),
                       sum(S[u]["yy"] for u in by_ds[d])) for d in train]
            dv, li = fit(groups, K=len(groups))
            decs["raw"][ln] = (L, dv)
            lam.append(dict(test=test, agg="raw", lag=ln, lam=LAMBDAS[li]))
            # avgdec: per-subject decoders (normalised to unit norm), averaged within then across datasets
            paths = {}
            for d in train:
                P = []
                for u in by_ds[d]:
                    g = norm_group(S[u], idx)
                    D = ridge_path(g[0], g[1])
                    P.append(D / np.linalg.norm(D, axis=0, keepdims=True))
                paths[d] = np.mean(P, 0)
            sc = []
            for hold in train:
                Dm = np.mean([paths[d] for d in train if d != hold], 0)
                g = ds_mean(S, by_ds[hold], idx)
                sc.append(corr_from_stats(Dm, *g))
            li = int(np.nanargmax(np.mean(sc, 0)))
            decs["avgdec"][ln] = (L, np.mean([paths[d] for d in train], 0)[:, li])
            lam.append(dict(test=test, agg="avgdec", lag=ln, lam=LAMBDAS[li]))
        rows = [r for _, r in ti[ti.ds == test].iterrows()]
        chunks = [rows[i::CFG["n_jobs"]] for i in range(CFG["n_jobs"])]
        for agg, dd in decs.items():
            Parallel(n_jobs=CFG["n_jobs"])(delayed(recon_rows)(cfg, f"lodo_{agg}", ch, dd) for ch in chunks)
            np.savez(os.path.join(CACHE, "recon", cfg, f"lodo_{agg}", f"_decoder_{test}.npz"),
                     **{ln: v[1] for ln, v in dd.items()})
        log.info(f"LODO test={test} done")
    return pd.DataFrame(lam)


def run_crosshalf(cfg, ti, C, LS, log):
    """H6: decoders trained on one half of the KUL-AV-GC trials of a subject, tested on the other half."""
    lam = []
    for u in sorted(ti[ti.ds == "AVGC"].subj_uid.unique()):
        z = np.load(os.path.join(CACHE, "stats", cfg, f"{u}.npz"))
        keys = list(z["half_keys"])
        rows = [r for _, r in ti[ti.subj_uid == u].iterrows()]
        decs = {"xh_h1": {}, "xh_h2B": {}, "xh_h2A": {}}
        for ln, L in LS.items():
            idx = sub_idx(C, L)
            g1 = [(z["half_R1"][i][np.ix_(idx, idx)], z["half_r0_1"][i][idx], z["half_yy0_1"][i]) for i in range(len(keys))]
            g2B = [(z["half_R2"][i][np.ix_(idx, idx)], z["half_r0_2"][i][idx], z["half_yy0_2"][i]) for i in range(len(keys))]
            g2A = [(z["half_R2"][i][np.ix_(idx, idx)], z["half_r1_2"][i][idx], z["half_yy1_2"][i]) for i in range(len(keys))]
            for name, g in (("xh_h1", g1), ("xh_h2B", g2B), ("xh_h2A", g2A)):
                dv, li = fit(g)
                decs[name][ln] = (L, dv)
                lam.append(dict(subject=u, dec=name, lag=ln, lam=LAMBDAS[li]))
        for name, dd in decs.items():
            recon_rows(cfg, name, rows, dd)
        log.info(f"crosshalf {u} done")
    return pd.DataFrame(lam)


if __name__ == "__main__":
    cfg, step = sys.argv[1], sys.argv[2]
    dry = len(sys.argv) > 3 and sys.argv[3] == "dry"
    log = get_logger(f"c2_{cfg}_{step}")
    ti = trial_index()
    if dry:
        ti = ti[ti.subject <= ti.groupby("ds", observed=True).subject.transform("min") + 2]   # 3 subjects/dataset
    v, m, car = parse_cfg(cfg)
    fs = fs_of(v)
    LS = lagsets(fs)
    from common import MONTAGES
    C = len(MONTAGES[m])
    if step == "stats":
        do_loto = cfg == MAIN_CFG
        groups = [[r for _, r in g.iterrows()] for _, g in ti.groupby("subj_uid")]
        # big subjects first for load balance
        groups.sort(key=lambda g: -sum(r["n"] for r in g))
        nj = CFG["n_jobs"] if not v.endswith("wide") else 3
        res = Parallel(n_jobs=nj, verbose=0)(delayed(job_stats)(cfg, g, do_loto) for g in groups)
        lam = [x for _, _, rows in res for x in rows]
        if lam:
            pd.DataFrame(lam).to_csv(os.path.join(TAB, f"lambda_{cfg}_loto{'_dry' if dry else ''}.csv"), index=False)
        log.info(f"stats: {sum(s == 'done' for _, s, _ in res)} subjects computed")
    elif step == "loso":
        S = load_subject_stats(cfg, ti)
        lam = run_loso(cfg, ti, S, C, LS, log)
        lam.to_csv(os.path.join(TAB, f"lambda_{cfg}_loso.csv"), index=False)
    elif step == "lodo":
        S = load_subject_stats(cfg, ti)
        lam = run_lodo(cfg, ti, S, C, LS, log)
        lam.to_csv(os.path.join(TAB, f"lambda_{cfg}_lodo.csv"), index=False)
    elif step == "crosshalf":
        lam = run_crosshalf(cfg, ti, C, LS, log)
        lam.to_csv(os.path.join(TAB, f"lambda_{cfg}_crosshalf.csv"), index=False)
    log.info("done")
