"""C1 neural tracking: envelope-EEG cross-correlation (attended vs ignored) and forward TRFs.

Pipeline: c19 causal (HP0.5 -> CAR(common57) -> BP1-9 -> AA -> 32 Hz -> running z), 1-s edges trimmed.
Lag convention: positive lag = EEG follows the envelope. Lags -250..+750 ms (-8..+24 samples).
Per trial: r[stream, lag, ch] with stream 0 = attended (per-sample label), 1 = ignored.
Null: circular shifts (>= 10 s from 0) of both envelopes, 100 shifts, on 40 random trials per dataset
(computed exactly for circular shifts with one FFT cross-correlation per channel).
Forward TRF (MNE is not installed; own ridge): EEG[t] = sum_k w[k] env[t-k], k = -100..400 ms, relative
ridge lambda = 1 (x trace/p), per subject from summed statistics; attended and ignored fitted jointly.
Outputs: cache/xcorr/<key>.npy, cache/xcorr_null.parquet, cache/trf/<subj>.npz, tables/xcorr_*.csv,
figures/c1_*.png
"""
import os
import sys

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import numpy as np  # noqa: E402
import pandas as pd  # noqa: E402
from joblib import Parallel, delayed  # noqa: E402
from scipy import stats  # noqa: E402

sys.path.insert(0, os.path.dirname(__file__))
from common import (CACHE, CFG, COLORS, DS_KEYS, LABELS, MONTAGES, SEED, TAB, boot_ci, get_logger,  # noqa: E402
                    load_pp, savefig, topo, trial_index)

log = get_logger("c1_xcorr")
FSO = 32
LAGS = np.arange(int(CFG["xcorr"]["lags_ms"][0] / 1000 * FSO), int(CFG["xcorr"]["lags_ms"][1] / 1000 * FSO) + 1)
TRF_LAGS = np.arange(-3, 14)          # -94..406 ms at 32 Hz
D = os.path.join(CACHE, "xcorr")
os.makedirs(D, exist_ok=True)


def zs(a):
    return (a - a.mean(0)) / (a.std(0) + 1e-12)


def xcorr(env2, eeg):
    """env2 (T,2) streams, eeg (T,C) -> r (2, nLags, C)."""
    T = len(eeg)
    out = np.zeros((2, len(LAGS), eeg.shape[1]))
    for i, k in enumerate(LAGS):
        if k >= 0:
            a, b = env2[: T - k], eeg[k:]
        else:
            a, b = env2[-k:], eeg[: T + k]
        a, b = zs(a), zs(b)
        out[:, i] = a.T @ b / len(a)
    return out


def att_ign(d):
    e, att = d["env"], d["att_col"].astype(int)
    idx = np.arange(len(e))
    return np.c_[e[idx, att], e[idx, 1 - att]]


def per_trial(row):
    p = os.path.join(D, f"{row['key']}.npy")
    if os.path.exists(p):
        return
    d = load_pp(row, "c19", "common57")
    np.save(p, xcorr(att_ign(d), d["eeg"]).astype(np.float32))


def null_trial(row, n_shift, seed):
    """Circular-shift null via FFT: for shift s and lag k, corr(roll(env, s), eeg[+k]) = c(k + s), where c is the
    circular cross-correlation of the z-scored signals (exact for circular shifts; edges wrap around)."""
    d = load_pp(row, "c19", "common57")
    ea, x = zs(att_ign(d)), zs(d["eeg"])
    T = len(x)
    X = np.fft.rfft(x, axis=0)
    rng = np.random.default_rng(seed)
    shifts = rng.integers(10 * FSO, max(T - 10 * FSO, 10 * FSO + 1), n_shift)
    rows = []
    cc = []
    for s_ in (0, 1):
        A = np.fft.rfft(ea[:, s_])
        cc.append(np.fft.irfft(np.conj(A)[:, None] * X, T, axis=0) / T)      # c[tau, ch] = mean_t a[t] x[t+tau]
    for i, sh in enumerate(shifts):
        idx = (LAGS + sh) % T
        rms = np.array([np.sqrt((cc[s_][idx] ** 2).mean(1)) for s_ in (0, 1)])   # (2, lags)
        rows.append(dict(key=row["key"], ds=row["ds"], shift=i, peak_rms_att=float(rms[0].max()),
                         peak_rms_ign=float(rms[1].max()), peak_diff=float((rms[0] - rms[1]).max())))
    return rows


def trf_subject(rows):
    uid = rows[0]["subj_uid"]
    p = os.path.join(CACHE, "trf", f"{uid}.npz")
    if os.path.exists(p):
        return
    K = len(TRF_LAGS)
    Ree = np.zeros((2 * K, 2 * K))
    Rex = None
    for row in rows:
        d = load_pp(row, "c19", "common57")
        ea, x = att_ign(d), d["eeg"]
        T = len(x)
        E = np.zeros((T, 2 * K))
        for j, k in enumerate(TRF_LAGS):
            for s in (0, 1):
                if k >= 0:
                    E[k:, s * K + j] = ea[: T - k, s]
                else:
                    E[: T + k, s * K + j] = ea[-k:, s]
        Ree += E.T @ E
        Rex = E.T @ x if Rex is None else Rex + E.T @ x
    lam = np.trace(Ree) / Ree.shape[0]
    W = np.linalg.solve(Ree + lam * np.eye(2 * K), Rex)
    os.makedirs(os.path.dirname(p), exist_ok=True)
    np.savez(p, W_att=W[:K], W_ign=W[K:], lags_ms=TRF_LAGS / FSO * 1000)


if __name__ == "__main__":
    ti = trial_index()
    if len(sys.argv) > 1 and sys.argv[1] == "dry":
        ti = ti[ti.subject == ti.groupby("ds", observed=True).subject.transform("min")]
    Parallel(n_jobs=CFG["n_jobs"])(delayed(per_trial)(r) for _, r in ti.iterrows())
    Parallel(n_jobs=CFG["n_jobs"])(delayed(trf_subject)([r for _, r in g.iterrows()]) for _, g in ti.groupby("subj_uid"))
    # null on a subset of trials
    nullp = os.path.join(CACHE, "xcorr_null.parquet")
    if not os.path.exists(nullp):
        rng = np.random.default_rng(SEED)
        sub = pd.concat([g.sample(min(len(g), CFG["xcorr"]["null_trials_per_dataset"]), random_state=SEED)
                         for _, g in ti.groupby("ds", observed=True)])
        res = Parallel(n_jobs=CFG["n_jobs"])(delayed(null_trial)(r, CFG["xcorr"]["n_null_shifts"], SEED + i)
                                             for i, (_, r) in enumerate(sub.iterrows()))
        pd.DataFrame([x for rs in res for x in rs]).to_parquet(nullp)
    null = pd.read_parquet(nullp)

    # ---------------- aggregate: trial -> subject (mean of Fisher z) -> dataset
    chs = MONTAGES["common57"]
    lag_ms = LAGS / FSO * 1000
    subj = {}
    for uid, g in ti.groupby("subj_uid"):
        R = np.array([np.load(os.path.join(D, f"{k}.npy")) for k in g.key])
        subj[uid] = (g.ds.iloc[0], np.tanh(np.arctanh(np.clip(R, -0.999, 0.999)).mean(0)))
    # observed trial-level peak RMS for the null subset (same statistic)
    obs = []
    for k in null.key.unique():
        r = np.load(os.path.join(D, f"{k}.npy"))
        rms = np.sqrt((r ** 2).mean(2))
        obs.append(dict(key=k, obs_peak_rms_att=float(rms[0].max()), obs_peak_rms_ign=float(rms[1].max()),
                        obs_peak_diff=float((rms[0] - rms[1]).max())))
    obs = pd.DataFrame(obs)
    nt = null.groupby("key").agg(null95_att=("peak_rms_att", lambda s: s.quantile(0.95)),
                                 null95_diff=("peak_diff", lambda s: s.quantile(0.95)),
                                 null_med_att=("peak_rms_att", "median")).reset_index()
    obs = obs.merge(nt, on="key").merge(ti[["key", "ds"]], on="key")
    obs["att_above_null95"] = obs.obs_peak_rms_att > obs.null95_att
    obs["diff_above_null95"] = obs.obs_peak_diff > obs.null95_diff
    obs.to_csv(os.path.join(TAB, "xcorr_null_trials.csv"), index=False)

    rows, curves = [], []
    for uid, (ds, R) in subj.items():
        rms = np.sqrt((R ** 2).mean(2))                      # (2, lags) RMS over channels
        # peak in the physiological range 0..500 ms
        m = (lag_ms >= 0) & (lag_ms <= 500)
        i = np.flatnonzero(m)[np.argmax(rms[0, m])]
        rows.append(dict(subj_uid=uid, ds=ds, peak_lag_ms=lag_ms[i], peak_rms_att=rms[0, i], rms_ign_at_peak=rms[1, i],
                         diff_rms=rms[0, i] - rms[1, i], peak_abs_r_att=np.abs(R[0, i]).max(),
                         peak_abs_r_ign=np.abs(R[1, i]).max()))
        for s, nm in ((0, "att"), (1, "ign")):
            curves.append(pd.DataFrame(dict(subj_uid=uid, ds=ds, stream=nm, lag_ms=lag_ms, rms=rms[s])))
    sr = pd.DataFrame(rows)
    sr.to_csv(os.path.join(TAB, "xcorr_per_subject.csv"), index=False)
    cv = pd.concat(curves)
    out = []
    for ds in DS_KEYS:
        g = sr[sr.ds == ds]
        o = obs[obs.ds == ds]
        out.append(dict(ds=ds, n_subjects=len(g), peak_lag_ms=boot_ci(g.peak_lag_ms), peak_rms_att=boot_ci(g.peak_rms_att),
                        rms_ign=boot_ci(g.rms_ign_at_peak), diff=boot_ci(g.diff_rms),
                        wilcoxon_att_vs_ign_p=stats.wilcoxon(g.peak_rms_att, g.rms_ign_at_peak).pvalue,
                        share_subjects_att_gt_ign=float((g.diff_rms > 0).mean()),
                        null_n_trials=len(o), share_trials_att_above_null95=float(o.att_above_null95.mean()),
                        share_trials_diff_above_null95=float(o.diff_above_null95.mean()),
                        median_null95_att=float(o.null95_att.median())))
    out = pd.DataFrame(out)
    flat = out.copy()
    for c in ("peak_lag_ms", "peak_rms_att", "rms_ign", "diff"):
        flat[c] = out[c].map(lambda t: t[0])
        flat[c + "_ci"] = out[c].map(lambda t: f"[{t[1]:.4g}, {t[2]:.4g}]")
    flat.to_csv(os.path.join(TAB, "xcorr_summary.csv"), index=False)
    log.info("\n" + flat.to_string())

    # figures: tracking curves
    fig, axs = plt.subplots(1, 5, figsize=(20, 3.6), sharey=True)
    for ax, ds in zip(axs, DS_KEYS):
        for nm, ls in (("att", "-"), ("ign", "--")):
            c = cv[(cv.ds == ds) & (cv.stream == nm)].pivot(index="subj_uid", columns="lag_ms", values="rms")
            q1, q2, q3 = np.quantile(c.values, [0.25, 0.5, 0.75], axis=0)
            ax.plot(c.columns, q2, ls, color=COLORS[ds], label=nm)
            ax.fill_between(c.columns, q1, q3, color=COLORS[ds], alpha=0.15 if nm == "att" else 0.07)
        ax.axvline(0, color="k", lw=0.5)
        ax.set_title(f"{LABELS[ds]} (n={int((sr.ds == ds).sum())} subjects)")
        ax.set_xlabel("lag (ms), EEG after envelope")
        ax.legend(fontsize=7)
    axs[0].set_ylabel("RMS over 57 channels of r")
    fig.suptitle("Subject-level tracking curves (median over subjects, IQR band). The circular-shift null is a "
                 "single-trial statistic and is compared per trial in tables/xcorr_null_trials.csv", fontsize=9)
    savefig(fig, "c1_tracking_curves", cv)

    # topographies at the dataset median peak lag
    pos = pd.read_csv(os.path.join(CACHE, "b64_pos.csv")).set_index("ch").loc[chs]
    fig, axs = plt.subplots(3, 5, figsize=(18, 10))
    tp = []
    for j, ds in enumerate(DS_KEYS):
        lag = float(sr[sr.ds == ds].peak_lag_ms.median())
        i = int(np.argmin(np.abs(lag_ms - lag)))
        A = np.median([R[0, i] for u, (d, R) in subj.items() if d == ds], 0)
        B = np.median([R[1, i] for u, (d, R) in subj.items() if d == ds], 0)
        v = max(np.abs(A).max(), np.abs(B).max(), np.abs(A - B).max(), 1e-3)
        for k, (vals, nm) in enumerate(((A, "attended"), (B, "ignored"), (A - B, "att - ign"))):
            topo(axs[k, j], pos.x.values, pos.y.values, vals, vmin=-v, vmax=v)
            axs[k, j].set_title(f"{LABELS[ds]} {nm}, lag {lag_ms[i]:.0f} ms", fontsize=9)
            tp.append(pd.DataFrame(dict(ds=ds, kind=nm, lag_ms=lag_ms[i], ch=chs, r=vals)))
    fig.suptitle("Median-over-subjects envelope-EEG correlation at the peak lag (c19 causal pipeline)")
    savefig(fig, "c1_topographies", pd.concat(tp))

    # TRFs
    fig, axs = plt.subplots(1, 5, figsize=(20, 3.5), sharey=False)
    tr = []
    for ax, ds in zip(axs, DS_KEYS):
        W = [np.load(os.path.join(CACHE, "trf", f"{u}.npz")) for u in sorted(ti[ti.ds == ds].subj_uid.unique())]
        la = W[0]["lags_ms"]
        ga = np.array([np.sqrt((w["W_att"] ** 2).mean(1)) for w in W])
        gi = np.array([np.sqrt((w["W_ign"] ** 2).mean(1)) for w in W])
        ax.plot(la, np.median(ga, 0), color=COLORS[ds], label="attended")
        ax.plot(la, np.median(gi, 0), "--", color=COLORS[ds], label="ignored")
        ax.set_title(f"{LABELS[ds]} forward TRF GFP (n={len(W)})")
        ax.set_xlabel("lag (ms)")
        ax.legend(fontsize=7)
        tr.append(pd.DataFrame(dict(ds=ds, lag_ms=la, gfp_att=np.median(ga, 0), gfp_ign=np.median(gi, 0))))
    savefig(fig, "c1_forward_trf_gfp", pd.concat(tr))
    log.info("done")
