"""EDA of the Muse sleep-onset dataset from the per-window cache (01_extract_features.py).

Writes ../cache/stats.json (every number quoted in the report) and ../figures/*.png.
Conventions: ttn = n2_onset - window_stop (s); evaluation windows are ttn >= 0;
target = clip(ttn, 0, 600). TP = mean of TP9/TP10, AF = mean of AF7/AF8.
A channel-window is BAD when max|x| > 200 uV or RMS 30-45 Hz > 5 uV.
"""
import json
import os

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.stats import mannwhitneyu, spearmanr
from sklearn.metrics import roc_auc_score

HERE = os.path.dirname(os.path.abspath(__file__))
BASE = os.path.dirname(HERE)
CACHE, FIG = f"{BASE}/cache", f"{BASE}/figures"
CH = ["TP9", "AF7", "AF8", "TP10"]
EDGES, WEIGHTS = (0, 40, 90, 300, 600), (10, 5, 3, 1)
BINS = ["0-40 s", "40-90 s", "90-300 s", "300-600 s"]

BLUE, ORANGE, AQUA, YELLOW, GRAY, VIOLET = "#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#898781", "#6250d6"
INK, INK2, GRID = "#0b0b0b", "#52514e", "#e1e0d9"
plt.rcParams.update({
    "font.family": "DejaVu Sans", "font.size": 8.5, "axes.edgecolor": "#c3c2b7",
    "axes.labelcolor": INK2, "xtick.color": GRAY, "ytick.color": GRAY, "xtick.labelcolor": INK2,
    "ytick.labelcolor": INK2, "axes.spines.top": False, "axes.spines.right": False,
    "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6, "axes.axisbelow": True,
    "axes.titlesize": 9.5, "axes.titleweight": "bold", "axes.titlecolor": INK,
    "axes.titlelocation": "left", "figure.dpi": 200, "savefig.bbox": "tight",
    "savefig.facecolor": "white", "legend.frameon": False, "legend.fontsize": 7.5,
})
S = {}


def save(fig, name):
    fig.savefig(f"{FIG}/{name}")
    plt.close(fig)


def rnd(x, k=3):
    return float(np.round(x, k))


# ------------------------------------------------------------------ load ---
rec = pd.read_csv(f"{CACHE}/recordings.csv")
z = np.load(f"{CACHE}/windows.npz")
W = {k: z[k] for k in z.files}
r_idx, ttn, t_stop = W["rec"], W["ttn"].astype(np.float64), W["t_stop"].astype(np.float64)
ev = ttn >= 0                                         # evaluation windows
y = np.clip(ttn, 0, 600)
bad_ch = (W["maxabs"] > 200) | (W["rms_hf"] > 5)      # (N, 4)
n_bad = bad_ch.sum(1)


def tp(k):
    return W[k][:, [0, 3]].mean(1)


def af(k):
    return W[k][:, [1, 2]].mean(1)


F = {  # window-level features (causal, single window)
    "beta TP": tp("log_beta"), "gamma TP": tp("log_gamma"), "alpha 8-11 TP": tp("log_alpha_lo"),
    "alpha 8-12 AF": af("log_alpha"), "delta TP": tp("log_delta"), "delta AF": af("log_delta"),
    "theta TP": tp("log_theta"), "spindle 11-15 TP": tp("log_spindle"),
    "sigma/beta TP": tp("log_sigma") - tp("log_beta"), "sigma/beta AF": af("log_sigma") - af("log_beta"),
    "theta/alpha TP": tp("log_theta") - tp("log_alpha"), "delta/beta TP": tp("log_delta") - tp("log_beta"),
    "1/f slope TP": tp("aper_slope"), "1/f slope AF": af("aper_slope"),
    "sigma residual TP": tp("sigma_resid"), "sigma residual AF": af("sigma_resid"),
    "spectral entropy TP": tp("spec_entropy"), "SEF95 TP": tp("sef95"),
    "log SD TP": np.log10(tp("sd") + 1e-6), "log SD AF": np.log10(af("sd") + 1e-6),
    "line length TP": tp("line_len"), "RMS 30-45 Hz TP": tp("rms_hf"),
    "n BAD channels": n_bad.astype(float), "corr TP9-TP10": W["corr_tp9_tp10"],
    "corr front-back": (W["corr_tp9_af7"] + W["corr_af8_tp10"]) / 2,
    "corr AF7-AF8 (broadband)": W["corr_af7_af8"],
}

# --------------------------------------------------------- A. structure ---
nights = rec.groupby("subject").size()
S["n_recordings"], S["n_participants"] = len(rec), int(rec.subject.nunique())
S["total_hours"] = rnd(rec.recording_duration.sum() / 3600, 2)
S["nights_per_participant"] = {int(k): int(v) for k, v in nights.value_counts().sort_index().items()}
S["onset_q"] = {q: rnd(rec.n2_onset.quantile(q), 1) for q in (0, .05, .25, .5, .75, .95, 1)}
S["onset_mean_sd"] = [rnd(rec.n2_onset.mean(), 1), rnd(rec.n2_onset.std(), 1)]
S["duration_q_min"] = {q: rnd(rec.recording_duration.quantile(q) / 60, 2) for q in (0, .5, 1)}
S["duration_minus_onset_unique"] = sorted(set(np.round(rec.recording_duration - rec.n2_onset, 6)))
S["samples_equal_duration_x128"] = bool(np.allclose(rec.recording_duration * 128, rec.sample_count))
S["share_onset_lt600"] = rnd(100 * (rec.n2_onset < 600).mean(), 1)
S["share_onset_lt300"] = rnd(100 * (rec.n2_onset < 300).mean(), 1)
S["split"] = rec.split.value_counts().to_dict()
test_subj = set(rec[rec.split == "test"].subject)
S["test_subjects"] = len(test_subj)
S["test_subjects_train_nights"] = {int(k): int(v) for k, v in
                                   rec[rec.subject.isin(test_subj) & (rec.split == "train")]
                                   .groupby("subject").size().value_counts().items()}
rec["offgrid"] = rec.n2_onset % 30 != 0
S["offgrid"] = dict(n=int(rec.offgrid.sum()), min_onset=rnd(rec[rec.offgrid].n2_onset.min(), 1),
                    share_train=rnd(100 * rec[rec.split == "train"].offgrid.mean(), 1),
                    share_test=rnd(100 * rec[rec.split == "test"].offgrid.mean(), 1),
                    share_among_onset_ge900=rnd(100 * rec[rec.n2_onset >= 900].offgrid.mean(), 1),
                    samples_not_multiple_640=int((rec[rec.offgrid].sample_count % 640 != 0).sum()),
                    ongrid_samples_not_multiple_640=int((rec[~rec.offgrid].sample_count % 640 != 0).sum()))
# ICC(1), one-way random effects, unbalanced (McGraw & Wong k0)
g = rec.groupby("subject").n2_onset
k, n = g.size(), g.ngroups
msb = (k * (g.mean() - rec.n2_onset.mean()) ** 2).sum() / (n - 1)
msw = sum(((rec[rec.subject == s].n2_onset - m) ** 2).sum() for s, m in g.mean().items()) / (k - 1).sum()
k0 = (k.sum() - (k ** 2).sum() / k.sum()) / (n - 1)
S["onset_icc1"] = rnd((msb - msw) / (msb + (k0 - 1) * msw))
S["onset_sd_between_within"] = [rnd(np.sqrt(max(msb - msw, 0) / k0), 0), rnd(np.sqrt(msw), 0)]
S["onset_train_test_median"] = [rnd(rec[rec.split == "train"].n2_onset.median()),
                                rnd(rec[rec.split == "test"].n2_onset.median())]
S["onset_train_test_p"] = rnd(mannwhitneyu(rec[rec.split == "train"].n2_onset,
                                           rec[rec.split == "test"].n2_onset).pvalue)

fig, ax = plt.subplots(1, 3, figsize=(10.5, 2.9), gridspec_kw=dict(wspace=0.35))
ax[0].hist(rec.n2_onset / 60, bins=np.arange(0, 26, 1), color=BLUE, edgecolor="white", lw=0.6)
ax[0].hist(rec[rec.offgrid].n2_onset / 60, bins=np.arange(0, 26, 1), color=ORANGE, edgecolor="white",
           lw=0.6, label=f"off-grid (n={S['offgrid']['n']})")
ax[0].axvline(10, color=INK2, lw=1, ls=(0, (4, 3)))
ax[0].set(title="Thời điểm N2 đầu tiên", xlabel="phút từ đầu bản ghi", ylabel="số bản ghi")
ax[0].legend(loc="upper right")
b = ax[1].bar(nights.value_counts().sort_index().index.astype(str), nights.value_counts().sort_index().values,
              color=BLUE, width=0.6)
ax[1].bar_label(b, fontsize=7.5, color=INK2, padding=2)
ax[1].set(title="Số đêm mỗi người", xlabel="số bản ghi / người", ylabel="số người")
for s, sub in rec.groupby("subject"):
    if len(sub) >= 4:
        ax[2].plot([sub.n2_onset.mean() / 60] * len(sub), sub.n2_onset / 60, "o", ms=2.2, color=BLUE, alpha=0.5)
ax[2].plot([3, 22], [3, 22], color=GRAY, lw=0.8)
ax[2].set(title=f"Đêm của cùng một người (ICC={S['onset_icc1']:.2f})",
          xlabel="onset trung bình của người (phút)", ylabel="onset từng đêm (phút)")
for a in ax[:2]:
    a.grid(axis="x", visible=False)
save(fig, "fig01_structure.png")

# ------------------------------------------------------------ B. target ---
yb = np.digitize(y[ev], EDGES[1:-1])
S["n_eval_windows"] = int(ev.sum())
S["windows_per_rec_median"] = rnd(np.median(np.bincount(r_idx[ev])), 0)
S["bin_share_windows"] = [rnd(100 * (yb == i).mean(), 1) for i in range(4)]
wshare = np.zeros(4)
for r in np.unique(r_idx[ev]):
    present = np.unique(yb[r_idx[ev] == r])
    for b_ in present:
        wshare[b_] += WEIGHTS[b_] / sum(WEIGHTS[p] for p in present)
S["bin_share_weight"] = [rnd(100 * w / wshare.sum(), 1) for w in wshare]
S["cap_share"] = rnd(100 * (y[ev] == 600).mean(), 1)
S["share_target_ge450"] = rnd(100 * (y[ev] >= 450).mean(), 1)
# hazard per minute of elapsed time
mins = np.arange(0, 25)
alive = np.array([(rec.n2_onset > m * 60).sum() for m in mins])
event = np.array([((rec.n2_onset > m * 60) & (rec.n2_onset <= (m + 1) * 60)).sum() for m in mins])
haz = np.where(alive > 0, event / np.maximum(alive, 1), np.nan)
S["hazard_per_min"] = {int(m): rnd(100 * h, 1) for m, h in zip(mins, haz)}
S["spearman_elapsed_target"] = rnd(spearmanr(t_stop[ev], y[ev]).statistic)

fig, ax = plt.subplots(1, 3, figsize=(10.5, 2.9), gridspec_kw=dict(wspace=0.35))
x_ = np.arange(4)
b1 = ax[0].bar(x_ - 0.2, S["bin_share_windows"], 0.36, color=BLUE, label="% cửa sổ")
b2 = ax[0].bar(x_ + 0.2, S["bin_share_weight"], 0.36, color=ORANGE, label="% trọng số W-bMAE")
ax[0].bar_label(b1, fmt="%.0f%%", fontsize=7, color=INK2, padding=1)
ax[0].bar_label(b2, fmt="%.0f%%", fontsize=7, color=INK2, padding=1)
ax[0].set_xticks(x_, ["0-40", "40-90", "90-300", "300-600"])
ax[0].set(title="Dữ liệu ở đâu, trọng số ở đâu", xlabel="target thật (s)", ylim=(0, 75))
ax[0].legend(loc="upper center")
ax[0].grid(axis="x", visible=False)
ax[1].hist(y[ev], bins=np.arange(0, 606, 10), color=AQUA, edgecolor="white", lw=0.3)
ax[1].set(title=f"Target ({S['cap_share']:.0f}% đúng bằng 600)", xlabel="target (s)",
          ylabel="số cửa sổ")
ax[1].set_yscale("log")
ax[2].bar(mins + 0.5, 100 * haz, width=0.85, color=BLUE)
ax[2].set(title="Hazard của N2 theo phút", xlabel="phút đã trôi từ đầu bản ghi",
          ylabel="% vào N2 trong phút đó")
ax[2].grid(axis="x", visible=False)
save(fig, "fig02_target.png")

# ----------------------------------------------------------- C. quality ---
pre = ttn > 0
S["sd_median_pre"] = {c: rnd(np.median(W["sd"][pre, i]), 1) for i, c in enumerate(CH)}
S["sd_p95_pre"] = {c: rnd(np.percentile(W["sd"][pre, i], 95), 0) for i, c in enumerate(CH)}
recmax = pd.Series(W["maxabs"].max(1)).groupby(r_idx).max()
S["recs_reaching_780uV"] = int((recmax >= 780).sum())
S["first_sample_zero_pct"] = rnd(100 * rec.first_sample_zero.mean(), 1)
S["window_any_bad_pre"] = rnd(100 * (n_bad[pre] > 0).mean(), 1)
S["window_all4_bad_pre"] = rnd(100 * (n_bad[pre] == 4).mean(), 1)
S["window_tp_and_af_ok_pre"] = rnd(100 * (((~bad_ch[pre][:, [0, 3]]).any(1)) & ((~bad_ch[pre][:, [1, 2]]).any(1))).mean(), 1)
grp = [("> 600 s", ttn > 600), ("300-600", (ttn > 300) & (ttn <= 600)), ("90-300", (ttn > 90) & (ttn <= 300)),
       ("40-90", (ttn > 40) & (ttn <= 90)), ("0-40", (ttn >= 0) & (ttn <= 40)), ("sau N2", ttn < 0)]
S["bad_rate_by_group"] = {gname: rnd(100 * (n_bad[m] > 0).mean(), 1) for gname, m in grp}
S["bad_rate_tp_af_by_group"] = {gname: [rnd(100 * bad_ch[m][:, [0, 3]].mean(), 1),
                                        rnd(100 * bad_ch[m][:, [1, 2]].mean(), 1)] for gname, m in grp}
first = {"0-30 s": t_stop <= 30, "30-60 s": (t_stop > 30) & (t_stop <= 60), "60-120 s": (t_stop > 60) & (t_stop <= 120),
         "120-300 s": (t_stop > 120) & (t_stop <= 300)}
S["bad_chwin_rate_first_minutes"] = {k_: rnd(100 * bad_ch[m & pre].mean(), 1) for k_, m in first.items()}
S["mains_50hz_recs"] = int((rec[[f"pk50_{c}" for c in CH]].max(1) > 10).sum())
S["mains_60hz_recs"] = int((rec[[f"pk60_{c}" for c in CH]].max(1) > 10).sum())
S["af7_fs3_line_recs"] = int((rec["pk42_AF7"] > 10).sum())
S["af7_fs6_line_recs"] = int((rec["pk21_AF7"] > 10).sum())
clean = n_bad == 0
S["corr_clean_median"] = {k_: rnd(np.median(W[k_][clean & pre]), 2) for k_ in
                          ("corr_tp9_tp10", "corr_af7_af8", "corr_tp9_af7", "corr_af8_tp10")}

fig, ax = plt.subplots(1, 3, figsize=(10.5, 2.9), gridspec_kw=dict(wspace=0.35))
bins_ = np.logspace(0, 3, 60)
for i, c in enumerate(CH):
    ax[0].hist(W["sd"][pre, i], bins=bins_, histtype="step", lw=1.3, color=[BLUE, ORANGE, YELLOW, AQUA][i], label=c)
ax[0].set_xscale("log")
ax[0].set(title="SD của cửa sổ 5 s (trước N2)", xlabel="SD (µV)", ylabel="số cửa sổ")
ax[0].legend()
labels = [g_[0] for g_ in grp]
tpv = [S["bad_rate_tp_af_by_group"][l][0] for l in labels]
afv = [S["bad_rate_tp_af_by_group"][l][1] for l in labels]
xx = np.arange(len(labels))
ax[1].plot(xx, tpv, "o-", color=BLUE, label="TP9/TP10")
ax[1].plot(xx, afv, "o-", color=ORANGE, label="AF7/AF8")
ax[1].set_xticks(xx, labels, rotation=20)
ax[1].set(title="BAD theo khoảng cách tới N2", ylabel="% kênh-cửa sổ BAD")
ax[1].legend()
fm = list(S["bad_chwin_rate_first_minutes"].items())
b = ax[2].bar([k_ for k_, _ in fm], [v for _, v in fm], color=GRAY, width=0.6)
ax[2].bar_label(b, fmt="%.0f%%", fontsize=7.5, color=INK2, padding=1)
ax[2].set(title="Những phút đầu rất nhiễu", xlabel="thời gian từ đầu bản ghi", ylabel="% kênh-cửa sổ BAD")
ax[2].tick_params(axis="x", labelsize=7)
ax[2].grid(axis="x", visible=False)
save(fig, "fig03_quality.png")

# ---------------------------------------------------------- D. spectra ---
sp = np.load(f"{CACHE}/spectra.npz")
fr, mlp, gnames = sp["freqs"], sp["mean_logpsd"], list(sp["groups"])
colors = [GRAY, BLUE, AQUA, YELLOW, ORANGE, VIOLET]
glab = {"gt600": "> 600 s", "300-600": "300-600 s", "90-300": "90-300 s", "40-90": "40-90 s",
        "0-40": "0-40 s", "postN2": "sau N2"}
fig, ax = plt.subplots(1, 3, figsize=(10.5, 3.0), gridspec_kw=dict(wspace=0.32))
mfr = (fr >= 0.5) & (fr <= 45)
for j, (title, chs) in enumerate((("Phổ thái dương (TP9/TP10)", [0, 3]), ("Phổ trán (AF7/AF8)", [1, 2]))):
    for gi, gname in enumerate(gnames):
        ax[j].plot(fr[mfr], 10 * mlp[gi][chs].mean(0)[mfr], color=colors[gi], lw=1.1, label=glab[gname])
    ax[j].set(title=title, xlabel="tần số (Hz)", ylabel="10·log10 PSD (dB µV²/Hz)")
    ax[j].set_xscale("log")
    ax[j].set_xticks([1, 2, 4, 8, 16, 32], ["1", "2", "4", "8", "16", "32"])
ax[0].legend(fontsize=7, ncols=2)
i3, i0, i5 = gnames.index("300-600"), gnames.index("0-40"), gnames.index("postN2")
for chs, lab, col in (([0, 3], "TP", BLUE), ([1, 2], "AF", ORANGE)):
    ax[2].plot(fr[mfr], 10 * (mlp[i0][chs].mean(0) - mlp[i3][chs].mean(0))[mfr], color=col, lw=1.2,
               label=f"{lab}: 0-40 s trừ 300-600 s")
    ax[2].plot(fr[mfr], 10 * (mlp[i5][chs].mean(0) - mlp[i3][chs].mean(0))[mfr], color=col, lw=1.0, ls="--",
               label=f"{lab}: sau N2 trừ 300-600 s")
ax[2].axhline(0, color=GRAY, lw=0.8)
ax[2].set(title="Phổ hiệu so với lúc còn xa N2", xlabel="tần số (Hz)", ylabel="chênh lệch (dB)")
ax[2].set_xscale("log")
ax[2].set_xticks([1, 2, 4, 8, 16, 32], ["1", "2", "4", "8", "16", "32"])
ax[2].legend(fontsize=6.5)
save(fig, "fig04_spectra.png")
diff_tp = 10 * (mlp[i0][[0, 3]].mean(0) - mlp[i3][[0, 3]].mean(0))
diff_post = 10 * (mlp[i5][[1, 2]].mean(0) - mlp[i3][[1, 2]].mean(0))
S["spec_diff_TP_0to40_vs_300to600_dB"] = {f"{lo}-{hi} Hz": rnd(diff_tp[(fr >= lo) & (fr < hi)].mean(), 2)
                                          for lo, hi in ((0.5, 4), (4, 8), (8, 11), (11, 15), (16, 30), (30, 45))}
mpk = (fr >= 9) & (fr <= 16)
S["post_N2_AF_peak_Hz"] = rnd(fr[mpk][np.argmax(diff_post[mpk])], 2)

# ------------------------------------------- E. onset-locked trajectories ---
late = rec.index[rec.n2_onset >= 600].values
lock_t = np.arange(-600, 305, 5)                     # time relative to N2 (window stop - onset)
rel = -ttn                                           # window stop relative to N2
traj_feats = ["beta TP", "gamma TP", "alpha 8-11 TP", "delta TP", "sigma/beta TP", "1/f slope TP",
              "sigma residual TP", "theta/alpha TP", "log SD TP", "n BAD channels",
              "corr front-back", "slow corr AF7-AF8"]
F["slow corr AF7-AF8"] = W["slowcorr_af7_af8"]
base_mask = (ttn >= 450) & (ttn <= 600)
traj, dz = {}, {}
for f_ in traj_feats:
    v = F[f_]
    M = np.full((len(late), len(lock_t)), np.nan)
    d_ = []
    for ii, r in enumerate(late):
        m = r_idx == r
        b_ = v[m & base_mask].mean()
        sub = (lock_t[None, :] == np.round(rel[m])[:, None])
        pos = sub.argmax(1)
        ok = sub.any(1)
        vv = v[m][ok] - (0 if f_ in ("slow corr AF7-AF8", "n BAD channels") else b_)
        M[ii, pos[ok]] = vv
        last40 = v[m & (ttn >= 0) & (ttn < 40)].mean()
        far = v[m & (ttn >= 300) & (ttn <= 600)].mean()
        d_.append(last40 - far)
    d_ = np.array(d_)
    if f_ == "n BAD channels":          # mostly 0: the median is uninformative, use the mean
        mm = np.nanmean(M, 0)
        traj[f_] = (mm, mm, mm)
    else:
        traj[f_] = (np.nanmedian(M, 0), np.nanpercentile(M, 25, 0), np.nanpercentile(M, 75, 0))
    dz[f_] = rnd(np.nanmean(d_) / np.nanstd(d_), 2)
S["dz_last40_vs_300to600"] = dz
S["n_late_recordings"] = int(len(late))

def halfway(f_):
    med = traj[f_][0]
    m = (lock_t <= 0)
    ref, fin = np.nanmean(med[(lock_t >= -600) & (lock_t <= -450)]), np.nanmean(med[(lock_t > -30) & (lock_t <= 0)])
    tgt = ref + 0.5 * (fin - ref)
    sgn = np.sign(fin - ref)
    idx_ = np.where(m & (sgn * (med - tgt) >= 0))[0]
    return int(lock_t[idx_[0]]) if len(idx_) else None
S["halfway_s_before_N2"] = {f_: halfway(f_) for f_ in traj_feats if f_ not in ("slow corr AF7-AF8",)}

fig, axs = plt.subplots(2, 4, figsize=(10.5, 4.8), sharex=True, gridspec_kw=dict(wspace=0.38, hspace=0.42))
show = ["beta TP", "gamma TP", "alpha 8-11 TP", "delta TP", "sigma/beta TP", "1/f slope TP",
        "n BAD channels", "slow corr AF7-AF8"]
units = {"n BAD channels": "số kênh BAD (trung bình)", "slow corr AF7-AF8": "tương quan 0.1-1 Hz",
         "1/f slope TP": "Δ độ dốc", "sigma/beta TP": "Δ log10 tỉ số"}
for a, f_ in zip(axs.ravel(), show):
    med, lo, hi = traj[f_]
    a.fill_between(lock_t, lo, hi, color=BLUE, alpha=0.15, lw=0)
    a.plot(lock_t, med, color=BLUE, lw=1.3)
    a.axvline(0, color=ORANGE, lw=1.1)
    a.axvline(-90, color=GRAY, lw=0.7, ls=(0, (3, 3)))
    a.set_title(f_ + (f"  dz={dz[f_]:+.2f}" if f_ in dz else ""), fontsize=8.5)
    a.set_ylabel(units.get(f_, "Δ log10 công suất"))
for a in axs[1]:
    a.set_xlabel("giây so với N2 đầu tiên")
save(fig, "fig05_onset_locked.png")

# --------------------------------------------- F. last 120 s and AUC ---
def ema_rec(v, tau_s):
    out = np.empty_like(v, dtype=np.float64)
    a = 1 - np.exp(-5.0 / tau_s)
    for r in np.unique(r_idx):
        m = np.where(r_idx == r)[0]
        s = v[m[0]]
        for j in m:
            s = (1 - a) * s + a * v[j]
            out[j] = s
    return out


def runmean_rec(v):
    out = np.empty_like(v, dtype=np.float64)
    for r in np.unique(r_idx):
        m = np.where(r_idx == r)[0]
        out[m] = np.cumsum(v[m]) / np.arange(1, len(m) + 1)
    return out

FE30 = {f_: ema_rec(v.astype(np.float64), 30) for f_, v in F.items()}
FRM = {f_: FE30[f_] - runmean_rec(v.astype(np.float64)) for f_, v in F.items()}
mA = ev & (y < 90)
labA = (y[mA] < 40).astype(int)
mB = ev & (y >= 40) & (y < 300)
labB = (y[mB] < 90).astype(int)
auc = {}
for f_ in F:
    for name, src in (("raw", F), ("ema30", FE30)):
        a1 = roc_auc_score(labA, src[f_][mA]); a2 = roc_auc_score(labB, src[f_][mB])
        auc[(f_, name)] = (max(a1, 1 - a1), max(a2, 1 - a2))
S["auc_A_0-40_vs_40-90"] = {f_: [rnd(auc[(f_, "raw")][0]), rnd(auc[(f_, "ema30")][0])] for f_ in F}
S["auc_B_40-90_vs_90-300"] = {f_: [rnd(auc[(f_, "raw")][1]), rnd(auc[(f_, "ema30")][1])] for f_ in F}
S["auc_A_elapsed"] = rnd(max(roc_auc_score(labA, t_stop[mA]), 1 - roc_auc_score(labA, t_stop[mA])))
topA = sorted(F, key=lambda f_: -auc[(f_, "ema30")][0])[:10]

fig, ax = plt.subplots(1, 2, figsize=(10.5, 3.1), gridspec_kw=dict(wspace=0.3, width_ratios=[1.1, 1]))
zl = (lock_t >= -150) & (lock_t <= 60)
for f_, col in (("sigma/beta TP", BLUE), ("1/f slope TP", AQUA), ("delta/beta TP", ORANGE), ("beta TP", GRAY)):
    if f_ not in traj:
        M = np.full((len(late), len(lock_t)), np.nan)
        for ii, r in enumerate(late):
            m = r_idx == r
            b_ = F[f_][m & base_mask].mean()
            sub = (lock_t[None, :] == np.round(rel[m])[:, None])
            ok = sub.any(1)
            M[ii, sub.argmax(1)[ok]] = F[f_][m][ok] - b_
        traj[f_] = (np.nanmedian(M, 0),)
    med = traj[f_][0]
    ref = np.nanmean(med[(lock_t >= -150) & (lock_t <= -120)])
    ax[0].plot(lock_t[zl], (med - ref)[zl], color=col, lw=1.4, label=f_)
ax[0].axvline(0, color=ORANGE, lw=1)
ax[0].axvspan(-40, 0, color=ORANGE, alpha=0.08, lw=0)
ax[0].axvspan(-90, -40, color=YELLOW, alpha=0.08, lw=0)
ax[0].set(title="150 s cuối: đổi dần, không có bậc ở -40 s",
          xlabel="giây so với N2", ylabel="Δ so với mức -150..-120 s")
ax[0].legend(fontsize=7)
yy = np.arange(len(topA))
ax[1].barh(yy - 0.18, [auc[(f_, "raw")][0] for f_ in topA], 0.34, color=GRAY, label="1 cửa sổ (thô)")
ax[1].barh(yy + 0.18, [auc[(f_, "ema30")][0] for f_ in topA], 0.34, color=BLUE, label="EMA causal 30 s")
ax[1].set_yticks(yy, topA)
ax[1].invert_yaxis()
ax[1].axvline(0.5, color=INK2, lw=0.8)
ax[1].set(xlim=(0.45, 0.8), title="AUC tách bin 0-40 s khỏi 40-90 s", xlabel="AUC (0.5 = ngẫu nhiên)")
ax[1].legend(loc="lower right")
save(fig, "fig06_last90s_auc.png")

# --------------------------------------------------- G. predictability ---
from scipy.stats import rankdata
ye, te = y[ev], t_stop[ev]
rt = rankdata(te)
ry = rankdata(ye)
def partial_rank(x):
    rx = rankdata(x)
    bx = np.polyfit(rt, rx, 1); by = np.polyfit(rt, ry, 1)
    ex, ey = rx - np.polyval(bx, rt), ry - np.polyval(by, rt)
    return np.corrcoef(ex, ey)[0, 1]
pred = []
for f_ in F:
    raw = spearmanr(F[f_][ev], ye).statistic
    e30 = spearmanr(FE30[f_][ev], ye).statistic
    rm = spearmanr(FRM[f_][ev], ye).statistic
    pr = partial_rank(FRM[f_][ev])
    bins_r = []
    for lo, hi in ((0, 40), (300, 600.1)):
        m = (ye >= lo) & (ye < hi)
        bins_r.append(spearmanr(FRM[f_][ev][m], ye[m]).statistic)
    pred.append(dict(feature=f_, raw=raw, ema30=e30, ema30_minus_runmean=rm, partial_elapsed=pr,
                     rho_bin0_40=bins_r[0], rho_bin300_600=bins_r[1]))
P = pd.DataFrame(pred)
P["best_abs"] = P[["raw", "ema30", "ema30_minus_runmean"]].abs().max(1)
P = P.sort_values("best_abs", ascending=False)
S["predictability"] = P.round(3).to_dict(orient="records")
S["median_abs_rho"] = {c: rnd(P[c].abs().median()) for c in ("raw", "ema30", "ema30_minus_runmean", "partial_elapsed")}
top = P.head(12)
fig, ax = plt.subplots(figsize=(10.5, 3.3))
yy = np.arange(len(top))
for j, (c, col, lab) in enumerate((("raw", GRAY, "1 cửa sổ"), ("ema30", AQUA, "EMA 30 s"),
                                   ("ema30_minus_runmean", BLUE, "EMA 30 s − running mean"),
                                   ("partial_elapsed", ORANGE, "như trên, đã khử elapsed time"))):
    ax.barh(yy + (j - 1.5) * 0.2, top[c].abs(), 0.19, color=col, label=lab)
ax.axvline(abs(S["spearman_elapsed_target"]), color=INK2, lw=1, ls=(0, (4, 3)))
ax.text(abs(S["spearman_elapsed_target"]) - 0.005, -0.7, "elapsed time một mình", fontsize=7, color=INK2, ha="right")
ax.set_yticks(yy, top.feature)
ax.invert_yaxis()
ax.set(title="Mức liên hệ |Spearman| giữa feature causal và target (80,975 cửa sổ đánh giá)", xlabel="|rho|")
ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.17), ncols=4, fontsize=7)
save(fig, "fig07_predictability.png")

# --------------------------------------- H. participants and nights ---
base_rec = {}
for f_ in ("alpha 8-11 TP", "beta TP", "log SD TP", "sigma/beta TP", "1/f slope TP", "delta TP"):
    v = pd.Series(F[f_][ttn >= 300]).groupby(r_idx[ttn >= 300]).median()
    d_ = pd.DataFrame({"v": v, "subject": rec.subject.values[v.index]})
    gg = d_.groupby("subject").v
    kk = gg.size()
    mb = (kk * (gg.mean() - d_.v.mean()) ** 2).sum() / (gg.ngroups - 1)
    mw = sum(((d_[d_.subject == s].v - m) ** 2).sum() for s, m in gg.mean().items()) / max((kk - 1).sum(), 1)
    k0_ = (kk.sum() - (kk ** 2).sum() / kk.sum()) / (gg.ngroups - 1)
    base_rec[f_] = dict(icc1=rnd((mb - mw) / (mb + (k0_ - 1) * mw)),
                        sd_between_dB=rnd(10 * np.sqrt(max(mb - mw, 0) / k0_), 2) if "slope" not in f_ else rnd(np.sqrt(max(mb - mw, 0) / k0_), 3),
                        sd_within_dB=rnd(10 * np.sqrt(mw), 2) if "slope" not in f_ else rnd(np.sqrt(mw), 3))
S["baseline_feature_icc"] = base_rec
# between-recording variance share, raw vs minus running mean
def between_share(v):
    s_ = pd.Series(v[ev]).groupby(r_idx[ev])
    return s_.mean().var() / pd.Series(v[ev]).var()
S["between_rec_share_median"] = [rnd(np.median([between_share(F[f_]) for f_ in F])),
                                 rnd(np.median([between_share(F[f_] - runmean_rec(F[f_].astype(np.float64))) for f_ in F]))]

# --------------------------------------------- I. train vs test, off-grid ---
first_min_bad = pd.Series((n_bad > 0)[t_stop <= 60].astype(float)).groupby(r_idx[t_stop <= 60]).mean()
rec["first_min_bad"] = first_min_bad.reindex(rec.index).values
tr, te_ = rec.split == "train", rec.split == "test"
S["first_min_bad_median_train_test"] = [rnd(rec[tr].first_min_bad.median()), rnd(rec[te_].first_min_bad.median())]
S["first_min_bad_p"] = rnd(mannwhitneyu(rec[tr].first_min_bad, rec[te_].first_min_bad).pvalue, 4)
late_on = rec.n2_onset >= 900
S["offgrid_vs_ongrid_late"] = dict(
    first_sample_zero_pct=[rnd(100 * rec[rec.offgrid].first_sample_zero.mean(), 1),
                           rnd(100 * rec[~rec.offgrid & late_on].first_sample_zero.mean(), 1)],
    first_min_bad_median=[rnd(rec[rec.offgrid].first_min_bad.median()),
                          rnd(rec[~rec.offgrid & late_on].first_min_bad.median())])
fig, ax = plt.subplots(1, 3, figsize=(10.5, 2.9), gridspec_kw=dict(wspace=0.35))
for lab, m, col in (("train", tr, BLUE), ("test", te_, ORANGE)):
    v = np.sort(rec[m].n2_onset / 60)
    ax[0].step(v, np.arange(1, len(v) + 1) / len(v), color=col, lw=1.3, label=f"{lab} (n={m.sum()})")
ax[0].set(title=f"Onset: train và test (p={S['onset_train_test_p']:.2f})", xlabel="phút", ylabel="tỉ lệ tích luỹ")
ax[0].legend()
groups_ = [("train", tr), ("test", te_), ("on-grid\n≥15 phút", ~rec.offgrid & late_on), ("off-grid", rec.offgrid)]
vals = [rec[m].first_min_bad.dropna().values for _, m in groups_]
bp = ax[1].boxplot(vals, tick_labels=[g_[0] for g_ in groups_], widths=0.5, patch_artist=True, showfliers=False)
for patch, col in zip(bp["boxes"], [BLUE, ORANGE, GRAY, AQUA]):
    patch.set_facecolor(col); patch.set_alpha(0.5)
ax[1].set(title="Tỉ lệ cửa sổ BAD trong phút đầu", ylabel="tỉ lệ cửa sổ có ≥1 kênh BAD")
ax[1].tick_params(axis="x", labelsize=7)
ax[1].grid(axis="x", visible=False)
labs = ["on-grid", "off-grid"]
v1 = [100 * rec[~rec.offgrid].first_sample_zero.mean(), 100 * rec[rec.offgrid].first_sample_zero.mean()]
b = ax[2].bar(labs, v1, color=[GRAY, AQUA], width=0.5)
ax[2].bar_label(b, fmt="%.0f%%", fontsize=7.5, color=INK2, padding=1)
ax[2].set(title="Bắt đầu bằng mẫu ≈0 (khởi động)", ylabel="% bản ghi", ylim=(0, 100))
ax[2].grid(axis="x", visible=False)
save(fig, "fig08_split_offgrid.png")

json.dump(S, open(f"{CACHE}/stats.json", "w"), indent=1, ensure_ascii=False, default=float)
print("done; figures:", sorted(os.listdir(FIG)))
