"""Caches for the E5-E10 batch, all on top of the best pipeline: causal low-pass 45 Hz + first second of each recording zeroed (E4).

One pass over the BrainVision files builds, per 5 s window:
  flags.npy           (N, 4) uint8  bad-channel flags from the RAW window: max |x| > 200 uV or RMS 30-45 Hz > 5 uV (E5)
  X_af7.npy           features with the AF7 interference lines (21.33 and 42.67 Hz) replaced by their neighbours (E7a)
  X_band40.npy        features computed on 0.5-40 Hz instead of 0.5-45 Hz (total power, gamma 30-40 Hz, spectrogram 1-39 Hz) (E7c)
  X_nfft512.npy       features with 4 s STFT frames (0.25 Hz resolution, 3 frames per window) (E9a)
  X_nfft128.npy       features with 1 s STFT frames (1 Hz resolution, 9 frames per window) (E9b)
From the existing caches (no file reading):
  X_relspec.npy       the 88 spectrogram features made relative (minus the log total power of the pair) (E7b)
  emarm.npy           (N, 42) EMA 30 s minus running mean of the 42 core features, causal per recording (E6)
  ea_raw_f16.npy      (N, 4, 640) filtered windows after causal Euclidean alignment (E10): running covariance per recording
                      (forgetting 60 s, prior = population covariance of the 500 public training recordings with a weight of one
                      window, updated only with windows without bad channels), whitening R^(-1/2), scaled by 10 so the CNN's
                      fixed 40 uV scale still applies
The three artifact feature groups always come from the unfiltered window (as in every earlier experiment).
"""
import os
import sys
import time

import mne
import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(HERE))
sys.path.insert(0, os.path.join(ROOT, "experiments/stft_tree"))
sys.path.insert(0, os.path.join(ROOT, "experiments/iir_filter"))
import run_stft_tree as R  # noqa: E402
import run_iir_filter as I  # noqa: E402

CACHE = os.path.join(HERE, "cache")
E4C = os.path.join(ROOT, "experiments/e4_start/cache")
FS, WIN, CH = 128, 640, R.CH
BANDS = R.BANDS
log = R.log


def window_features(xw, nfft=256, hop=64, fmax=45.0, af7_interp=False):
    """Parametrised copy of run_stft_tree.window_features (identical output with the defaults)."""
    freqs = np.fft.rfftfreq(nfft, 1 / FS); df = freqs[1]
    hann = np.hanning(nfft); u = (hann ** 2).sum()
    fidx = np.arange((WIN - nfft) // hop + 1)[:, None] * hop + np.arange(nfft)
    dm = xw - xw.mean(axis=2, keepdims=True)
    P = np.abs(np.fft.rfft(dm[:, :, fidx] * hann, axis=-1)) ** 2 / (FS * u)
    P[..., 1:-1] *= 2
    if af7_interp:                                        # AF7 is channel 1; replace the bins around 21.33 and 42.67 Hz
        for f0 in (21 + 1 / 3, 42 + 2 / 3):
            bad = np.where(np.abs(freqs - f0) < max(df, 0.5))[0]
            lo, hi = bad.min() - 1, bad.max() + 1
            fill = 0.5 * (P[:, 1, :, lo] + P[:, 1, :, hi])                # (n, frames)
            for b in bad:
                P[:, 1, :, b] = fill
    Pm = P.mean(axis=2)
    f, names = [], []

    def add(arr, name):
        if arr.ndim == 2:
            for c, ch in enumerate(CH):
                f.append(arr[:, c]); names.append(f"{name}_{ch}")
        else:
            f.append(arr); names.append(name)

    bands = dict(BANDS); bands["gamma"] = (30, fmax)
    tot_m = (freqs >= 0.5) & (freqs < fmax)
    tot = Pm[..., tot_m].sum(-1) * df + 1e-6
    add(np.log10(tot), "logtot")
    bp, bp_fr = {}, {}
    for b, (lo, hi) in bands.items():
        m = (freqs >= lo) & (freqs < hi)
        bp[b] = Pm[..., m].sum(-1) * df + 1e-6
        bp_fr[b] = np.log10(P[..., m].sum(-1) * df + 1e-6)
        add(np.log10(bp[b]), f"log_{b}"); add(np.log10(bp[b] / tot), f"rel_{b}")
    add(np.log10(bp["theta"] / bp["alpha"]), "theta_alpha")
    add(np.log10(bp["sigma"] / bp["beta"]), "sigma_beta")
    add(np.log10(bp["delta"] / bp["beta"]), "delta_beta")
    Pt = Pm[..., tot_m]; Pn = Pt / (Pt.sum(-1, keepdims=True) + 1e-12)
    add(-(Pn * np.log(Pn + 1e-12)).sum(-1) / np.log(tot_m.sum()), "spec_entropy")
    cum = np.cumsum(Pn, axis=-1)
    add(freqs[tot_m][np.argmax(cum >= 0.95, axis=-1)], "sef95")
    add(freqs[tot_m][np.argmax(cum >= 0.5, axis=-1)], "sef50")
    fm = (freqs >= 2) & (freqs <= min(40, fmax)) & ~((freqs > 7) & (freqs < 17))
    lf = np.log10(freqs[fm]); lfc = lf - lf.mean()
    ly = np.log10(Pm[..., fm] + 1e-6)
    slope = ((ly - ly.mean(-1, keepdims=True)) * lfc).sum(-1) / (lfc ** 2).sum()
    add(slope, "aper_slope")
    sm = (freqs >= 11) & (freqs < 15)
    fit = ly.mean(-1, keepdims=True) + slope[..., None] * (np.log10(freqs[sm]) - lf.mean())
    add((np.log10(Pm[..., sm] + 1e-6) - fit).mean(-1), "sigma_resid")
    am = (freqs >= 7) & (freqs <= 13)
    add(freqs[am][np.argmax(Pm[..., am], axis=-1)], "alpha_peak")
    for b in ("delta", "alpha", "beta", "gamma"):
        add(bp_fr[b].std(-1), f"frstd_{b}")
    add(np.log10(P[..., tot_m].sum(-1) * df + 1e-6).std(-1), "frstd_tot")
    sd = dm.std(-1)
    add(np.log10(sd + 1e-3), "log_sd")
    add(np.log10(np.abs(np.diff(dm, axis=-1)).mean(-1) + 1e-3), "log_linelen")
    mx = np.abs(xw).max(-1)
    add(np.log10(mx + 1e-3), "log_maxabs")
    add((np.abs(xw) >= 780).mean(-1), "clip_frac")
    g = (freqs >= 30) & (freqs < 45)
    rms_hf = np.sqrt(Pm[..., g].sum(-1) * df)
    bad = (mx > 200) | (rms_hf > 5)
    add(bad.sum(-1).astype(np.float64), "n_bad")
    z = dm / (sd[..., None] + 1e-6)
    for a, b in ((0, 3), (1, 2), (0, 1), (2, 3)):
        add((z[:, a] * z[:, b]).mean(-1), f"corr_{CH[a]}_{CH[b]}")
    lp = np.log10(Pm + 1e-6)
    for pair, idx in (("TP", [0, 3]), ("AF", [1, 2])):
        for j in range(1, int(fmax)):
            m = (freqs >= j) & (freqs < j + 1)
            add(lp[:, idx][..., m].mean(axis=(1, 2)), f"spec{j:02d}_{pair}")
    return np.stack(f, 1).astype(np.float32), names, bad


SETS = {"X_af7": dict(af7_interp=True), "X_band40": dict(fmax=40.0), "X_nfft512": dict(nfft=512), "X_nfft128": dict(nfft=128)}


def art_cols(names):
    return [i for i, n in enumerate(names) if n.startswith(("clip_frac_", "log_maxabs_")) or n == "n_bad"]


def main():
    os.makedirs(CACHE, exist_ok=True)
    X0, names, win, rec = R.extract()
    names = list(names)
    Xm = np.load(f"{E4C}/mask_X.npy")                                  # base features of the best pipeline (217)
    # sanity: the parametrised function reproduces the base features with its defaults
    x_test = np.random.default_rng(0).standard_normal((3, 4, WIN)) * 20
    assert np.allclose(window_features(x_test)[0], R.window_features(x_test)[0], atol=1e-5), "feature port mismatch"
    counts = win.groupby("rec").size(); n_all = len(win)
    todo = [k for k in SETS if not os.path.exists(f"{CACHE}/{k}.npy")]
    if todo or not os.path.exists(f"{CACHE}/flags.npy"):
        out = {k: None for k in todo}; flags = np.zeros((n_all, 4), np.uint8); nm = {}
        sos = I.design(None); pos, t0 = 0, time.time()
        for i, r in rec.iterrows():
            vhdr = f"{R.BIDS}/{r.subject}/{r.session_id}/eeg/{r.rec_id}_task-sleeponset_eeg.vhdr"
            x = mne.io.read_raw_brainvision(vhdr, preload=True, verbose="ERROR").get_data(picks=CH) * 1e6
            n = int(counts[i]); x = x[:, : n * WIN]
            rw = x.reshape(4, n, WIN).transpose(1, 0, 2)
            flags[pos:pos + n] = window_features(rw)[2].astype(np.uint8)
            xm = x.copy(); xm[:, :128] = 0.0
            xf, _ = I.filter_stream(sos, xm)
            fw = xf.reshape(4, n, WIN).transpose(1, 0, 2)
            for k in todo:
                F, nm[k], _ = window_features(fw, **SETS[k])
                if out[k] is None:
                    out[k] = np.empty((n_all, F.shape[1]), np.float32)
                out[k][pos:pos + n] = F
            pos += n
            if (i + 1) % 100 == 0:
                log(f"build: {i + 1}/{len(rec)} recordings, {time.time() - t0:.0f} s")
        assert pos == n_all
        np.save(f"{CACHE}/flags.npy", flags)
        for k in todo:
            a_new, a_old = art_cols(nm[k]), art_cols(names)
            out[k][:, a_new] = Xm[:, a_old]                            # artifact features from the raw signal, as before
            np.save(f"{CACHE}/{k}.npy", out[k]); np.save(f"{CACHE}/{k}_names.npy", np.array(nm[k]))
        log(f"feature sets {todo} and flags built ({time.time() - t0:.0f} s); share of bad channel-windows {flags.mean():.3f}")
    if not os.path.exists(f"{CACHE}/X_relspec.npy"):
        X = Xm.copy()
        lt = {p: X[:, [names.index(f"logtot_{c}") for c in cs]].mean(1) for p, cs in (("TP", ("TP9", "TP10")), ("AF", ("AF7", "AF8")))}
        for j, n in enumerate(names):
            if n.startswith("spec") and n[4:6].isdigit():
                X[:, j] = X[:, j] - lt[n.split("_")[-1]]
        np.save(f"{CACHE}/X_relspec.npy", X)
        log("relative spectrogram features built")
    if not os.path.exists(f"{CACHE}/emarm.npy"):
        C, cn = R.core_matrix(Xm, names)
        k = len(cn)
        out = np.zeros((n_all, k), np.float32)
        for r, idx in win.groupby("rec").indices.items():
            out[idx] = R.context_one(C[idx])[:, 3 * k:4 * k]               # block "ema30_minus_rm"
        assert R.context_names(cn)[3 * k].startswith("ema30_minus_rm_")
        np.save(f"{CACHE}/emarm.npy", out)
        log(f"EMA30 - running mean features built ({k})")
    if not os.path.exists(f"{CACHE}/ea_raw_f16.npy"):
        raw = np.load(f"{E4C}/mask_raw_f16.npy")
        flags = np.load(f"{CACHE}/flags.npy").astype(bool)
        idx = win.groupby("rec").indices
        train_rows = np.concatenate([idx[r] for r in rec.index[rec.split == "train"]])
        good = train_rows[~flags[train_rows].any(1)]
        acc = np.zeros((4, 4)); nwin = 0
        for s in range(0, len(good), 4096):
            xb = raw[good[s:s + 4096]].astype(np.float64); xb -= xb.mean(-1, keepdims=True)
            acc += np.einsum("nct,ndt->cd", xb, xb) / WIN; nwin += len(xb)
        R0 = acc / nwin
        lam = np.exp(-5.0 / 60.0)
        out = np.empty_like(raw)
        for r, ii in idx.items():
            S, W = R0.copy(), 1.0
            for k in ii:
                x = raw[k].astype(np.float64); x = x - x.mean(-1, keepdims=True)
                if not flags[k].any():
                    S = lam * S + x @ x.T / WIN; W = lam * W + 1.0
                Rm = S / W
                ev, evec = np.linalg.eigh(Rm)
                Wh = evec @ np.diag(1.0 / np.sqrt(np.maximum(ev, 1e-6))) @ evec.T
                out[k] = (10.0 * Wh @ x).astype(np.float16)
        np.save(f"{CACHE}/ea_raw_f16.npy", out)
        log(f"Euclidean-aligned windows built; population covariance diagonal {np.round(np.diag(R0), 1).tolist()} uV^2")


if __name__ == "__main__":
    main()
