"""Convert SparrKULee (Accou et al. 2024, KU Leuven; derivatives/) to the decoder_lodo cache format.

Single-speaker listening (audiobooks / podcasts, ~85 subjects, ~15-min runs). Used as EXTRA TRAINING / PRETRAINING data
only (never held out), so there is no ignored stream: column 1 of `env` is a MISMATCHED negative = the same run's
envelope circularly shifted by half its length (other speech, same speaker / statistics), att = 0 everywhere.

Input (download_sparrkulee.py -> raw/):
  preprocessed_eeg/sub-*/ses-*/*_desc-preproc-audio-<stim>_eeg.npy   (64, T) float64, 64 Hz, BioSemi-64 order,
                                                                      authors' preprocessing (high-pass, MWF, CAR)
  preprocessed_stimuli/<stim>_-_envelope.npy                          (T, 1) 64 Hz, gammatone (28 bands) |.|^0.6
  (EEG and envelope are aligned at stimulus onset; checked: ridge-decoder r peaks at lag 0.)
Pipeline (= prep 'mat_reac'):
  EEG : filtfilt Butterworth order 2, 1-9 Hz -> 32 Hz -> 58 montage channels (b64ustc58, by name) -> CAR -> full-trial
        z-score -> causal running EA carried across the subject's runs (session, run order) -> 1-s edge trim
  ENV : filtfilt order 2, 1-9 Hz -> 32 Hz -> z-score -> trim; mismatched column as above
Output: cache/mat_reac/SPK_S{sub:03d}_{ses}_R{run:02d}.npz (env, att, t0) + <key>_eeg.npy (float16, (T, 58); loaded
with mmap by run_experiment.load_external so parallel jobs share the page cache); cache/index.csv; cache/.done
usage: python convert_sparrkulee.py [mat_reac|mat]"""
import glob
import os
import re
import sys

import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from scipy.signal import butter, sosfiltfilt

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
sys.path.insert(0, os.path.join(ROOT, "experiments", "decoder_lodo"))
from prepare_data import causal_ea, zscore_trial  # noqa: E402
from common import MONTAGES  # noqa: E402

RAW = os.path.join(HERE, "raw")
PREP = sys.argv[1] if len(sys.argv) > 1 else "mat_reac"
OUT = os.path.join(HERE, "cache", PREP)
B64 = ['FP1', 'AF7', 'AF3', 'F1', 'F3', 'F5', 'F7', 'FT7', 'FC5', 'FC3', 'FC1', 'C1', 'C3', 'C5', 'T7', 'TP7', 'CP5',
       'CP3', 'CP1', 'P1', 'P3', 'P5', 'P7', 'P9', 'PO7', 'PO3', 'O1', 'IZ', 'OZ', 'POZ', 'PZ', 'CPZ', 'FPZ', 'FP2', 'AF8',
       'AF4', 'AFZ', 'FZ', 'F2', 'F4', 'F6', 'F8', 'FT8', 'FC6', 'FC4', 'FC2', 'FCZ', 'CZ', 'C2', 'C4', 'C6', 'T8', 'TP8',
       'CP6', 'CP4', 'CP2', 'P2', 'P4', 'P6', 'P8', 'P10', 'PO8', 'PO4', 'O2']
SEL = [B64.index(c.upper()) for c in MONTAGES["b64ustc58"]]
FS0, TRIM = 64, 32
BP = butter(2, [1, 9], "bandpass", fs=FS0, output="sos")
PAT = re.compile(r"sub-(\d+)_(ses-[^_]+)_task-[^_]+_run-(\d+)_desc-preproc-audio-(.+)_eeg\.npy$")


def subject(sub, files):
    R, rows = None, []
    for f in files:                                        # sorted by (session, run) = recording order
        m = PAT.search(os.path.basename(f))
        ses, run, stim = m.group(2), int(m.group(3)), m.group(4)
        key = f"SPK_S{sub:03d}_{ses.replace('ses-', '')}_R{run:02d}"
        envf = os.path.join(RAW, "preprocessed_stimuli", f"{stim}_-_envelope.npy")
        if not os.path.exists(envf):
            rows.append(dict(key=key, subject=sub, session=ses, run=run, stim=stim, n=0, note="no envelope"))
            continue
        x = np.load(f).astype(np.float64)
        if x.shape[0] == 64 and x.shape[1] != 64:
            x = x.T
        x = sosfiltfilt(BP, x, axis=0)[::2][:, SEL]
        x -= x.mean(1, keepdims=True)
        eeg = zscore_trial(x)
        if PREP == "mat_reac":
            eeg, R = causal_ea(eeg, 32, tau_s=60.0, R0=R, return_R=True)
        v = sosfiltfilt(BP, np.load(envf).astype(np.float64).reshape(-1), axis=0)[::2]
        n = min(len(eeg), len(v))
        v = zscore_trial(v[:n, None])[:, 0]
        env = np.stack([v, np.roll(v, n // 2)], 1)         # attended, mismatched negative
        np.save(os.path.join(OUT, key + "_eeg.npy"), eeg[TRIM:n - TRIM].astype(np.float16))
        np.savez(os.path.join(OUT, key + ".npz"), env=env[TRIM:n - TRIM].astype(np.float32),
                 att=np.zeros(n - 2 * TRIM, np.int8), t0=TRIM / 32)
        rows.append(dict(key=key, subject=sub, session=ses, run=run, stim=stim, n=n - 2 * TRIM, note=""))
    return rows


if __name__ == "__main__":
    os.makedirs(OUT, exist_ok=True)
    files = sorted(glob.glob(os.path.join(RAW, "preprocessed_eeg", "sub-*", "ses-*", "*_eeg.npy")))
    files = [f for f in files if os.path.exists(f + ".ok")]
    by = {}
    for f in files:
        m = PAT.search(os.path.basename(f))
        if m:
            by.setdefault(int(m.group(1)), []).append((m.group(2), int(m.group(3)), f))
    jobs = [(s, [f for _, _, f in sorted(v)]) for s, v in sorted(by.items())]
    rows = sum(Parallel(n_jobs=3)(delayed(subject)(s, fs) for s, fs in jobs), [])
    I = pd.DataFrame(rows)
    I.to_csv(os.path.join(HERE, "cache", f"index_{PREP}.csv"), index=False)
    ok = I[I.n > 0]
    print(f"SparrKULee [{PREP}]: {ok.subject.nunique()} subjects, {len(ok)} runs, {ok.n.sum() / 32 / 3600:.1f} h; "
          f"skipped {len(I) - len(ok)} runs without envelope ({sorted(I[I.n == 0].stim.unique())[:10]})")
    open(os.path.join(HERE, "cache", f".done_{PREP}"), "w").close()
