"""Submission step 1: convert one CND test set (organisers' dataCND folder) into the external-set cache read by
eval_external.ext_trials, with EXACTLY the corpus preprocessing of prep eaP_n5_t60 (prepare_data.build, variant 'sup'):

  EEG   supplied pre_dataSub<N>.mat (32 Hz, organisers' 1-9 Hz zero-phase filter + bad-channel repair: exempt, once)
        -> 64 B64 channels by label -> 58-channel montage b64ustc58 -> CAR over the present channels
        -> 1-s edge trim -> full-trial z-score (exempt, once) -> causal running EA (tau 60 s, start = fixed Fuglsang
        prior weighted as 5 s, shrinkage 0.1; restarts in every trial: nothing is carried across trials)
  env   supplied pre_dataStim<N>.mat envelopes (32 Hz), same trim, full-trial z-score
  dom / orel / odiff   causal dominance and onset targets of build_glimpse.py ('onorm'; 'norm' = dom only) from the
        supplied 128-Hz envelopes dataStim<N>.mat, 128 -> 32 Hz, same trim
Stream order = the organisers' order (envelope_stream1, envelope_stream2): column k of env is stream k+1; nothing is
re-ordered. No labels exist: att / att128 are zeros (placeholders, never read by the predictor).
Every trial is processed on its own (no subject or trial identity enters the preprocessing).

Output <out>/<name>/cache/{index.csv, eaP_n5_t60/<key>.npz, glimpsenorm/<key>.npz, glimpseonorm/<key>.npz},
key = <name>_S<subject:03d>_T<trial index:02d> (trial index 0-based, the column of the CND cell array).

Check mode (--verify DS): convert a corpus dataset from data/cnd/<folder>/dataCND (subjects --subjects) to a scratch
folder and compare with cache/eaP_n5_t60 and cache/glimpse{norm,onorm} of the corpus (where column 0 is the
majority-attended stream: the comparison allows the column swap and reports it).
usage: python -I sub_convert.py --data_dir <set>/dataCND --name TEST_NJU [--out DIR]
       python -I sub_convert.py --verify DTU --subjects 1 2 --out <scratch>
"""
import argparse
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)                                    # python -I drops the script directory

import numpy as np                                          # noqa: E402
import pandas as pd                                         # noqa: E402
import scipy.io as sio                                      # noqa: E402

from prepare_data import CFG, causal_ea, ea_prior, zscore_trial  # noqa: E402
from common import B64U, MONTAGES                           # noqa: E402

_argv, sys.argv = sys.argv, [sys.argv[0], "onorm"]          # build_glimpse reads its variant from argv at import
import build_glimpse as BG                                  # noqa: E402
sys.argv = _argv

PREP = "eaP_n5_t60"
FS = 32
TRIM = int(round(1.0 * FS))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
EXT = os.path.abspath(os.path.join(HERE, "..", "external_data"))
CORPUS_CND = {"DTU": "DTU2018", "KUL": "KUL2016", "AVGC": "AVGCAAD2024", "UHD": "KUL_AAD_UltraHD2021", "USTC": "USTC2026"}


def _str(x):
    while isinstance(x, np.ndarray):
        if x.size == 0:
            return ""
        x = x.ravel()[0]
    return str(x)


def load_cnd(path, key):
    s = sio.loadmat(path)[key][0, 0]
    return s


def stream_rows(stim):
    names = [_str(x) for x in stim["names"].ravel()]
    rows = [(int(n.rsplit("_stream", 1)[1]), i) for i, n in enumerate(names) if "_stream" in n and n.startswith("envelope")]
    if len(rows) != 2:
        raise ValueError(f"expected 2 envelope streams, got {names}")
    return [i for _, i in sorted(rows)]                     # by stream number: stream1, stream2


def subject_trials(d, sid):
    """Per trial: (eeg32 (T, C), labels, env32 (T, 2), env128 (N, 2))."""
    ne = load_cnd(os.path.join(d, f"pre_dataSub{sid}.mat"), "neural")
    st = load_cnd(os.path.join(d, f"pre_dataStim{sid}.mat"), "stim")
    st128 = load_cnd(os.path.join(d, f"dataStim{sid}.mat"), "stim")
    assert float(ne["fs"].ravel()[0]) == FS and float(st["fs"].ravel()[0]) == FS and float(st128["fs"].ravel()[0]) == 128
    labels = [_str(c) for c in ne["chanlocs"]["labels"].ravel()]
    r32, r128 = stream_rows(st), stream_rows(st128)
    out = []
    for t in range(ne["data"].shape[1]):
        eeg = np.asarray(ne["data"][0, t], np.float64)
        env = np.stack([np.asarray(st["data"][i, t], np.float64).ravel() for i in r32], 1)
        e128 = np.stack([np.asarray(st128["data"][i, t], np.float64).ravel() for i in r128], 1)
        out.append((eeg, labels, env, e128))
    return out


def preprocess(eeg, labels, env, e128):
    """-> dict(eeg (T, 58), env (T, 2), t0, n128, dom/orel/odiff (T, 2)), exactly prepare_data.build + build_glimpse."""
    up = [c.upper() for c in labels]
    pos = {c: i for i, c in enumerate(up)}
    src = np.array([pos.get(c, -1) for c in B64U])
    present = src >= 0
    x = np.zeros((eeg.shape[0], 64))
    x[:, present] = eeg[:, src[present]]
    sel = np.array([B64U.index(c) for c in MONTAGES[CFG["montage"]]])
    x, pres = x[:, sel], present[sel]
    x[:, pres] = x[:, pres] - x[:, pres].mean(1, keepdims=True)
    n128 = len(e128)
    n = min(len(x), len(env), -(-n128 // 4))
    x, env = x[TRIM:n - TRIM], env[TRIM:n - TRIM]
    x, env = zscore_trial(x), zscore_trial(env)
    P = CFG["preps"][PREP]
    x = causal_ea(x, FS, tau_s=P.get("ea_tau", 60.0), R0=ea_prior(P, x), n0=P.get("ea_n0"), shrink=P.get("ea_shrink", 0.1))
    T = len(x)
    g = BG.features(e128)
    out = dict(eeg=x.astype(np.float32), env=env.astype(np.float32), t0=TRIM / FS, n128=n128, missing=int((~pres).sum()))
    for k in ("dom", "orel", "odiff"):
        out[k] = g[k][::4][TRIM:TRIM + T].astype(np.float32)
        assert len(out[k]) == T, (k, len(out[k]), T)
    return out


def write(name, root, items):
    c = os.path.join(root, name, "cache")
    for sub in (PREP, "glimpsenorm", "glimpseonorm"):
        os.makedirs(os.path.join(c, sub), exist_ok=True)
    rows = []
    for key, sid, t, p in items:
        T = len(p["eeg"])
        np.savez(os.path.join(c, PREP, key + ".npz"), eeg=p["eeg"], env=p["env"], att=np.zeros(T, np.int8), t0=p["t0"],
                 att128=np.zeros(p["n128"], np.int8), n128=p["n128"])
        np.savez(os.path.join(c, "glimpsenorm", key + ".npz"), dom=p["dom"])
        np.savez(os.path.join(c, "glimpseonorm", key + ".npz"), dom=p["dom"], orel=p["orel"], odiff=p["odiff"])
        rows.append(dict(key=key, subject=sid, trial=t, n=p["n128"], n32=T, missing_channels=p["missing"]))
    pd.DataFrame(rows).to_csv(os.path.join(c, "index.csv"), index=False)
    return pd.DataFrame(rows)


def subjects_in(d):
    return sorted(int(f[len("pre_dataSub"):-4]) for f in os.listdir(d)
                  if f.startswith("pre_dataSub") and f.endswith(".mat") and f[len("pre_dataSub"):-4].isdigit())


def verify(ds, subs, root):
    from common_exp import rows_of
    d = os.path.join(ROOT, "data", "cnd", CORPUS_CND[ds], "dataCND")
    rows = rows_of(ds)
    worst = dict(eeg=0.0, env=0.0, dom=0.0, orel=0.0)
    n = 0
    for sid in subs:
        rs = rows[rows.subject == sid].sort_values("trial")
        for t, tr in enumerate(subject_trials(d, sid)):
            p = preprocess(*tr)
            r = rs.iloc[t]
            z = np.load(os.path.join(HERE, "cache", PREP, r["key"] + ".npz"))
            go = np.load(os.path.join(HERE, "cache", "glimpseonorm", r["key"] + ".npz"))
            T = min(len(z["eeg"]), len(p["eeg"]))
            assert abs(len(z["eeg"]) - len(p["eeg"])) <= 1, (r["key"], len(z["eeg"]), len(p["eeg"]))
            sw = np.abs(z["env"][:T] - p["env"][:T]).max() > np.abs(z["env"][:T] - p["env"][:T, ::-1]).max()
            pe = p["env"][:T, ::-1] if sw else p["env"][:T]
            worst["eeg"] = max(worst["eeg"], float(np.abs(z["eeg"][:T] - p["eeg"][:T]).max()))
            worst["env"] = max(worst["env"], float(np.abs(z["env"][:T] - pe).max()))
            for k in ("dom", "orel"):
                pk = p[k][:T, ::-1] if sw else p[k][:T]
                worst[k] = max(worst[k], float(np.abs(go[k][:T] - pk).max()))
            n += 1
    print(f"verify {ds} subjects {subs}: {n} trials, max |difference| vs corpus cache: "
          + ", ".join(f"{k} {v:.2e}" for k, v in worst.items()), flush=True)
    return worst


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--data_dir")
    ap.add_argument("--name")
    ap.add_argument("--out", default=EXT)
    ap.add_argument("--verify", choices=sorted(CORPUS_CND))
    ap.add_argument("--subjects", nargs="*", type=int)
    a = ap.parse_args()
    if a.verify:
        w = verify(a.verify, a.subjects or [1], a.out)
        sys.exit(0 if max(w.values()) < 1e-3 else 1)
    items = []
    for sid in (a.subjects or subjects_in(a.data_dir)):
        for t, tr in enumerate(subject_trials(a.data_dir, sid)):
            items.append((f"{a.name}_S{sid:03d}_T{t:02d}", sid, t, preprocess(*tr)))
        print(f"subject {sid}: {t + 1} trials", flush=True)
    df = write(a.name, a.out, items)
    print(f"{a.name}: {len(df)} trials, {df.subject.nunique()} subjects, missing montage channels per trial: "
          f"{sorted(df.missing_channels.unique().tolist())}; written to {os.path.join(a.out, a.name, 'cache')}", flush=True)
