"""Turn data/cnd, Broderick2018 and Fuglsang2020 into the (T, 4) logit tables
the causal transformer already trains on, and load AASD / Carta2026 the same way.

The transformer never reads EEG. Each trial is four direct-classifier logits
(one per seed) at 4 Hz. Those four networks are the published all-five
checkpoints ``all5_direct_clf_w10b3sw2P_s0..s3`` (window 10 s, 3 blocks,
targets env/onset/dom). The same weights produced
``ext_clf_{AASD,Carta2026}_w10b3sw2P_evidence_s*.pkl``, so train and test
share one front-end.

Corpus trials are read from the eaP_n5_t60 cache (built from data/cnd) plus
glimpsenorm dominance. Fuglsang2020 already has that same prep. Broderick2018
was only converted to mat_reac and has no glimpse cache: dominance is rebuilt
here from the released 128 Hz envelopes with the same causal formula as
build_glimpse.py 'norm', aligned to the trimmed 32 Hz trial.

Training trials get one synthetic-switch schedule applied to the stimulus
before the classifier, as direct_classifier.run_fold does. Corpus trials use
the challenge schedule (mean interval 60 s, minimum segment 30 s). Broderick
and Fuglsang trials are only about 50-60 s, so that schedule almost never
fires and every label would stay on column 0; they use a 20 s mean interval
and an 8 s minimum segment, in the same range as the real switches in Carta2026.
AASD and Carta2026 keep the real key-press / cue labels already stored in the
evidence pickles.
"""
import os
import pickle
import time
import zlib

import numpy as np
import pandas as pd
import scipy.io as sio
import torch
from scipy.signal import butter, lfilter, sosfilt, sosfiltfilt

from common import SEED
from common_exp import DATA_EXP, EXP, EXTERNAL_DATA, FS
from decoders import D, DEV, _targets, apply_overrides
from e1_switch import draw_switch_times
from prepare_data import zscore_trial
import direct_classifier as DC

CACHE = os.path.join(EXP, "cache", "logit_v2")
CND_ID = {"001": "AVGC", "003": "DTU", "004": "KUL", "005": "UHD", "007": "USTC"}
TRAIN_DATASETS = ("AVGC", "DTU", "KUL", "UHD", "USTC", "BRO", "FUG")
TEST_DATASETS = ("AASD", "Carta2026")
EXT_TAG = "_w10b3sw2P"
TRIM = 32
BP128 = butter(2, [1.0, 9.0], "bandpass", fs=128, output="sos")
SMOOTH_A = np.exp(-1.0 / (0.025 * 128))
MEAN_A = np.exp(-1.0 / (10.0 * 128))
BRO_ROOT = os.path.join(EXTERNAL_DATA, "Broderick2018")
CHUNK = int(os.environ.get("LOGIT_CHUNK", "256"))


def _rng(key):
    return np.random.default_rng(np.uint32(SEED + zlib.crc32(key.encode("utf-8"))))


def _switch_mask(key, n, mean_int=60.0, min_seg=30.0):
    times = draw_switch_times(_rng(key), n / float(FS), mean_int=mean_int, min_seg=min_seg)
    st = np.zeros(n, np.int8)
    cur, prev = 0, 0
    for tt in times:
        s0 = min(n - 1, int(round(float(tt) * FS)))
        st[prev:s0] = cur
        cur, prev = 1 - cur, s0
    st[prev:] = cur
    return st


def _past_mean(x):
    run = lfilter([1.0 - MEAN_A], [1.0, -MEAN_A], x)
    return np.concatenate([[x[0]], run[:-1]])


def _dom_from_raw128(raw):
    """Causal glimpsenorm dominance at 128 Hz. raw is (N, 2), non-negative envelopes."""
    s = lfilter([1.0 - SMOOTH_A], [1.0, -SMOOTH_A], np.maximum(raw, 1e-9), axis=0)
    tot = s.sum(1)
    g = (s[:, 0] - s[:, 1]) / (tot + 0.05 * _past_mean(tot))
    return sosfilt(BP128, np.stack([g, -g], 1), axis=0)


def configure_front():
    apply_overrides(["model_blocks=3"], [])
    D["targets"] = ["env", "onset", "dom"]
    DC.W = int(10 * FS)
    DC.HOP = int(0.25 * FS)
    DC.FUT = int(0.5 * FS)
    DC.POS0 = DC.W - DC.FUT - DC.HOP
    DC.LAB = DC.W - FS


def load_nets():
    configure_front()
    nets = []
    for i in range(4):
        path = os.path.join(DATA_EXP, "models", f"all5_direct_clf_w10b3sw2P_s{i}.pt")
        net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
        try:
            state = torch.load(path, map_location=DEV, weights_only=True)
        except Exception:
            state = torch.load(path, map_location=DEV, weights_only=False)
        net.load_state_dict(state)
        net.eval()
        nets.append(net)
        print(f"loaded front {i} {path} temporal {tuple(net.tw.shape)}", flush=True)
    if tuple(nets[0].tw.shape) != (DC.W,):
        raise RuntimeError(f"checkpoint window {tuple(nets[0].tw.shape)} != {DC.W}")
    return nets


@torch.inference_mode()
def _logits(net, x, S):
    T = len(x)
    xp = np.concatenate([
        np.zeros((DC.POS0, 58), np.float32),
        np.asarray(x, np.float32),
        np.zeros((DC.FUT + DC.HOP, 58), np.float32),
    ])
    Sp = np.concatenate([
        np.zeros((DC.POS0,) + S.shape[1:], np.float32),
        np.asarray(S, np.float32),
        np.zeros((DC.FUT + DC.HOP,) + S.shape[1:], np.float32),
    ])
    starts = np.arange(0, T, DC.HOP)
    out = np.empty(len(starts), np.float32)
    for i in range(0, len(starts), CHUNK):
        a = starts[i:i + CHUNK]
        xb = torch.from_numpy(np.stack([xp[k:k + DC.W] for k in a])).to(DEV, non_blocking=True)
        sb = torch.from_numpy(np.stack([Sp[k:k + DC.W] for k in a])).to(DEV, non_blocking=True)
        out[i:i + len(a)] = net(xb, sb).detach().float().cpu().numpy()
    return out, starts


def decision_row(nets, key, eeg, env, att, dom, switch=(60.0, 30.0)):
    n = min(len(eeg), len(env), len(att), len(dom))
    eeg, env, att, dom = eeg[:n], env[:n], att[:n], dom[:n]
    S = _targets(dict(eeg=eeg, env=env, att=att, dom=dom))
    n = min(n, len(S))
    eeg, att, S = eeg[:n], att[:n], S[:n]
    st = _switch_mask(key, n, mean_int=switch[0], min_seg=switch[1])
    Ssw = np.where(st[:, None, None], np.ascontiguousarray(S[:, ::-1]), S)
    att_s = (att.astype(np.int16) ^ st.astype(np.int16)).astype(np.int16)
    cols, starts = [], None
    for net in nets:
        lg, starts = _logits(net, eeg, Ssw)
        cols.append(lg)
    D = np.stack(cols, 1).astype(np.float32)
    truth = att_s[np.maximum(starts - DC.HOP, 0)].astype(np.int8)
    if len(truth) != len(D):
        raise RuntimeError(f"{key}: label {len(truth)} != logits {len(D)}")
    return D, truth


def _dump(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    tmp = path + ".tmp"
    with open(tmp, "wb") as fh:
        pickle.dump(obj, fh, protocol=4)
    os.replace(tmp, path)


def _load_ds(name):
    path = os.path.join(CACHE, name + ".pkl")
    with open(path, "rb") as fh:
        blob = pickle.load(fh)
    return blob["logits"], blob["att"]


def _summarize(name, logits, att):
    y = np.concatenate([att[k] for k in att])
    n_sw = 0
    for a in att.values():
        if len(a) > 1 and np.any(a[1:] != a[:-1]):
            n_sw += 1
    print(f"  {name}: {len(logits)} trials, {len(y)} frames, "
          f"P(att=0) {float((y == 0).mean()):.3f}, trials with a switch {n_sw}", flush=True)


def _build_dataset(name, keys, load_trial, nets, switch=(60.0, 30.0)):
    path = os.path.join(CACHE, name + ".pkl")
    if os.path.exists(path) and os.environ.get("REBUILD_LOGITS") != "1":
        logits, att = _load_ds(name)
        _summarize(name, logits, att)
        return
    logits, att = {}, {}
    t0 = time.time()
    for i, key in enumerate(keys):
        eeg, env, a, dom = load_trial(key)
        D, truth = decision_row(nets, key, eeg, env, a, dom, switch=switch)
        logits[key] = D
        att[key] = truth
        if (i + 1) % 25 == 0 or i + 1 == len(keys):
            rate = (i + 1) / max(time.time() - t0, 1e-6)
            print(f"  {name} {i + 1}/{len(keys)}  {rate:.2f} trials/s", flush=True)
    _dump(path, {"logits": logits, "att": att})
    _summarize(name, logits, att)
    print(f"  wrote {path} ({time.time() - t0:.0f}s)", flush=True)


def _cnd_trial(key):
    z = np.load(os.path.join(DATA_EXP, "cache", "eaP_n5_t60", key + ".npz"))
    g = np.load(os.path.join(DATA_EXP, "cache", "glimpsenorm", key + ".npz"))
    return z["eeg"], z["env"], z["att"], g["dom"]


def _fug_trial(key):
    z = np.load(os.path.join(EXTERNAL_DATA, "Fuglsang2020", "cache", "eaP_n5_t60", key + ".npz"))
    g = np.load(os.path.join(EXTERNAL_DATA, "Fuglsang2020", "cache", "glimpsenorm", key + ".npz"))
    return z["eeg"], z["env"], z["att"], g["dom"]


def _broderick_raw(story, run, cache):
    hit = cache.get((story, run))
    if hit is not None:
        return hit
    path = os.path.join(BRO_ROOT, "raw", "Cocktail Party", "Stimuli", "Envelopes", story, f"{story}_{run}_env.mat")
    d = sio.loadmat(path)
    if int(np.asarray(d["fsEnv"]).ravel()[0]) != 128:
        raise RuntimeError(f"unexpected envelope rate in {path}")
    raw = np.asarray(d["envelope"], np.float64).ravel()
    cache[(story, run)] = raw
    return raw


def _check_broderick_alignment(row, raw_cache):
    """Band-passed envelope must match the converted npz. Dominance uses that same grid."""
    key = row.name
    subject, run = int(row.subject), int(row.run)
    att_story, other = ("20000", "Journey") if subject <= 17 else ("Journey", "20000")
    raw = np.stack([_broderick_raw(att_story, run, raw_cache), _broderick_raw(other, run, raw_cache)], 1)
    n32 = int(row.n32)
    bp = sosfiltfilt(BP128, raw, axis=0)[::4]
    got = zscore_trial(bp[:n32])[TRIM:n32 - TRIM].astype(np.float32)
    ref = np.load(os.path.join(BRO_ROOT, "cache", "mat_reac", key + ".npz"))["env"]
    if got.shape != ref.shape:
        raise RuntimeError(f"{key}: rebuilt env {got.shape} != cache {ref.shape}")
    err = float(np.max(np.abs(got - ref)))
    print(f"Broderick envelope alignment {key}: max abs err {err:.3e}", flush=True)
    if err > 1e-3:
        raise RuntimeError(f"Broderick envelope rebuild does not match the cache ({err})")
    return err


def _broderick_loader(index, raw_cache):
    def load_trial(key):
        row = index.loc[key]
        subject, run = int(row.subject), int(row.run)
        att_story, other = ("20000", "Journey") if subject <= 17 else ("Journey", "20000")
        raw = np.stack([
            _broderick_raw(att_story, run, raw_cache),
            _broderick_raw(other, run, raw_cache),
        ], 1)
        n32 = int(row.n32)
        z = np.load(os.path.join(BRO_ROOT, "cache", "mat_reac", key + ".npz"))
        dom = _dom_from_raw128(raw)[::4][:n32][TRIM:n32 - TRIM].astype(np.float32)
        return z["eeg"], z["env"], z["att"], dom
    return load_trial


def ensure_train(nets=None):
    os.makedirs(CACHE, exist_ok=True)
    own = nets is None
    if own:
        nets = load_nets()
    raw_cache = {}
    index = pd.read_csv(os.path.join(BRO_ROOT, "cache", "index.csv")).set_index("key")
    if not os.path.exists(os.path.join(CACHE, "BRO.pkl")) or os.environ.get("REBUILD_LOGITS") == "1":
        _check_broderick_alignment(index.iloc[0], raw_cache)
    _build_dataset("BRO", list(index.index), _broderick_loader(index, raw_cache), nets, switch=(20.0, 8.0))
    fug_dir = os.path.join(EXTERNAL_DATA, "Fuglsang2020", "cache", "eaP_n5_t60")
    fug_keys = sorted(f[:-4] for f in os.listdir(fug_dir) if f.endswith(".npz"))
    _build_dataset("FUG", fug_keys, _fug_trial, nets, switch=(20.0, 8.0))
    by_ds = {ds: [] for ds in CND_ID.values()}
    cnd_dir = os.path.join(DATA_EXP, "cache", "eaP_n5_t60")
    for fn in sorted(os.listdir(cnd_dir)):
        if not fn.endswith(".npz"):
            continue
        did = fn.split("-")[1]
        if did not in CND_ID:
            raise RuntimeError(f"unknown CND dataset id in {fn}")
        by_ds[CND_ID[did]].append(fn[:-4])
    for ds, keys in by_ds.items():
        if not keys:
            raise RuntimeError(f"no eaP_n5_t60 trials for {ds}")
        _build_dataset(ds, keys, _cnd_trial, nets)
    if own and DEV.type == "cuda":
        torch.cuda.empty_cache()


def load_train():
    logits, att = {}, {}
    for name in TRAIN_DATASETS:
        lg, a = _load_ds(name)
        logits[name] = lg
        att[name] = a
    return logits, att


def load_test():
    logits, meta = {}, {}
    for ds in TEST_DATASETS:
        evs = []
        for i in range(4):
            path = os.path.join(DATA_EXP, "results", "research", f"ext_clf_{ds}{EXT_TAG}_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            D = np.stack(cols, 1)
            e0 = evs[0][key]
            if len(D) != len(e0["t_end"]) or len(D) != len(e0["att"]):
                raise RuntimeError(f"{ds}/{key}: logit length does not match evidence")
            logits[ds][key] = D
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]),
                t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]),
                n128=int(e0["n128"]),
                subject=int(e0["subject"]),
            )
        print(f"  test {ds}: {len(logits[ds])} trials", flush=True)
    return logits, meta


def probe(n_cnd=1, n_fug=1):
    """Time a handful of front-end passes and check the Broderick envelope grid."""
    nets = load_nets()
    index = pd.read_csv(os.path.join(BRO_ROOT, "cache", "index.csv")).set_index("key")
    raw_cache = {}
    _check_broderick_alignment(index.iloc[0], raw_cache)
    load_bro = _broderick_loader(index, raw_cache)
    jobs = [("BRO", index.index[0], load_bro)]
    fug_dir = os.path.join(EXTERNAL_DATA, "Fuglsang2020", "cache", "eaP_n5_t60")
    fug_keys = sorted(f[:-4] for f in os.listdir(fug_dir) if f.endswith(".npz"))
    for key in fug_keys[:n_fug]:
        jobs.append(("FUG", key, _fug_trial))
    cnd_dir = os.path.join(DATA_EXP, "cache", "eaP_n5_t60")
    cnd_keys = sorted(f[:-4] for f in os.listdir(cnd_dir) if f.startswith("dataset-001-"))
    for key in cnd_keys[:n_cnd]:
        jobs.append(("AVGC", key, _cnd_trial))
    switch = {"BRO": (20.0, 8.0), "FUG": (20.0, 8.0), "AVGC": (60.0, 30.0)}
    built = {}
    for name, key, loader in jobs:
        eeg, env, att, dom = loader(key)
        t0 = time.time()
        D, truth = decision_row(nets, key, eeg, env, att, dom, switch=switch[name])
        built.setdefault(name, {"logits": {}, "att": {}})
        built[name]["logits"][key] = D
        built[name]["att"][key] = truth
        print(f"probe {name} {key}: eeg {tuple(eeg.shape)} logits {D.shape} "
              f"P(att=0) {float((truth == 0).mean()):.3f} ({time.time() - t0:.1f}s)", flush=True)
    return built
