"""Cheap screen of the CBraMod embedding as a view (first gate before D2 / D3): can a LINEAR backward model read the attended envelope from it, and is the result
different from the classifier? Ridge from the 32-dim embedding at 0.25-s hops to the attended envelope (mean over 0.25 s) at three delays; trained on the other corpus sets
+ Fuglsang (LODO) or on all corpus sets + Fuglsang (AASD, Carta2026); never on the evaluated set. Evidence = causal 10-s correlation with stream 0 minus stream 1, mean over
the delays. Reported per set: d' = mean / sd of the sign-corrected evidence, d' per subject (mean +- SE), correlation with the 4-seed sw2 classifier evidence, d' of the best
2-way linear mix (fitted on the same data: optimistic), null = envelopes circularly shifted by >= 20 s. Control: the same ridge on the plain EEG at 4 Hz (58 channels, 4 lags).
A linear read-out at 4 Hz is a low bound for what DirectNet could do with the embedding; a negative result does not prove there is nothing inside.
usage: python fm_screen.py -> results/research/fm_screen.txt"""
import glob
import os
import pickle
import sys

import numpy as np
import pandas as pd

from common_exp import EXP, fold_file, rows_of
from common import DS_KEYS

XT = os.path.join(EXP, "..", "external_data")
R = os.path.join(EXP, "results", "research")
CLF = ["direct_clf__eaP_n5_t60_w10b3_ptSPK" + t + "_aug-sw2" for t in ("", "_s1", "_s2", "_s3")]
DELAYS, N, LAM = (1, 2, 3), 40, 1e-2
lines = []


def say(s=""):
    print(s, flush=True)
    lines.append(s)


NAME = sys.argv[1] if len(sys.argv) > 1 else "cbramod"
FEATS = ("Z", "A") if NAME == "cbramod" else ("Z",)


def zpath(key):
    return os.path.join(EXP, "cache", "fm_" + NAME, key + ".npy")


def bin8(x):
    n = len(x) // 8
    return x[:n * 8].reshape((n, 8) + x.shape[1:]).mean(1)


def load(key, path=None):
    if path is None:
        z = np.load(os.path.join(EXP, "cache", "eaP_n5_t60", key + ".npz"))
    else:
        z = np.load(path)
    eeg, env, att = z["eeg"], z["env"], z["att"]
    n = min(len(np.load(zpath(key), mmap_mode="r")), len(eeg) // 8)
    return dict(key=key, n=n, E=bin8(eeg.astype(np.float32))[:n] if NAME == "cbramod" else None, env=bin8(env.astype(np.float32))[:n], att=(bin8(att.astype(np.float32))[:n] > 0.5).astype(int))


def feat(t, f):
    if f == "Z":
        return np.load(zpath(t["key"]))[:t["n"]]
    A = np.concatenate([np.roll(t["E"], L, 0) for L in range(4)], 1)            # 4 lags of the 4-Hz EEG
    A[:3] = 0
    return A


def stats(trials, f):
    """Sums of one group of trials: n, sum x, sum y, x'x, x'y (nothing per trial is kept)."""
    st = None
    for t in trials:
        e = t["env"][np.arange(len(t["att"])), t["att"]]
        y = np.stack([np.roll(e, L) for L in DELAYS], 1)[4:].astype(np.float64)
        x = feat(t, f)[4:].astype(np.float64)
        v = [len(x), x.sum(0), y.sum(0), x.T @ x, x.T @ y]
        st = v if st is None else [a + b for a, b in zip(st, v)]
    return st


def ridge(groups):
    n, sx, sy, xx, xy_ = [sum(g[i] for g in groups) for i in range(5)]
    m, my = sx / n, sy / n
    C = xx / n - np.outer(m, m)
    s = np.sqrt(np.diag(C)) + 1e-6
    W = np.linalg.solve(C / np.outer(s, s) + LAM * np.eye(len(s)), (xy_ / n - np.outer(m, my)) / s[:, None])
    return m, s, W


def wcorr(a, b):
    """Causal correlation over the last N steps."""
    c = lambda v: np.r_[0.0, np.cumsum(v)]
    sum_ = lambda v: c(v)[N:] - c(v)[:-N]
    sa, sb, saa, sbb, sab = sum_(a), sum_(b), sum_(a * a), sum_(b * b), sum_(a * b)
    r = (N * sab - sa * sb) / np.sqrt(np.maximum((N * saa - sa ** 2) * (N * sbb - sb ** 2), 1e-12))
    return np.r_[np.zeros(N - 1), r]


def evidence(t, model, f, shift=0):
    m, s, W = model
    Rc = (feat(t, f) - m) / s @ W
    d = np.zeros(len(Rc))
    env = np.roll(t["env"], shift, 0) if shift else t["env"]
    for j, L in enumerate(DELAYS):
        d += wcorr(Rc[:, j], np.roll(env[:, 0], L)) - wcorr(Rc[:, j], np.roll(env[:, 1], L))
    return d / len(DELAYS)


def clf_real(ds):
    Es = [pickle.load(open(os.path.join(R, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl"), "rb")) for i in range(4)]
    return {k: np.mean([(e[k]["r0"] - e[k]["r1"]).astype(float) for e in Es], 0) * np.where(Es[0][k]["att"] == 0, 1.0, -1.0) for k in Es[0]}


def report(name, trials, models, c, subj):
    rs = np.random.default_rng(0)
    pack = {f: {} for f in models}
    nul = []
    for t in trials:
        sg = np.where(t["att"] == 0, 1.0, -1.0)
        v = c[t["key"]][: max(len(c[t["key"]]) - 2, 0)]                          # classifier hop i uses the EEG of embedding patch i + 2
        for f, mdl in models.items():
            d = evidence(t, mdl, f) * sg
            n = min(len(d) - 2 - N, len(v) - N)
            if n > 0:
                pack[f].setdefault("z", []).append(d[N + 2:N + 2 + n])
                pack[f].setdefault("c", []).append(v[N:N + n])
                pack[f].setdefault("s", []).append(np.full(n, subj(t["key"])))
        nul.append(evidence(t, models["Z"], "Z", int(rs.integers(80, max(81, len(t["att"]) - 80))))[N:] * sg[N:])
    nz = np.concatenate(nul)
    row = dict(set=name, null_d=nz.mean() / nz.std())
    for f in models:
        z, cl, sb = (np.concatenate(pack[f][k]) for k in ("z", "c", "s"))
        ds_ = [z[sb == u].mean() / z[sb == u].std() for u in np.unique(sb)]
        X = np.stack([z / z.std(), cl / cl.std()], 1)
        w = np.linalg.solve(np.cov(X.T), X.mean(0))
        row.update({f + "_d": z.mean() / z.std(), f + "_d_subj": np.mean(ds_), f + "_se": np.std(ds_) / np.sqrt(len(ds_)), f + "_corr": np.corrcoef(z, cl)[0, 1],
                    f + "_mix": (X @ w).mean() / (X @ w).std()})
        row["clf_d"] = cl.mean() / cl.std()
    return row


if __name__ == "__main__":
    corpus = {ds: [load(r["key"]) for _, r in rows_of(ds).iterrows()] for ds in DS_KEYS}
    print("corpus loaded", flush=True)
    fug = [load(os.path.basename(f)[:-4], f) for f in sorted(glob.glob(os.path.join(XT, "Fuglsang2020", "cache", "eaP_n5_t60", "*.npz")))]
    real = {ds: [load(os.path.basename(f)[:-4], f) for f in sorted(glob.glob(os.path.join(XT, ds, "cache", "eaP_n5_t60", "*.npz")))] for ds in ("AASD", "Carta2026")}
    groups = {d: {f: stats(corpus[d], f) for f in FEATS} for d in DS_KEYS}
    groups["FUG"] = {f: stats(fug, f) for f in FEATS}
    print("statistics done", flush=True)
    rows = []
    for ds in DS_KEYS:
        M = {f: ridge([groups[d][f] for d in groups if d != ds]) for f in FEATS}
        c, att = {}, {}
        for e in CLF:
            with open(fold_file(e, ds), "rb") as fh:
                ev = pickle.load(fh)["evidence"]["causal"]
            for k, v in ev.items():
                c[k] = c.get(k, 0) + (v["r0"] - v["r1"]).astype(np.float32) / len(CLF)
                att[k] = v["att"]
            del ev
        c = {k: v * np.where(att[k] == 0, 1.0, -1.0) for k, v in c.items()}
        sid = dict(zip(rows_of(ds)["key"], pd.factorize(rows_of(ds)["subj_uid"])[0]))
        rows.append(report("synthetic " + ds, corpus[ds], M, c, sid.get))
        print(rows[-1], flush=True)
    M = {f: ridge([groups[d][f] for d in groups]) for f in FEATS}
    for ds in real:
        rows.append(report("real " + ds, real[ds], M, clf_real(ds), lambda k: int(k.split("_")[1][1:])))
        print(rows[-1], flush=True)
    df = pd.DataFrame(rows).set_index("set")
    df.to_csv(os.path.join(R, f"fm_screen_{NAME}.csv"))
    pd.set_option("display.width", 220)
    say("d' linear read-out: Z = CBraMod embedding, A = plain 4-Hz EEG (control); null = shifted envelopes (embedding); clf = 4-seed sw2 classifier; corr = with clf; mix = best 2-way mix")
    say(df[[c for c in ("null_d", "Z_d", "Z_d_subj", "Z_se", "A_d", "A_d_subj", "A_se", "clf_d", "Z_corr", "A_corr", "Z_mix", "A_mix") if c in df]].round(3).to_string())
    open(os.path.join(R, f"fm_screen_{NAME}.txt"), "w").write("\n".join(lines) + "\n")
