"""Two follow-up checks prompted by the label-free test-set diagnostics (test_checks.py). Training / labelled data only,
except the label-free part A3 on TEST_DLAAD. Classifiers only (DirectNet sw2, BMMS3; seeds 0-1): the decoders read
EEG only and cannot hold the stimulus.

A  stimulus memorisation. 14 % of the TEST_DLAAD envelopes occur in KUL / AVGC / UHD. A classifier trained on those
   trials might recognise a stimulus and output "the stream that was attended in training" without the EEG.
   A1 corpus: the SAME trials scored by the LODO fold model (never saw them) and by the all-five model (trained on
      them), with real EEG, zero EEG and mismatched EEG (another trial of the same dataset); usual synthetic switch
      schedule. Memorisation = all-five accuracy above 50 % without (matching) EEG while the fold model stays at 50 %.
   A2 AASD (stimuli unseen by every model): all-five, real / zero EEG.
   A3 TEST_DLAAD, label-free: zero-EEG logits of the all-five models; every test stream is matched against the
      training envelopes (normalised cross-correlation of a 20-s snippet, KUL / AVGC / UHD). For matched streams
      (peak NCC > 0.8): is the zero-EEG output biased towards the stream whose training copy was the attended one?
B  level imbalance. NJU streams differ by 4.2 dB (training sets 0.6-1.7). The dominance target dom depends on it
   (env and onset are z-scored per trial: unaffected). LODO fold models on their held-out dataset with one random
   stream per trial made L dB louder before dom is computed (L = 0, 4, 8; L = 0 is checked against the cached dom).
   Frame accuracy overall and split by whether the boosted stream is the attended one at that frame.
Output results/research/test_followups_summary.txt (+ .csv).
"""
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import numpy as np  # noqa: E402
import pandas as pd  # noqa: E402
import torch  # noqa: E402
from scipy.signal import fftconvolve  # noqa: E402

import decoders  # noqa: E402
import direct_classifier as DC  # noqa: E402
import eval_external as E  # noqa: E402
from bmms_3feature_model import BMMS3Enhanced  # noqa: E402
from common import DS_KEYS, SEED, load_env  # noqa: E402
from common_exp import CFG, EXP, FS, load_trial, rows_of  # noqa: E402
from decoders import D, _targets  # noqa: E402
from e1_switch import draw_switch_times  # noqa: E402

_argv, sys.argv = sys.argv, [sys.argv[0], "norm"]
import build_glimpse as BG  # noqa: E402
sys.argv = _argv

RES = os.path.join(EXP, "results", "research")
N_TR = 16
FAM = {"directnet": (lambda i: f"direct_clf__eaP_n5_t60_w10b3_ptSPK{'' if i == 0 else f'_s{i}'}_aug-sw2_wts",
                     lambda i: f"all5_direct_clf_w10b3sw2P_s{i}.pt", DC.DirectNet),
       "bmms3": (lambda i: f"direct_clf__eaP_n5_t60_w10_bmms3enh_ptSPK_s{i}",
                 lambda i: f"all5_direct_clf_w10bmms3enhP_s{i}.pt", BMMS3Enhanced)}
SEEDS = (0, 1)
L = []


def say(*a):
    s = " ".join(str(x) for x in a); L.append(s); print(s, flush=True)


def setup():
    DC.W = int(10 * FS); DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS
    D.update(targets=["env", "onset", "dom"], augment={**CFG["dnn_augment"], **CFG["aug_variants"]["cluster"]})
    decoders.apply_overrides(["model_blocks=3", "infer_bs=8"], ["clf_p_switch2=0.5"])
    E.CONF["prep"] = "eaP_n5_t60"
    torch.set_num_threads(2)


def net(fam, path):
    m = FAM[fam][2](58, 3, "vlaai_ws").to(DC.DEV)
    m.load_state_dict(torch.load(path, map_location=DC.DEV, weights_only=True)); m.eval()
    return m


def schedule(T128, n32, t0, rng):
    times = draw_switch_times(rng, T128 / 128); rng.integers(0, 2)
    st128, cur, prev = np.zeros(T128, bool), False, 0
    for tt in times:
        s0 = min(T128 - 1, int(round(tt * 128))); st128[prev:s0] = cur; cur, prev = not cur, s0
    st128[prev:] = cur
    return st128[np.minimum(T128 - 1, np.round((np.arange(n32) / FS + t0) * 128).astype(int))]


@torch.no_grad()
def run(m, x, S, st):
    lg, starts = DC.logits(m, x, np.where(st[:, None, None], S[:, ::-1], S))
    return lg, np.maximum(starts - DC.HOP, 0)


def part_a(rng):
    rows = []
    for ds in DS_KEYS:
        r = rows_of(ds); pick = r.iloc[np.sort(rng.choice(len(r), N_TR, replace=False))]
        trials = [(row, load_trial("eaP_n5_t60", row["key"])) for _, row in pick.iterrows()]
        sch = [schedule(int(row["n"]), len(t["eeg"]), t["t0"], np.random.default_rng(SEED + i)) for i, (row, t) in enumerate(trials)]
        for fam, (fold, all5, _) in FAM.items():
            for seed in SEEDS:
                for kind, path in (("fold", os.path.join(EXP, "models", f"fold_{fold(seed)}__{ds}.pt")),
                                   ("all5", os.path.join(EXP, "models", all5(seed)))):
                    m = net(fam, path); acc = {}
                    for i, (row, t) in enumerate(trials):
                        S, x, st = _targets(t), t["eeg"], sch[i]
                        other = trials[(i + 1) % len(trials)][1]["eeg"]
                        xm = np.resize(other, x.shape).astype(np.float32)
                        for cond, xx in (("real", x), ("zero", np.zeros_like(x)), ("mismatch", xm)):
                            lg, idx = run(m, xx, S, st)
                            truth = (t["att"].astype(int) ^ st)[idx]
                            acc.setdefault(cond, []).append((lg * np.where(truth == 0, 1., -1.), np.abs(lg)))
                    for cond, v in acc.items():
                        sg = np.concatenate([a for a, _ in v]); ab = np.concatenate([b for _, b in v])
                        rows.append(dict(part="A1", ds=ds, family=fam, seed=seed, model=kind, cond=cond,
                                         acc=float((sg > 0).mean() * 100), signed=float(sg.mean()), absl=float(ab.mean())))
                    del m; torch.cuda.empty_cache()
        print("A1", ds, flush=True)
    # A2 AASD: all-five only (no model saw AASD)
    tr = E.ext_trials("AASD"); tr = [tr[i] for i in np.sort(rng.choice(len(tr), 60, replace=False))]
    for fam, (_, all5, _) in FAM.items():
        for seed in SEEDS:
            m = net(fam, os.path.join(EXP, "models", all5(seed))); acc = {}
            for t in tr:
                S = _targets(t); st = np.zeros(len(t["eeg"]), bool)
                for cond, xx in (("real", t["eeg"]), ("zero", np.zeros_like(t["eeg"]))):
                    lg, idx = run(m, xx, S, st)
                    acc.setdefault(cond, []).append((lg * np.where(t["att"].astype(int)[idx] == 0, 1., -1.), np.abs(lg)))
            for cond, v in acc.items():
                sg = np.concatenate([a for a, _ in v]); ab = np.concatenate([b for _, b in v])
                rows.append(dict(part="A2", ds="AASD", family=fam, seed=seed, model="all5", cond=cond,
                                 acc=float((sg > 0).mean() * 100), signed=float(sg.mean()), absl=float(ab.mean())))
            del m
    print("A2 done", flush=True)
    return rows


def part_a3(rng):
    """TEST_DLAAD zero-EEG bias towards the training-attended copy of a matched stream (label-free)."""
    bank, owner = [], []
    for s in ("KUL", "AVGC", "UHD"):
        for k in rows_of(s).key:
            e = np.load(os.path.join(EXP, "cache", "eaP_n5_t60", k + ".npz"))["env"].astype(np.float32)
            for c in (0, 1):                                   # corpus column 0 = majority-attended stream
                bank += [e[:, c], np.zeros(64, np.float32)]; owner += [c] * len(e) + [-1] * 64
    B = np.concatenate(bank); owner = np.array(owner, np.int8)
    en = np.sqrt(np.maximum(fftconvolve(B ** 2, np.ones(640, np.float32), "valid"), 1e-6))
    tr = E.ext_trials("TEST_DLAAD"); tr = [tr[i] for i in np.sort(rng.choice(len(tr), 80, replace=False))]
    match = []
    for t in tr:
        m_ = []
        for c in (0, 1):
            e = t["env"][:, c].astype(np.float32); s = e[len(e) // 2 - 320:len(e) // 2 + 320]; s = s - s.mean()
            ncc = fftconvolve(B, s[::-1], "valid") / (en * np.linalg.norm(s))
            j = int(np.argmax(np.abs(ncc))); m_.append((float(abs(ncc[j])), int(owner[j])))
        match.append(m_)
    del B, en
    rows = []
    for fam, (_, all5, _) in FAM.items():
        for seed in SEEDS:
            m = net(fam, os.path.join(EXP, "models", all5(seed)))
            for t, mm in zip(tr, match):
                S = _targets(t); st = np.zeros(len(t["eeg"]), bool)
                lz, _ = run(m, np.zeros_like(t["eeg"]), S, st); lr, _ = run(m, t["eeg"], S, st)
                # predicted training-attended test stream: c with owner 0 (attended copy); -1 if unmatched / ambiguous
                att_c = [c for c in (0, 1) if mm[c][0] > .8 and mm[c][1] == 0]
                ign_c = [c for c in (0, 1) if mm[c][0] > .8 and mm[c][1] == 1]
                tgt = att_c[0] if len(att_c) == 1 else (1 - ign_c[0] if len(ign_c) == 1 and not att_c else -1)
                rows.append(dict(part="A3", family=fam, seed=seed, key=t["key"], ncc0=mm[0][0], ncc1=mm[1][0],
                                 matched=int(max(mm[0][0], mm[1][0]) > .8), target=tgt,
                                 zero_abs=float(np.abs(lz).mean()), real_abs=float(np.abs(lr).mean()),
                                 zero_mean=float(lz.mean()),
                                 zero_toward=float(np.mean(lz * (1. if tgt == 0 else -1.)) if tgt >= 0 else np.nan)))
            del m
    print("A3 done", flush=True)
    return rows


def part_b(rng):
    rows = []
    for ds in DS_KEYS:
        r = rows_of(ds); pick = r.iloc[np.sort(rng.choice(len(r), N_TR, replace=False))]
        trials = [(row, load_trial("eaP_n5_t60", row["key"])) for _, row in pick.iterrows()]
        sch = [schedule(int(row["n"]), len(t["eeg"]), t["t0"], np.random.default_rng(SEED + 7 + i)) for i, (row, t) in enumerate(trials)]
        boost = rng.integers(0, 2, len(trials))
        doms = {}
        for i, (row, t) in enumerate(trials):
            e = np.asarray(load_env(row["key"]), np.float64)
            for lv in (0, 4, 8):
                ee = e.copy(); ee[:, boost[i]] *= 10 ** (lv * 0.6 / 20)
                d = BG.features(ee)["dom"][::4][32:32 + len(t["eeg"])].astype(np.float32)
                if lv == 0:
                    assert np.abs(d - t["dom"]).max() < 1e-4, ("dom recomputation differs", row["key"])
                doms[(i, lv)] = d
        for fam, (fold, _, _) in FAM.items():
            for seed in SEEDS:
                m = net(fam, os.path.join(EXP, "models", f"fold_{fold(seed)}__{ds}.pt"))
                for lv in (0, 4, 8):
                    sg_all, sg_att, sg_ign = [], [], []
                    for i, (row, t) in enumerate(trials):
                        S = _targets(dict(t, dom=doms[(i, lv)]))
                        lg, idx = run(m, t["eeg"], S, sch[i])
                        truth = (t["att"].astype(int) ^ sch[i])[idx]
                        # column of the boosted stream at the network input (columns swap with the schedule)
                        bcol = boost[i] ^ sch[i][idx].astype(int)
                        sg = lg * np.where(truth == 0, 1., -1.)
                        sg_all.append(sg); sg_att.append(sg[bcol == truth]); sg_ign.append(sg[bcol != truth])
                    f = lambda v: float((np.concatenate(v) > 0).mean() * 100)
                    rows.append(dict(part="B", ds=ds, family=fam, seed=seed, level_db=lv, acc=f(sg_all),
                                     acc_boost_attended=f(sg_att), acc_boost_ignored=f(sg_ign)))
                del m; torch.cuda.empty_cache()
        print("B", ds, flush=True)
    return rows


def main():
    assert os.environ.get("GLIMPSE_THR") == "norm"
    setup(); rng = np.random.default_rng(0)
    ra, rb, r3 = part_a(rng), part_b(rng), part_a3(rng)
    df = pd.DataFrame(ra + rb); df.to_csv(os.path.join(RES, "test_followups.csv"), index=False)
    a3 = pd.DataFrame(r3); a3.to_csv(os.path.join(RES, "test_followups_a3.csv"), index=False)
    A = df[df.part.isin(["A1", "A2"])]
    say("A1/A2 stimulus memorisation: frame accuracy % (mean over seeds 0-1), synthetic schedule on the corpus, real labels on AASD")
    say(A.pivot_table(index=["family", "ds"], columns=["model", "cond"], values="acc").round(2).to_string())
    say("\nmean |logit| by condition:")
    say(A.pivot_table(index=["family", "ds"], columns=["model", "cond"], values="absl").round(3).to_string())
    say("\nA3 TEST_DLAAD (80 trials), zero-EEG all-five logits; matched = a stream with peak NCC > 0.8 to a training envelope")
    g = a3.groupby(["family", "matched"]).agg(trials=("key", "count"), zero_abs=("zero_abs", "mean"), real_abs=("real_abs", "mean"),
                                              zero_mean=("zero_mean", "mean"))
    say(g.round(3).to_string())
    t = a3[a3.target >= 0].groupby("family").agg(n=("key", "count"), toward_mean=("zero_toward", "mean"),
                                                   share_toward=("zero_toward", lambda v: float((v > 0).mean() * 100)))
    say("bias of the zero-EEG logit towards the stream whose training copy was attended (> 0 = towards; 50 % = none):")
    say(t.round(3).to_string())
    B = df[df.part == "B"]
    say("\nB level imbalance: LODO fold models on the held-out dataset, frame accuracy % (mean over seeds)")
    for col in ("acc", "acc_boost_attended", "acc_boost_ignored"):
        say(f"\n{col}:")
        p = B.pivot_table(index=["family", "ds"], columns="level_db", values=col)
        p.loc[("mean", "LODO"), :] = p.groupby(level=0).mean().mean()
        say(p.round(2).to_string())
    open(os.path.join(RES, "test_followups_summary.txt"), "w").write("\n".join(L) + "\n")


if __name__ == "__main__":
    main()
