"""R2: is AASD (and Carta2026) accuracy fixable, or near chance by nature?  Diagnostics only, no tuning.

Part A, from the saved evidence of the reference pipeline (nested 4 decoder + 4 DirectNet sw2, all-five models,
fusion a = 0.5, HMM emissions fitted on the five corpus datasets; same as r1_acc_decision_grid.py):
  * per-subject Acc and a trial-level sign-flip null (flipping a trial's evidence flips its HMM decisions exactly,
    so the null keeps the within-trial temporal structure): how many subjects are significantly above / below 50 %;
  * per-subject lag scan: Acc of the 128-Hz predictions against labels shifted by -2..+10 s (label lag diagnosis);
  * Acc of the frame-wise evidence sign (no HMM).
Part B, an independent linear backward decoder (ridge, EEG lags 0-250 ms, prep eaP_n5_t60, 32 Hz) scored by
5-s / 10-s window match accuracy, in three training regimes:
  XDS   trained on the five corpus datasets (the competition situation);
  LOSO  same dataset, held-out subjects AND held-out stimuli (eval_external_clf.cv_split, k = 3; corpus: subjects);
  SS    subject-specific, held-out stimuli (3 folds over T); an upper bound that needs the subject's labels.
DTU and KUL are run the same way as references. Ridge strength is chosen on an inner split of the training trials.
"""
import os
import numpy as np
import pandas as pd
import torch

import eval_external as E
import r1_acc_decision_grid as R
from common import DS_KEYS, SEED
from common_exp import EXP, load_trial, rows_of
from e1_switch import expand, integ_hmm
from eval_external_clf import cv_split

RES = os.path.join(EXP, 'results', 'research')
PREP = 'eaP_n5_t60'
FS32 = 32
LAGS = 9                      # EEG samples t .. t+8 (0-250 ms after the stimulus sample)
ALPHAS = (1e-4, 1e-3, 1e-2, 1e-1, 1., 10.)
REAL = ('AASD', 'Carta2026')
P_RULES = (0.998, 0.95)
SHIFTS = np.arange(-2., 10.5, 0.5)
N_PERM = 20000


# ----------------------------------------------------------------------------------------------- part A
def part_a():
    torch.set_num_threads(2)
    clf, dec, la = R.load_reference()
    tab = R.tables('ALL', clf, dec)
    tr = R.components([t for d in DS_KEYS for t in tab[d].values()])
    parts = {d: {k: tr(t['x']) for k, t in tab[d].items()} for d in DS_KEYS}
    mu, sd = R.emissions(parts, tab, list(DS_KEYS), 0.5)
    subj_rows, lag_rows = [], []
    rng_null = np.random.default_rng(SEED)
    for ds in REAL:
        real = R.real_inputs(ds)
        for p in P_RULES:
            rng = np.random.default_rng(SEED)            # same guesses as eval_external.score
            per = {}
            for k in sorted(real):
                x, e = real[k]
                d_, c_ = tr(x); z = 0.5 * d_ + 0.5 * c_
                dec_ = integ_hmm(z, dict(p_stay=p, mu=mu, sd=sd))
                lab = expand(dec_, e['t_end'].astype(float), la, int(e['n128']), int(rng.integers(0, 2)))
                gt = e['att128'].astype(float)
                frame_ok = np.mean((z > 0) == (np.asarray(e['att']) == 0))
                per.setdefault(e['subject'], []).append((lab, gt, frame_ok))
            for s, lst in per.items():
                n = np.array([len(g) for _, g, _ in lst], float)
                c = np.array([np.sum(l == g) for l, g, _ in lst], float)
                acc = c.sum() / n.sum()
                flips = rng_null.integers(0, 2, size=(N_PERM, len(n))).astype(bool)
                null = np.where(flips, n - c, c).sum(1) / n.sum()
                pval = float((np.abs(null - .5) >= abs(acc - .5) - 1e-12).mean())
                subj_rows.append(dict(ds=ds, p_stay=p, subject=s, n_trials=len(n), acc=acc * 100,
                                      null_sd=null.std() * 100, p_two_sided=pval,
                                      frame_sign_acc=np.mean([f for _, _, f in lst]) * 100))
                for sh in SHIFTS:
                    m = int(round(sh * 128)); ok = tot = 0
                    for l, g, _ in lst:
                        if m >= 0:
                            a, b = l[m:], g[:len(g) - m]
                        else:
                            a, b = l[:len(l) + m], g[-m:]
                        ok += np.sum(a == b); tot += len(a)
                    lag_rows.append(dict(ds=ds, p_stay=p, subject=s, shift_s=sh, acc=ok / tot * 100))
    sr, lr = pd.DataFrame(subj_rows), pd.DataFrame(lag_rows)
    sr.to_csv(os.path.join(RES, 'r2_subject_null.csv'), index=False)
    lr.to_csv(os.path.join(RES, 'r2_lag_scan.csv'), index=False)
    return sr, lr


# ----------------------------------------------------------------------------------------------- part B
def lagged(eeg):
    T = len(eeg) - LAGS + 1
    return np.concatenate([eeg[j:j + T] for j in range(LAGS)], axis=1).astype(np.float64), T


def prepare(eeg, env, att, subject, stim):
    """Raw float32 storage; the 522-column lagged matrix is built on demand (37 GB for the whole corpus)."""
    env = np.asarray(env, np.float32)
    env = (env - env.mean(0)) / (env.std(0) + 1e-8)
    return dict(eeg=np.asarray(eeg, np.float32), env=env, att=np.asarray(att).astype(np.int8),
                subject=str(subject), stim=str(stim))


def xy(t):
    X, T = lagged(t['eeg'])
    env = t['env'][:T].astype(np.float64); att = t['att'][:T].astype(int)
    return X, env[np.arange(T), att], env, att


def corpus_iter(ds):
    for _, r in rows_of(ds).iterrows():
        t = load_trial(PREP, r['key'])
        yield prepare(t['eeg'], t['env'], t['att'], r['subj_uid'], r['trial'])


def real_trials(ds):
    c = os.path.join(E.EXT, ds, 'cache')
    I = pd.read_csv(os.path.join(c, 'index.csv'))
    out = []
    for _, r in I.iterrows():
        z = np.load(os.path.join(c, PREP, r['key'] + '.npz'))
        out.append(prepare(z['eeg'], z['env'], z['att'], r['key'].split('_')[1], r['key'].split('_')[2]))
    return out


def accumulate(trials):
    XtX = np.zeros((58 * LAGS, 58 * LAGS)); Xty = np.zeros(58 * LAGS)
    for t in trials:
        X, y, _, _ = xy(t)
        XtX += X.T @ X; Xty += X.T @ y
    return XtX, Xty


def solve(XtX, Xty, alpha):
    lam = alpha * np.trace(XtX) / len(XtX)
    return np.linalg.solve(XtX + lam * np.eye(len(XtX)), Xty)


def window_acc(trials, w, W):
    """Per-subject accuracy of the window decision (corr with env 0 vs env 1) against the majority label."""
    per = {}
    for t in trials:
        X, _, env, att = xy(t)
        rec = X @ w
        for s in range(0, len(rec) - W + 1, W // 2):
            r = rec[s:s + W]; e = env[s:s + W]
            c0 = np.corrcoef(r, e[:, 0])[0, 1]; c1 = np.corrcoef(r, e[:, 1])[0, 1]
            lab = int(np.mean(att[s:s + W]) > .5)
            per.setdefault(t['subject'], []).append(int(c1 > c0) == lab)
    return {s: float(np.mean(v)) * 100 for s, v in per.items()}


def pick(XtX_in, Xty_in, val, XtX_all, Xty_all):
    best = max(ALPHAS, key=lambda al: np.mean(list(window_acc(val, solve(XtX_in, Xty_in, al), 10 * FS32).values())))
    return solve(XtX_all, Xty_all, best), best


def fit(train, inner_seed=0):
    """Small sets: choose alpha on an inner 80/20 split of the training trials (10-s windows), refit on all."""
    rng = np.random.default_rng(inner_seed)
    idx = rng.permutation(len(train)); cut = max(1, int(.8 * len(train)))
    a_tr = [train[i] for i in idx[:cut]]; a_va = [train[i] for i in idx[cut:]] or a_tr
    XtX, Xty = accumulate(a_tr)
    XtX_all, Xty_all = accumulate(a_va)
    return pick(XtX, Xty, a_va, XtX + XtX_all, Xty + Xty_all)


def regime_rows(ds, regime, test, w, alpha):
    rows = []
    for win in (5, 10):
        for s, a in window_acc(test, w, win * FS32).items():
            rows.append(dict(ds=ds, regime=regime, window_s=win, subject=s, acc=a, alpha=alpha))
    return rows


def corpus_sums():
    """Per corpus dataset: XtX / Xty of even and odd trials, a capped validation sample of odd trials, and the raw
    trials of DTU and KUL (needed as test sets)."""
    sums, val, keep = {}, {}, {}
    for d in DS_KEYS:
        ev = [np.zeros((58 * LAGS, 58 * LAGS)), np.zeros(58 * LAGS)]; od = [np.zeros_like(ev[0]), np.zeros_like(ev[1])]
        vlist, store, budget = [], [], 2 * 3600 * FS32          # <= 2 h of validation EEG per dataset
        for i, t in enumerate(corpus_iter(d)):
            X, y, _, _ = xy(t); acc = ev if i % 2 == 0 else od
            acc[0] += X.T @ X; acc[1] += X.T @ y
            if i % 2 == 1 and budget > 0:
                vlist.append(t); budget -= len(t['eeg'])
            if d in ('DTU', 'KUL'):
                store.append(t)
        sums[d], val[d] = (ev, od), vlist
        if store:
            keep[d] = store
        print('corpus sums', d, flush=True)
    return sums, val, keep


def part_b():
    sums, val, keep = corpus_sums()
    data = {ds: real_trials(ds) for ds in REAL}
    data.update(keep)
    rows = []
    for ds in list(REAL) + ['DTU', 'KUL']:
        trials = data[ds]
        # XDS: the corpus datasets other than ds; alpha chosen with even-trial fits on odd-trial windows
        tr = [d for d in DS_KEYS if d != ds]
        XtX_in = sum(sums[d][0][0] for d in tr); Xty_in = sum(sums[d][0][1] for d in tr)
        XtX_all = XtX_in + sum(sums[d][1][0] for d in tr); Xty_all = Xty_in + sum(sums[d][1][1] for d in tr)
        w, al = pick(XtX_in, Xty_in, [t for d in tr for t in val[d]], XtX_all, Xty_all)
        rows += regime_rows(ds, 'XDS', trials, w, al)
        # LOSO: held-out subjects (and held-out stimuli for the real sets)
        keys = [f"{ds}_{t['subject']}_{t['stim']}" for t in trials]
        for i in range(3):
            if ds in REAL:
                hs, ht = cv_split(keys, 3, i)
                trn = [t for t in trials if t['subject'] not in hs and t['stim'] not in ht]
                te = [t for t in trials if t['subject'] in hs and t['stim'] in ht]
            else:
                subs = sorted({t['subject'] for t in trials}); hs = set(subs[i::3])
                trn = [t for t in trials if t['subject'] not in hs]; te = [t for t in trials if t['subject'] in hs]
            w, al = fit(trn, i)
            rows += regime_rows(ds, 'LOSO', te, w, al)
        # SS: subject-specific, held-out stimuli
        for s in sorted({t['subject'] for t in trials}):
            mine = [t for t in trials if t['subject'] == s]
            stims = sorted({t['stim'] for t in mine})
            for i in range(3):
                ht = set(stims[i::3])
                trn = [t for t in mine if t['stim'] not in ht]; te = [t for t in mine if t['stim'] in ht]
                if len(trn) < 2 or not te:
                    continue
                w, al = fit(trn, i)
                rows += regime_rows(ds, 'SS', te, w, al)
        print(ds, 'done', flush=True)
    df = pd.DataFrame(rows)
    # SS: the three folds of a subject are pooled by averaging their window accuracies
    df = df.groupby(['ds', 'regime', 'window_s', 'subject'], as_index=False).acc.mean()
    df.to_csv(os.path.join(RES, 'r2_linear_ceiling.csv'), index=False)
    return df


def summary(sr, lr, lin):
    L = ['R2 diagnostics: AASD / Carta2026 (real switches); kappa-era accuracy focus', '']
    for (ds, p), g in sr.groupby(['ds', 'p_stay']):
        L.append(f'{ds} p_stay {p}: subjects {len(g)}, Acc mean {g.acc.mean():.2f} (min {g.acc.min():.1f}, '
                 f'max {g.acc.max():.1f}); significant > 50 % (p < 0.05): {int(((g.acc > 50) & (g.p_two_sided < .05)).sum())}, '
                 f'< 50 %: {int(((g.acc < 50) & (g.p_two_sided < .05)).sum())}; trial-flip null sd {g.null_sd.mean():.2f}; '
                 f'frame-sign Acc {g.frame_sign_acc.mean():.2f}')
    L.append('')
    for (ds, p), g in lr.groupby(['ds', 'p_stay']):
        m = g.groupby('shift_s').acc.mean(); b = g.loc[g.groupby('subject').acc.idxmax()]
        L.append(f'{ds} p_stay {p}: lag scan mean Acc at 0 s {m.loc[0.]:.2f}, best shift {m.idxmax():+.1f} s -> {m.max():.2f}; '
                 f'per-subject best shift median {b.shift_s.median():+.1f} s (range {b.shift_s.min():+.1f}..{b.shift_s.max():+.1f})')
    L.append('')
    L.append('Linear backward decoder, window accuracy (mean over subjects, %):')
    piv = lin.groupby(['ds', 'regime', 'window_s']).acc.mean().unstack(['regime', 'window_s']).round(2)
    L.append(piv.to_string())
    txt = '\n'.join(L) + '\n'
    open(os.path.join(RES, 'r2_summary.txt'), 'w').write(txt)
    print(txt, flush=True)


if __name__ == '__main__':
    sr, lr = part_a()
    lin = part_b()
    summary(sr, lr, lin)
