"""Label-free check of EEG-stimulus tracking on the blind test sets (DIAGNOSTIC ONLY; nothing here feeds the decoder).

A linear backward decoder (ridge, EEG lags 0-250 ms, prep eaP_n5_t60, r2_aasd_diagnostics helpers) is trained on a
sample of corpus trials: leave-dataset-out weights for the five corpus datasets, all-five weights for AASD, Carta2026
and the test sets. Per window (10 s / 5 s, hop half a window) the reconstruction is correlated with both streams:
c0, c1, d = c1 - c0. The same with MISMATCHED stimuli (both envelopes rolled by 1/3 and 2/3 of the trial: same trial,
same statistics, no relation to the EEG) gives the noise of d.
Label-free quantities, per subject then averaged:
  excess    mean max(c0, c1) minus the same under the mismatch          (is any stream tracked at all)
  est_acc   Phi(Delta / sigma), Delta^2 = E[d^2] - E[d_null^2], sigma^2 = E[d_null^2]: the window accuracy this decoder
            would have if d = +-Delta + noise. Validated against the TRUE window accuracy on the seven labelled sets.
  lag scan  est_acc with the stimuli shifted by s samples at 32 Hz (S'[t] = S[t - s]); true accuracy where labels exist.
            A shift must not be chosen from the test sets (that would be tuning on test data).
Output results/research/test_tracking.csv, test_tracking_subjects.csv, test_tracking_summary.txt.
"""
import os
import numpy as np
import pandas as pd
from numpy.lib.stride_tricks import sliding_window_view
from scipy.stats import norm

import eval_external as E
import r2_aasd_diagnostics as R2
from common import DS_KEYS
from common_exp import EXP, load_trial, rows_of

RES = os.path.join(EXP, 'results', 'research')
SHIFTS = (-3, -2, -1, 0, 1, 2, 3)
SETS = list(DS_KEYS) + ['AASD', 'Carta2026', 'TEST_DLAAD', 'TEST_NJU']
N_TRAIN, N_VAL, N_EVAL = 80, 8, 60


def corpus(ds, keys):
    r = rows_of(ds).set_index('key')
    for k in keys:
        t = load_trial(R2.PREP, k)
        yield R2.prepare(t['eeg'], t['env'], t['att'], r.loc[k, 'subj_uid'], k)


def external(ds, n, rng):
    c = os.path.join(E.EXT, ds, 'cache')
    I = pd.read_csv(os.path.join(c, 'index.csv'))
    if n and len(I) > n:
        I = I.iloc[np.sort(rng.choice(len(I), n, replace=False))]
    for _, r in I.iterrows():
        z = np.load(os.path.join(c, R2.PREP, r['key'] + '.npz'))
        yield R2.prepare(z['eeg'], z['env'], z['att'], r['subject'], r['key'])


def shift(S, s):
    if s == 0:
        return S
    out = np.empty_like(S)
    if s > 0:
        out[s:] = S[:-s]; out[:s] = S[:1]
    else:
        out[:s] = S[-s:]; out[s:] = S[-1:]
    return out


def wcorr(a, B, W):
    v = sliding_window_view(a, W)[::W // 2]
    v = v - v.mean(1, keepdims=True)
    out = []
    for c in range(B.shape[1]):
        u = sliding_window_view(B[:, c], W)[::W // 2]
        u = u - u.mean(1, keepdims=True)
        out.append((v * u).sum(1) / np.sqrt((v ** 2).sum(1) * (u ** 2).sum(1) + 1e-20))
    return np.stack(out, 1)


def trial_stats(t, w, W):
    X, T = R2.lagged(t['eeg'])
    rec = X @ w
    env, att = t['env'][:T].astype(np.float64), t['att'][:T].astype(float)
    if T < W:
        return None
    out = {}
    for s in SHIFTS:
        c = wcorr(rec, shift(env, s), W)
        out[s] = c[:, 1] - c[:, 0]
        if s == 0:
            out['cmax'] = c.max(1)
    dn, cn = [], []
    for f in (1 / 3, 2 / 3):
        c = wcorr(rec, np.roll(env, int(T * f), axis=0), W)
        dn.append(c[:, 1] - c[:, 0]); cn.append(c.max(1))
    out['null'], out['cmax_null'] = np.concatenate(dn), np.concatenate(cn)
    out['lab'] = (sliding_window_view(att, W)[::W // 2].mean(1) > .5).astype(int)
    return out


def main():
    rng = np.random.default_rng(0)
    keys = {ds: rng.permutation(rows_of(ds).key.values) for ds in DS_KEYS}
    sums, val = {}, {}
    for ds in DS_KEYS:
        n = min(N_TRAIN, int(.6 * len(keys[ds])))
        sums[ds] = R2.accumulate(corpus(ds, keys[ds][:n]))
        val[ds] = list(corpus(ds, keys[ds][n:n + N_VAL]))
        print('sums', ds, n, flush=True)
    W8 = {}
    for held in list(DS_KEYS) + ['ALL']:
        tr = [d for d in DS_KEYS if d != held]
        XtX, Xty = sum(sums[d][0] for d in tr), sum(sums[d][1] for d in tr)
        W8[held] = R2.pick(XtX, Xty, [t for d in tr for t in val[d]], XtX, Xty)
        print('weights', held, 'alpha', W8[held][1], flush=True)
    rows, srows = [], []
    for ds in SETS:
        labelled = not ds.startswith('TEST')
        if ds in DS_KEYS:
            trials, w = corpus(ds, keys[ds][-N_EVAL:]), W8[ds][0]
        else:
            trials, w = external(ds, None if not labelled else 300, rng), W8['ALL'][0]
        per = {}
        for t in trials:
            for win in (10, 5):
                st = trial_stats(t, w, win * R2.FS32)
                if st is not None:
                    per.setdefault((win, t['subject']), []).append(st)
        for win in (10, 5):
            subj = {}
            for (wn, s), lst in per.items():
                if wn != win:
                    continue
                cat = lambda k: np.concatenate([x[k] for x in lst])
                sig2 = float(np.mean(cat('null') ** 2))
                r = dict(ds=ds, window_s=win, subject=s, n_windows=len(cat(0)),
                         excess=float(cat('cmax').mean() - cat('cmax_null').mean()), sigma=np.sqrt(sig2))
                for sh in SHIFTS:
                    d = cat(sh)
                    delta2 = max(float(np.mean(d ** 2)) - sig2, 0.)
                    r[f'est{sh:+d}'] = float(norm.cdf(np.sqrt(delta2 / sig2)) * 100)
                    if labelled:
                        r[f'acc{sh:+d}'] = float(np.mean((d > 0) == (cat('lab') == 1)) * 100)
                subj[s] = r
            S = pd.DataFrame(subj.values()); srows.append(S)
            a = dict(ds=ds, window_s=win, subjects=len(S), excess=S.excess.mean(), sigma=S.sigma.mean(),
                     est_min=S['est+0'].min(), est_med=S['est+0'].median(), est_max=S['est+0'].max())
            for sh in SHIFTS:
                a[f'est{sh:+d}'] = S[f'est{sh:+d}'].mean()
                a[f'acc{sh:+d}'] = S[f'acc{sh:+d}'].mean() if labelled else np.nan
            rows.append(a)
            print({k: (round(v, 2) if isinstance(v, float) else v) for k, v in a.items() if k in
                   ('ds', 'window_s', 'excess', 'est+0', 'acc+0', 'est-1', 'acc-1')}, flush=True)
    df = pd.DataFrame(rows)
    df.to_csv(os.path.join(RES, 'test_tracking.csv'), index=False)
    pd.concat(srows).to_csv(os.path.join(RES, 'test_tracking_subjects.csv'), index=False)
    L = ['Label-free tracking check, linear backward decoder (diagnostic only)', '']
    for win in (10, 5):
        t = df[df.window_s == win].set_index('ds')
        L += [f'window {win} s: label-free estimate (est) vs true window accuracy (acc), % mean over subjects; excess = '
              'mean max(c0, c1) minus mismatch; est_min/med/max over subjects',
              t[['subjects', 'excess', 'est+0', 'acc+0', 'est_min', 'est_med', 'est_max']].round(2).to_string(), '']
    t = df[df.window_s == 10].set_index('ds')
    L += ['lag scan, 10-s windows: label-free estimate per stimulus shift (samples at 32 Hz; negative = stimulus earlier):',
          t[[f'est{s:+d}' for s in SHIFTS]].round(2).to_string(), '',
          'true accuracy per shift (labelled sets):', t[[f'acc{s:+d}' for s in SHIFTS]].round(2).to_string(), '']
    txt = '\n'.join(L)
    open(os.path.join(RES, 'test_tracking_summary.txt'), 'w').write(txt); print(txt, flush=True)


if __name__ == '__main__':
    main()
