"""Label-free checks of the blind test sets against the corpus datasets and AASD / Carta (DIAGNOSTIC ONLY).

1 integrity   NaN, flat channels, channel-amplitude outliers, artifact share (1-s blocks with global field power > 3x
              the trial median), amplitude; supplied 32-Hz files, 58-channel montage, CAR (before the z-score)
2 pipeline    is pre_* the same function of the raw 128-Hz file as in the corpus: per-channel correlation of
              filtfilt(Butterworth 2, 1-9 Hz)(raw)[::4] with pre_*
3 stimuli     raw 128-Hz envelopes: level difference between the streams (dB), share of near-silence, modulation
              peak (2-8 Hz), scale; number of distinct envelopes in a test set and reuse across subjects
4 overlap     do test stimuli occur in the same-language training sets (normalised cross-correlation of a 20-s
              snippet of every distinct test envelope against all training envelopes at 32 Hz)
5 EA          output of the causal Euclidean alignment (prep eaP_n5_t60): mean variance and spread of the log
              eigenvalues of the channel covariance (0 = white), whole trial and first 10 s
6 domain      multinomial logistic regression on per-trial features (delta/theta log ratio per channel + 30 PCs of
              the channel correlation matrix, first 50 s of a trial), trained on the seven labelled sets
              (subject-grouped CV accuracy), applied to the test trials: which training domain do they look like
Output results/research/test_checks_summary.txt (+ test_checks_subjects.csv).
"""
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import numpy as np  # noqa: E402
import pandas as pd  # noqa: E402
import scipy.io as sio  # noqa: E402
from scipy.signal import butter, fftconvolve, filtfilt, welch  # noqa: E402

import sub_convert as SC  # noqa: E402
from common import B64U, MONTAGES, load_eeg, load_env, load_pp  # noqa: E402
from common import DS_KEYS  # noqa: E402
from common_exp import EXP, rows_of  # noqa: E402
from prepare_data import zscore_trial  # noqa: E402

RES = os.path.join(EXP, 'results', 'research')
EXT = os.path.join(EXP, '..', 'external_data')
TP = os.path.join(EXP, '..', '..', 'test_phase')
TEST = {'TEST_DLAAD': os.path.join(TP, 'dlaad', 'dataCND'), 'TEST_NJU': os.path.join(TP, 'nju', 'dataCND')}
REAL = ('AASD', 'Carta2026')
SETS = list(DS_KEYS) + list(REAL) + list(TEST)
rng = np.random.default_rng(0)
L = []


def say(*a):
    s = ' '.join(str(x) for x in a)
    L.append(s); print(s, flush=True)


def pre58(tr):
    """test trial -> (x (T, 58) CAR, trimmed, NOT z-scored; present mask; env32 trimmed)."""
    eeg, labels, env, _ = tr
    pos = {c.upper(): i for i, c in enumerate(labels)}
    src = np.array([pos.get(c, -1) for c in B64U]); present = src >= 0
    x = np.zeros((eeg.shape[0], 64)); x[:, present] = eeg[:, src[present]]
    sel = np.array([B64U.index(c) for c in MONTAGES['b64ustc58']]); x, pres = x[:, sel], present[sel]
    x[:, pres] -= x[:, pres].mean(1, keepdims=True)
    n = min(len(x), len(env))
    return x[32:n - 32], pres, env[32:n - 32]


def integrity(x, pres):
    xp = x[:, pres]
    sd = xp.std(0); med = np.median(sd[sd > 0]) if (sd > 0).any() else 1.
    gfp = xp.std(1); nb = len(gfp) // 32
    blk = gfp[:nb * 32].reshape(nb, 32).mean(1)
    return dict(nan=int(~np.isfinite(xp).all()), flat=int((sd == 0).sum()), amp=float(med),
                bad_ch=int(((sd > 3 * med) | (sd < med / 3)).sum()), art=float(np.mean(blk > 3 * np.median(blk)) * 100))


def feats(x):
    """per-trial domain features from z-scored 58-ch EEG (first 50 s)."""
    x = x[:1600]
    pres = x.std(0) > 0
    f, P = welch(x, fs=32, nperseg=256, axis=0)
    with np.errstate(divide='ignore', invalid='ignore'):
        dt = np.log(P[(f >= 1) & (f < 4)].sum(0) / P[(f >= 4) & (f < 8)].sum(0))
    dt[~pres] = 0.
    C = np.zeros((58, 58)); C[np.ix_(pres, pres)] = np.corrcoef(x[:, pres].T)
    return dt, C[np.triu_indices(58, 1)]


def ea_stats(x):
    pres = x.std(0) > 0
    out = []
    for seg in (x, x[:320]):
        C = seg[:, pres].T @ seg[:, pres] / len(seg)
        w = np.linalg.eigvalsh(C + 1e-9 * np.eye(pres.sum()))
        out += [float(np.diag(C).mean()), float(np.log(np.maximum(w, 1e-9)).std())]
    return out


def env_stats(e):
    e = np.maximum(np.asarray(e, float), 0)
    m = e.mean(0) + 1e-12
    f, P = welch(e - e.mean(0), fs=128, nperseg=1024, axis=0); P = P.mean(1)
    band = (f >= 2) & (f <= 8)
    return dict(level_db=float(abs((20 / 0.6) * np.log10(m[0] / m[1]))),
                silence=float(np.mean(e < 0.1 * np.median(e, 0)) * 100),
                mod_peak=float(f[band][np.argmax(P[band])]), scale=float(np.median(e)))


def main():
    X = {ds: dict(integ=[], env=[], ea=[], feat=[], subj=[]) for ds in SETS}
    subj_rows = []
    # ---------------- corpus
    for ds in DS_KEYS:
        r = rows_of(ds)
        for _, row in r.iloc[np.sort(rng.choice(len(r), min(40, len(r)), replace=False))].iterrows():
            d = load_pp(row, 'sup', 'b64ustc58', do_car=True, do_runz=False, do_trim=True)
            X[ds]['integ'].append(integrity(d['eeg'], d['present']))
            X[ds]['feat'].append(feats(zscore_trial(d['eeg']))); X[ds]['subj'].append(f"{ds}_{row['subject']}")
            X[ds]['env'].append(env_stats(load_env(row['key'])))
            X[ds]['ea'].append(ea_stats(np.load(os.path.join(EXP, 'cache', 'eaP_n5_t60', row['key'] + '.npz'))['eeg'].astype(float)))
        print('loaded', ds, flush=True)
    # ---------------- real-switch sets (z-scored cache only)
    for ds in REAL:
        c = os.path.join(EXT, ds, 'cache'); I = pd.read_csv(os.path.join(c, 'index.csv'))
        for _, row in I.iloc[np.sort(rng.choice(len(I), 60, replace=False))].iterrows():
            x = np.load(os.path.join(c, 'mat', row['key'] + '.npz'))['eeg'].astype(float)
            X[ds]['feat'].append(feats(x)); X[ds]['subj'].append(f"{ds}_{row['subject']}")
            g = integrity(x, x.std(0) > 0); g.update(amp=np.nan, bad_ch=np.nan); X[ds]['integ'].append(g)
            X[ds]['ea'].append(ea_stats(np.load(os.path.join(c, 'eaP_n5_t60', row['key'] + '.npz'))['eeg'].astype(float)))
        fs_ = sorted(os.listdir(os.path.join(c, 'env_raw')))
        for f in [fs_[i] for i in np.sort(rng.choice(len(fs_), min(20, len(fs_)), replace=False))]:
            X[ds]['env'].append(env_stats(np.load(os.path.join(c, 'env_raw', f))))
        print('loaded', ds, flush=True)
    # ---------------- test sets: every trial
    finger = {}
    for ds, d in TEST.items():
        finger[ds] = {}
        for sid in SC.subjects_in(d):
            per = []
            for t, tr in enumerate(SC.subject_trials(d, sid)):
                x, pres, _ = pre58(tr)
                g = integrity(x, pres); per.append(g); X[ds]['integ'].append(g)
                X[ds]['feat'].append(feats(zscore_trial(x))); X[ds]['subj'].append(f'{ds}_{sid}')
                X[ds]['env'].append(env_stats(tr[3]))
                key = f'{ds}_S{sid:03d}_T{t:02d}'
                X[ds]['ea'].append(ea_stats(np.load(os.path.join(EXT, ds, 'cache', 'eaP_n5_t60', key + '.npz'))['eeg'].astype(float)))
                for c in range(2):
                    e = tr[3][:, c]
                    fp = tuple(np.round(e[::64][:60] / (e.mean() + 1e-12), 2))
                    finger[ds].setdefault(fp, []).append((sid, t, c))
            p = pd.DataFrame(per)
            subj_rows.append(dict(ds=ds, subject=sid, trials=len(p), flat=int(p.flat.max()), bad_ch_max=int(p.bad_ch.max()),
                                  bad_ch_mean=float(p.bad_ch.mean()), art_mean=float(p.art.mean()), art_max=float(p.art.max()),
                                  amp=float(p.amp.median()), nan=int(p.nan.sum())))
        print('loaded', ds, flush=True)
    SJ = pd.DataFrame(subj_rows); SJ.to_csv(os.path.join(RES, 'test_checks_subjects.csv'), index=False)

    say('=== 1 integrity (median over trials; bad_ch = channels with std > 3x or < 1/3 of the trial median; art = % of 1-s blocks with GFP > 3x median)')
    t = pd.DataFrame({ds: pd.DataFrame(X[ds]['integ']).agg(dict(nan='sum', flat='max', amp='median', bad_ch='mean', art='mean'))
                      for ds in SETS}).T
    t['art_p95'] = [pd.DataFrame(X[ds]['integ']).art.quantile(.95) for ds in SETS]
    t['trials'] = [len(X[ds]['integ']) for ds in SETS]
    say(t.round(2).to_string())
    say('\ntest subjects (trials, worst values):'); say(SJ.round(2).to_string(index=False))

    say('\n=== 2 pre_* vs band-passed raw (per-channel correlation; median / 5th percentile over channels and trials)')
    bp = butter(2, [1, 9], 'bandpass', fs=128)
    rows = []
    for ds in ('DTU', 'KUL', 'AVGC', 'USTC'):
        r = rows_of(ds); cs = []
        for _, row in r.iloc[rng.choice(len(r), 6, replace=False)].iterrows():
            raw, pre = np.asarray(load_eeg(row['key'], 'raw'), float), np.asarray(load_eeg(row['key'], 'pre'), float)
            if raw.shape[1] != pre.shape[1]:
                continue
            y = filtfilt(bp[0], bp[1], raw, axis=0)[::4]; n = min(len(y), len(pre))
            cs += [np.corrcoef(y[32:n - 32, c], pre[32:n - 32, c])[0, 1] for c in range(pre.shape[1]) if pre[:, c].std() > 0]
        if cs:
            rows.append(dict(ds=ds, median=np.nanmedian(cs), p05=np.nanquantile(cs, .05), n=len(cs)))
    for ds, d in TEST.items():
        cs = []
        for sid in SC.subjects_in(d)[:3]:
            raw = sio.loadmat(os.path.join(d, f'dataSub{sid}.mat'))['neural'][0, 0]['data']
            pre = sio.loadmat(os.path.join(d, f'pre_dataSub{sid}.mat'))['neural'][0, 0]['data']
            for tr in range(min(2, raw.shape[1])):
                a, b = np.asarray(raw[0, tr], float), np.asarray(pre[0, tr], float)
                y = filtfilt(bp[0], bp[1], a, axis=0)[::4]; n = min(len(y), len(b))
                cs += [np.corrcoef(y[32:n - 32, c], b[32:n - 32, c])[0, 1] for c in range(b.shape[1]) if b[:, c].std() > 0]
        rows.append(dict(ds=ds, median=np.nanmedian(cs), p05=np.nanquantile(cs, .05), n=len(cs)))
    say(pd.DataFrame(rows).round(4).to_string(index=False))

    say('\n=== 3 stimuli, raw 128-Hz envelopes (median over trials / files)')
    t = pd.DataFrame({ds: pd.DataFrame(X[ds]['env']).median() for ds in SETS}).T
    t['level_db_p90'] = [pd.DataFrame(X[ds]['env']).level_db.quantile(.9) for ds in SETS]
    say(t.round(3).to_string())
    for ds in TEST:
        f = finger[ds]; n = sum(len(v) for v in f.values())
        multi = sum(len({s for s, _, _ in v}) > 1 for v in f.values())
        both = sum(len({c for _, _, c in v}) > 1 for v in f.values())
        say(f'{ds}: {n} stream envelopes, {len(f)} distinct; {multi} distinct ones used by more than one subject; '
            f'{both} occur both as stream 1 and as stream 2; uses per distinct envelope: median {np.median([len(v) for v in f.values()]):.0f}, max {max(len(v) for v in f.values())}')

    say('\n=== 4 overlap of test stimuli with same-language training stimuli (NCC of a 20-s snippet, 32 Hz)')
    def bank(sets):
        out = []
        for s in sets:
            if s in DS_KEYS:
                fs_ = [os.path.join(EXP, 'cache', 'eaP_n5_t60', k + '.npz') for k in rows_of(s).key]
            else:
                fs_ = [os.path.join(EXT, s, 'cache', 'eaP_n5_t60', f) for f in sorted(os.listdir(os.path.join(EXT, s, 'cache', 'eaP_n5_t60')))]
            for p in fs_:
                e = np.load(p)['env'].astype(np.float32)
                out += [e[:, 0], np.zeros(64, np.float32), e[:, 1], np.zeros(64, np.float32)]
        return np.concatenate(out)
    for ds, sets in (('TEST_DLAAD', ('KUL', 'AVGC', 'UHD')), ('TEST_NJU', ('USTC', 'AASD'))):
        B = bank(sets); en = np.sqrt(np.maximum(fftconvolve(B ** 2, np.ones(640, np.float32), 'valid'), 1e-6))
        keys = list(finger[ds].values()); pk = []
        for i in rng.choice(len(keys), min(120, len(keys)), replace=False):
            sid, t, c = keys[i][0]
            e = np.load(os.path.join(EXT, ds, 'cache', 'eaP_n5_t60', f'{ds}_S{sid:03d}_T{t:02d}.npz'))['env'][:, c].astype(np.float32)
            s = e[len(e) // 2 - 320:len(e) // 2 + 320]; s = s - s.mean()
            ncc = fftconvolve(B, s[::-1], 'valid') / (en * np.linalg.norm(s))
            pk.append(float(np.abs(ncc).max()))
        pk = np.array(pk)
        # positive control: a training snippet must be found
        ctl = B[100000:100640] - B[100000:100640].mean()
        c1 = float(np.abs(fftconvolve(B, ctl[::-1], 'valid') / (en * np.linalg.norm(ctl))).max())
        say(f'{ds} vs {sets} ({len(B) / 32 / 3600:.1f} h of envelopes): {len(pk)} distinct test envelopes, peak NCC median {np.median(pk):.3f}, '
            f'max {pk.max():.3f}, share > 0.8: {np.mean(pk > .8) * 100:.1f} %; control (training snippet) {c1:.3f}')
        del B, en

    say('\n=== 5 causal EA output (prep eaP_n5_t60): mean variance / std of log eigenvalues; whole trial and first 10 s')
    t = pd.DataFrame({ds: np.median(np.array(X[ds]['ea']), 0) for ds in SETS}, index=['var_all', 'logeig_sd_all', 'var_10s', 'logeig_sd_10s']).T
    say(t.round(3).to_string())

    say('\n=== 6 domain classifier (trained on the seven labelled sets; rows = share of test trials assigned to each set, %)')
    from sklearn.decomposition import PCA
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import GroupKFold
    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import StandardScaler
    lab = [ds for ds in SETS if ds not in TEST]
    DT = np.array([f[0] for ds in lab for f in X[ds]['feat']]); CC = np.array([f[1] for ds in lab for f in X[ds]['feat']])
    y = np.array([ds for ds in lab for _ in X[ds]['feat']]); g = np.array([s for ds in lab for s in X[ds]['subj']])
    def design(pca, dt, cc):
        return np.column_stack([dt, pca.transform(cc)])
    accs = []
    for tr, te in GroupKFold(5).split(DT, y, g):
        pca = PCA(30, random_state=0).fit(CC[tr])
        m = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000, C=0.1)).fit(design(pca, DT[tr], CC[tr]), y[tr])
        accs.append(m.score(design(pca, DT[te], CC[te]), y[te]))
    say(f'subject-grouped 5-fold accuracy of telling the seven labelled sets apart: {np.mean(accs) * 100:.1f} % ({len(y)} trials)')
    pca = PCA(30, random_state=0).fit(CC)
    m = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000, C=0.1)).fit(design(pca, DT, CC), y)
    rows = {}
    for ds in TEST:
        dt = np.array([f[0] for f in X[ds]['feat']]); cc = np.array([f[1] for f in X[ds]['feat']])
        P = m.predict_proba(design(pca, dt, cc))
        pred = m.classes_[P.argmax(1)]
        rows[ds] = {c: float(np.mean(pred == c) * 100) for c in m.classes_}
        rows[ds + ' (mean prob)'] = dict(zip(m.classes_, P.mean(0) * 100))
    say(pd.DataFrame(rows).T[lab].round(1).to_string())
    open(os.path.join(RES, 'test_checks_summary.txt'), 'w').write('\n'.join(L) + '\n')


if __name__ == '__main__':
    main()
