"""R0 + R1: accuracy-first decision grid for the nested 4 decoder + 4 DirectNet (sw2) reference.

Goal (user, 2026-11-07): raise raw per-subject accuracy. This rescans the decision stage only (no training):
  a      = DirectNet share of the fused evidence, (1 - a) * decoder-LDA + a * DirectNet, both standardised on the
           outer-training datasets exactly as nested_stack_fusion.baseline (a = 0.5 reproduces it);
  p_stay = HMM stay probability (0.5 = frame-wise sign of the evidence, no smoothing).
Everything that is fitted (LDA target weights, scales, HMM emissions) uses only the four outer-training datasets of
each LODO fold (pair-excluded first-stage evidence, as score_balanced_baselines_kappa.py), and only the five corpus
datasets for AASD / Carta2026. The selection rule is fixed in advance: best mean LODO Acc (and, separately, best mean
LODO H); AASD / Carta are only reported, never used to choose.
Scorer: scorer_kappa_next_inclusive (provisional kappa / next-switch metric). Per-subject accuracies are written too.
Not covered (needs re-inference, not a decision-stage change): DEC_OFFSET and the decoder / classifier lookahead.
"""
import os
import numpy as np
import pandas as pd
import torch

import eval_external as E
import nested_schedule_eval as N
import nested_stack_fusion as F
import scorer_kappa_next_inclusive as K
import score as S
import e1_switch
from common import DS_KEYS
from common_exp import EXP, rows_of
from nested_stack_base import ROOT as CR
from nested_stack_decoder import ROOT as DR

A_GRID = (0.0, 0.3, 0.5, 0.7, 0.85, 1.0)
P_GRID = (0.5, 0.8, 0.9, 0.95, 0.98, 0.99, 0.995, 0.998, 0.999, 0.9995)
RES = os.path.join(EXP, 'results', 'research')
OUT = os.path.join(RES, 'r1_acc_grid.csv')
OUT_SUBJ = os.path.join(RES, 'r1_acc_grid_subjects.csv')
OUT_TXT = os.path.join(RES, 'r1_acc_grid_summary.txt')

_CAP = []


def _agg(per):
    """K.aggregate that also keeps the per-subject accuracies of the last call (same subject order as the scorer)."""
    _CAP.clear()
    _CAP.extend(float(p['acc']) for p in per)
    return K.aggregate(per)


def install_scorer():
    S.fast_score_subject = K.fast_score_subject
    E.fast_score_subject = K.fast_score_subject
    S.aggregate = _agg
    E.aggregate = _agg


def components(trials):
    """Decoder-LDA and DirectNet parts of nested_stack_fusion.baseline, each standardised (fit on `trials` only)."""
    x = np.concatenate([t['x'] * np.where(t['y'] == 0, 1., -1.)[:, None] for t in trials])
    weights, scales = [], []
    for i in range(4):
        v = x[:, 4 + 3 * i:7 + 3 * i].astype(float)
        w = np.linalg.solve(np.cov(v.T) + np.eye(3) * 1e-8, v.mean(0))
        weights.append(w / max((v @ w).std(), 1e-8))
        scales.append(max(x[:, i].std(), 1e-8))
    zd = np.mean([x[:, 4 + 3 * i:7 + 3 * i] @ weights[i] for i in range(4)], axis=0)
    zc = np.mean([x[:, i] / scales[i] for i in range(4)], axis=0)
    sd, sc = max(zd.std(), 1e-8), max(zc.std(), 1e-8)

    def transform(z):
        d = np.mean([z[:, 4 + 3 * i:7 + 3 * i] @ weights[i] for i in range(4)], axis=0) / sd
        c = np.mean([z[:, i] / scales[i] for i in range(4)], axis=0) / sc
        return d, c
    return transform


def load_reference():
    clf, info = zip(*[S.load_folds(e, 'causal') for e in F.REF])
    dec, _ = zip(*[S.load_folds(e, 'causal') for e in F.DEC])
    return clf, dec, float(info[0][DS_KEYS[0]]['lookahead_s'])


def tables(outer, clf, dec):
    tab = {}
    for ds in DS_KEYS:
        if outer in ('ALL', ds):
            ec = [v[ds] for v in clf]; ed = [v[ds] for v in dec]
        else:
            ec = [F.load_pair(CR, outer, ds, i) for i in range(4)]; ed = [F.load_pair(DR, outer, ds, i) for i in range(4)]
        tab[ds] = N.original_table(ed, ec, ds, True)
    return tab


def real_inputs(ds):
    """Joined (x, ev) per trial of a real set: 4 all-five decoders (bestP) + 4 all-five DirectNet sw2."""
    dn = 'aasd_c1_bestP' if ds == 'AASD' else f'ext_c1_{ds}_bestP'
    ed = [N.load(os.path.join(RES, f'{dn}_evidence_s{i}.pkl')) for i in range(4)]
    ec = [N.load(os.path.join(RES, f'ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl')) for i in range(4)]
    return {k: F.join(ed, ec, k) for k in ed[0]}


def emissions(parts, tab, train, a):
    signed = np.concatenate([((1 - a) * parts[d][k][0] + a * parts[d][k][1]) * np.where(t['y'] == 0, 1., -1.)
                             for d in train for k, t in tab[d].items()])
    mu, sd = float(signed.mean()), float(signed.std())
    if mu <= 0 or sd < 1e-8:
        raise ValueError('invalid emissions')
    return mu, sd


def subject_order_real(ev):
    return list(dict.fromkeys(ev[k]['subject'] for k in sorted(ev)))


def main(outers=None):
    torch.set_num_threads(2)
    install_scorer()
    clf, dec, la = load_reference()
    rows, srows = [], []
    for outer in (outers or list(DS_KEYS) + ['ALL']):
        tab = tables(outer, clf, dec)
        train = [d for d in DS_KEYS if d != outer]
        tr = components([t for d in train for t in tab[d].values()])
        parts = {d: {k: tr(t['x']) for k, t in tab[d].items()} for d in DS_KEYS}
        real = {ds: real_inputs(ds) for ds in ('AASD', 'Carta2026')} if outer == 'ALL' else {}
        real_parts = {ds: {k: tr(x) for k, (x, e) in v.items()} for ds, v in real.items()}
        for a in A_GRID:
            mu, sd = emissions(parts, tab, train, a)
            mix = lambda pr: (1 - a) * pr[0] + a * pr[1]
            base = {d: {k: dict(t['ev'], r0=mix(parts[d][k]), r1=np.zeros(len(t['y'])), pre_switched=True)
                        for k, t in tab[d].items()} for d in DS_KEYS}
            for p in P_GRID:
                def integ(d, prm, p=p):
                    return e1_switch.integ_hmm(d, dict(p_stay=p, mu=prm['mu'], sd=prm['sd']))
                S.integ_hmm = integ; E.integ_hmm = integ
                if outer == 'ALL':
                    for ds in real:
                        ev = {k: dict(e, r0=mix(real_parts[ds][k]), r1=np.zeros(len(e['att'])))
                              for k, (x, e) in real[ds].items()}
                        out = E.score(ev, mu, sd, p, la)
                        subs = subject_order_real(ev)
                        rows.append(dict(a=a, p_stay=p, ds=ds, kind='real', **out,
                                         Kappa=max(0., 2 * out['Acc'] / 100 - 1)))
                        srows += [dict(a=a, p_stay=p, ds=ds, subject=str(s), acc=v) for s, v in zip(subs, _CAP)]
                else:
                    out = S.score_dataset(outer, base, la, emis=(mu, sd), p_stay=p)
                    subs = list(dict.fromkeys(rows_of(outer)['subj_uid']))
                    rows.append(dict(a=a, p_stay=p, ds=outer, kind='synthetic', **out,
                                     Kappa=max(0., 2 * out['Acc'] / 100 - 1)))
                    srows += [dict(a=a, p_stay=p, ds=outer, subject=str(s), acc=v) for s, v in zip(subs, _CAP)]
                print({k: rows[-1][k] for k in ('a', 'p_stay', 'ds', 'Acc', 'H', 'SDT')}, flush=True)
        pd.DataFrame(rows).to_csv(OUT.replace('.csv', '_partial.csv'), index=False)
    df = pd.DataFrame(rows).assign(metric='kappa_next_inclusive_provisional')
    df.to_csv(OUT, index=False)
    pd.DataFrame(srows).to_csv(OUT_SUBJ, index=False)
    if outers is None:
        summarise(df)


def summarise(df):
    lodo = df[df.kind == 'synthetic']
    g = lodo.groupby(['a', 'p_stay'])
    s = pd.DataFrame(dict(LODO_Acc=g.Acc.mean(), LODO_H=g.H.mean(),
                          AVGC_Acc=lodo[lodo.ds == 'AVGC'].set_index(['a', 'p_stay']).Acc))
    for ds in ('AASD', 'Carta2026'):
        r = df[df.ds == ds].set_index(['a', 'p_stay'])
        s[f'{ds}_Acc'] = r.Acc; s[f'{ds}_H'] = r.H
    lines = ['R1 accuracy-first decision grid (nested 4+4 sw2; kappa_next_inclusive_provisional)',
             'Selection fixed in advance on LODO only; real sets reported, never used to choose.', '',
             s.round(4).to_string(), '']
    for crit in ('LODO_Acc', 'LODO_H'):
        best = s[crit].idxmax()
        lines.append(f'best by {crit}: a={best[0]} p_stay={best[1]} -> ' +
                     ', '.join(f'{c} {s.loc[best, c]:.4f}' for c in s.columns))
    ref = (0.5, 0.998)
    lines.append('reference a=0.5 p_stay=0.998 -> ' + ', '.join(f'{c} {s.loc[ref, c]:.4f}' for c in s.columns))
    open(OUT_TXT, 'w').write('\n'.join(lines) + '\n')
    print('\n'.join(lines), flush=True)


if __name__ == '__main__':
    main()
