"""G1b: DirectNet variants pooled INSIDE the DirectNet member (user 2026-11-12: a higher LODO is also a goal).

In G1 an extra variant as a fourth member raised LODO (+0.2..0.44) but lowered Carta because BMMS3 dropped from 1/3
to 1/4. Here the DirectNet member `clf` is the mean of the standardised logits of sw2 and of the chosen variants
(4 seeds each), so the weights of decoder / Transformer / BMMS3 stay those of the reference:
  eq3       dec 1/3, clf 1/3, bm 1/3            (main ensemble)
  ref_t     dec .375, clf .375, tr .25          (highest LODO without stem)
  ref_t_bm  dec .3375, clf .3375, tr .225, bm .1
DirectNet groups: sw2 (reference), sw2+swfd, sw2+c6warp, sw2+recipe, sw2+c1xfade, sw2+swfd+c6warp+recipe (all3),
all3+c1xfade (all4). Transformer input = sw2 + decoder streams as trained. Classic LODO calibration, kappa scorer,
HMM p 0.998 and 0.95; real sets with the all-five models. Output results/research/g1b_groups.csv / _summary.txt.
"""
import os
import numpy as np
import pandas as pd

import eval_external as E
import e1_switch
import nested_schedule_eval as N
import nested_stack_fusion as F
import r1_acc_decision_grid as R
import score as S
import tb_members as TB
from common import DS_KEYS
from common_exp import EXP

RES = os.path.join(EXP, 'results', 'research')
SEEDS = (0, 1, 2, 3)
sfx = lambda i: '' if i == 0 else f'_s{i}'
VARS = {'sw2': ('aug-sw2', '_w10b3sw2P'), 'swfd': ('aug-swfd', '_w10b3swfdP'), 'c6warp': ('aug-c6warp', '_w10b3c6warpP'),
        'recipe': ('aug-sw2recipe', '_w10b3recipeP'), 'c1xfade': ('aug-c1xfade', '_w10b3c1xfadeP')}
ORDER = list(VARS)
GROUPS = {'sw2': ['sw2'], 'sw2+swfd': ['sw2', 'swfd'], 'sw2+c6warp': ['sw2', 'c6warp'], 'sw2+recipe': ['sw2', 'recipe'],
          'sw2+c1xfade': ['sw2', 'c1xfade'], 'all3': ['sw2', 'swfd', 'c6warp', 'recipe'],
          'all4': ['sw2', 'swfd', 'c6warp', 'recipe', 'c1xfade']}
FRAMES = {'eq3': {'dec': 1 / 3, 'clf': 1 / 3, 'bm': 1 / 3}, 'ref_t': {'dec': .375, 'clf': .375, 'tr': .25},
          'ref_t_bm': {'dec': .3375, 'clf': .3375, 'tr': .225, 'bm': .1}}
NV, NS = len(ORDER), len(SEEDS)
NC = NV * NS                                    # clf columns: variant-major, seed-minor; then 3 x 4 decoder columns
col = lambda v, i: ORDER.index(v) * NS + i
X16 = dict(clf_cols=[col('sw2', i) for i in range(NS)], dec_cols=[NC + 3 * i for i in range(NS)])
P_GRID = (0.998, 0.95)
REAL = ('AASD', 'Carta2026')


def fit(trials):
    x = np.concatenate([t['x'] * np.where(t['y'] == 0, 1., -1.)[:, None] for t in trials])
    Wd = []
    for i in range(NS):
        v = x[:, NC + 3 * i:NC + 3 * i + 3].astype(float)
        w = np.linalg.solve(np.cov(v.T) + np.eye(3) * 1e-8, v.mean(0)); Wd.append(w / max((v @ w).std(), 1e-8))
    Sc = [max(x[:, j].std(), 1e-8) for j in range(NC)]
    dec = lambda z: np.mean([z[:, NC + 3 * i:NC + 3 * i + 3] @ Wd[i] for i in range(NS)], axis=0)
    grp = {g: (lambda cols: lambda z: np.mean([z[:, j] / Sc[j] for j in cols], axis=0))([col(v, i) for v in vs for i in range(NS)])
           for g, vs in GROUPS.items()}
    nd = max(dec(x).std(), 1e-8); ng = {g: max(f(x).std(), 1e-8) for g, f in grp.items()}
    return lambda z: dict(dec=dec(z) / nd, **{f'clf:{g}': f(z) / ng[g] for g, f in grp.items()})


def main():
    R.install_scorer()
    exps = [f'direct_clf__eaP_n5_t60_w10b3_ptSPK{sfx(i)}_{VARS[v][0]}' for v in ORDER for i in SEEDS]
    clf = [S.load_folds(e, 'causal') for e in exps]
    la = float(clf[0][1][DS_KEYS[0]]['lookahead_s'])
    clfs = [c[0] for c in clf]
    dec = [S.load_folds(f'vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK{sfx(i)}', 'causal')[0] for i in SEEDS]
    bm = [S.load_folds(f'direct_clf__eaP_n5_t60_w10_bmms3enh_ptSPK_s{i}', 'causal')[0] for i in SEEDS]
    tab = {ds: F.corpus_table([v[ds] for v in dec], [v[ds] for v in clfs], ds) for ds in DS_KEYS}
    BMV = {d: {k: TB.bm_mean([b[d][k] for b in bm], t['ev']['t_end'], t['y'], SEEDS) for k, t in tab[d].items()} for d in DS_KEYS}
    real = {}
    for ds in REAL:
        pre = 'aasd_c1' if ds == 'AASD' else f'ext_c1_{ds}'
        ed = [N.load(os.path.join(RES, f'{pre}_bestP_evidence_s{i}.pkl')) for i in SEEDS]
        ec = [N.load(os.path.join(RES, f'ext_clf_{ds}{VARS[v][1]}_evidence_s{i}.pkl')) for v in ORDER for i in SEEDS]
        rb = [N.load(os.path.join(RES, f'ext_clf_{ds}_w10bmms3enhP_evidence_s{i}.pkl')) for i in SEEDS]
        real[ds] = {k: F.join(ed, ec, k) for k in ed[0]}
        for k, (x, e) in real[ds].items():
            e['_bm'] = TB.bm_mean([o[k] for o in rb], e['t_end'], e['att'], SEEDS)
    rows = []
    for outer in list(DS_KEYS) + ['ALL']:
        train = [d for d in DS_KEYS if d != outer] if outer != 'ALL' else list(DS_KEYS)
        parts = fit([t for d in train for t in tab[d].values()])
        P = {d: {k: dict(parts(t['x']), bm=BMV[d][k], tr=TB.t_score(outer, TB.x16(t['x'], **X16))) for k, t in tab[d].items()} for d in DS_KEYS}
        RP = ({ds: {k: dict(parts(x), bm=e['_bm'], tr=TB.t_score(outer, TB.x16(x, **X16))) for k, (x, e) in v.items()} for ds, v in real.items()}
              if outer == 'ALL' else {})
        for m in ('bm', 'tr'):
            sm = max(float(np.concatenate([P[d][k][m] * np.where(t['y'] == 0, 1., -1.) for d in train for k, t in tab[d].items()]).std()), 1e-8)
            for Q in (P, RP):
                for v in Q.values():
                    for pr in v.values():
                        pr[m] = pr[m] / sm
        for g in GROUPS:
            for fr, w in FRAMES.items():
                ww = {(f'clf:{g}' if m == 'clf' else m): c for m, c in w.items()}
                mix = lambda p: sum(c * p[m] for m, c in ww.items())
                sg = np.concatenate([mix(P[d][k]) * np.where(t['y'] == 0, 1., -1.) for d in train for k, t in tab[d].items()])
                mu, sd = float(sg.mean()), float(sg.std())
                base = {d: {k: dict(t['ev'], r0=mix(P[d][k]), r1=np.zeros(len(t['y'])), pre_switched=True) for k, t in tab[d].items()} for d in DS_KEYS}
                for p in P_GRID:
                    def integ(d, prm, p=p):
                        return e1_switch.integ_hmm(d, dict(p_stay=p, mu=prm['mu'], sd=prm['sd']))
                    S.integ_hmm = integ; E.integ_hmm = integ
                    if outer == 'ALL':
                        for ds, v in real.items():
                            ev = {k: dict(e, r0=mix(RP[ds][k]), r1=np.zeros(len(e['att']))) for k, (x, e) in v.items()}
                            rows.append(dict(group=g, frame=fr, p_stay=p, ds=ds, kind='real', **E.score(ev, mu, sd, p, la)))
                    else:
                        rows.append(dict(group=g, frame=fr, p_stay=p, ds=outer, kind='synthetic', **S.score_dataset(outer, base, la, emis=(mu, sd), p_stay=p)))
        print(outer, 'scored', flush=True)
        pd.DataFrame(rows).to_csv(os.path.join(RES, 'g1b_groups_partial.csv'), index=False)
    df = pd.DataFrame(rows).assign(metric='kappa_next_inclusive_provisional', calibration='classic_lodo')
    df.to_csv(os.path.join(RES, 'g1b_groups.csv'), index=False)
    lo = df[df.kind == 'synthetic'].groupby(['frame', 'group', 'p_stay'])
    s = pd.DataFrame(dict(LODO_Acc=lo.Acc.mean(), LODO_H=lo.H.mean(), LODO_SDT=lo.SDT.mean()))
    s['AVGC_Acc'] = df[df.ds == 'AVGC'].set_index(['frame', 'group', 'p_stay']).Acc
    for ds in REAL:
        r = df[df.ds == ds].set_index(['frame', 'group', 'p_stay']); s[f'{ds}_Acc'] = r.Acc; s[f'{ds}_H'] = r.H
    s = s.reset_index()
    L = ['G1b: DirectNet variants pooled inside the DirectNet member (kappa_next_inclusive_provisional, classic LODO)', '']
    for p in P_GRID:
        L += [f'HMM p_stay {p}:', s[s.p_stay == p].drop(columns='p_stay').round(3).to_string(index=False), '']
    txt = '\n'.join(L)
    open(os.path.join(RES, 'g1b_groups_summary.txt'), 'w').write(txt + '\n'); print(txt, flush=True)


if __name__ == '__main__':
    main()
