"""Single-target decoder branch (ST): 4 seeds each of vlaai_ws trained on env only, onset only, dom only
(runners/run_st_seed.sh), alone and added to the reference ensemble.

Members, each standardised on the calibration datasets like nested_stack_fusion.baseline:
  dec    reference 3-target decoders (LDA over env/onset/dom per seed), 4 seeds
  clf    DirectNet sw2, 4 seeds
  env / onset / dom   single-target decoders, mean of 4 seeds
  st     the single-target branch: LDA over (env, onset, dom) branch scores, fitted on the calibration datasets
Combinations (fixed in advance): ref = 0.5 dec + 0.5 clf; env / onset / dom / st alone; dec_st = 0.5 dec + 0.5 st;
clf_st = 0.5 clf + 0.5 st (the ST branch replaces the 3-target decoder); ref_st = dec / clf / st in thirds.
With the current methods (user, 07/11): tr = rate-aux Transformer (4 stage-2 seeds, input = the 16 streams of the
reference 4+4, checkpoints of nested_schedule_rate_aux trained without the outer dataset), bm = BMMS3 (4 seeds);
both standardised by their std on the calibration datasets. Combinations ref_t, ref_bm, ref_t_bm (~ the 4+4+4+4 of
job 387), and the same with st added (ref_st_t, ref_st_bm, ref_st_t_bm, st_t_bm), plus clf_st_bm, clf_st_t.
HMM p_stay 0.998 and 0.95. Classic LODO calibration (other four datasets' out-of-fold evidence; no pair-excluded
fits exist for the new decoders), so ref is re-scored the same way. Real sets: all-five models, corpus calibration.
Scorer: scorer_kappa_next_inclusive (provisional).
"""
import os
import numpy as np
import pandas as pd
import torch

import eval_external as E
import e1_switch
import nested_schedule_eval as N
import nested_stack_fusion as F
import r1_acc_decision_grid as R
import tb_members as TB
import score as S
from common import DS_KEYS
from common_exp import EXP, rows_of

RES = os.path.join(EXP, 'results', 'research')
OUT = os.path.join(RES, 'st_ensemble.csv')
OUT_TXT = os.path.join(RES, 'st_ensemble_summary.txt')
SEEDS = (0, 1, 2, 3)
TARGETS = ('env', 'onset', 'dom')
sfx = lambda i: '' if i == 0 else f'_s{i}'
CLF = [f'direct_clf__eaP_n5_t60_w10b3_ptSPK{sfx(i)}_aug-sw2' for i in SEEDS]
DEC = [f'vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK{sfx(i)}' for i in SEEDS]
ST = {t: [f'vlaai_ws__eaP_n5_t60_aug_cluster_T{t}_xFUG_ptSPK{sfx(i)}' for i in SEEDS] for t in TARGETS}
NC, ND = len(SEEDS), len(SEEDS)                      # F.join columns: NC logits, 3*ND multi-target, then ST singles
COL = {t: NC + 3 * ND + j * len(SEEDS) for j, t in enumerate(TARGETS)}
COMBOS = {'ref': {'dec': .5, 'clf': .5}, 'env': {'env': 1.}, 'onset': {'onset': 1.}, 'dom': {'dom': 1.},
          'st': {'st': 1.}, 'dec_st': {'dec': .5, 'st': .5}, 'clf_st': {'clf': .5, 'st': .5},
          'ref_st': {'dec': 1 / 3, 'clf': 1 / 3, 'st': 1 / 3},
          'ref_t': {'dec': .375, 'clf': .375, 'tr': .25},
          'ref_bm': {'dec': 1 / 3, 'clf': 1 / 3, 'bm': 1 / 3},
          'ref_t_bm': {'dec': .3375, 'clf': .3375, 'tr': .225, 'bm': .1},
          'ref_st_t': {'dec': .25, 'clf': .25, 'st': .25, 'tr': .25},
          'ref_st_bm': {'dec': .25, 'clf': .25, 'st': .25, 'bm': .25},
          'ref_st_t_bm': {'dec': .2, 'clf': .2, 'st': .2, 'tr': .2, 'bm': .2},
          'st_t_bm': {'st': 1 / 3, 'tr': 1 / 3, 'bm': 1 / 3},
          'clf_st_bm': {'clf': 1 / 3, 'st': 1 / 3, 'bm': 1 / 3},
          'clf_st_t': {'clf': 1 / 3, 'st': 1 / 3, 'tr': 1 / 3},
          # both Transformer and BMMS3 (user, 07/11)
          'ref_t_bm25': {'dec': .28125, 'clf': .28125, 'tr': .1875, 'bm': .25},
          'ref_st_t_bm10': {'dec': .225, 'clf': .225, 'st': .225, 'tr': .225, 'bm': .1},
          'ref_st_t_bm25': {'dec': .1875, 'clf': .1875, 'st': .1875, 'tr': .1875, 'bm': .25},
          'clf_st_t_bm': {'clf': .25, 'st': .25, 'tr': .25, 'bm': .25},
          'dec_st_t_bm': {'dec': .25, 'st': .25, 'tr': .25, 'bm': .25},
          # baseline + BMMS3 + Transformer + env-only + onset-only + dom-only as separate members (user, 07/11)
          'ref_t_bm_eod': {m: 1 / 7 for m in ('dec', 'clf', 'bm', 'tr', 'env', 'onset', 'dom')}}
BM = [f'direct_clf__eaP_n5_t60_w10_bmms3enh_ptSPK_s{i}' for i in SEEDS]
X16 = dict(clf_cols=range(NC), dec_cols=[NC + 3 * i for i in range(ND)])
P_GRID = (0.998, 0.95)


def with_rf(ev_by_key):
    """common_exp stores per-target r0f / r1f only for >= 2 targets; one target: r0 / r1 are that target."""
    for e in ev_by_key.values():
        if 'r0f' not in e:
            e['r0f'] = np.asarray(e['r0'], np.float32)[:, None]; e['r1f'] = np.asarray(e['r1'], np.float32)[:, None]
    return ev_by_key


def fit_parts(trials):
    x = np.concatenate([t['x'] * np.where(t['y'] == 0, 1., -1.)[:, None] for t in trials])
    Wd, Sc = [], []
    for i in range(ND):
        v = x[:, NC + 3 * i:NC + 3 * i + 3].astype(float)
        w = np.linalg.solve(np.cov(v.T) + np.eye(3) * 1e-8, v.mean(0)); Wd.append(w / max((v @ w).std(), 1e-8))
    Sc = [max(x[:, i].std(), 1e-8) for i in range(NC)]
    Ss = {t: [max(x[:, COL[t] + i].std(), 1e-8) for i in range(len(SEEDS))] for t in TARGETS}
    raw = {'dec': lambda z: np.mean([z[:, NC + 3 * i:NC + 3 * i + 3] @ Wd[i] for i in range(ND)], axis=0),
           'clf': lambda z: np.mean([z[:, i] / Sc[i] for i in range(NC)], axis=0)}
    for t in TARGETS:
        raw[t] = (lambda t: lambda z: np.mean([z[:, COL[t] + i] / Ss[t][i] for i in range(len(SEEDS))], axis=0))(t)
    norm = {k: max(f(x).std(), 1e-8) for k, f in raw.items()}
    V = np.stack([raw[t](x) / norm[t] for t in TARGETS], 1)
    wst = np.linalg.solve(np.cov(V.T) + np.eye(3) * 1e-8, V.mean(0))
    wst = wst / max((V @ wst).std(), 1e-8)

    def parts(z):
        p = {k: f(z) / norm[k] for k, f in raw.items()}
        p['st'] = np.stack([p[t] for t in TARGETS], 1) @ wst
        return p
    return parts, dict(zip(TARGETS, np.round(wst / np.abs(wst).sum(), 3)))


def main(outers=None):
    torch.set_num_threads(2)
    R.install_scorer()
    clf, info = zip(*[S.load_folds(e, 'causal') for e in CLF])
    decs = [S.load_folds(e, 'causal')[0] for e in DEC + [n for t in TARGETS for n in ST[t]]]
    for v in decs:
        for ds in DS_KEYS:
            with_rf(v[ds])
    bm = [S.load_folds(e, 'causal')[0] for e in BM]
    la = float(info[0][DS_KEYS[0]]['lookahead_s'])
    tab = {ds: F.corpus_table([v[ds] for v in decs], [v[ds] for v in clf], ds) for ds in DS_KEYS}
    real = {}
    for ds in ('AASD', 'Carta2026'):
        pre = 'aasd_c1' if ds == 'AASD' else f'ext_c1_{ds}'
        ed = [N.load(os.path.join(RES, f'{pre}_bestP_evidence_s{i}.pkl')) for i in SEEDS]
        ed += [with_rf(N.load(os.path.join(RES, f'{pre}_T{t}P_evidence_s{i}.pkl'))) for t in TARGETS for i in SEEDS]
        ec = [N.load(os.path.join(RES, f'ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl')) for i in SEEDS]
        real[ds] = {k: F.join(ed, ec, k) for k in ed[0]}
        rbm = [N.load(os.path.join(RES, f'ext_clf_{ds}_w10bmms3enhP_evidence_s{i}.pkl')) for i in SEEDS]
        for k, (x, e) in real[ds].items():
            e['_bm'] = TB.bm_mean([o[k] for o in rbm], e['t_end'], e['att'], SEEDS)
    BMV = {d: {k: TB.bm_mean([b[d][k] for b in bm], t['ev']['t_end'], t['y'], SEEDS) for k, t in tab[d].items()}
           for d in DS_KEYS}
    rows, wrows = [], []
    for outer in (outers or list(DS_KEYS) + ['ALL']):
        train = [d for d in DS_KEYS if d != outer] if outer != 'ALL' else list(DS_KEYS)
        parts, wst = fit_parts([t for d in train for t in tab[d].values()])
        wrows.append(dict(outer=outer, **wst))
        P = {d: {k: parts(t['x']) for k, t in tab[d].items()} for d in DS_KEYS}
        RP = {ds: {k: parts(x) for k, (x, e) in v.items()} for ds, v in real.items()} if outer == 'ALL' else {}
        # Transformer of this outer fold and BMMS3, standardised on the calibration datasets (signed std)
        for d in DS_KEYS:
            for k, t in tab[d].items():
                P[d][k]['tr'] = TB.t_score(outer, TB.x16(t['x'], **X16)); P[d][k]['bm'] = BMV[d][k]
        for ds, v in (real.items() if outer == 'ALL' else ()):
            for k, (x, e) in v.items():
                RP[ds][k]['tr'] = TB.t_score(outer, TB.x16(x, **X16)); RP[ds][k]['bm'] = e['_bm']
        for m in ('tr', 'bm'):
            sm = max(float(np.concatenate([P[d][k][m] * np.where(t['y'] == 0, 1., -1.) for d in train
                                           for k, t in tab[d].items()]).std()), 1e-8)
            for Q in (P, RP):
                for v in Q.values():
                    for pr in v.values():
                        pr[m] = pr[m] / sm
        for name, w in COMBOS.items():
            mix = lambda p: sum(c * p[m] for m, c in w.items())
            sg = np.concatenate([mix(P[d][k]) * np.where(t['y'] == 0, 1., -1.) for d in train for k, t in tab[d].items()])
            mu, sd = float(sg.mean()), float(sg.std())
            if mu <= 0 or sd < 1e-8:
                raise ValueError('invalid emissions')
            base = {d: {k: dict(t['ev'], r0=mix(P[d][k]), r1=np.zeros(len(t['y'])), pre_switched=True)
                        for k, t in tab[d].items()} for d in DS_KEYS}
            for p in P_GRID:
                def integ(d, prm, p=p):
                    return e1_switch.integ_hmm(d, dict(p_stay=p, mu=prm['mu'], sd=prm['sd']))
                S.integ_hmm = integ; E.integ_hmm = integ
                if outer == 'ALL':
                    for ds, v in real.items():
                        ev = {k: dict(e, r0=mix(RP[ds][k]), r1=np.zeros(len(e['att']))) for k, (x, e) in v.items()}
                        rows.append(dict(combo=name, p_stay=p, ds=ds, kind='real', **E.score(ev, mu, sd, p, la)))
                else:
                    rows.append(dict(combo=name, p_stay=p, ds=outer, kind='synthetic',
                                     **S.score_dataset(outer, base, la, emis=(mu, sd), p_stay=p)))
                print({k: rows[-1][k] for k in ('combo', 'p_stay', 'ds', 'Acc', 'H')}, flush=True)
        pd.DataFrame(rows).to_csv(OUT.replace('.csv', '_partial.csv'), index=False)
    df = pd.DataFrame(rows).assign(metric='kappa_next_inclusive_provisional', calibration='classic_lodo')
    df.to_csv(OUT, index=False)
    lodo = df[df.kind == 'synthetic'].groupby(['combo', 'p_stay'])
    s = pd.DataFrame(dict(LODO_H=lodo.H.mean(), LODO_Acc=lodo.Acc.mean()))
    for ds in ('AVGC', 'AASD', 'Carta2026'):
        r = df[df.ds == ds].set_index(['combo', 'p_stay']); s[f'{ds}_H'] = r.H; s[f'{ds}_Acc'] = r.Acc
    txt = ('Single-target decoder branch (classic LODO calibration; kappa_next_inclusive_provisional)\n\n'
           + s.round(4).to_string() + '\n\nST branch LDA weights (sum |w| = 1) per calibration set:\n'
           + pd.DataFrame(wrows).to_string(index=False) + '\n')
    open(OUT_TXT, 'w').write(txt); print(txt, flush=True)


if __name__ == '__main__':
    main()
