"""No-decoder combinations of the four direct-classifier seeds, on the existing LODO folds.

The seeds are already encoder (vlaai_ws) + linear head. This script never loads a reconstruction
decoder and never refits those networks. It only recombines the saved causal logits.

Weights that are learned (lda, logreg) use the other four datasets only; the held-out dataset
is scored by score.score_dataset at the same p_stay grid as ensemble_clf.py. median needs no fit.

  invstd  weighted mean, weight = 1 / std of label-signed (r0-r1) on the other four datasets
  median  median of the four (r0-r1) streams
  lda     Gaussian LDA on the four logits (Sigma^{-1} mu of label-signed evidence)
  logreg  L2 logistic regression on the four logits (decision function is the evidence)

usage: python ensemble_clf_meta.py
writes results/research/ensemble_clf_meta.csv and results/research/no_decoder_splits.csv
"""
import os

import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from sklearn.linear_model import LogisticRegression

import ensemble_clf as E
import score as S
from common_exp import CFG, EXP, rows_of
from common import DS_KEYS

P_STAY = E.P_STAY
OUT = os.path.join(EXP, "results", "research", "ensemble_clf_meta.csv")
SPLIT_OUT = os.path.join(EXP, "results", "research", "no_decoder_splits.csv")
LOGREG_MAX = 400_000


def split_table():
    """Validation subject of every training dataset, same RNG as direct_classifier.run_fold."""
    rows = []
    for held in DS_KEYS:
        rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
        for ds in DS_KEYS:
            if ds == held:
                continue
            sub = rows_of(ds)
            subjects = sorted(sub.subject.unique())
            vs = rng.choice(subjects)
            rows.append(dict(held_out=held, train_ds=ds, val_subject=str(vs),
                             n_train_subjects=len(subjects), n_trials=int(len(sub)),
                             seed=int(CFG["seed"] + DS_KEYS.index(held))))
    return pd.DataFrame(rows)


def stack(evs, datasets, signed):
    parts = []
    for d in datasets:
        keys = evs[0][d].keys()
        for k in keys:
            D = np.stack([E.delta(ev[d][k]) for ev in evs], 1)
            if signed:
                D = D * np.where(evs[0][d][k]["att"] == 0, 1.0, -1.0)[:, None]
            parts.append(D)
    return np.concatenate(parts, 0)


def invstd_weights(evs, held):
    train = [d for d in DS_KEYS if d != held]
    w = []
    for ev in evs:
        parts = [E.delta(e) * np.where(e["att"] == 0, 1.0, -1.0) for d in train for e in ev[d].values()]
        w.append(1.0 / max(float(np.concatenate(parts).std()), 1e-12))
    w = np.asarray(w, np.float64)
    return w / w.sum()


def lda_weights(evs, held):
    X = stack(evs, [d for d in DS_KEYS if d != held], signed=True)
    C = np.cov(X.T)
    cond = float(np.linalg.cond(C))
    ridge = 0.0
    if cond > 1e8:
        ridge = 1e-3 * float(np.trace(C)) / C.shape[0]
        C = C + ridge * np.eye(C.shape[0])
    w = np.linalg.solve(C, X.mean(0))
    w = w / np.abs(w).sum()
    return w, dict(n=int(len(X)), cond=cond, ridge=ridge, intercept=0.0)


def logreg_weights(evs, held):
    train = [d for d in DS_KEYS if d != held]
    X = stack(evs, train, signed=False)
    y_parts = []
    for d in train:
        for e in evs[0][d].values():
            y_parts.append((e["att"] == 0).astype(np.int8))
    y = np.concatenate(y_parts)
    n_all = int(len(X))
    if n_all > LOGREG_MAX:
        rng = np.random.default_rng(CFG["seed"])
        take = rng.choice(n_all, LOGREG_MAX, replace=False)
        X, y = X[take], y[take]
    clf = LogisticRegression(C=1.0, solver="lbfgs", max_iter=400)
    clf.fit(X, y)
    return clf.coef_.ravel().astype(np.float64), dict(n=n_all, n_fit=int(len(X)), intercept=float(clf.intercept_[0]),
                                                      cond=np.nan, ridge=np.nan)


def combine_median(evs):
    ref = evs[0]
    out = {}
    for d in DS_KEYS:
        out[d] = {}
        for k, e0 in ref[d].items():
            acc = np.median(np.stack([E.delta(ev[d][k]) for ev in evs], 0), 0)
            out[d][k] = dict(r0=acc, r1=np.zeros_like(acc), att=e0["att"], t_end=e0["t_end"], pre_switched=True)
    return out


def combine_affine(evs, w, intercept):
    ev = E.combine(evs, w)
    if intercept:
        for d in ev.values():
            for e in d.values():
                e["r0"] = e["r0"] + intercept
    return ev


def acc1_of(ds, evd):
    per = {}
    for _, r in rows_of(ds).iterrows():
        e = evd[r["key"]]
        d = (e["r0"] - e["r1"])[::4]
        sgn = np.where(e["att"][::4] == 0, 1.0, -1.0)
        per.setdefault(r["subj_uid"], []).append(d * sgn)
    return float(np.median([np.mean(np.concatenate(v) > 0) for v in per.values()]))


def score_job(method, ds, ev, lookahead, extra):
    acc1 = acc1_of(ds, ev[ds])
    rows = []
    for p in P_STAY:
        r = S.score_dataset(ds, ev, lookahead, p_stay=p)
        rec = dict(method=method, p_stay=p, ds=ds, H=r["H"], Acc=r["Acc"], SDT=r["SDT"], L=r["L"], acc1=acc1)
        rec.update(extra)
        print(f"{method:7s} p_stay={p} {ds:4s}  H {r['H']:.4f}  Acc {r['Acc']:.2f}  SDT {r['SDT']:.2f}  L {r['L']:.3f}",
              flush=True)
        rows.append(rec)
    return rows


def pack(method, w, info):
    extra = dict(intercept=float(info.get("intercept", 0.0)), n=info.get("n", 0),
                 n_fit=info.get("n_fit", info.get("n", 0)), cond=info.get("cond", np.nan), ridge=info.get("ridge", 0.0))
    if w is None:
        for i in range(4):
            extra[f"w{i}"] = np.nan
    else:
        for i, wi in enumerate(w):
            extra[f"w{i}"] = float(wi)
    return extra


def main():
    loaded, lookahead = [], None
    for seed, exp in E.SEEDS:
        ev, info = S.load_folds(exp, "causal")
        for ds in DS_KEYS:
            la = float(info[ds]["lookahead_s"])
            if lookahead is None:
                lookahead = la
            elif abs(la - lookahead) > 1e-9:
                raise RuntimeError(f"lookahead mismatch on {exp} {ds}: {la} vs {lookahead}")
            for e in ev[ds].values():
                if not e.get("pre_switched"):
                    raise RuntimeError(f"{exp} {ds} evidence is not pre_switched")
        loaded.append(ev)
        print(f"loaded {exp}", flush=True)
    evs = loaded

    splits = split_table()
    os.makedirs(os.path.dirname(SPLIT_OUT), exist_ok=True)
    splits.to_csv(SPLIT_OUT, index=False)
    print(f"wrote {SPLIT_OUT}", flush=True)

    jobs = []
    ev_med = combine_median(evs)
    for ds in DS_KEYS:
        jobs.append(dict(method="median", ds=ds, ev=ev_med, extra=pack("median", None, {})))
    for held in DS_KEYS:
        w = invstd_weights(evs, held)
        jobs.append(dict(method="invstd", ds=held, ev=combine_affine(evs, w, 0.0), extra=pack("invstd", w, {})))
        w, info = lda_weights(evs, held)
        print(f"lda {held} w {np.round(w, 4)} cond {info['cond']:.1f} n {info['n']}", flush=True)
        jobs.append(dict(method="lda", ds=held, ev=combine_affine(evs, w, 0.0), extra=pack("lda", w, info)))
        w, info = logreg_weights(evs, held)
        print(f"logreg {held} w {np.round(w, 4)} b {info['intercept']:.4f} n {info['n']}", flush=True)
        jobs.append(dict(method="logreg", ds=held, ev=combine_affine(evs, w, info["intercept"]),
                         extra=pack("logreg", w, info)))

    n_jobs = int(os.environ.get("SCORE_JOBS", 4))
    parts = Parallel(n_jobs=n_jobs)(delayed(score_job)(j["method"], j["ds"], j["ev"], lookahead, j["extra"]) for j in jobs)
    df = pd.DataFrame([r for part in parts for r in part])
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    df.to_csv(OUT, index=False)
    mean = df.groupby(["method", "p_stay"], sort=False)[["H", "Acc", "SDT", "L", "acc1"]].mean().reset_index()
    pd.set_option("display.width", 220)
    pd.set_option("display.max_rows", 100)
    print("\nmean over the five held-out datasets:")
    print(mean.round(4).to_string(index=False))
    print(f"\nwrote {OUT}")


if __name__ == "__main__":
    main()
