"""Retrain stem_ms96 under the competition leave-one-dataset-out subject split.

The finished stem_ms96 run (experiments/transformer_v2.log, family_stem_ms96) trained
on every trial of the four non-held-out competition datasets and chose p_stay / emission
scale on a whole inner dataset, then refit on all four datasets.

This script keeps that network, loss, seed, step count and feature pipeline, and changes
only the competition split:

  test        one competition dataset, every subject
  validation  one subject from each of the other four datasets
              (same RNG as direct_classifier.run_fold / results/research/no_decoder_splits.csv)
  train       the remaining trials of those four datasets

p_stay and emission scale are chosen on the four validation subjects only.
The scored network is the 1000-step EMA trained without those subjects (no refit).

Rows of the image protocol that have no stem_ms96 features, so they are not applied:

  Fuglsang2020     would be train on every fold, but the four DirectNet seeds saved
                   held-out competition logits only. Fold weights were not kept, and
                   there is no Fuglsang logit stream in the 4-seed feature space.
  SparrKULee       pretrain weights are DirectNet (models/pretrain_spk_direct_clf_w10_b3_s*.pt
                   under the decoder_lodo tree), already used as init_from of those seeds.
                   LogitTransformer cannot load them. SparrKULee has no eaP_n5_t60 cache
                   and no 4-seed logits, so its four validation subjects are not in this val set.
  AASD, Carta2026  true-switch evaluation outside LODO. Not scored here.

usage:
  STEPS=1000 python stem_ms96_image_split.py
  SMOKE=1 STEPS=40 FOLD=AVGC python stem_ms96_image_split.py
"""
import os
import time

import numpy as np
import pandas as pd
import torch

import ensemble_clf_transformer_v2 as V
import ensemble_clf_nested as N
import score as S
from common_exp import CFG, EXTERNAL_DATA, rows_of
from common import DS_KEYS
from e1_switch import expand
from scorer import aggregate, fast_score_subject

EXP = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(EXP, "results", "research", "stem_ms96_image_split.csv")
MANIFEST = os.path.join(EXP, "results", "research", "stem_ms96_image_split_manifest.csv")
CKPT = os.path.join(EXP, "models", "stem_ms96_image_split")
LOG = os.path.join(EXP, "experiments", "stem_ms96_image_split.log")
SPLIT_CSV = os.path.join(EXP, "results", "research", "no_decoder_splits.csv")

STEM = ("stem_ms96", "ms", 96, 64, 2, True, 8e-4)
STEPS = int(os.environ.get("STEPS", "1000"))
THR = V.THR


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def competition_split(held):
    """One validation subject per training dataset. Identical draws to direct_classifier.run_fold."""
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
    val_subj = {}
    val_keys = {}
    train_keys = {}
    for ds in DS_KEYS:
        if ds == held:
            continue
        rows = rows_of(ds)
        subjects = sorted(rows.subject.unique())
        vs = int(rng.choice(subjects))
        val_subj[ds] = vs
        is_val = rows.subject.to_numpy() == vs
        val_keys[ds] = rows.loc[is_val, "key"].tolist()
        train_keys[ds] = rows.loc[~is_val, "key"].tolist()
        if not val_keys[ds] or not train_keys[ds]:
            raise RuntimeError(f"empty train/val split for held={held} ds={ds} subject={vs}")
    return val_subj, train_keys, val_keys


def assert_matches_saved_split(held, val_subj):
    if not os.path.exists(SPLIT_CSV):
        return
    saved = pd.read_csv(SPLIT_CSV)
    saved = saved[saved.held_out == held]
    for _, row in saved.iterrows():
        got = val_subj[row.train_ds]
        if int(row.val_subject) != got:
            raise RuntimeError(
                f"val subject mismatch held={held} {row.train_ds}: csv {int(row.val_subject)} vs draw {got}")


def filter_feat(feat, keys_by_ds):
    out = {}
    for family, table in feat.items():
        out[family] = {}
        for ds, keys in keys_by_ds.items():
            missing = [k for k in keys if k not in table[ds]]
            if missing:
                raise RuntimeError(f"missing features for {ds}: {missing[:3]}")
            out[family][ds] = {k: table[ds][k] for k in keys}
    return out


def trial_stats(keys_by_ds):
    rows = []
    for ds, keys in keys_by_ds.items():
        att = N.G["att"][ds]
        index = rows_of(ds).set_index("key")
        for key in keys:
            y = att[key]
            sub = int(index.loc[key, "subject"])
            uid = str(index.loc[key, "subj_uid"])
            rows.append(dict(dataset=ds, key=key, subject=sub, subj_uid=uid,
                             n_frames=int(len(y)), pos_rate=float((y == 0).mean())))
    return pd.DataFrame(rows)


def emis_on(streams, keys_by_ds):
    parts = []
    for ds, keys in keys_by_ds.items():
        att = N.G["att"][ds]
        for key in keys:
            d = np.asarray(streams[ds][key], np.float64)
            parts.append(d * np.where(att[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def evaluate_keys(ds, dmap, mu, sd, p_stay, keys):
    keys = set(keys)
    per = {}
    lookahead = N.G["lookahead"]
    for item in N.G["prep"][ds]:
        if item["key"] not in keys:
            continue
        dec = N.integ_nb(np.ascontiguousarray(dmap[item["key"]], np.float64), p_stay, mu, sd)
        labs = expand(dec.astype(np.float64), item["t_end"], lookahead, item["T"], item["guess"])
        per.setdefault(item["subj"], []).append((labs, item["gt"]))
    if not per:
        raise RuntimeError(f"no scored subjects in {ds}")
    res = []
    for lst in per.values():
        pred = np.concatenate([a for a, _ in lst])
        gt = np.concatenate([b for _, b in lst])
        trial = np.concatenate([np.full(len(a), i, np.int64) for i, (a, _) in enumerate(lst)])
        res.append(fast_score_subject(pred, gt, trial, fs=128))
    a = aggregate(res)
    per_acc = {}
    att = N.G["att"][ds]
    for item in N.G["prep"][ds]:
        if item["key"] not in keys:
            continue
        d = dmap[item["key"]]
        sgn = np.where(att[item["key"]][::4] == 0, 1.0, -1.0)
        per_acc.setdefault(item["subj"], []).append(d[::4] * sgn)
    acc1 = float(np.median([np.mean(np.concatenate(v) > 0) for v in per_acc.values()]))
    return dict(H=float(a["H"]), Acc=float(a["Acc"]), SDT=float(a["SDT"]), L=float(a["L"]), acc1=acc1,
                n_subjects=len(per))


def pick_setting(rows):
    df = pd.DataFrame(rows)
    ok = df[df.Acc >= THR]
    src = ok if len(ok) else df
    best = src.sort_values(["H", "Acc", "SDT"], ascending=[False, False, True]).iloc[0]
    return best, int(len(ok) > 0)


def official_test(streams, held, mu, sd, p_stay, scale):
    ev = N.to_ev(streams)
    r = S.score_dataset(held, ev, N.G["lookahead"], emis=(mu, sd * scale), p_stay=float(p_stay))
    dmap = {k: e["r0"] - e["r1"] for k, e in ev[held].items()}
    acc1 = evaluate_keys(held, dmap, mu, sd * scale, float(p_stay), list(dmap))["acc1"]
    return dict(H=float(r["H"]), Acc=float(r["Acc"]), SDT=float(r["SDT"]), L=float(r["L"]), acc1=acc1)


def external_inventory():
    """Counts from converted indexes. These corpora are not stem features."""
    lines = []
    fug = os.path.join(EXTERNAL_DATA, "Fuglsang2020", "cache", "index.csv")
    if os.path.exists(fug):
        df = pd.read_csv(fug)
        lines.append(f"Fuglsang2020 index: {df.subject.nunique()} subjects, {len(df)} trials, "
                     f"sum n32={int(df.n32.sum())} (32 Hz samples). Not in stem train: no 4-seed logits.")
    else:
        lines.append("Fuglsang2020 index missing")
    spk = os.path.join(EXTERNAL_DATA, "SparrKULee", "cache", "index_mat_reac.csv")
    if os.path.exists(spk):
        df = pd.read_csv(spk)
        ids = sorted({str(k).split("_")[1] for k in df.key})
        chosen = sorted(np.random.default_rng(1234).choice(ids, 4, replace=False).tolist())
        lines.append(f"SparrKULee index: {df.subject.nunique()} subjects, {len(df)} runs. "
                     f"pretrain val ids by rng 1234 (not used here): {chosen}. "
                     f"cache has mat_reac only, no eaP_n5_t60, no 4-seed logits.")
    else:
        lines.append("SparrKULee index missing")
    for name in ("AASD", "Carta2026"):
        path = os.path.join(EXTERNAL_DATA, name, "cache", "index.csv")
        if not os.path.exists(path):
            lines.append(f"{name} index missing")
            continue
        df = pd.read_csv(path)
        lines.append(f"{name} index: {df.subject.nunique()} subjects, {len(df)} trials. "
                     f"Outside LODO; stem_ms96 is not scored on it.")
    pre = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/models"
    weights = [f"pretrain_spk_direct_clf_w10_b3_s{i}.pt" for i in range(4)]
    present = [w for w in weights if os.path.exists(os.path.join(pre, w))]
    lines.append(f"SparrKULee DirectNet pretrain weights present ({len(present)}/4): {present}. "
                 f"Used already as init_from of the four logit seeds. Not loadable into LogitTransformer.")
    return lines


def main():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    if os.environ.get("SMOKE"):
        open(LOG, "a").close()
    else:
        os.makedirs(os.path.dirname(LOG), exist_ok=True)
        open(LOG, "w").close()
    log_line(f"device {device}  steps {STEPS}  batch {V.BATCH}  seed {V.TRAIN_SEED}  cfg {STEM}")
    log_line("protocol: LODO test = 1 competition dataset; val = 1 subject x 4 train datasets; "
             "train = remaining competition trials. Fuglsang/SparrKULee/AASD/Carta are not stem features.")
    for line in external_inventory():
        log_line(line)
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    N.load_all()
    checks = N.check_fast_matches_official()
    base_H = float(np.mean([c[1]["H"] for c in checks]))
    base_Acc = float(np.mean([c[1]["Acc"] for c in checks]))
    log_line(f"reproduced lda4 @ 0.998  H {base_H:.4f}  Acc {base_Acc:.2f}")
    if abs(base_H - 0.7965) > 5e-4 or abs(base_Acc - 73.01) > 0.02:
        raise RuntimeError("baseline reproduction does not match the published LDA numbers")
    V.assert_causal(device)
    feat = V.build_features()
    os.makedirs(CKPT, exist_ok=True)

    only = os.environ.get("FOLD", "").strip()
    helds = [only] if only else list(DS_KEYS)
    if only and only not in DS_KEYS:
        raise SystemExit(f"unknown FOLD {only}")

    manifest_rows = []
    out_rows = []
    for held in helds:
        val_subj, train_keys, val_keys = competition_split(held)
        assert_matches_saved_split(held, val_subj)
        test_keys = rows_of(held).key.tolist()
        train_df = trial_stats(train_keys)
        val_df = trial_stats(val_keys)
        test_df = trial_stats({held: test_keys})
        for role, frame in (("train", train_df), ("val", val_df), ("test", test_df)):
            for _, r in frame.iterrows():
                manifest_rows.append(dict(held_out=held, role=role, **r.to_dict()))
        log_line(
            f"fold {held}: train trials {len(train_df)} frames {int(train_df.n_frames.sum())} "
            f"subjects {train_df.subj_uid.nunique()} pos {train_df.pos_rate.mul(train_df.n_frames).sum() / train_df.n_frames.sum():.4f} | "
            f"val trials {len(val_df)} frames {int(val_df.n_frames.sum())} subjects {dict(val_subj)} "
            f"pos {val_df.pos_rate.mul(val_df.n_frames).sum() / val_df.n_frames.sum():.4f} | "
            f"test trials {len(test_df)} frames {int(test_df.n_frames.sum())} subjects {test_df.subj_uid.nunique()}")
        all4_keys = {ds: train_keys[ds] + val_keys[ds] for ds in train_keys}
        all4 = trial_stats(all4_keys)
        log_line(f"  vs previous stem refit (all 4 datasets, no subject holdout): "
                 f"trials {len(all4)} frames {int(all4.n_frames.sum())} "
                 f"(this train drops {len(all4) - len(train_df)} trials, "
                 f"{int(all4.n_frames.sum()) - int(train_df.n_frames.sum())} frames)")

        train_feat = filter_feat(feat, train_keys)
        t0 = time.time()
        model, mu, sd = V.train_model(STEM, list(train_keys), train_feat, device, STEPS)
        log_line(f"  fit {held} {time.time() - t0:.0f}s  norm-mu-abs {float(np.abs(mu).mean()):.4f}")
        if not os.environ.get("SMOKE"):
            path = os.path.join(CKPT, f"stem_ms96__held-{held}.pt")
            torch.save({"state": model.state_dict(), "mu": mu, "sd": sd, "cfg": STEM,
                        "train_keys": train_keys, "val_subject": val_subj, "steps": STEPS,
                        "seed": V.TRAIN_SEED}, path)
            log_line(f"  wrote {path}")
        streams = V.infer_all(model, feat[STEM[1]], mu, sd, STEM[2], device)
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()

        mu_tr, sd_tr = emis_on(streams, train_keys)
        mu_all, sd_all = emis_on(streams, all4_keys)
        val_rows = []
        for p in V.P_STAY:
            for scale in V.SCALES:
                merged = {}
                for ds, keys in val_keys.items():
                    part = evaluate_keys(ds, streams[ds], mu_tr, sd_tr * scale, p, keys)
                    merged[ds] = part
                # equal mean over the four validation datasets (one subject each), same reduction
                # the previous nested run used when it averaged inner datasets
                rec = dict(p_stay=p, scale=scale,
                           H=float(np.mean([v["H"] for v in merged.values()])),
                           Acc=float(np.mean([v["Acc"] for v in merged.values()])),
                           SDT=float(np.mean([v["SDT"] for v in merged.values()])),
                           L=float(np.mean([v["L"] for v in merged.values()])),
                           acc1=float(np.mean([v["acc1"] for v in merged.values()])))
                val_rows.append(rec)
        best, reached = pick_setting(val_rows)
        log_line(f"  val select reached_acc74={bool(reached)}  p {best.p_stay} sc {best.scale}  "
                 f"val H {best.H:.4f} Acc {best.Acc:.2f} SDT {best.SDT:.2f} L {best.L:.3f}")

        if os.environ.get("SMOKE"):
            off = official_test(streams, held, mu_tr, sd_tr, best.p_stay, best.scale)
            log_line(f"smoke {held} H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f}")
            return

        for role, mu_e, sd_e in (
            ("image_train_emis", mu_tr, sd_tr),
            ("image_all4_emis", mu_all, sd_all),
        ):
            off = official_test(streams, held, mu_e, sd_e, best.p_stay, best.scale)
            out_rows.append(dict(
                role=role, ds=held, model="stem_ms96", p_stay=float(best.p_stay), scale=float(best.scale),
                H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"], acc1=off["acc1"],
                val_H=float(best.H), val_Acc=float(best.Acc), val_SDT=float(best.SDT),
                val_reached_74=bool(reached),
                n_train_trials=int(len(train_df)), n_val_trials=int(len(val_df)),
                n_test_trials=int(len(test_df)),
                n_train_frames=int(train_df.n_frames.sum()), n_val_frames=int(val_df.n_frames.sum()),
                n_test_frames=int(test_df.n_frames.sum()),
                emis_mu=mu_e, emis_sd=sd_e, scorer="official"))
            log_line(f"{role} {held} stem_ms96 p {best.p_stay} sc {best.scale}  "
                     f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f} acc1 {off['acc1']:.3f}")

    man = pd.DataFrame(manifest_rows)
    df = pd.DataFrame(out_rows)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    man.to_csv(MANIFEST, index=False)
    df.to_csv(OUT, index=False)
    log_line("means:")
    for role, sub in df.groupby("role"):
        m = sub[["H", "Acc", "SDT", "L", "acc1"]].mean()
        log_line(f"  {role:20s}  H {m.H:.4f}  Acc {m.Acc:.2f}  SDT {m.SDT:.2f}  L {m.L:.3f}  acc1 {m.acc1:.3f}")
    log_line(f"wrote {OUT}")
    log_line(f"wrote {MANIFEST}")


if __name__ == "__main__":
    main()
