"""Label-free CORAL / diagonal-affine on the *4 raw-seed* channels only
inside the 24-d stem ms feature stack (NOT global 24-d CORAL).

Verified layout in ensemble_clf_transformer_v2.build_features:
  ms = concat([D, hp60, hp120, z60, slope1s, mean2s], axis=1)
  D is the 4 raw seed logits → indices 0:4. Dims 4:24 stay identity.

Reuse fit/score patterns from ms_feat_coral.py. Apply through frozen
stem_ms96 SWA; select by LODO Acc then H; score AASD+Carta @ 0.994/0.8.
Never use AASD/Carta labels for fit or selection.

usage:
  flock /root/disk_3tb/eeg-thao/multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u ms_partial_coral.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np
import torch

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

# Reuse CORAL math from the global 24-d script.
import ms_feat_coral as M

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "ms_partial_coral", "cache")
CKPT_DIR = os.path.join(EXP, "models", "ms_partial_coral")
LOG = os.path.join(EXP, "experiments", "ms_partial_coral.log")
OUT = os.path.join(EXP, "results", "research", "ms_partial_coral.json")
FEAT_SRC_CACHE = os.path.join(EXP, "models", "ms_feat_coral", "cache")
MOE_CACHE = M.MOE_CACHE

SWA_DIR = F.SWA_DIR
IRM_DIR = F.IRM_DIR
WINDOW = F.WINDOW
P_STAY = F.P_STAY
SCALE = F.SCALE
TEST_NAMES = F.TEST_NAMES
W_FUSE = 0.5
EPS = M.EPS
MAX_MOM = M.MAX_MOM
SEED = 20261029
LINEAR_HP30_BASELINE = F.LINEAR_HP30_BASELINE
SWA_LODO_REF = F.SWA_LODO
FUSE_LODO_REF = dict(H=0.8725, Acc=82.58)
FUSE_EXT_REF = dict(AASD=dict(H=0.6179, Acc=50.94), Carta2026=dict(H=0.5461, Acc=48.38))

# First 4 of 24-d ms stack = raw seed logits D (see v2.build_features).
RAW_SLICE = slice(0, 4)
RAW_D = 4
MS_D = 24
MS_LAYOUT = ("raw_seed×4", "hp60×4", "hp120×4", "z60×4", "slope1s×4", "mean2s×4")

VARIANTS = [
    "identity",
    "coral_AASD",
    "coral_t2s_AASD",
    "coral_whiten",
    "affine_diag_AASD",
    "affine_diag_AASD_scale",  # scale-only on raw-4 (mean fixed)
]


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def save_pkl(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as fh:
        pickle.dump(obj, fh, protocol=pickle.HIGHEST_PROTOCOL)


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def ensure_feat_cache():
    """Prefer shared ms_feat_coral 24-d caches; fall back to local build."""
    path = os.path.join(CACHE_DIR, "ms24_lodo.pkl")
    ext_path = os.path.join(CACHE_DIR, "ms24_external.pkl")
    src_path = os.path.join(FEAT_SRC_CACHE, "ms24_lodo.pkl")
    src_ext = os.path.join(FEAT_SRC_CACHE, "ms24_external.pkl")
    if os.path.isfile(path) and os.path.isfile(ext_path):
        log_line(f"feat cache hit {path}")
        return load_pkl(path), load_pkl(ext_path)
    if os.path.isfile(src_path) and os.path.isfile(src_ext):
        log_line(f"feat cache reuse from ms_feat_coral {src_path}")
        feat_all = load_pkl(src_path)
        ext_blob = load_pkl(src_ext)
        # hardlink/copy into our cache for self-contained re-runs
        try:
            os.link(src_path, path)
            os.link(src_ext, ext_path)
        except OSError:
            save_pkl(path, feat_all)
            save_pkl(ext_path, ext_blob)
        return feat_all, ext_blob
    log_line("building 24-d ms features (v2.build_features + external)")
    t0 = time.time()
    feat_all = V.build_features()["ms"]
    any_ds = next(iter(feat_all))
    any_key = next(iter(feat_all[any_ds]))
    d = feat_all[any_ds][any_key].shape[1]
    if d != MS_D:
        raise RuntimeError(f"expected {MS_D}-d ms features, got {d}")
    ext_logits, ext_meta = F.load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    save_pkl(path, feat_all)
    save_pkl(ext_path, dict(feat=ext_feat, meta=ext_meta))
    log_line(f"feat cache write ({time.time() - t0:.0f}s) d={d}")
    return feat_all, dict(feat=ext_feat, meta=ext_meta)


def verify_raw4_layout(feat_all):
    any_ds = next(iter(feat_all))
    any_key = next(iter(feat_all[any_ds]))
    X = np.asarray(feat_all[any_ds][any_key])
    if X.shape[1] != MS_D:
        raise RuntimeError(f"ms width {X.shape[1]} != {MS_D}")
    # Cross-check vs base raw logits for one key if available
    base = N.G["base"]["id"][any_ds][any_key]
    D = np.asarray(base, np.float64)
    if D.shape[1] != RAW_D:
        raise RuntimeError(f"raw seed width {D.shape[1]} != {RAW_D}")
    n = min(len(D), len(X), 64)
    err = float(np.max(np.abs(X[:n, :RAW_D] - D[:n])))
    log_line(f"layout ok: ms={MS_D} raw_idx=0:{RAW_D} ({', '.join(MS_LAYOUT)}); "
             f"raw vs base max|Δ|={err:.3e} on {any_ds}/{any_key}")
    if err > 1e-4:
        raise RuntimeError(f"raw-4 mismatch vs base logits: {err}")


def stack_raw4(feat, datasets, max_n=MAX_MOM, rng=None):
    X = M.stack_feat_frames(feat, datasets, max_n=max_n, rng=rng)
    return X[:, RAW_SLICE]


def embed_partial_params(p4):
    """Lift a 4-d linear map into a 24-d block-diagonal map (I on dims 4:24)."""
    if p4 is None:
        return None
    A24 = np.eye(MS_D, dtype=np.float64)
    A24[:RAW_D, :RAW_D] = np.asarray(p4["A"], np.float64)
    mu_s = np.zeros(MS_D, dtype=np.float64)
    mu_out = np.zeros(MS_D, dtype=np.float64)
    mu_s[:RAW_D] = np.asarray(p4["mu_s"], np.float64)
    mu_out[:RAW_D] = np.asarray(p4["mu_out"], np.float64)
    out = dict(
        kind=f"partial_raw4_{p4['kind']}",
        mu_s=mu_s,
        mu_out=mu_out,
        A=A24,
        match_mean=bool(p4.get("match_mean", True)),
        d=MS_D,
        raw_d=RAW_D,
        raw_slice=[0, RAW_D],
        inner=M.params_to_jsonable(p4),
    )
    if "scale" in p4:
        out["scale"] = np.asarray(p4["scale"])
    return out


def apply_partial_feat(feat, params24):
    """Apply block map; dims 4:24 unchanged (identity block)."""
    if params24 is None:
        return feat
    return M.apply_map_feat(feat, params24)


def fit_variant(name, feat_src, src_ds, feat_tgt_pools, tgt_name):
    if name == "identity":
        return None
    Xs = stack_raw4(feat_src, src_ds)
    if name == "coral_whiten":
        p4 = M.fit_coral_whiten(Xs)
    elif name.startswith("coral_t2s_"):
        Xt = stack_raw4(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        p4 = M.fit_coral_t2s(Xs, Xt, match_mean=True)
    elif name.startswith("coral_"):
        Xt = stack_raw4(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        p4 = M.fit_coral(Xs, Xt, match_mean=True)
    elif name.startswith("affine_diag_"):
        Xt = stack_raw4(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        match_mean = not name.endswith("_scale")
        p4 = M.fit_affine_diag(Xs, Xt, match_mean=match_mean)
    else:
        raise ValueError(name)
    return embed_partial_params(p4)


def variant_target(name):
    return M.variant_target(name)


def params_to_jsonable(p):
    if p is None:
        return None
    out = {k: v for k, v in p.items() if k not in ("mu_s", "mu_out", "A", "scale", "inner")}
    for key in ("mu_s", "mu_out", "A", "scale"):
        if key in p:
            out[key] = np.asarray(p[key]).tolist()
    if "inner" in p:
        out["inner"] = p["inner"]
    return out


def _infer_one(ckpt_path, feat, device, tag):
    return M._infer_one(ckpt_path, feat, device, tag)


def try_reuse_identity_held(held):
    # Prefer our cache, then ms_feat_coral identity, then moe
    for base in (CACHE_DIR, FEAT_SRC_CACHE):
        path = os.path.join(base, f"swa__identity__held-{held}.pkl")
        if os.path.isfile(path):
            log_line(f"identity SWA reuse {path}")
            return load_pkl(path)
    return M.load_identity_swa_held(held)


def try_reuse_identity_all5(kind):
    for base in (CACHE_DIR, FEAT_SRC_CACHE):
        path = os.path.join(base, f"swa__identity__all5_{kind}.pkl")
        if os.path.isfile(path):
            log_line(f"identity SWA reuse {path}")
            return load_pkl(path)
    return M.load_identity_swa_all5(kind)


def score_lodo_swa(streams, held):
    return N.official(held, N.to_ev(streams), P_STAY, SCALE)


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()

    prefer_cpu = os.environ.get("FORCE_CPU", "").strip() in ("1", "true", "yes")
    if prefer_cpu:
        os.environ["CUDA_VISIBLE_DEVICES"] = ""
    if "CUDA_VISIBLE_DEVICES" in os.environ and os.environ["CUDA_VISIBLE_DEVICES"].strip() == "":
        device = torch.device("cpu")
    else:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "8")))
    if device.type == "cuda":
        torch.backends.cuda.matmul.allow_tf32 = True
        torch.backends.cudnn.allow_tf32 = True
    log_line(f"ms-partial-coral device={device} threads={torch.get_num_threads()} "
             f"p_stay={P_STAY} scale={SCALE} max_mom={MAX_MOM} raw_slice=0:{RAW_D}")
    log_line("honesty: unlabeled AASD/Carta *raw-4* moments only; dims 4:24 identity; "
             "no label leakage; select on LODO Acc/H only")

    # Route parent infer logs into this experiment's log.
    M.LOG = LOG

    t_all = time.time()
    N.load_all()
    feat_all, ext_blob = ensure_feat_cache()
    verify_raw4_layout(feat_all)
    ext_feat = ext_blob["feat"]
    ext_meta = ext_blob["meta"]

    tgt_pools = {
        "AASD": {"AASD": ext_feat["AASD"]},
        "Carta2026": {"Carta2026": ext_feat["Carta2026"]},
        "pool": {"AASD": ext_feat["AASD"], "Carta2026": ext_feat["Carta2026"]},
    }

    results = dict(
        method=("label-free CORAL/affine on raw-4 seed channels only inside "
                "24-d stem ms features → frozen SWA/fuse (other 20 dims identity)"),
        ms_layout=list(MS_LAYOUT),
        raw_slice=[0, RAW_D],
        p_stay=P_STAY, scale=SCALE,
        honesty=("Uses unlabeled AASD/Carta *raw-4* feature moments; "
                 "dims 4:24 identity; no label leakage; select LODO Acc/H."),
        baselines=dict(linear_hp30=LINEAR_HP30_BASELINE, swa_lodo=SWA_LODO_REF,
                       fuse_lodo=FUSE_LODO_REF, fuse_ext=FUSE_EXT_REF,
                       global_ms_feat_coral_selected="identity AASD Acc 51.08"),
        variants={},
    )

    log_line("=== LODO SWA (select by mean Acc then H) ===")
    lodo_summary = {}
    for vname in VARIANTS:
        per = {}
        params_by_held = {}
        for held in DS_KEYS:
            train_ds = [d for d in DS_KEYS if d != held]
            tgt = variant_target(vname)
            params = fit_variant(vname, feat_all, train_ds, tgt_pools, tgt)
            params_by_held[held] = params
            cache_key = f"swa__{vname}__held-{held}.pkl"
            cache_path = os.path.join(CACHE_DIR, cache_key)
            if os.path.isfile(cache_path):
                log_line(f"stream cache hit {cache_key}")
                streams = load_pkl(cache_path)
            elif vname == "identity":
                streams = try_reuse_identity_held(held)
                if streams is None:
                    ckpt = os.path.join(SWA_DIR, f"held-{held}.pt")
                    streams = _infer_one(ckpt, feat_all, device, f"held-{held}-SWA-id")
                save_pkl(cache_path, streams)
            else:
                feat_use = apply_partial_feat(feat_all, params)
                ckpt = os.path.join(SWA_DIR, f"held-{held}.pt")
                streams = _infer_one(ckpt, feat_use, device, f"held-{held}-SWA-{vname}")
                save_pkl(cache_path, streams)
            off = score_lodo_swa(streams, held)
            per[held] = off
            log_line(f"LODO {vname:28s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")

        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        avgc = per["AVGC"]
        log_line(f"LODO_MEAN {vname:28s}  H {mean_h:.4f} Acc {mean_a:.2f}  "
                 f"AVGC {avgc['H']:.4f}/{avgc['Acc']:.2f}")
        lodo_summary[vname] = dict(
            per=per, mean=dict(H=mean_h, Acc=mean_a),
            avgc=avgc,
            params_held={h: params_to_jsonable(params_by_held[h]) for h in DS_KEYS},
        )
        results["variants"][vname] = dict(lodo=lodo_summary[vname])

    best = max(VARIANTS, key=lambda n: (lodo_summary[n]["mean"]["Acc"],
                                        lodo_summary[n]["mean"]["H"]))
    selected = best
    log_line(f"SELECTED {selected} LODO mean H {lodo_summary[selected]['mean']['H']:.4f} "
             f"Acc {lodo_summary[selected]['mean']['Acc']:.2f}")

    log_line("=== external all5 SWA ===")
    att = N.G["att"]
    for vname in VARIANTS:
        tgt = variant_target(vname)
        params = fit_variant(vname, feat_all, list(DS_KEYS), tgt_pools, tgt)
        corp_cache = os.path.join(CACHE_DIR, f"swa__{vname}__all5_corpus.pkl")
        ext_cache = os.path.join(CACHE_DIR, f"swa__{vname}__all5_external.pkl")
        if os.path.isfile(corp_cache):
            corp = load_pkl(corp_cache)
        elif vname == "identity":
            corp = try_reuse_identity_all5("corpus")
            if corp is None:
                corp = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_all, device,
                                  f"all5-corp-SWA-{vname}")
            save_pkl(corp_cache, corp)
        else:
            feat_c = apply_partial_feat(feat_all, params)
            corp = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_c, device,
                              f"all5-corp-SWA-{vname}")
            save_pkl(corp_cache, corp)

        if os.path.isfile(ext_cache):
            ext_st = load_pkl(ext_cache)
        elif vname == "identity":
            ext_st = try_reuse_identity_all5("external")
            if ext_st is None:
                feat_e = apply_partial_feat(ext_feat, params)
                ext_st = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                                    f"all5-ext-SWA-{vname}")
            save_pkl(ext_cache, ext_st)
        else:
            feat_e = apply_partial_feat(ext_feat, params)
            ext_st = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                                f"all5-ext-SWA-{vname}")
            save_pkl(ext_cache, ext_st)

        mu_e, sd_e = F.emissions_from(corp, att, list(DS_KEYS))
        ext_rows = {}
        for ds in TEST_NAMES:
            r = F.score_external(ext_st[ds], ext_meta[ds], mu_e, sd_e)
            dacc = r["Acc"] - LINEAR_HP30_BASELINE[ds]["Acc"]
            ext_rows[ds] = r
            log_line(f"EXT_SWA {vname:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs hp30 {dacc:+.2f}")
        results["variants"][vname]["external_swa"] = ext_rows
        results["variants"][vname]["emis_swa"] = dict(mu=mu_e, sd=sd_e)
        results["variants"][vname]["params_all5"] = params_to_jsonable(params)

    log_line("=== external all5 fuse (selected + identity) ===")
    fuse_names = ["identity"] if selected == "identity" else ["identity", selected]
    for vname in fuse_names:
        tgt = variant_target(vname)
        params = fit_variant(vname, feat_all, list(DS_KEYS), tgt_pools, tgt)
        f_corp_c = os.path.join(CACHE_DIR, f"fuse__{vname}__all5_corpus.pkl")
        f_ext_c = os.path.join(CACHE_DIR, f"fuse__{vname}__all5_external.pkl")
        if os.path.isfile(f_corp_c) and os.path.isfile(f_ext_c):
            corp_f = load_pkl(f_corp_c)
            ext_f = load_pkl(f_ext_c)
        elif vname == "identity":
            id_corp = os.path.join(FEAT_SRC_CACHE, "fuse__identity__all5_corpus.pkl")
            id_ext = os.path.join(FEAT_SRC_CACHE, "fuse__identity__all5_external.pkl")
            if os.path.isfile(id_corp) and os.path.isfile(id_ext):
                corp_f = load_pkl(id_corp)
                ext_f = load_pkl(id_ext)
            else:
                corp_f = load_pkl(os.path.join(MOE_CACHE, "fuse_w05__all5_corpus.pkl"))
                ext_f = load_pkl(os.path.join(MOE_CACHE, "fuse_w05__all5_external.pkl"))
            save_pkl(f_corp_c, corp_f)
            save_pkl(f_ext_c, ext_f)
        else:
            feat_c = apply_partial_feat(feat_all, params)
            feat_e = apply_partial_feat(ext_feat, params)
            irm_c = _infer_one(os.path.join(IRM_DIR, "all5.pt"), feat_c, device,
                               f"all5-corp-IRM-{vname}")
            swa_c = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_c, device,
                               f"all5-corp-SWA-fuse-{vname}")
            corp_f = F.blend_streams(irm_c, swa_c, W_FUSE)
            irm_e = _infer_one(os.path.join(IRM_DIR, "all5.pt"), feat_e, device,
                               f"all5-ext-IRM-{vname}")
            swa_e = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                               f"all5-ext-SWA-fuse-{vname}")
            ext_f = {ds: F.blend_one(irm_e[ds], swa_e[ds], W_FUSE) for ds in irm_e}
            save_pkl(f_corp_c, corp_f)
            save_pkl(f_ext_c, ext_f)

        mu_e, sd_e = F.emissions_from(corp_f, att, list(DS_KEYS))
        ext_rows = {}
        for ds in TEST_NAMES:
            r = F.score_external(ext_f[ds], ext_meta[ds], mu_e, sd_e)
            dacc = r["Acc"] - LINEAR_HP30_BASELINE[ds]["Acc"]
            ext_rows[ds] = r
            log_line(f"EXT_FUSE {vname:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs hp30 {dacc:+.2f}")
        results["variants"][vname]["external_fuse"] = ext_rows
        results["variants"][vname]["emis_fuse"] = dict(mu=mu_e, sd=sd_e)

    sel = results["variants"][selected]
    aasd = sel["external_swa"]["AASD"]
    carta = sel["external_swa"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["selected"] = selected
    results["headline"] = dict(
        selected=selected,
        backend="SWA",
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=lodo_summary[selected]["mean"],
        avgc=lodo_summary[selected]["avgc"],
        fuse_external=sel.get("external_fuse"),
    )
    log_line(f"HEADLINE selected={selected} (LODO Acc/H) SWA AASD "
             f"H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} (Δ {d_aasd:+.2f} vs 53.12)  "
             f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {lodo_summary[selected]['mean']['H']:.4f} "
             f"Acc {lodo_summary[selected]['mean']['Acc']:.2f}  "
             f"AVGC {lodo_summary[selected]['avgc']['H']:.4f}/"
             f"{lodo_summary[selected]['avgc']['Acc']:.2f}")
    if sel.get("external_fuse"):
        fa = sel["external_fuse"]["AASD"]
        log_line(f"HEADLINE fuse AASD H {fa['H']:.4f} Acc {fa['Acc']:.2f} "
                 f"(Δ {fa['Acc'] - LINEAR_HP30_BASELINE['AASD']['Acc']:+.2f} vs 53.12)")

    with open(os.path.join(CKPT_DIR, "fit_params.json"), "w") as fh:
        json.dump({k: v.get("params_all5") for k, v in results["variants"].items()},
                  fh, indent=1)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
