"""Label-free CORAL / diagonal-affine on 24-d stem ms features (not 1-D logits).

Distinct from ext_coral_logits.py (4-seed / 1-D decision streams) and from
1-D online-moment match. Fit train→unlabeled external *feature* moments, then
run frozen stem_ms96 SWA (and fuse IRM↔SWA w=0.5 for the selected map) on
adapted features.

Honesty: using unlabeled AASD/Carta feature moments is stronger than train-only
whitening; att/labels are never used for loss, selection, or moment fit.

Select transform by mean LODO Acc then H at fixed HMM 0.994 / 0.8.
Score AASD/Carta at the same preset. Never use external labels for selection.

GPU: hold multi_agent_opt/gpu.lock for stem forward.

usage:
  flock /root/disk_3tb/eeg-thao/multi_agent_opt/gpu.lock \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u ms_feat_coral.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np
import torch

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import stem_ms96_irm_swa_fuse as F
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(EXP, "models", "ms_feat_coral", "cache")
CKPT_DIR = os.path.join(EXP, "models", "ms_feat_coral")
LOG = os.path.join(EXP, "experiments", "ms_feat_coral.log")
OUT = os.path.join(EXP, "results", "research", "ms_feat_coral.json")
MOE_CACHE = os.path.join(EXP, "models", "causal_moe_fuse", "cache")

SWA_DIR = F.SWA_DIR
IRM_DIR = F.IRM_DIR
WINDOW = F.WINDOW
P_STAY = F.P_STAY
SCALE = F.SCALE
TEST_NAMES = F.TEST_NAMES
W_FUSE = 0.5
EPS = 1e-5
MAX_MOM = int(os.environ.get("CORAL_MAX_MOM", "600000"))
SEED = 20261028
LINEAR_HP30_BASELINE = F.LINEAR_HP30_BASELINE
SWA_LODO_REF = F.SWA_LODO
FUSE_LODO_REF = dict(H=0.8725, Acc=82.58)
FUSE_EXT_REF = dict(AASD=dict(H=0.6179, Acc=50.94), Carta2026=dict(H=0.5461, Acc=48.38))


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def save_pkl(path, obj):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as fh:
        pickle.dump(obj, fh, protocol=pickle.HIGHEST_PROTOCOL)


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def cov_stats(X, eps=EPS):
    X = np.asarray(X, np.float64)
    if X.ndim == 1:
        X = X[:, None]
    mu = X.mean(0)
    Xc = X - mu
    n = max(len(X) - 1, 1)
    C = (Xc.T @ Xc) / n + eps * np.eye(X.shape[1])
    return mu, C


def matrix_power(C, p):
    w, V = np.linalg.eigh(np.asarray(C, np.float64))
    w = np.maximum(w, 1e-12)
    return (V * (w ** p)) @ V.T


def fit_coral(Xs, Xt, match_mean=True):
    """Source→target CORAL: z'=(z-μ_s) C_s^{-1/2} C_t^{1/2} + μ_out."""
    mu_s, Cs = cov_stats(Xs)
    mu_t, Ct = cov_stats(Xt)
    A = matrix_power(Cs, -0.5) @ matrix_power(Ct, 0.5)
    mu_out = mu_t if match_mean else mu_s
    return dict(kind="coral", mu_s=mu_s, mu_out=mu_out, A=A,
                match_mean=bool(match_mean), d=int(A.shape[0]))


def fit_coral_t2s(Xs, Xt, match_mean=True):
    """Target→source CORAL (standard DA direction when applied to target)."""
    mu_s, Cs = cov_stats(Xs)
    mu_t, Ct = cov_stats(Xt)
    A = matrix_power(Ct, -0.5) @ matrix_power(Cs, 0.5)
    mu_out = mu_s if match_mean else mu_t
    return dict(kind="coral_t2s", mu_s=mu_t, mu_out=mu_out, A=A,
                match_mean=bool(match_mean), d=int(A.shape[0]))


def fit_coral_whiten(Xs):
    mu_s, Cs = cov_stats(Xs)
    A = matrix_power(Cs, -0.5)
    return dict(kind="coral_whiten", mu_s=mu_s, mu_out=mu_s, A=A,
                match_mean=False, d=int(A.shape[0]))


def fit_affine_diag(Xs, Xt, match_mean=True):
    Xs = np.asarray(Xs, np.float64)
    Xt = np.asarray(Xt, np.float64)
    ms, ss = Xs.mean(0), Xs.std(0) + 1e-8
    mt, st = Xt.mean(0), Xt.std(0) + 1e-8
    scale = st / ss
    A = np.diag(scale)
    mu_out = mt if match_mean else ms
    return dict(kind="affine_diag", mu_s=ms, mu_out=mu_out, A=A,
                match_mean=bool(match_mean), d=int(A.shape[0]), scale=scale)


def apply_linear_map(D, params):
    D = np.asarray(D, np.float64)
    if D.ndim == 1:
        D = D[:, None]
        squeeze = True
    else:
        squeeze = False
    out = (D - params["mu_s"]) @ params["A"] + params["mu_out"]
    if squeeze:
        return out.reshape(-1)
    return out


def stack_feat_frames(feat, datasets, max_n=MAX_MOM, rng=None):
    rng = rng or np.random.default_rng(SEED)
    parts = []
    for ds in datasets:
        for _key, X in feat[ds].items():
            parts.append(np.asarray(X, np.float64))
    X = np.concatenate(parts, 0)
    if len(X) > max_n:
        idx = rng.choice(len(X), size=max_n, replace=False)
        X = X[idx]
    return X


def apply_map_feat(feat, params):
    if params is None:
        return feat
    out = {}
    for ds, table in feat.items():
        out[ds] = {
            k: apply_linear_map(X, params).astype(np.float32)
            for k, X in table.items()
        }
    return out


def params_to_jsonable(p):
    if p is None:
        return None
    out = {k: v for k, v in p.items() if k not in ("mu_s", "mu_out", "A", "scale")}
    for key in ("mu_s", "mu_out", "A", "scale"):
        if key in p:
            out[key] = np.asarray(p[key]).tolist()
    return out


def ensure_feat_cache():
    path = os.path.join(CACHE_DIR, "ms24_lodo.pkl")
    ext_path = os.path.join(CACHE_DIR, "ms24_external.pkl")
    if os.path.isfile(path) and os.path.isfile(ext_path):
        log_line(f"feat cache hit {path}")
        return load_pkl(path), load_pkl(ext_path)
    log_line("building 24-d ms features (v2.build_features + external)")
    t0 = time.time()
    feat_all = V.build_features()["ms"]
    # sanity: 24-d
    any_ds = next(iter(feat_all))
    any_key = next(iter(feat_all[any_ds]))
    d = feat_all[any_ds][any_key].shape[1]
    if d != 24:
        raise RuntimeError(f"expected 24-d ms features, got {d}")
    ext_logits, ext_meta = F.load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    save_pkl(path, feat_all)
    save_pkl(ext_path, dict(feat=ext_feat, meta=ext_meta))
    log_line(f"feat cache write ({time.time() - t0:.0f}s) d={d}")
    return feat_all, dict(feat=ext_feat, meta=ext_meta)


def _infer_one(ckpt_path, feat, device, tag):
    t0 = time.time()
    log_line(f"  infer {tag} {os.path.basename(ckpt_path)}")
    net, mu, sd = F.load_ckpt(ckpt_path, device)
    st = V.infer_all(net, feat, mu, sd, WINDOW, device)
    del net
    if device.type == "cuda":
        torch.cuda.empty_cache()
    log_line(f"  infer {tag} done ({time.time() - t0:.0f}s)")
    return st


def load_identity_swa_held(held):
    """Reuse moe cache when present (raw features, identity map)."""
    path = os.path.join(MOE_CACHE, f"_tmp_swa__held-{held}.pkl")
    if os.path.isfile(path):
        log_line(f"identity SWA cache hit held-{held}")
        return load_pkl(path)
    return None


def load_identity_swa_all5(kind):
    path = os.path.join(MOE_CACHE, f"_tmp_swa__all5_{kind}.pkl")
    if os.path.isfile(path):
        log_line(f"identity SWA cache hit all5_{kind}")
        return load_pkl(path)
    return None


def load_identity_fuse_held(held):
    path = os.path.join(MOE_CACHE, f"fuse_w05__held-{held}.pkl")
    if os.path.isfile(path):
        return load_pkl(path)
    return None


def fit_variant(name, feat_src, src_ds, feat_tgt_pools, tgt_name):
    if name == "identity":
        return None
    Xs = stack_feat_frames(feat_src, src_ds)
    if name == "coral_whiten":
        return fit_coral_whiten(Xs)
    if name.startswith("coral_t2s_"):
        Xt = stack_feat_frames(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        return fit_coral_t2s(Xs, Xt, match_mean=True)
    if name.startswith("coral_") and name.endswith("_covonly"):
        Xt = stack_feat_frames(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        return fit_coral(Xs, Xt, match_mean=False)
    if name.startswith("coral_"):
        Xt = stack_feat_frames(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        return fit_coral(Xs, Xt, match_mean=True)
    if name.startswith("affine_diag_"):
        Xt = stack_feat_frames(feat_tgt_pools[tgt_name], list(feat_tgt_pools[tgt_name]))
        match_mean = not name.endswith("_scale")
        return fit_affine_diag(Xs, Xt, match_mean=match_mean)
    raise ValueError(name)


def variant_target(name):
    if name in ("identity", "coral_whiten"):
        return None
    for key in ("pool", "AASD", "Carta2026"):
        if key in name:
            return key
    return "AASD"


# Pre-specified variants. Selection uses LODO Acc/H only (never AASD/Carta Acc).
# Trimmed: full s2t CORAL already tanks LODO (see coral_AASD caches); keep
# whiten / diag / t2s + one s2t for external report.
VARIANTS = [
    "identity",
    "coral_AASD",
    "coral_t2s_AASD",
    "coral_whiten",
    "affine_diag_AASD",
]


def score_lodo_swa(streams, held):
    return N.official(held, N.to_ev(streams), P_STAY, SCALE)


def main():
    os.makedirs(CACHE_DIR, exist_ok=True)
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()

    if "CUDA_VISIBLE_DEVICES" in os.environ and os.environ["CUDA_VISIBLE_DEVICES"].strip() == "":
        log_line("WARNING: CUDA_VISIBLE_DEVICES empty; unsetting for GPU stem infer")
        del os.environ["CUDA_VISIBLE_DEVICES"]
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "8")))
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    log_line(f"ms-feat-coral device={device} threads={torch.get_num_threads()} "
             f"p_stay={P_STAY} scale={SCALE} max_mom={MAX_MOM}")
    log_line("honesty: unlabeled AASD/Carta 24-d ms feature moments for adaptation; "
             "no label leakage; select on LODO Acc/H only")

    t_all = time.time()
    N.load_all()
    feat_all, ext_blob = ensure_feat_cache()
    ext_feat = ext_blob["feat"]
    ext_meta = ext_blob["meta"]

    tgt_pools = {
        "AASD": {"AASD": ext_feat["AASD"]},
        "Carta2026": {"Carta2026": ext_feat["Carta2026"]},
        "pool": {"AASD": ext_feat["AASD"], "Carta2026": ext_feat["Carta2026"]},
    }

    results = dict(
        method="label-free CORAL/affine on 24-d stem ms features → frozen SWA/fuse",
        p_stay=P_STAY, scale=SCALE,
        honesty=("Uses unlabeled AASD/Carta *feature* moments for adaptation; "
                 "stronger than train-only whitening; no label leakage."),
        baselines=dict(linear_hp30=LINEAR_HP30_BASELINE, swa_lodo=SWA_LODO_REF,
                       fuse_lodo=FUSE_LODO_REF, fuse_ext=FUSE_EXT_REF),
        variants={},
    )

    # --- LODO select on SWA ---
    log_line("=== LODO SWA (select by mean Acc then H) ===")
    lodo_summary = {}
    for vname in VARIANTS:
        per = {}
        params_by_held = {}
        for held in DS_KEYS:
            train_ds = [d for d in DS_KEYS if d != held]
            tgt = variant_target(vname)
            params = fit_variant(vname, feat_all, train_ds, tgt_pools, tgt)
            params_by_held[held] = params
            cache_key = f"swa__{vname}__held-{held}.pkl"
            cache_path = os.path.join(CACHE_DIR, cache_key)
            if os.path.isfile(cache_path):
                log_line(f"stream cache hit {cache_key}")
                streams = load_pkl(cache_path)
            elif vname == "identity":
                streams = load_identity_swa_held(held)
                if streams is None:
                    feat_use = feat_all
                    ckpt = os.path.join(SWA_DIR, f"held-{held}.pt")
                    streams = _infer_one(ckpt, feat_use, device, f"held-{held}-SWA-id")
                    save_pkl(cache_path, streams)
                else:
                    save_pkl(cache_path, streams)
            else:
                feat_use = apply_map_feat(feat_all, params)
                ckpt = os.path.join(SWA_DIR, f"held-{held}.pt")
                streams = _infer_one(ckpt, feat_use, device, f"held-{held}-SWA-{vname}")
                save_pkl(cache_path, streams)
            off = score_lodo_swa(streams, held)
            per[held] = off
            log_line(f"LODO {vname:28s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")

        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        avgc = per["AVGC"]
        log_line(f"LODO_MEAN {vname:28s}  H {mean_h:.4f} Acc {mean_a:.2f}  "
                 f"AVGC {avgc['H']:.4f}/{avgc['Acc']:.2f}")
        lodo_summary[vname] = dict(
            per=per, mean=dict(H=mean_h, Acc=mean_a),
            avgc=avgc,
            params_held={h: params_to_jsonable(params_by_held[h]) for h in DS_KEYS},
        )
        results["variants"][vname] = dict(lodo=lodo_summary[vname])

    # Select: max Acc, then H. Prefer non-identity only if it beats identity.
    best = max(VARIANTS, key=lambda n: (lodo_summary[n]["mean"]["Acc"],
                                        lodo_summary[n]["mean"]["H"]))
    selected = best
    log_line(f"SELECTED {selected} LODO mean H {lodo_summary[selected]['mean']['H']:.4f} "
             f"Acc {lodo_summary[selected]['mean']['Acc']:.2f}")

    # --- External: all5 SWA (+ fuse for selected) ---
    log_line("=== external all5 SWA ===")
    # Score all variants on external (honest report); selection already locked to LODO.
    att = N.G["att"]
    for vname in VARIANTS:
        tgt = variant_target(vname)
        params = fit_variant(vname, feat_all, list(DS_KEYS), tgt_pools, tgt)
        corp_cache = os.path.join(CACHE_DIR, f"swa__{vname}__all5_corpus.pkl")
        ext_cache = os.path.join(CACHE_DIR, f"swa__{vname}__all5_external.pkl")
        if os.path.isfile(corp_cache):
            corp = load_pkl(corp_cache)
        elif vname == "identity":
            corp = load_identity_swa_all5("corpus")
            if corp is None:
                corp = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_all, device,
                                  f"all5-corp-SWA-{vname}")
            save_pkl(corp_cache, corp)
        else:
            feat_c = apply_map_feat(feat_all, params)
            corp = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_c, device,
                              f"all5-corp-SWA-{vname}")
            save_pkl(corp_cache, corp)

        if os.path.isfile(ext_cache):
            ext_st = load_pkl(ext_cache)
        elif vname == "identity":
            ext_st = load_identity_swa_all5("external")
            if ext_st is None:
                feat_e = apply_map_feat(ext_feat, params)
                ext_st = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                                    f"all5-ext-SWA-{vname}")
            save_pkl(ext_cache, ext_st)
        else:
            feat_e = apply_map_feat(ext_feat, params)
            ext_st = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                                f"all5-ext-SWA-{vname}")
            save_pkl(ext_cache, ext_st)

        mu_e, sd_e = F.emissions_from(corp, att, list(DS_KEYS))
        ext_rows = {}
        for ds in TEST_NAMES:
            r = F.score_external(ext_st[ds], ext_meta[ds], mu_e, sd_e)
            dacc = r["Acc"] - LINEAR_HP30_BASELINE[ds]["Acc"]
            ext_rows[ds] = r
            log_line(f"EXT_SWA {vname:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs hp30 {dacc:+.2f}")
        results["variants"][vname]["external_swa"] = ext_rows
        results["variants"][vname]["emis_swa"] = dict(mu=mu_e, sd=sd_e)
        results["variants"][vname]["params_all5"] = params_to_jsonable(params)

    # Fuse path for selected + identity (sanity)
    log_line("=== external all5 fuse (selected + identity) ===")
    fuse_names = ["identity", selected] if selected != "identity" else ["identity"]
    # Always also score coral_AASD fuse if selected something else — keep lean: selected+id
    for vname in fuse_names:
        tgt = variant_target(vname)
        params = fit_variant(vname, feat_all, list(DS_KEYS), tgt_pools, tgt)
        f_corp_c = os.path.join(CACHE_DIR, f"fuse__{vname}__all5_corpus.pkl")
        f_ext_c = os.path.join(CACHE_DIR, f"fuse__{vname}__all5_external.pkl")
        if os.path.isfile(f_corp_c) and os.path.isfile(f_ext_c):
            corp_f = load_pkl(f_corp_c)
            ext_f = load_pkl(f_ext_c)
        elif vname == "identity":
            corp_f = load_pkl(os.path.join(MOE_CACHE, "fuse_w05__all5_corpus.pkl"))
            ext_f = load_pkl(os.path.join(MOE_CACHE, "fuse_w05__all5_external.pkl"))
            save_pkl(f_corp_c, corp_f)
            save_pkl(f_ext_c, ext_f)
        else:
            feat_c = apply_map_feat(feat_all, params)
            feat_e = apply_map_feat(ext_feat, params)
            irm_c = _infer_one(os.path.join(IRM_DIR, "all5.pt"), feat_c, device,
                               f"all5-corp-IRM-{vname}")
            swa_c = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_c, device,
                               f"all5-corp-SWA-fuse-{vname}")
            corp_f = F.blend_streams(irm_c, swa_c, W_FUSE)
            irm_e = _infer_one(os.path.join(IRM_DIR, "all5.pt"), feat_e, device,
                               f"all5-ext-IRM-{vname}")
            swa_e = _infer_one(os.path.join(SWA_DIR, "all5.pt"), feat_e, device,
                               f"all5-ext-SWA-fuse-{vname}")
            ext_f = {ds: F.blend_one(irm_e[ds], swa_e[ds], W_FUSE) for ds in irm_e}
            save_pkl(f_corp_c, corp_f)
            save_pkl(f_ext_c, ext_f)

        mu_e, sd_e = F.emissions_from(corp_f, att, list(DS_KEYS))
        ext_rows = {}
        for ds in TEST_NAMES:
            r = F.score_external(ext_f[ds], ext_meta[ds], mu_e, sd_e)
            dacc = r["Acc"] - LINEAR_HP30_BASELINE[ds]["Acc"]
            ext_rows[ds] = r
            log_line(f"EXT_FUSE {vname:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs hp30 {dacc:+.2f}")
        results["variants"][vname]["external_fuse"] = ext_rows
        results["variants"][vname]["emis_fuse"] = dict(mu=mu_e, sd=sd_e)

    # Headline: selected by LODO; report SWA external (and fuse if present)
    sel = results["variants"][selected]
    aasd = sel["external_swa"]["AASD"]
    carta = sel["external_swa"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["selected"] = selected
    results["headline"] = dict(
        selected=selected,
        backend="SWA",
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=lodo_summary[selected]["mean"],
        avgc=lodo_summary[selected]["avgc"],
        fuse_external=sel.get("external_fuse"),
    )
    log_line(f"HEADLINE selected={selected} (LODO Acc/H) SWA AASD "
             f"H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} (Δ {d_aasd:+.2f} vs 53.12)  "
             f"Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    log_line(f"HEADLINE LODO mean H {lodo_summary[selected]['mean']['H']:.4f} "
             f"Acc {lodo_summary[selected]['mean']['Acc']:.2f}  "
             f"AVGC {lodo_summary[selected]['avgc']['H']:.4f}/"
             f"{lodo_summary[selected]['avgc']['Acc']:.2f}")
    if sel.get("external_fuse"):
        fa = sel["external_fuse"]["AASD"]
        log_line(f"HEADLINE fuse AASD H {fa['H']:.4f} Acc {fa['Acc']:.2f} "
                 f"(Δ {fa['Acc'] - LINEAR_HP30_BASELINE['AASD']['Acc']:+.2f} vs 53.12)")

    with open(os.path.join(CKPT_DIR, "fit_params.json"), "w") as fh:
        json.dump({k: v.get("params_all5") for k, v in results["variants"].items()},
                  fh, indent=1)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
