"""Label-free CORAL / affine alignment of frozen 4-seed logits onto external moments.

CPU-only. Uses cached 4-seed classifier logits (same front-end as linear_hp30).
No new network train. Never uses AASD/Carta att/labels for loss, selection, or
moment estimation — only unlabeled frame means/covariances.

Source: CORPUS (5 LODO train sets) 4-seed logits, optionally high-passed (tau 30 s).
Target moments: AASD and/or Carta unlabeled streams (and pooled).
Map: classic CORAL  z'=(z-μ_s) A + μ_out  with A = C_s^{-1/2} C_t^{1/2},
or 1-D mean/std affine on the mean-of-4 decision stream.

Distinct from entropy-min TTA (results_tta.md) and Platt/temp (results_calib.md).
Distinct from train-time CORAL weight in v3 gate_robust (ALREADY_TRIED).

Honesty: adapting with unlabeled *external* moments is stronger than pure
train-only whitening; still no label leakage.

HMM fixed p_stay 0.994 scale 0.8. Score LODO 5 + AASD + Carta; also report
Acc delta vs linear_hp30 AASD 53.12.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH python -u ext_coral_logits.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "ext_coral_logits.log")
OUT = os.path.join(EXP, "results", "research", "ext_coral_logits.json")
CKPT = os.path.join(EXP, "models", "ext_coral_logits")

CORPUS = ("AVGC", "DTU", "KUL", "UHD", "USTC")
TEST_NAMES = ("AASD", "Carta2026")
P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = 0.75
EPS = 1e-5
MAX_MOM = int(os.environ.get("CORAL_MAX_MOM", "800000"))
SEED = 20270101

LINEAR_HP30_BASELINE = dict(
    AASD=dict(H=0.6422, Acc=53.12),
    Carta2026=dict(H=0.6520, Acc=60.54),
    LODO_mean=dict(H=0.7899, Acc=69.21),
)


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def cov_stats(X, eps=EPS):
    X = np.asarray(X, np.float64)
    if X.ndim == 1:
        X = X[:, None]
    mu = X.mean(0)
    Xc = X - mu
    n = max(len(X) - 1, 1)
    C = (Xc.T @ Xc) / n
    d = C.shape[0]
    C = C + eps * np.eye(d)
    return mu, C


def matrix_power(C, p):
    w, V = np.linalg.eigh(np.asarray(C, np.float64))
    w = np.maximum(w, 1e-12)
    return (V * (w ** p)) @ V.T


def fit_coral(Xs, Xt, match_mean=True):
    mu_s, Cs = cov_stats(Xs)
    mu_t, Ct = cov_stats(Xt)
    A = matrix_power(Cs, -0.5) @ matrix_power(Ct, 0.5)
    mu_out = mu_t if match_mean else mu_s
    return dict(kind="coral", mu_s=mu_s, mu_out=mu_out, A=A,
                match_mean=bool(match_mean), d=int(A.shape[0]))


def fit_coral_whiten(Xs):
    """Train-only: z'=(z-μ) C^{-1/2}+μ (cov→I, mean kept). No external moments."""
    mu_s, Cs = cov_stats(Xs)
    A = matrix_power(Cs, -0.5)
    return dict(kind="coral_whiten", mu_s=mu_s, mu_out=mu_s, A=A,
                match_mean=False, d=int(A.shape[0]))


def apply_linear_map(D, params):
    D = np.asarray(D, np.float64)
    if D.ndim == 1:
        D = D[:, None]
        squeeze = True
    else:
        squeeze = False
    out = (D - params["mu_s"]) @ params["A"] + params["mu_out"]
    if squeeze:
        return out.reshape(-1)
    return out


def fit_affine1(xs, xt, match_mean=True):
    xs = np.asarray(xs, np.float64).reshape(-1)
    xt = np.asarray(xt, np.float64).reshape(-1)
    ms, ss = float(xs.mean()), float(xs.std()) + 1e-8
    mt, st = float(xt.mean()), float(xt.std()) + 1e-8
    scale = st / ss
    return dict(kind="affine1", ms=ms, scale=scale,
                bias=mt if match_mean else ms, match_mean=bool(match_mean))


def apply_affine1(x, params):
    x = np.asarray(x, np.float64)
    return (x - params["ms"]) * params["scale"] + params["bias"]


def stack_frames(logits_4d, datasets, max_n=MAX_MOM, rng=None):
    """Concatenate (T,4) frames; no labels."""
    rng = rng or np.random.default_rng(SEED)
    parts = []
    for ds in datasets:
        for _key, D in logits_4d[ds].items():
            parts.append(np.asarray(D, np.float64))
    X = np.concatenate(parts, 0)
    if len(X) > max_n:
        idx = rng.choice(len(X), size=max_n, replace=False)
        X = X[idx]
    return X


def stack_1d(streams, datasets, max_n=MAX_MOM, rng=None):
    rng = rng or np.random.default_rng(SEED + 1)
    parts = []
    for ds in datasets:
        for _key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64).reshape(-1))
    x = np.concatenate(parts)
    if len(x) > max_n:
        idx = rng.choice(len(x), size=max_n, replace=False)
        x = x[idx]
    return x


def maybe_hp(logits_4d, use_hp):
    if not use_hp:
        return logits_4d
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {k: V4.highpass(np.asarray(D, np.float64)) for k, D in table.items()}
    return out


def mean_stream(logits_4d):
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {k: np.asarray(D, np.float64).mean(1) for k, D in table.items()}
    return out


def apply_coral_4d(logits_4d, params):
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {k: apply_linear_map(D, params) for k, D in table.items()}
    return out


def apply_affine_streams(streams, params):
    out = {}
    for ds, table in streams.items():
        out[ds] = {k: apply_affine1(d, params) for k, d in table.items()}
    return out


def emissions_from(streams, att, datasets):
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
                       att128=m["att128"], n128=m["n128"], subject=m["subject"])
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, LOOKAHEAD)


def load_lodo_4d():
    """Four LODO direct-classifier seed logits per corpus trial (T,4)."""
    evs = []
    for _seed, exp in EC.SEEDS:
        ev, _info = S.load_folds(exp, "causal")
        evs.append(ev)
    logits = {}
    for ds in DS_KEYS:
        logits[ds] = {}
        for k in evs[0][ds].keys():
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            logits[ds][k] = np.stack(cols, 1).astype(np.float64)
    return logits


def params_to_jsonable(p):
    out = {k: v for k, v in p.items() if k not in ("mu_s", "mu_out", "A")}
    for key in ("mu_s", "mu_out", "A"):
        if key in p:
            out[key] = np.asarray(p[key]).tolist()
    return out


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    log_line(f"ext-coral-logits CPU-only  p_stay {P_STAY} scale {SCALE} max_mom {MAX_MOM}")
    log_line("honesty: unlabeled external moments used for adaptation (no labels)")

    N.load_all()

    log_line("loading CACHE train logits + external 4-seed evidence")
    cache_logits, cache_att = V4.load_train()
    # CORPUS only for source tables from cache (BRO/FUG unused for emissions).
    # Emissions MUST use cache_att (same synthetic-switch labels as linear_hp30
    # published 53.12), not N.G['att'] from LODO folds.
    src_cache = {ds: cache_logits[ds] for ds in CORPUS}
    train_att = {ds: cache_att[ds] for ds in CORPUS}
    test_logits, test_meta = V4.load_test()

    log_line("loading LODO fold 4-seed logits")
    lodo_4d = load_lodo_4d()

    # Target pools (unlabeled): AASD, Carta, pooled
    tgt_pools = {
        "AASD": {"AASD": test_logits["AASD"]},
        "Carta2026": {"Carta2026": test_logits["Carta2026"]},
        "pool": {"AASD": test_logits["AASD"], "Carta2026": test_logits["Carta2026"]},
    }

    # Pre-specified variants (no Acc-based selection on AASD/Carta)
    # primary: coral4 hp30 → AASD moments with mean match (external Acc push)
    variants = [
        ("linear_mean", False, "baseline", None, True),
        ("linear_hp30", True, "baseline", None, True),
        ("coral4_hp30_AASD", True, "coral4", "AASD", True),
        ("coral4_hp30_AASD_covonly", True, "coral4", "AASD", False),
        ("coral4_hp30_Carta", True, "coral4", "Carta2026", True),
        ("coral4_hp30_pool", True, "coral4", "pool", True),
        ("coral4_raw_AASD", False, "coral4", "AASD", True),
        ("coral4_hp30_whiten", True, "coral4", "whiten", False),
        ("affine1_hp30_AASD", True, "affine1", "AASD", True),
        ("affine1_hp30_AASD_scale", True, "affine1", "AASD", False),
        ("affine1_hp30_pool", True, "affine1", "pool", True),
    ]
    PRIMARY = "coral4_hp30_AASD"

    results = dict(
        method="label-free CORAL/affine on frozen 4-seed logits → unlabeled external moments",
        p_stay=P_STAY, scale=SCALE, primary=PRIMARY,
        honesty="Uses unlabeled AASD/Carta logit moments for adaptation; stronger than train-only; no label leakage.",
        baselines=LINEAR_HP30_BASELINE,
        variants={},
    )

    # --- External scores (source moments from CORPUS cache; target as named) ---
    log_line("=== external ===")
    for name, use_hp, mode, tgt, match_mean in variants:
        # Build combined table: CORPUS cache + test
        combined = {}
        combined.update(src_cache)
        combined.update(test_logits)
        if mode == "baseline":
            streams = mean_stream(maybe_hp(combined, use_hp))
            params = None
        elif mode == "coral4":
            src_h = maybe_hp(src_cache, use_hp)
            Xs = stack_frames(src_h, list(CORPUS))
            if tgt == "whiten":
                params = fit_coral_whiten(Xs)
            else:
                tgt_h = maybe_hp(tgt_pools[tgt], use_hp)
                Xt = stack_frames(tgt_h, list(tgt_h.keys()))
                params = fit_coral(Xs, Xt, match_mean=match_mean)
            mapped = apply_coral_4d(maybe_hp(combined, use_hp), params)
            streams = mean_stream(mapped)
        else:  # affine1
            base = mean_stream(maybe_hp(combined, use_hp))
            xs = stack_1d(base, list(CORPUS))
            tgt_h = maybe_hp(tgt_pools[tgt], use_hp)
            xt = stack_1d(mean_stream(tgt_h), list(tgt_h.keys()))
            params = fit_affine1(xs, xt, match_mean=match_mean)
            streams = apply_affine_streams(base, params)

        mu_e, sd_e = emissions_from(streams, train_att, list(CORPUS))
        ext = {}
        for ds in TEST_NAMES:
            r = score_external(streams[ds], test_meta[ds], mu_e, sd_e)
            ext[ds] = r
            base_acc = LINEAR_HP30_BASELINE[ds]["Acc"]
            dacc = r["Acc"] - base_acc
            log_line(f"EXT {name:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
                     f"ΔAcc vs linear_hp30 {dacc:+.2f}")
        results["variants"][name] = dict(
            external=ext,
            emis=dict(mu=mu_e, sd=sd_e),
            params=None if params is None else params_to_jsonable(params),
            use_hp=use_hp, mode=mode, target=tgt, match_mean=match_mean,
        )

    # --- LODO at fixed HMM (primary + baselines + a few key coral variants) ---
    log_line("=== LODO (fixed HMM) ===")
    lodo_names = [
        "linear_mean", "linear_hp30",
        "coral4_hp30_AASD", "coral4_hp30_AASD_covonly", "coral4_hp30_pool",
        "coral4_hp30_whiten", "affine1_hp30_AASD",
    ]
    for name, use_hp, mode, tgt, match_mean in variants:
        if name not in lodo_names:
            continue
        # Fit on CORPUS excluding held for source moments; target moments fixed (external unlabeled)
        per = {}
        for held in DS_KEYS:
            train_ds = [d for d in DS_KEYS if d != held]
            # combined LODO 4d + external test for applying map to held
            if mode == "baseline":
                streams = mean_stream(maybe_hp(lodo_4d, use_hp))
                params = None
            elif mode == "coral4":
                src_h = maybe_hp({d: lodo_4d[d] for d in train_ds}, use_hp)
                Xs = stack_frames(src_h, train_ds)
                if tgt == "whiten":
                    params = fit_coral_whiten(Xs)
                else:
                    tgt_h = maybe_hp(tgt_pools[tgt], use_hp)
                    Xt = stack_frames(tgt_h, list(tgt_h.keys()))
                    params = fit_coral(Xs, Xt, match_mean=match_mean)
                mapped = apply_coral_4d(maybe_hp(lodo_4d, use_hp), params)
                streams = mean_stream(mapped)
            else:
                base = mean_stream(maybe_hp(lodo_4d, use_hp))
                xs = stack_1d(base, train_ds)
                tgt_h = maybe_hp(tgt_pools[tgt], use_hp)
                xt = stack_1d(mean_stream(tgt_h), list(tgt_h.keys()))
                params = fit_affine1(xs, xt, match_mean=match_mean)
                streams = apply_affine_streams(base, params)

            off = N.official(held, N.to_ev(streams), P_STAY, SCALE)
            per[held] = off
            log_line(f"LODO {name:28s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")

        mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        log_line(f"LODO_MEAN {name:28s}  H {mean_h:.4f} Acc {mean_a:.2f}")
        results["variants"][name]["lodo"] = per
        results["variants"][name]["lodo_mean"] = dict(H=mean_h, Acc=mean_a)

    # Headline summary
    prim = results["variants"][PRIMARY]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["headline"] = dict(
        primary=PRIMARY,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=prim.get("lodo_mean"),
    )
    log_line(f"HEADLINE {PRIMARY} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
             f"(Δ {d_aasd:+.2f} vs 53.12)  Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}")
    if prim.get("lodo_mean"):
        log_line(f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}")

    with open(os.path.join(CKPT, "fit_params.json"), "w") as fh:
        json.dump({k: v.get("params") for k, v in results["variants"].items()}, fh, indent=1)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
