"""Causal past-only rank→Gaussianize on 4-seed classifier logits + hp30.

CPU-only. No new nets. No fuse↔stem / MoE / CORAL / seed-reliability /
seed-disagree / multi-τ / online-moment.

Per seed, map each frame through a causal past-only empirical CDF (full past
or rolling window) → N(0,1) via norm.ppf. Then mean and/or causal one-pole
hp30 (τ=30). Also try rank-gauss *after* per-seed hp30.

Variants:
  linear_hp30              baseline (equal mean after hp30)
  equal_mean               raw equal mean (selectable reference)
  rg_full_mean             full-past RG → mean
  rg_full_hp30             full-past RG → per-seed hp30 → mean
  rg_full_mean_hp30        full-past RG → mean → 1-D hp30
  hp30_rg_full_mean        per-seed hp30 → full-past RG → mean
  rg_w{W}_mean             rolling-W RG → mean          W∈{15,30,60,120}s
  rg_w{W}_hp30             rolling-W RG → hp30 → mean
  rg_w{W}_mean_hp30        rolling-W RG → mean → hp30
  hp30_rg_w{W}_mean        hp30 → rolling-W RG → mean

Selection: mean LODO Acc then H @ HMM p_stay=0.994 scale=0.8.
Never use AASD/Carta labels. Score LODO + AASD + Carta. ΔAcc vs linear_hp30 53.12.

usage:
  PYTHONPATH=.../reports/eda_full/scripts:$PYTHONPATH \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u rank_gauss_hp30.py
"""
from __future__ import annotations

import json
import os
import time

import numpy as np
from numba import njit
from scipy.stats import norm

import ensemble_clf as EC
import ensemble_clf_nested as N
import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
LOG = os.path.join(EXP, "experiments", "rank_gauss_hp30.log")
OUT = os.path.join(EXP, "results", "research", "rank_gauss_hp30.json")
CKPT = os.path.join(EXP, "models", "rank_gauss_hp30")

CORPUS = ("AVGC", "DTU", "KUL", "UHD", "USTC")
TEST_NAMES = ("AASD", "Carta2026")
P_STAY = 0.994
SCALE = 0.8
LOOKAHEAD = 0.75
HOP = 0.25
HP_TAU = 30.0
WIN_S = (15, 30, 60, 120)
PPF_LO = 1e-6
PPF_HI = 1.0 - 1e-6
NBIN = 100_001
PPF_TABLE = norm.ppf(np.linspace(PPF_LO, PPF_HI, NBIN)).astype(np.float64)

LINEAR_HP30_BASELINE = dict(
    AASD=dict(H=0.6422, Acc=53.12),
    Carta2026=dict(H=0.6520, Acc=60.54),
    LODO_mean=dict(H=0.7899, Acc=69.21),
)


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def highpass(D, tau=HP_TAU, hop=HOP):
    return V4.highpass(np.asarray(D, np.float64), tau=float(tau), hop=hop).astype(np.float64)


@njit(cache=True)
def _insert_sorted(buf, n, val):
    i = n
    while i > 0 and buf[i - 1] > val:
        buf[i] = buf[i - 1]
        i -= 1
    buf[i] = val
    return n + 1


@njit(cache=True)
def _remove_sorted(buf, n, val):
    lo, hi = 0, n
    while lo < hi:
        mid = (lo + hi) // 2
        if buf[mid] < val:
            lo = mid + 1
        else:
            hi = mid
    idx = lo
    if idx >= n or buf[idx] != val:
        found = -1
        for j in range(max(0, idx - 2), min(n, idx + 3)):
            if buf[j] == val:
                found = j
                break
        if found < 0:
            return n
        idx = found
    for j in range(idx, n - 1):
        buf[j] = buf[j + 1]
    return n - 1


@njit(cache=True)
def _bisect_right(buf, n, val):
    lo, hi = 0, n
    while lo < hi:
        mid = (lo + hi) // 2
        if buf[mid] <= val:
            lo = mid + 1
        else:
            hi = mid
    return lo


@njit(cache=True)
def causal_rg_numba(x, window, ppf_table):
    """Past-only empirical CDF → N(0,1). window<=0 means full past."""
    T = x.shape[0]
    out = np.zeros(T, dtype=np.float64)
    if T == 0:
        return out
    maxn = T if window <= 0 else window
    buf = np.empty(maxn + 1, dtype=np.float64)
    n = 0
    nbin = ppf_table.shape[0]
    span = (1.0 - 2e-6) / (nbin - 1)
    for t in range(T):
        if n > 0:
            k = _bisect_right(buf, n, x[t])
            r = (k + 0.5) / (n + 1.0)
            if r < 1e-6:
                r = 1e-6
            if r > 1.0 - 1e-6:
                r = 1.0 - 1e-6
            idx = int((r - 1e-6) / span + 0.5)
            if idx < 0:
                idx = 0
            if idx >= nbin:
                idx = nbin - 1
            out[t] = ppf_table[idx]
        n = _insert_sorted(buf, n, x[t])
        if window > 0 and n > window:
            n = _remove_sorted(buf, n, x[t - window])
    return out


def causal_rank_gauss_1d(x, window_frames=None):
    x = np.ascontiguousarray(np.asarray(x, np.float64).reshape(-1))
    w = 0 if window_frames is None else int(window_frames)
    return causal_rg_numba(x, w, PPF_TABLE)


def rank_gauss_cols(D, window_frames=None):
    D = np.asarray(D, np.float64)
    if D.ndim == 1:
        return causal_rank_gauss_1d(D, window_frames)
    out = np.empty_like(D)
    for k in range(D.shape[1]):
        out[:, k] = causal_rank_gauss_1d(D[:, k], window_frames)
    return out


def win_frames(win_s):
    if win_s is None:
        return None
    return max(1, int(round(float(win_s) / HOP)))


def transform_table(logits_4d, kind, window_frames=None):
    """kind: 'raw' | 'hp30' | 'rg' | 'hp30_rg'. Returns (T,K) tables."""
    out = {}
    for ds, table in logits_4d.items():
        out[ds] = {}
        for k, D0 in table.items():
            D = np.asarray(D0, np.float64)
            if kind == "raw":
                out[ds][k] = D
            elif kind == "hp30":
                out[ds][k] = highpass(D)
            elif kind == "rg":
                out[ds][k] = rank_gauss_cols(D, window_frames)
            elif kind == "hp30_rg":
                out[ds][k] = rank_gauss_cols(highpass(D), window_frames)
            else:
                raise ValueError(kind)
    return out


def mean_stream(table_4d):
    out = {}
    for ds, table in table_4d.items():
        out[ds] = {k: np.asarray(D, np.float64).mean(1) for k, D in table.items()}
    return out


def hp_then_mean_1d(mean_streams):
    """1-D causal hp30 on already-averaged streams."""
    out = {}
    for ds, table in mean_streams.items():
        out[ds] = {k: highpass(np.asarray(d, np.float64).reshape(-1)).reshape(-1)
                   for k, d in table.items()}
    return out


def build_variants():
    variants = [
        dict(name="linear_hp30", family="baseline", win_s=None, selectable=True),
        dict(name="equal_mean", family="baseline", win_s=None, selectable=True),
        dict(name="rg_full_mean", family="rg_mean", win_s=None, selectable=True),
        dict(name="rg_full_hp30", family="rg_hp30", win_s=None, selectable=True),
        dict(name="rg_full_mean_hp30", family="rg_mean_hp30", win_s=None, selectable=True),
        dict(name="hp30_rg_full_mean", family="hp30_rg_mean", win_s=None, selectable=True),
    ]
    for w in WIN_S:
        variants.append(dict(
            name=f"rg_w{w}_mean", family="rg_mean", win_s=float(w), selectable=True,
        ))
        variants.append(dict(
            name=f"rg_w{w}_hp30", family="rg_hp30", win_s=float(w), selectable=True,
        ))
        variants.append(dict(
            name=f"rg_w{w}_mean_hp30", family="rg_mean_hp30", win_s=float(w), selectable=True,
        ))
        variants.append(dict(
            name=f"hp30_rg_w{w}_mean", family="hp30_rg_mean", win_s=float(w), selectable=True,
        ))
    return variants


def emissions_from(streams, att, datasets):
    parts = []
    for ds in datasets:
        labels = att[ds]
        for key, d in streams[ds].items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_external(stream, meta, mu_e, sd_e):
    ev = {}
    for key, d in stream.items():
        m = meta[key]
        d = np.asarray(d, np.float64)
        ev[key] = dict(
            r0=d, r1=np.zeros(len(d)), att=m["att"], t_end=m["t_end"],
            att128=m["att128"], n128=m["n128"], subject=m["subject"],
        )
    return E.score(ev, mu_e, sd_e * SCALE, P_STAY, LOOKAHEAD)


def load_lodo_4d():
    import score as S
    evs = []
    for _seed, exp in EC.SEEDS:
        ev, _info = S.load_folds(exp, "causal")
        evs.append(ev)
    logits = {}
    for ds in DS_KEYS:
        logits[ds] = {}
        for k in evs[0][ds].keys():
            cols = [EC.delta(ev[ds][k]) for ev in evs]
            logits[ds][k] = np.stack(cols, 1).astype(np.float64)
    return logits


def pick_variant(lodo_means, variants):
    best_name, best_key = None, None
    selectable = {v["name"] for v in variants if v.get("selectable", True)}
    for name, m in lodo_means.items():
        if name not in selectable:
            continue
        key = (float(m["Acc"]), float(m["H"]))
        if best_key is None or key > best_key:
            best_key, best_name = key, name
    return best_name


def score_lodo(name, streams, results, lodo_means):
    t0 = time.time()
    per = {}
    for held in DS_KEYS:
        off = N.official(held, N.to_ev(streams), P_STAY, SCALE)
        per[held] = off
        log_line(f"LODO {name:28s} {held:5s}  H {off['H']:.4f} Acc {off['Acc']:.2f}")
    mean_h = float(np.mean([per[d]["H"] for d in DS_KEYS]))
    mean_a = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
    log_line(f"LODO_MEAN {name:28s}  H {mean_h:.4f} Acc {mean_a:.2f}  ({time.time()-t0:.0f}s)")
    lodo_means[name] = dict(H=mean_h, Acc=mean_a)
    results["variants"][name] = dict(
        lodo=per, lodo_mean=dict(H=mean_h, Acc=mean_a),
    )
    return mean_h, mean_a


def score_ext(name, train_streams, test_streams, train_att, test_meta, results):
    mu_e, sd_e = emissions_from(train_streams, train_att, list(CORPUS))
    ext = {}
    for ds in TEST_NAMES:
        r = score_external(test_streams[ds], test_meta[ds], mu_e, sd_e)
        ext[ds] = r
        base_acc = LINEAR_HP30_BASELINE[ds]["Acc"]
        dacc = r["Acc"] - base_acc
        log_line(
            f"EXT {name:28s} {ds:10s}  H {r['H']:.4f} Acc {r['Acc']:.2f}  "
            f"ΔAcc vs linear_hp30 {dacc:+.2f}"
        )
    results["variants"][name]["external"] = ext
    results["variants"][name]["emis"] = dict(mu=mu_e, sd=sd_e)
    return ext


def build_all_streams(logits_4d, tag):
    """Build all variant 1-D streams for one logits source. Cache RG transforms."""
    t0 = time.time()
    streams = {}
    # baselines
    streams["equal_mean"] = mean_stream(logits_4d)
    hp = transform_table(logits_4d, "hp30")
    streams["linear_hp30"] = mean_stream(hp)

    win_list = [None] + [float(w) for w in WIN_S]
    for win_s in win_list:
        wf = win_frames(win_s)
        tag_w = "full" if win_s is None else f"w{int(win_s)}"
        t1 = time.time()
        rg = transform_table(logits_4d, "rg", window_frames=wf)
        log_line(f"  [{tag}] rg_{tag_w} in {time.time()-t1:.1f}s")
        streams[f"rg_{tag_w}_mean"] = mean_stream(rg)
        streams[f"rg_{tag_w}_hp30"] = mean_stream(
            {ds: {k: highpass(D) for k, D in table.items()} for ds, table in rg.items()}
        )
        streams[f"rg_{tag_w}_mean_hp30"] = hp_then_mean_1d(streams[f"rg_{tag_w}_mean"])

        t1 = time.time()
        hprg = transform_table(logits_4d, "hp30_rg", window_frames=wf)
        log_line(f"  [{tag}] hp30_rg_{tag_w} in {time.time()-t1:.1f}s")
        streams[f"hp30_rg_{tag_w}_mean"] = mean_stream(hprg)

    log_line(f"  [{tag}] all streams ready in {time.time()-t0:.1f}s ({len(streams)} variants)")
    return streams


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(CKPT, exist_ok=True)
    open(LOG, "w").close()
    t_all = time.time()
    variants = build_variants()
    log_line(f"rank-gauss-hp30 CPU-only  p_stay {P_STAY} scale {SCALE}")
    log_line(f"n_variants={len(variants)}  win_s={WIN_S}  hp_tau={HP_TAU}")
    log_line("honesty: LODO Acc then H selection only; never AASD/Carta labels")
    log_line("distinct from: multi-tau, seed-reliability, seed-disagree, online-moment, CORAL, fuse↔hp30")

    # Warm up numba once
    _ = causal_rank_gauss_1d(np.linspace(-1, 1, 64), window_frames=16)
    _ = causal_rank_gauss_1d(np.linspace(-1, 1, 64), window_frames=None)

    N.load_all()

    log_line("loading CACHE train + external 4-seed evidence")
    cache_logits, cache_att = V4.load_train()
    src_cache = {ds: cache_logits[ds] for ds in CORPUS}
    train_att = {ds: cache_att[ds] for ds in CORPUS}
    test_logits, test_meta = V4.load_test()

    log_line("loading LODO fold 4-seed logits")
    lodo_4d = load_lodo_4d()

    results = dict(
        method="causal past-only rank→Gaussianize on 4-seed logits + mean/hp30",
        p_stay=P_STAY, scale=SCALE, hop=HOP, hp_tau=HP_TAU, win_s=list(WIN_S),
        honesty="Selection on LODO Acc then H only; never AASD/Carta labels. Causal past-only RG.",
        baselines=LINEAR_HP30_BASELINE,
        variants={},
    )
    lodo_means = {}

    log_line("=== building streams ===")
    lodo_s = build_all_streams(lodo_4d, "lodo")
    cache_s = build_all_streams(src_cache, "cache")
    test_s = build_all_streams(test_logits, "test")

    # Sanity: variant names match
    names = [v["name"] for v in variants]
    for name in names:
        if name not in lodo_s:
            raise KeyError(f"missing stream {name}; have {sorted(lodo_s)}")

    log_line("=== LODO: all variants ===")
    for v in variants:
        name = v["name"]
        score_lodo(name, lodo_s[name], results, lodo_means)
        results["variants"][name]["family"] = v["family"]
        results["variants"][name]["win_s"] = v["win_s"]
        results["variants"][name]["win_frames"] = win_frames(v["win_s"])

    selected = pick_variant(lodo_means, variants)
    log_line(
        f"SELECTED (LODO Acc then H): {selected}  "
        f"H {lodo_means[selected]['H']:.4f} Acc {lodo_means[selected]['Acc']:.2f}"
    )
    results["selected"] = selected

    ranked = sorted(
        lodo_means.items(), key=lambda kv: (kv[1]["Acc"], kv[1]["H"]), reverse=True,
    )
    results["lodo_rank"] = [
        dict(name=n, Acc=m["Acc"], H=m["H"]) for n, m in ranked
    ]
    log_line("LODO Acc rank (top): " + ", ".join(
        f"{n}({m['Acc']:.2f}/{m['H']:.4f})" for n, m in ranked[:8]
    ))

    log_line("=== external (all variants; headline = selected) ===")
    for name in names:
        score_ext(name, cache_s[name], test_s[name], train_att, test_meta, results)

    prim = results["variants"][selected]
    aasd = prim["external"]["AASD"]
    carta = prim["external"]["Carta2026"]
    d_aasd = aasd["Acc"] - LINEAR_HP30_BASELINE["AASD"]["Acc"]
    results["headline"] = dict(
        selected=selected,
        AASD=aasd, Carta2026=carta,
        AASD_Acc_delta_vs_linear_hp30=d_aasd,
        lodo_mean=prim["lodo_mean"],
        lodo_per_fold=prim["lodo"],
    )
    log_line(
        f"HEADLINE {selected} AASD H {aasd['H']:.4f} Acc {aasd['Acc']:.2f} "
        f"(Δ {d_aasd:+.2f} vs 53.12)  Carta H {carta['H']:.4f} Acc {carta['Acc']:.2f}"
    )
    log_line(
        f"HEADLINE LODO mean H {prim['lodo_mean']['H']:.4f} Acc {prim['lodo_mean']['Acc']:.2f}"
    )

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(results, fh, indent=1, default=float)
    with open(os.path.join(CKPT, "selection.json"), "w") as fh:
        json.dump(
            dict(selected=selected, lodo_means=lodo_means, lodo_rank=results["lodo_rank"]),
            fh, indent=1, default=float,
        )
    log_line(f"wrote {OUT} ({time.time() - t_all:.0f}s)")


if __name__ == "__main__":
    main()
