"""ELM / random-feature residual with HARD HP — tip DC preserved.

out = tip + HP_τ( ridge( tanh(X @ W_rand) ) )
X = [tip, stem, stem_hp30, tip_hp30] per frame (causal window stats optional).

Distinct from light TF / GRU soft-anchor / soft_hp50 / AC-transplant.
Train ridge on LODO train only; no GPU. Select Acc>75 then H.
"""
from __future__ import annotations

import json
import os
import time
from pathlib import Path

import numpy as np

import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS
from common_exp import EXP
from dual_hp30_sw2_bmms import LOOKAHEAD, TEST_NAMES, blend_ev, score_lodo, signed
from accfuse_hp30_hselect import align_hp, load_accfuse
from stem_hp30_accfuse import (
    TIP_ALPHA,
    apply_hp30_signed,
    emis_from,
    ev_to_signed,
    load_stem_ext,
    load_stem_honest,
)
from ac_transplant_tip_stem import build_ext_tip_and_stem, signed_to_ev
from light_hp_resid_tf import build_feat_tables

RUN_ID = "elm_hp_resid_20261109"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
OUT_JSON = os.path.join(EXP, "results", "research", "elm_hp_resid.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_elm_hp_resid.md")
SEED = 20261109
H = 64
REG = 1.0
HP_TAU = 30.0
P_CANDS = [(0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)]
ALPHAS = (0.25, 0.5, 0.75, 1.0)  # scale of HP(ELM) residual


def log(m):
    print(f"{time.strftime('%F %T')} {m}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(os.path.join(RUN_DIR, "run.log"), "a") as fh:
        fh.write(m + "\n")


def make_W(d_in=4, h=H, seed=SEED):
    rng = np.random.RandomState(seed)
    W = rng.normal(0, 1.0 / np.sqrt(d_in), size=(d_in, h)).astype(np.float64)
    b = rng.uniform(-1, 1, size=(h,)).astype(np.float64)
    return W, b


def phi(X, W, b):
    return np.tanh(X @ W + b)


def fit_ridge(Phi, y, reg=REG):
    # Phi: (N,H), y: (N,)
    A = Phi.T @ Phi + reg * np.eye(Phi.shape[1])
    return np.linalg.solve(A, Phi.T @ y)


def collect_xy(feats, tip_ev, datasets, max_frames=200_000):
    Xs, ys = [], []
    n = 0
    rng = np.random.RandomState(SEED)
    for ds in datasets:
        keys = list(feats[ds].keys())
        rng.shuffle(keys)
        for k in keys:
            if k not in tip_ev[ds]:
                continue
            f = feats[ds][k]
            e = tip_ev[ds][k]
            att = np.asarray(e["att"], np.int64)
            tip = signed(e)
            m = min(len(f), len(att), len(tip))
            # target residual in tip space: want tip+resid ≈ correct-signed large
            # Train resid to push toward correct direction beyond tip:
            # y* = where(att==0, +1, -1)  — target signed evidence
            # resid_target = y* - tip  (then HP applied at apply time)
            ystar = np.where(att[:m] == 0, 1.0, -1.0)
            resid_t = ystar - tip[:m]
            Xs.append(f[:m])
            ys.append(resid_t)
            n += m
            if n >= max_frames:
                break
        if n >= max_frames:
            break
    X = np.concatenate(Xs, 0)
    y = np.concatenate(ys, 0)
    return X, y


def apply_model(feats, tip_s, W, b, beta, alpha, tau=HP_TAU):
    out = {}
    for ds in feats:
        out[ds] = {}
        for k, f in feats[ds].items():
            tip = tip_s[ds][k]
            n = min(len(f), len(tip))
            r = phi(f[:n].astype(np.float64), W, b) @ beta
            r_hp = V4.highpass(r[:, None], tau=tau, hop=0.25)[:, 0]
            out[ds][k] = tip[:n] + alpha * r_hp
    return out


def main():
    t0 = time.time()
    os.makedirs(RUN_DIR, exist_ok=True)
    open(os.path.join(RUN_DIR, "run.log"), "w").close()
    log("=== ELM HP-residual on tip ===")

    af = load_accfuse()
    _, lin_hp, _ = V4._mean_and_hp_from_folds()
    lin_al = align_hp(af, lin_hp)
    tip_ev = {ds: blend_ev(af[ds], lin_al[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    stem_s = load_stem_honest()
    feats, _, _ = build_feat_tables(tip_s, stem_s)
    W, b = make_W()

    # LODO
    lodo = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        train_ds = [d for d in DS_KEYS if d != held]
        X, y = collect_xy(feats, tip_ev, train_ds)
        Phi = phi(X, W, b)
        beta = fit_ridge(Phi, y)
        log(f"  held-{held} N={len(y)} ||beta||={np.linalg.norm(beta):.3f}")
        # pick alpha via quick LODO-inner? Use fixed grid later on held streams
        # Store beta per held; evaluate alphas after
        for a in ALPHAS:
            pred = apply_model({held: feats[held]}, {held: tip_s[held]}, W, b, beta, a)
            # stash under name
            if a not in lodo:
                lodo[a] = {ds: {} for ds in DS_KEYS}
            lodo[a][held] = pred[held]

    tip_ext, tip_ext_s, stem_ext_s = build_ext_tip_and_stem()
    feats_ext, _, _ = build_feat_tables(tip_ext_s, stem_ext_s)
    X_all, y_all = collect_xy(feats, tip_ev, list(DS_KEYS))
    beta_all = fit_ridge(phi(X_all, W, b), y_all)

    rows = []
    best_joint = None
    for a in ALPHAS:
        ev = signed_to_ev(lodo[a], af)
        grid = []
        for p, sc in P_CANDS:
            per, mH, mA = score_lodo(ev, p_stay=p, scale=sc)
            grid.append(dict(p=p, sc=sc, H=mH, Acc=mA))
        ok = [r for r in grid if r["H"] > 0.85 and r["Acc"] > 75]
        mild = next(r for r in grid if r["p"] == 0.996 and r["sc"] == 0.8)
        best = mild if (mild["H"] > 0.85 and mild["Acc"] > 75) else (
            sorted(ok if ok else grid, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
        )
        ext_signed = apply_model(feats_ext, tip_ext_s, W, b, beta_all, a)
        ext_ev = signed_to_ev(ext_signed, tip_ext)
        mu, sd = emis_from(ev)
        ext = {}
        for ds in TEST_NAMES:
            r = E.score(ext_ev[ds], mu, sd * best["sc"], best["p"], LOOKAHEAD)
            ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        gate = (
            best["H"] > 0.85 and best["Acc"] > 75
            and ext["AASD"]["H"] >= 0.65 and ext["Carta2026"]["H"] >= 0.65
        )
        row = dict(alpha=a, lodo=best, grid=grid, external=ext, pass_joint=gate)
        rows.append(row)
        log(
            f"  a={a} LODO {best['H']:.4f}/{best['Acc']:.2f} "
            f"AASD {ext['AASD']['H']:.4f} Carta {ext['Carta2026']['H']:.4f} pass={gate}"
        )
        if gate:
            best_joint = row

    out = dict(method="elm_hp_resid", rows=rows, best_joint=best_joint,
               pass_joint=best_joint is not None, elapsed_s=time.time() - t0)
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    lines = [
        "# results_elm_hp_resid\n\n",
        f"out=tip+α·HP_{HP_TAU}(ELM tanh ridge); H={H} reg={REG}.\n\n",
        "| α | LODO | AASD | Carta | Pass |\n",
        "|---:|---:|---:|---:|:---:|\n",
    ]
    for r in rows:
        lo, e = r["lodo"], r["external"]
        lines.append(
            f"| {r['alpha']} | {lo['H']:.4f}/{lo['Acc']:.2f} | "
            f"{e['AASD']['H']:.4f} | {e['Carta2026']['H']:.4f} | "
            f"{'YES' if r['pass_joint'] else 'NO'} |\n"
        )
    Path(RESULTS_MD).write_text("".join(lines))
    log(f"DONE pass={best_joint is not None} elapsed={time.time()-t0:.0f}s")
    return 0 if best_joint else 1


if __name__ == "__main__":
    raise SystemExit(main())
