"""Light causal transformer residual with HARD HP on residual — tip DC preserved.

out = tip + highpass_τ( TinyCausalTF(feats) )
feats = [tip, stem, stem_hp30, tip_hp30]  (4-d per frame)

Distinct from:
  - light_hp_anchor_resid (GRU + λ‖out−tip‖² soft anchor; deleted)
  - soft_hp50 / AC-transplant (fixed linear mixes)
  - logit transformers v1–v4 (train on 4-seed logits only)

Train lightly (≤600 steps/fold) on LODO train folds; EMA; zero-init head.
Selection: Acc>75 then H on LODO; external acceptance.
"""
from __future__ import annotations

import json
import os
import time
from pathlib import Path

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS
from common_exp import EXP
from dual_hp30_sw2_bmms import (
    LOOKAHEAD,
    TEST_NAMES,
    as_sym,
    blend_ev,
    load_ext_mean4,
    score_lodo,
    signed,
)
from accfuse_hp30_hselect import align_hp, load_accfuse
from stem_hp30_accfuse import (
    TIP_ALPHA,
    apply_hp30_signed,
    emis_from,
    ev_to_signed,
    load_stem_ext,
    load_stem_honest,
)
from ac_transplant_tip_stem import build_ext_tip_and_stem, signed_to_ev

RUN_ID = "light_hp_resid_tf_20261109"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
CKPT_DIR = os.path.join(EXP, "models", "light_hp_resid_tf")
OUT_JSON = os.path.join(EXP, "results", "research", "light_hp_resid_tf.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_light_hp_resid_tf.md")
SEED = 20261109
STEPS = int(os.environ.get("STEPS", "500"))
BATCH = int(os.environ.get("BATCH", "64"))
WIN = int(os.environ.get("WIN", "64"))  # 16 s @ 4 Hz
D_MODEL = 32
N_HEAD = 4
N_LAYERS = 1
HP_TAU = float(os.environ.get("HP_TAU", "30"))
LR = 1e-3
EMA = 0.995
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
P_CANDS = [(0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)]


def log(m):
    print(f"{time.strftime('%F %T')} {m}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(os.path.join(RUN_DIR, "run.log"), "a") as fh:
        fh.write(m + "\n")


class TinyCausalTF(nn.Module):
    def __init__(self, d_in=4, d_model=D_MODEL, n_head=N_HEAD, n_layers=N_LAYERS):
        super().__init__()
        self.proj = nn.Linear(d_in, d_model)
        layer = nn.TransformerEncoderLayer(
            d_model=d_model, nhead=n_head, dim_feedforward=d_model * 2,
            dropout=0.1, batch_first=True, activation="gelu", norm_first=True,
        )
        self.enc = nn.TransformerEncoder(layer, num_layers=n_layers)
        self.head = nn.Linear(d_model, 1)
        nn.init.zeros_(self.head.weight)
        nn.init.zeros_(self.head.bias)

    def forward(self, x):
        # x: (B, T, C)
        h = self.proj(x)
        T = h.size(1)
        mask = torch.triu(torch.ones(T, T, device=h.device, dtype=torch.bool), diagonal=1)
        h = self.enc(h, mask=mask)
        return self.head(h).squeeze(-1)  # (B, T)


def causal_hp_torch(x, tau=HP_TAU, hop=0.25):
    """Differentiable causal one-pole HP along time dim. x: (B, T)"""
    alpha = 1.0 - np.exp(-hop / tau)
    # EMA recursion
    ema = torch.zeros_like(x)
    ema[:, 0] = alpha * x[:, 0]
    a = alpha
    b = 1.0 - alpha
    for t in range(1, x.size(1)):
        ema[:, t] = a * x[:, t] + b * ema[:, t - 1]
    return x - ema


def build_feat_tables(tip_s, stem_s):
    tip_hp = apply_hp30_signed(tip_s, tau=30)
    stem_hp = apply_hp30_signed(stem_s, tau=30)
    feats = {}
    for ds in tip_s:
        feats[ds] = {}
        for k in tip_s[ds]:
            if k not in stem_s[ds]:
                continue
            n = min(
                len(tip_s[ds][k]), len(stem_s[ds][k]),
                len(tip_hp[ds][k]), len(stem_hp[ds][k]),
            )
            f = np.stack([
                tip_s[ds][k][:n],
                stem_s[ds][k][:n],
                stem_hp[ds][k][:n],
                tip_hp[ds][k][:n],
            ], axis=-1).astype(np.float32)
            feats[ds][k] = f
    return feats, tip_hp, stem_hp


def collect_windows(feats, tip_s, datasets, win=WIN):
    Xs, Ys, Tips = [], [], []
    for ds in datasets:
        for k, f in feats[ds].items():
            tip = tip_s[ds][k]
            # labels from tip evidence template att via sign of tip? Use fuse labels from af
            # We need binary labels — get from tip evidence att
            n = min(len(f), len(tip))
            if n < win:
                continue
            # sample several windows
            step = win // 2
            for t0 in range(0, n - win + 1, step):
                Xs.append(f[t0:t0 + win])
                Tips.append(tip[t0:t0 + win])
                # y = sign convention: tip already signed as r0-r1 with att
                # BCE vs soft labels from tip direction is wrong — need true att.
                # Use tip evidence's att from af when building.
    return Xs, Tips


def make_labeled_windows(feats, tip_ev, datasets, win=WIN, max_per_ds=400):
    Xs, Ys, Tips = [], [], []
    rng = np.random.RandomState(SEED)
    for ds in datasets:
        keys = list(feats[ds].keys())
        rng.shuffle(keys)
        count = 0
        for k in keys:
            if k not in tip_ev[ds]:
                continue
            f = feats[ds][k]
            e = tip_ev[ds][k]
            tip = signed(e)
            att = np.asarray(e["att"], np.int64)
            n = min(len(f), len(tip), len(att))
            if n < win:
                continue
            step = win // 2
            for t0 in range(0, n - win + 1, step):
                sl = slice(t0, t0 + win)
                # target: antisymmetric label — when att==0, positive tip means speaker0
                # Use frame att as 0/1 for BCE on logit = out
                # out should match preferred speaker: y=1 if att==0 else 0 for "prefer r0"?
                # Our signed stream is already (r0-r1)*sign; score uses signed * ±1 by att.
                # Train BCE on sigmoid(out) vs 1.0 with out already in "correct-sign" space:
                # correct_signed = tip * where(att==0, 1, -1) should be positive when correct.
                # So train to predict correct_signed direction: y = (att==0).float for raw r0-r1?
                # Simpler: regress toward tip's sign-corrected oracle labels via BCE on
                # logits that are att-aligned.
                y = (att[sl] == 0).astype(np.float32)
                # But our output is a signed score like tip — convert: model predicts r0-r1 style
                # BCE(sigmoid(out), y) with y=1 when att=0
                Xs.append(f[sl])
                Ys.append(y)
                Tips.append(tip[sl].astype(np.float32))
                count += 1
                if count >= max_per_ds:
                    break
            if count >= max_per_ds:
                break
    X = torch.tensor(np.stack(Xs), dtype=torch.float32)
    Y = torch.tensor(np.stack(Ys), dtype=torch.float32)
    T = torch.tensor(np.stack(Tips), dtype=torch.float32)
    return X, Y, T


def ema_update(ema, model, decay=EMA):
    with torch.no_grad():
        for ep, p in zip(ema.parameters(), model.parameters()):
            ep.data.mul_(decay).add_(p.data, alpha=1 - decay)


def train_fold(feats, tip_ev, train_ds, tag):
    torch.manual_seed(SEED)
    model = TinyCausalTF().to(DEVICE)
    ema = TinyCausalTF().to(DEVICE)
    ema.load_state_dict(model.state_dict())
    opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
    X, Y, Tip = make_labeled_windows(feats, tip_ev, train_ds)
    if len(X) == 0:
        raise RuntimeError(f"no windows for {tag}")
    n = len(X)
    log(f"  train {tag}: {n} windows device={DEVICE}")
    model.train()
    for step in range(STEPS):
        idx = torch.randint(0, n, (min(BATCH, n),))
        xb = X[idx].to(DEVICE)
        yb = Y[idx].to(DEVICE)
        tipb = Tip[idx].to(DEVICE)
        resid = model(xb)
        resid_hp = causal_hp_torch(resid, tau=HP_TAU)
        out = tipb + resid_hp
        loss = F.binary_cross_entropy_with_logits(out, yb)
        # tiny L2 on resid before HP to keep residual small
        loss = loss + 0.01 * (resid ** 2).mean()
        opt.zero_grad()
        loss.backward()
        opt.step()
        ema_update(ema, model)
        if step % 100 == 0 or step == STEPS - 1:
            log(f"    step {step} loss={loss.item():.4f}")
    return ema


@torch.no_grad()
def infer_streams(model, feats, tip_s):
    model.eval()
    out = {}
    for ds in feats:
        out[ds] = {}
        for k, f in feats[ds].items():
            tip = tip_s[ds][k]
            n = min(len(f), len(tip))
            # run in chunks
            x = torch.tensor(f[:n][None], dtype=torch.float32, device=DEVICE)
            resid = model(x)[0].cpu().numpy()
            # apply numpy HP for exact match with eval path
            resid_hp = V4.highpass(resid[:, None], tau=HP_TAU, hop=0.25)[:, 0]
            out[ds][k] = tip[:n] + resid_hp.astype(np.float64)
    return out


def main():
    t0 = time.time()
    os.makedirs(RUN_DIR, exist_ok=True)
    os.makedirs(CKPT_DIR, exist_ok=True)
    open(os.path.join(RUN_DIR, "run.log"), "w").close()
    log(f"=== light HP-residual TF τ={HP_TAU} steps={STEPS} ===")

    af = load_accfuse()
    _, lin_hp, _ = V4._mean_and_hp_from_folds()
    lin_al = align_hp(af, lin_hp)
    tip_ev = {ds: blend_ev(af[ds], lin_al[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    stem_s = load_stem_honest()
    feats, _, _ = build_feat_tables(tip_s, stem_s)

    # LODO: train on other 4, infer held
    lodo_streams = {ds: {} for ds in DS_KEYS}
    for held in DS_KEYS:
        train_ds = [d for d in DS_KEYS if d != held]
        log(f"=== held-{held} ===")
        ema = train_fold(feats, tip_ev, train_ds, f"held-{held}")
        path = os.path.join(CKPT_DIR, f"held-{held}.pt")
        torch.save(ema.state_dict(), path)
        # infer only held
        held_feat = {held: feats[held]}
        held_tip = {held: tip_s[held]}
        pred = infer_streams(ema, held_feat, held_tip)
        lodo_streams[held] = pred[held]

    # all5 for external
    log("=== all5 ===")
    ema_all = train_fold(feats, tip_ev, list(DS_KEYS), "all5")
    torch.save(ema_all.state_dict(), os.path.join(CKPT_DIR, "all5.pt"))

    tip_ext, tip_ext_s, stem_ext_s = build_ext_tip_and_stem()
    feats_ext, _, _ = build_feat_tables(tip_ext_s, stem_ext_s)
    ext_signed = infer_streams(ema_all, feats_ext, tip_ext_s)
    ext_ev = signed_to_ev(ext_signed, tip_ext)

    lodo_ev = signed_to_ev(lodo_streams, af)
    grid = []
    for p, sc in P_CANDS:
        per, mH, mA = score_lodo(lodo_ev, p_stay=p, scale=sc)
        grid.append(dict(p=p, sc=sc, H=mH, Acc=mA,
                         per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"])) for d in DS_KEYS}))
        log(f"  p={p} sc={sc} LODO {mH:.4f}/{mA:.2f}")
    ok = [r for r in grid if r["H"] > 0.85 and r["Acc"] > 75]
    mild = next(r for r in grid if r["p"] == 0.996 and r["sc"] == 0.8)
    best = mild if (mild["H"] > 0.85 and mild["Acc"] > 75) else (
        sorted(ok if ok else grid, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
    )
    mu, sd = emis_from(lodo_ev)
    ext = {}
    for ds in TEST_NAMES:
        r = E.score(ext_ev[ds], mu, sd * best["sc"], best["p"], LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        log(f"  EXT {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")
    gate = (
        best["H"] > 0.85 and best["Acc"] > 75
        and ext["AASD"]["H"] >= 0.65 and ext["Carta2026"]["H"] >= 0.65
    )
    out = dict(
        method="light_hp_resid_tf",
        hp_tau=HP_TAU, steps=STEPS, win=WIN,
        grid=grid, selected=best, external=ext, pass_joint=gate,
        elapsed_s=time.time() - t0,
    )
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    Path(RESULTS_MD).write_text(
        "# results_light_hp_resid_tf\n\n"
        f"out = tip + HP_τ={HP_TAU}(TinyCausalTF([tip,stem,stem_hp,tip_hp])); "
        f"STEPS={STEPS}.\n\n"
        f"LODO {best['H']:.4f}/{best['Acc']:.2f} @ p={best['p']} sc={best['sc']}\n"
        f"AASD {ext['AASD']['H']:.4f}/{ext['AASD']['Acc']:.2f}; "
        f"Carta {ext['Carta2026']['H']:.4f}/{ext['Carta2026']['Acc']:.2f}\n"
        f"pass_joint={gate}\n"
    )
    log(f"DONE pass={gate} elapsed={time.time()-t0:.0f}s")
    if not gate:
        # delete failing checkpoints
        import shutil
        if os.path.isdir(CKPT_DIR):
            shutil.rmtree(CKPT_DIR)
            log(f"deleted failing ckpts {CKPT_DIR}")
    return 0 if gate else 1


if __name__ == "__main__":
    raise SystemExit(main())
