"""Train-time dual BCE: raw + causal-HP(logits) aux — emit debiased stream.

Distinct from failed hp30_center_ft (BCE on HP-only target; loss exploded) and
from post-hoc tip↔stem soft mixes / AC-transplant / light TF residual.

loss = BCE(raw, y) + λ · BCE(HP_τ(raw), y)   [same labels, switch weights]
At eval emit: raw | hp | soft_hp50(=0.5 raw+0.5 hp). Select Acc>75 then H on LODO.
Init SWA stem_ms96. EMA 0.995. Never AASD/Carta labels for selection.

usage:
  SMOKE=1 STEPS=40 python -u stem_dual_hp_aux.py
  STEPS=800 flock multi_agent_opt/gpu.lock python -u stem_dual_hp_aux.py
"""
from __future__ import annotations

import copy
import json
import math
import os
import pickle
import shutil
import time

import numpy as np
import torch
import torch.nn.functional as F

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS
from stem_ms96_layerwise_lr import (
    CFG,
    EMA_DECAY,
    WINDOW,
    load_external,
    load_swa,
    param_groups,
    set_lr,
    weighted_bce,
)

EXP = os.path.dirname(os.path.abspath(__file__))
SWA_DIR = os.path.join(EXP, "models", "stem_ms96_swa")
CKPT_DIR = os.path.join(EXP, "models", "stem_dual_hp_aux")
LOG = os.path.join(EXP, "experiments", "stem_dual_hp_aux.log")
OUT = os.path.join(EXP, "results", "research", "stem_dual_hp_aux.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_stem_dual_hp_aux.md")
TEST_NAMES = ("AASD", "Carta2026")

STEPS = int(os.environ.get("STEPS", "800"))
BATCH = int(os.environ.get("BATCH", "512"))
SMOKE = os.environ.get("SMOKE", "0") == "1"
LR = float(os.environ.get("LR", "8e-4"))
BACKBONE_LR_MULT = float(os.environ.get("BACKBONE_LR_MULT", "0.1"))
HP_TAU = float(os.environ.get("HP_TAU", "30"))
LAM = float(os.environ.get("LAM", "0.5"))  # aux HP BCE weight; smoke can override
P_GRID = [(0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)]
CHANCE_LOSS = 0.685
ABORT_AFTER = 50


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def causal_hp_torch(x, tau=HP_TAU, hop=0.25):
    """x: (B, T) logits → causal one-pole HP along time."""
    alpha = 1.0 - math.exp(-hop / tau)
    ema = torch.zeros_like(x)
    ema[:, 0] = alpha * x[:, 0]
    a, b = alpha, 1.0 - alpha
    for t in range(1, x.size(1)):
        ema[:, t] = a * x[:, t] + b * ema[:, t - 1]
    return x - ema


def dual_loss(logit, y, m, pos_weight, lam=LAM):
    raw = weighted_bce(logit, y, m, pos_weight)
    hp = causal_hp_torch(logit)
    aux = weighted_bce(hp, y, m, pos_weight)
    return raw + lam * aux, raw, aux


def train_dual(model, groups, y_all, mu, sd, device, steps, batch, tag):
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    groups_cfg, n_head, n_bb = param_groups(model, LR, BACKBONE_LR_MULT)
    opt = torch.optim.AdamW(groups_cfg, weight_decay=1e-4)
    log_line(
        f"    {tag} dual-HP τ={HP_TAU} λ={LAM} head={n_head} bb={n_bb} "
        f"LR_h={LR:.2e} LR_bb={LR * BACKBONE_LR_MULT:.2e}"
    )
    rng = np.random.default_rng(V.TRAIN_SEED)
    ema = copy.deepcopy(model).eval()
    model.train()
    t0 = time.time()
    last = None
    early = []
    for step in range(steps):
        set_lr(opt, step, steps)
        xb, yb, mb = V.sample_batch(rng, groups, WINDOW, batch, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        m = torch.from_numpy(mb).to(device, non_blocking=True)
        x = V.augment(x)
        opt.zero_grad(set_to_none=True)
        logit = model(x)
        loss, raw, aux = dual_loss(logit, y, m, pos_weight, LAM)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss step {step} {tag}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        V.ema_update(model, ema, EMA_DECAY)
        last = float(loss.item())
        if step < ABORT_AFTER:
            early.append(last)
            if step + 1 == ABORT_AFTER:
                mean_e = float(np.mean(early))
                log_line(f"    {tag} early mean_loss@{ABORT_AFTER}={mean_e:.4f}")
                if mean_e >= CHANCE_LOSS:
                    raise RuntimeError(f"early abort {tag}: {mean_e:.4f} >= chance")
        if step == 0 or (step + 1) % 200 == 0 or step + 1 == steps:
            log_line(
                f"    {tag} step {step + 1}/{steps} loss {last:.4f} "
                f"raw {float(raw.detach()):.4f} hp_aux {float(aux.detach()):.4f}"
            )
    ema.eval()
    acc = V.train_frame_acc(ema, groups, WINDOW, mu, sd, device)
    log_line(f"  trained {tag} loss {last:.4f} frame-acc {acc:.3f} ({time.time()-t0:.0f}s)")
    return ema, dict(loss=last, frame_acc=acc, lam=LAM, hp_tau=HP_TAU)


def fit_oom(path, groups, y_all, device, steps, batch, tag, halved=False):
    model, mu, sd, _ = load_swa(path, device)
    try:
        ema, stats = train_dual(model, groups, y_all, mu, sd, device, steps, batch, tag)
        return ema, mu, sd, batch, stats
    except torch.cuda.OutOfMemoryError:
        if halved:
            raise
        log_line(f"  OOM {tag} batch {batch}; halving")
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
        return fit_oom(path, groups, y_all, device, steps, batch // 2, tag, True)


def stream_variants(streams):
    """raw, hp30, soft_hp50 from signed 1-d streams dict[ds][key]->d."""
    out = {"raw": streams, "hp30": {}, "soft_hp50": {}}
    for ds, table in streams.items():
        out["hp30"][ds] = {}
        out["soft_hp50"][ds] = {}
        for k, d in table.items():
            d = np.asarray(d, np.float64)
            h = V4.highpass(d[:, None], tau=HP_TAU, hop=0.25)[:, 0]
            out["hp30"][ds][k] = h
            out["soft_hp50"][ds][k] = 0.5 * d + 0.5 * h
    return out


def score_stream_lodo(streams, p_stay, scale):
    ev = N.to_ev(streams)
    Hs, Accs = [], []
    per = {}
    for held in DS_KEYS:
        r = N.official(held, ev, p_stay, scale)
        per[held] = r
        Hs.append(r["H"])
        Accs.append(r["Acc"])
    return per, float(np.mean(Hs)), float(np.mean(Accs))


def emis_from_streams(streams, att):
    parts = []
    for ds, table in streams.items():
        for key, d in table.items():
            parts.append(np.asarray(d, np.float64) * np.where(att[ds][key] == 0, 1.0, -1.0))
    A = np.concatenate(parts)
    return float(A.mean()), float(A.std())


def score_ext(streams_ext, meta, mu, sd, p_stay, scale):
    rows = {}
    for ds in TEST_NAMES:
        ev = {}
        for key, d in streams_ext[ds].items():
            mk = meta[ds][key]
            d = np.asarray(d, np.float64)
            ev[key] = dict(
                r0=d, r1=np.zeros(len(d)), att=mk["att"], t_end=mk["t_end"],
                att128=mk["att128"], n128=mk["n128"], subject=mk["subject"],
            )
        r = E.score(ev, mu, sd * scale, p_stay, 0.75)
        rows[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
        log_line(f"  EXT {ds} H={rows[ds]['H']:.4f} Acc={rows[ds]['Acc']:.2f}")
    return rows


def select_variant(lodo_by_var):
    """lodo_by_var[name] = list of {p,sc,H,Acc} grid rows → pick Acc>75 then H."""
    cands = []
    for name, grid in lodo_by_var.items():
        ok = [r for r in grid if r["H"] > 0.85 and r["Acc"] > 75]
        mild = next((r for r in grid if r["p"] == 0.996 and r["sc"] == 0.8), None)
        if mild and mild["H"] > 0.85 and mild["Acc"] > 75:
            best = mild
        elif ok:
            best = sorted(ok, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
        else:
            best = sorted(grid, key=lambda r: (r["Acc"], r["H"]), reverse=True)[0]
        cands.append((name, best))
    # prefer LODO-pass then H
    cands.sort(
        key=lambda x: (1 if x[1]["H"] > 0.85 and x[1]["Acc"] > 75 else 0, x[1]["H"], x[1]["Acc"]),
        reverse=True,
    )
    return cands[0]


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    log_line(
        f"=== stem_dual_hp_aux τ={HP_TAU} λ={LAM} steps={STEPS} device={device} smoke={SMOKE} ==="
    )
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    V.assert_causal(device)
    N.load_all()
    feat_all = V.build_features()["ms"]
    os.makedirs(CKPT_DIR, exist_ok=True)

    if SMOKE:
        held = "AVGC"
        path = os.path.join(SWA_DIR, f"held-{held}.pt")
        train_ds = [d for d in DS_KEYS if d != held]
        groups, y_all = V.trial_groups(feat_all, train_ds)
        smoke_steps = int(os.environ.get("SMOKE_STEPS", "40"))
        ema, mu, sd, used, stats = fit_oom(
            path, groups, y_all, device, smoke_steps, BATCH, f"smoke-{held}"
        )
        with open(OUT.replace(".json", "_smoke.json"), "w") as fh:
            json.dump(dict(smoke=stats, batch=used), fh, indent=1, default=float)
        log_line(f"SMOKE OK frame={stats['frame_acc']:.3f} loss={stats['loss']:.4f}")
        return

    # LODO folds
    per_streams = {}
    per_stats = {}
    for held in DS_KEYS:
        path = os.path.join(SWA_DIR, f"held-{held}.pt")
        train_ds = [d for d in DS_KEYS if d != held]
        groups, y_all = V.trial_groups(feat_all, train_ds)
        log_line(f"fold held-out={held}")
        ema, mu, sd, used, stats = fit_oom(
            path, groups, y_all, device, STEPS, BATCH, f"held-{held}"
        )
        torch.save(
            dict(state=ema.state_dict(), mu=mu, sd=sd, train=train_ds, steps=STEPS,
                 batch=used, stats=stats, cfg=CFG, lam=LAM, hp_tau=HP_TAU),
            os.path.join(CKPT_DIR, f"held-{held}.pt"),
        )
        streams = V.infer_all(ema, feat_all, mu, sd, WINDOW, device)
        # keep only held for LODO assembly
        per_streams[held] = streams[held]
        per_stats[held] = stats
        del ema
        if device.type == "cuda":
            torch.cuda.empty_cache()

    # assemble LODO-honest streams
    lodo_raw = {ds: per_streams[ds] for ds in DS_KEYS}
    variants = stream_variants(lodo_raw)

    lodo_by_var = {}
    for name, st in variants.items():
        grid = []
        for p, sc in P_GRID:
            per, mH, mA = score_stream_lodo(st, p, sc)
            grid.append(dict(p=p, sc=sc, H=mH, Acc=mA,
                             per={d: dict(H=float(per[d]["H"]), Acc=float(per[d]["Acc"]))
                                  for d in DS_KEYS}))
            log_line(f"  LODO {name} p={p} sc={sc} H={mH:.4f} Acc={mA:.2f}")
        lodo_by_var[name] = grid

    vname, best = select_variant(lodo_by_var)
    log_line(f"SELECTED variant={vname} p={best['p']} sc={best['sc']} "
             f"H={best['H']:.4f} Acc={best['Acc']:.2f}")

    # all5 for external
    log_line("all5 dual-HP FT")
    groups, y_all = V.trial_groups(feat_all, list(DS_KEYS))
    ema, mu, sd, used, stats = fit_oom(
        os.path.join(SWA_DIR, "all5.pt"), groups, y_all, device, STEPS, BATCH, "all5"
    )
    torch.save(
        dict(state=ema.state_dict(), mu=mu, sd=sd, train=list(DS_KEYS), steps=STEPS,
             batch=used, stats=stats, cfg=CFG, lam=LAM, hp_tau=HP_TAU),
        os.path.join(CKPT_DIR, "all5.pt"),
    )
    corpus = V.infer_all(ema, feat_all, mu, sd, WINDOW, device)
    corpus_vars = stream_variants(corpus)
    ext_logits, ext_meta = load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    ext_raw = V.infer_all(ema, ext_feat, mu, sd, WINDOW, device)
    ext_vars = stream_variants(ext_raw)

    # emissions from selected variant on corpus
    mu_e, sd_e = emis_from_streams(corpus_vars[vname], N.G["att"])
    ext = score_ext(ext_vars[vname], ext_meta, mu_e, sd_e, best["p"], best["sc"])
    # also score all variants @ selected HMM for bookkeeping
    ext_all = {}
    for name in ext_vars:
        mu_n, sd_n = emis_from_streams(corpus_vars[name], N.G["att"])
        ext_all[name] = score_ext(ext_vars[name], ext_meta, mu_n, sd_n, best["p"], best["sc"])

    gate = (
        best["H"] > 0.85 and best["Acc"] > 75
        and ext["AASD"]["H"] >= 0.65 and ext["Carta2026"]["H"] >= 0.65
    )
    out = dict(
        method="stem_dual_hp_aux",
        lam=LAM, hp_tau=HP_TAU, steps=STEPS,
        selected=dict(variant=vname, **best),
        lodo_by_var=lodo_by_var,
        external=ext,
        external_all_variants=ext_all,
        pass_joint=gate,
        lodo_stats=per_stats,
        all5_stats=stats,
    )
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    Path = __import__("pathlib").Path
    Path(RESULTS_MD).write_text(
        "# results_stem_dual_hp_aux\n\n"
        f"Train-time BCE(raw)+λ·BCE(HP_τ={HP_TAU}); λ={LAM}; init SWA; emit {vname}.\n\n"
        f"LODO {best['H']:.4f}/{best['Acc']:.2f} @ p={best['p']} sc={best['sc']}\n"
        f"AASD {ext['AASD']['H']:.4f}/{ext['AASD']['Acc']:.2f}; "
        f"Carta {ext['Carta2026']['H']:.4f}/{ext['Carta2026']['Acc']:.2f}\n"
        f"pass_joint={gate}\n"
        "≠ hp30_center_ft (HP-only BCE). ≠ tip↔stem soft mixes.\n"
    )
    log_line(f"DONE pass={gate} wrote {OUT}")
    if not gate:
        if os.path.isdir(CKPT_DIR):
            shutil.rmtree(CKPT_DIR)
            log_line(f"deleted failing ckpts {CKPT_DIR}")
    return 0 if gate else 1


if __name__ == "__main__":
    raise SystemExit(main() or 0)
