"""hp30-centered fine-tune of published SW2 DirectNet (inherits linear_hp30).

NEW vs ALREADY_TRIED:
  - Train BCE on causal τ=30 one-pole HIGH-PASS of trial-streamed logits
    (same transform as linear_hp30), so the net learns A/B evidence that is
    already drift-robust train→test. Inference uses the same online HP.
  - ≠ post-hoc multi-τ / soft_hp grids on frozen logits
  - ≠ rel_hp transformer on hp30 features
  - ≠ online-moment / CORAL affine on hp30
  - ≠ switch-schedule FT / CF-Null / weight-soup

Protocol: init from published SW2 fold / all5 weights; fine-tune with
trial-ordered hops; LODO + AASD/Carta @ HMM 0.994/0.8. Screen seed0 AVGC
first (ABORT if held H does not beat published seed0).

usage:
  GLIMPSE_THR=onorm SCREEN=1 flock multi_agent_opt/gpu.lock \\
    .../python -u hp30_center_ft_lodo.py
  FULL=1 ...  # all 5 folds + all5 after screen passes
"""
from __future__ import annotations

import json
import os
import pickle
import time
from types import SimpleNamespace

import numpy as np
import torch

os.environ.setdefault("GLIMPSE_THR", "onorm")

import direct_classifier as DC
import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
from common import DS_KEYS, SEED
from common_exp import CFG, DATA_EXP, EXP, FS, load_trial, rows_of, save_pickle
from decoders import D, DEV, _targets
from e1_switch import draw_switch_times
from run_experiment import load_external

RUN_ID = "hp30_center_ft_20261107"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
LOG = os.path.join(RUN_DIR, "run.log")
OUT_JSON = os.path.join(EXP, "results", "research", "hp30_center_ft.json")
OUT_MD = os.path.join(EXP, "multi_agent_opt", "results_hp30_center_ft.md")
CKPT_DIR = os.path.join(EXP, "models", "hp30_center_ft")
FOLD_EXP = "direct_clf__eaP_n5_t60_w10b3_ptSPK_hp30c_aug-sw2"

P_STAY, SCALE = 0.994, 0.8
LOOKAHEAD = 0.75
WINDOW_S = 10.0
HP_TAU = 30.0
HOP_S = 0.25
ALPHA = 1.0 - np.exp(-HOP_S / HP_TAU)
STEPS = int(os.environ.get("STEPS", "400"))  # trial-stream steps (each = one trial pass)
LR = float(os.environ.get("LR", "2e-4"))
SCREEN = os.environ.get("SCREEN", "1") == "1"
FULL = os.environ.get("FULL", "0") == "1"
TEST_NAMES = ("AASD", "Carta2026")

PUB_FOLD = {
    0: "fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2_wts__{held}.pt",
    1: "fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s1_aug-sw2_wts__{held}.pt",
    2: "fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s2_aug-sw2_wts__{held}.pt",
    3: "fold_direct_clf__eaP_n5_t60_w10b3_ptSPK_s3_aug-sw2_wts__{held}.pt",
}
PUB_ALL5 = [f"all5_direct_clf_w10b3sw2P_s{i}.pt" for i in range(4)]
# Published seed0 AVGC H for gate (from inventory / typical SW2)
PUB_AVGC_H = 0.726  # README_BEST-ish clf contribution; screen must not collapse


def log(msg: str) -> None:
    print(f"{time.strftime('%F %T')} {msg}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(msg + "\n")


def configure() -> None:
    D["window_s"] = WINDOW_S
    D["model_blocks"] = 3
    D["targets"] = ["env", "onset", "dom"]
    D["causal_future_s"] = float(CFG["dnn"].get("causal_future_s", 0.5))
    DC.HOP = int(D.get("infer_hop_s", HOP_S) * FS)
    DC.FUT = int(D["causal_future_s"] * FS)
    DC.W = int(WINDOW_S * FS)
    DC.POS0, DC.LAB = DC.W - DC.FUT - DC.HOP, DC.W - FS


def model_path(name: str) -> str:
    for root in (os.path.join(DATA_EXP, "models"), os.path.join(EXP, "models")):
        p = os.path.join(root, name)
        if os.path.exists(p):
            return p
    raise FileNotFoundError(name)


def load_net(path: str):
    net = DC.DirectNet(58, 3, "vlaai_ws").to(DEV)
    st = torch.load(path, map_location=DEV, weights_only=False)
    net.load_state_dict(st)
    return net


def trial_stream_loss(net, tr, opt=None):
    """Causal hop logits → one-pole HP → BCE vs attended column (att==0 → +)."""
    x = tr["eeg"].astype(np.float32)
    S = _targets(tr)
    att = tr["att"].astype(int)
    T = len(x)
    starts = np.arange(0, T, DC.HOP)
    # Build padded buffers once
    xp = np.concatenate([np.zeros((DC.POS0, 58), np.float32), x, np.zeros((DC.FUT + DC.HOP, 58), np.float32)])
    Sp = np.concatenate([np.zeros((DC.POS0,) + S.shape[1:], np.float32), S,
                         np.zeros((DC.FUT + DC.HOP,) + S.shape[1:], np.float32)])
    # Process in chunks for GPU
    logits = []
    bs = 256
    train = opt is not None
    if train:
        net.train()
    else:
        net.eval()
    ema = None
    total_loss = 0.0
    n = 0
    for i0 in range(0, len(starts), bs):
        a = starts[i0:i0 + bs]
        X = torch.from_numpy(np.stack([xp[k:k + DC.W] for k in a])).to(DEV)
        Sv = torch.from_numpy(np.stack([Sp[k:k + DC.W] for k in a])).to(DEV)
        if train:
            lg = net(X, Sv)  # (B,) antisymmetric logit for att==0
        else:
            with torch.no_grad():
                lg = net(X, Sv)
        # online HP in torch for differentiability
        hp = []
        for t in range(lg.shape[0]):
            v = lg[t]
            if ema is None:
                ema = v.detach()  # init EMA without grad spike
                y = v - ema
            else:
                ema = ALPHA * v.detach() + (1.0 - ALPHA) * ema
                y = v - ema
            hp.append(y)
        y = torch.stack(hp)
        lab_idx = np.maximum(a - DC.HOP, 0)
        y_lab = torch.from_numpy((att[lab_idx] == 0).astype(np.float32)).to(DEV)
        loss = torch.nn.functional.binary_cross_entropy_with_logits(y, y_lab)
        if train:
            opt.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0)
            opt.step()
        total_loss += float(loss.detach()) * len(a)
        n += len(a)
        logits.append(y.detach().cpu().numpy() if not train else None)
    return total_loss / max(n, 1)


def fit_fold(held: str, seed: int, steps: int):
    init = model_path(PUB_FOLD[seed].format(held=held))
    log(f"fold held={held} seed={seed} init={init}")
    net = load_net(init)
    opt = torch.optim.AdamW(net.parameters(), lr=LR, weight_decay=5e-4)
    # build train pool (other datasets + Fuglsang)
    rng = np.random.default_rng(CFG["seed"] + 100 + DS_KEYS.index(held) + 17 * seed)
    train = []
    for ds in DS_KEYS:
        if ds == held:
            continue
        for _, r in rows_of(ds).iterrows():
            train.append(load_trial("eaP_n5_t60", r["key"]))
    train += load_external(["Fuglsang2020"], "eaP_n5_t60", log)
    for tr in train:
        tr["_S"] = _targets(tr)
    log(f"  train trials={len(train)} steps={steps}")
    losses = []
    for step in range(steps):
        tr = train[int(rng.integers(0, len(train)))]
        # optional light synthetic switch on stimulus for robustness
        if rng.random() < 0.5:
            tr = dict(tr)
            S = tr["_S"].copy()
            T128 = int(round(len(tr["eeg"]) / FS * 128))
            times = draw_switch_times(rng, T128 / 128.0)
            st128, cur, prev = np.zeros(T128, bool), False, 0
            for tt in times:
                s0 = min(T128 - 1, int(round(tt * 128)))
                st128[prev:s0] = cur
                cur, prev = not cur, s0
            st128[prev:] = cur
            st = st128[np.minimum(T128 - 1, np.round((np.arange(len(tr["eeg"])) / FS + tr["t0"]) * 128).astype(int))]
            tr["_S"] = np.where(st[:, None, None], S[:, ::-1], S)
            tr["att"] = (tr["att"].astype(int) ^ st.astype(int))
        loss = trial_stream_loss(net, tr, opt)
        losses.append(loss)
        if (step + 1) % 50 == 0 or step == 0:
            log(f"  step {step + 1}/{steps} loss={np.mean(losses[-50:]):.4f}")
    out = os.path.join(CKPT_DIR, f"seed{seed}_held-{held}.pt")
    torch.save(net.state_dict(), out)
    log(f"  saved {out}")
    return net, out


@torch.no_grad()
def held_evidence(net, held: str):
    net.eval()
    srng = np.random.default_rng(SEED + DS_KEYS.index(held))
    ev = {}
    for _, r in rows_of(held).iterrows():
        tr = load_trial("eaP_n5_t60", r["key"])
        S, x, att, T128 = _targets(tr), tr["eeg"], tr["att"].astype(int), int(r["n"])
        times = draw_switch_times(srng, T128 / 128)
        srng.integers(0, 2)
        st128, cur, prev = np.zeros(T128, bool), False, 0
        for tt in times:
            s0 = min(T128 - 1, int(round(tt * 128)))
            st128[prev:s0] = cur
            cur, prev = not cur, s0
        st128[prev:] = cur
        st = st128[np.minimum(T128 - 1, np.round((np.arange(len(x)) / FS + tr["t0"]) * 128).astype(int))]
        Ssw = np.where(st[:, None, None], S[:, ::-1], S)
        lg, starts = DC.logits(net, x, Ssw)
        # apply same HP as training / linear_hp30
        hp = V4.highpass(lg[:, None], tau=HP_TAU, hop=HOP_S)[:, 0]
        truth = (att ^ st)[np.maximum(starts - DC.HOP, 0)]
        ev[r["key"]] = dict(
            r0=hp / 2, r1=-hp / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32), pre_switched=True,
        )
    return ev


@torch.no_grad()
def ext_evidence(net, ds: str):
    net.eval()
    EV = {}
    for i, tr in enumerate(E.ext_trials(ds)):
        lg, starts = DC.logits(net, tr["eeg"], _targets(tr))
        hp = V4.highpass(lg[:, None], tau=HP_TAU, hop=HOP_S)[:, 0]
        truth = tr["att"].astype(int)[np.maximum(starts - DC.HOP, 0)]
        EV[tr["key"]] = dict(
            r0=hp / 2, r1=-hp / 2, att=truth.astype(np.int8),
            t_end=(starts / FS + tr["t0"]).astype(np.float32),
            att128=tr["att128"], n128=tr["n128"], subject=tr["subject"],
        )
        if (i + 1) % 100 == 0:
            log(f"    {ds} {i + 1}")
    return EV


def score_lodo(ev):
    per = {}
    for ds in DS_KEYS:
        A = np.concatenate([
            (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
            for d in DS_KEYS if d != ds for e in ev[d].values()
        ])
        mu, sd = float(A.mean()), float(A.std())
        r = S.score_dataset(ds, ev, LOOKAHEAD, emis=(mu, sd * SCALE), p_stay=P_STAY)
        per[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT")}
        log(f"  LODO {ds} H={per[ds]['H']:.4f} Acc={per[ds]['Acc']:.2f}")
    mean = {k: float(np.mean([per[d][k] for d in DS_KEYS])) for k in ("H", "Acc", "L", "SDT")}
    log(f"  LODO MEAN H={mean['H']:.4f} Acc={mean['Acc']:.2f}")
    return per, mean


def main():
    t0 = time.time()
    os.makedirs(CKPT_DIR, exist_ok=True)
    os.makedirs(os.path.join(EXP, "folds", FOLD_EXP), exist_ok=True)
    configure()
    do_full = FULL
    log(f"=== hp30-centered FT  SCREEN={SCREEN} FULL={do_full} steps={STEPS} ===")

    # ---- SCREEN: seed0 held-AVGC ----
    if SCREEN or not do_full:
        net, _ = fit_fold("AVGC", 0, STEPS)
        ev_avgc = held_evidence(net, "AVGC")
        # temporary 1-fold score using published other folds' emissions approx from self
        A = np.concatenate([
            (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0) for e in ev_avgc.values()
        ])
        # Use train-pool emissions from published SW2 seed0 non-AVGC if available
        try:
            pub_ev, _ = S.load_folds("direct_clf__eaP_n5_t60_w10b3_ptSPK_aug-sw2", "causal")
            A_em = np.concatenate([
                (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
                for d in DS_KEYS if d != "AVGC" for e in pub_ev[d].values()
            ])
            # replace AVGC evidence with ours for scoring
            mix = {d: pub_ev[d] for d in DS_KEYS}
            mix["AVGC"] = ev_avgc
            mu, sd = float(A_em.mean()), float(A_em.std())
            r = S.score_dataset("AVGC", mix, LOOKAHEAD, emis=(mu, sd * SCALE), p_stay=P_STAY)
        except Exception as ex:
            log(f"  fallback self-emis: {ex}")
            mu, sd = float(A.mean()), float(A.std())
            mix = {"AVGC": ev_avgc}
            # can't score properly; use frame proxy
            r = dict(H=0.0, Acc=50.0, L=0.0, SDT=0.0)
        log(f"SCREEN AVGC H={r['H']:.4f} Acc={r['Acc']:.2f} (pub bar ~{PUB_AVGC_H})")
        screen = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT")}
        if screen["H"] < PUB_AVGC_H - 0.02:
            log("SCREEN FAIL: AVGC H collapsed vs published — abort full run")
            payload = dict(method="hp30_center_ft", screen=screen, verdict="FAIL_SCREEN",
                           elapsed_s=time.time() - t0)
            with open(OUT_JSON, "w") as fh:
                json.dump(payload, fh, indent=2)
            with open(OUT_MD, "w") as fh:
                fh.write(
                    "# results_hp30_center_ft\n\n"
                    f"SCREEN AVGC H={screen['H']:.4f} Acc={screen['Acc']:.2f} "
                    f"(pub≈{PUB_AVGC_H}). ABORT — collapse.\n"
                    "Do not rerun identical hp30-centered FT recipe without new loss design.\n"
                )
            if screen["H"] < 0.5:
                for p in Path_walk_ckpts():
                    os.remove(p)
            return 1
        log("SCREEN PASS threshold — continuing FULL seed0 LODO+external")
        do_full = True
        # fall through: re-use AVGC ckpt; train remaining held-outs

    if not do_full:
        log("nothing to do (SCREEN=0 FULL=0)")
        return 0

    # ---- FULL LODO (seed0 only first — expand if promising) ----
    seeds = [0] if os.environ.get("SEEDS", "0") == "0" else [0, 1, 2, 3]
    lodo = {ds: [] for ds in DS_KEYS}
    for seed in seeds:
        for held in DS_KEYS:
            pt = os.path.join(CKPT_DIR, f"seed{seed}_held-{held}.pt")
            if os.path.exists(pt) and held != "AVGC":
                net = load_net(pt)
            elif os.path.exists(pt) and held == "AVGC" and seed == 0:
                net = load_net(pt)
            else:
                net, pt = fit_fold(held, seed, STEPS)
            ev = held_evidence(net, held)
            save_pickle(
                dict(evidence={"causal": ev},
                     info=dict(decoder="hp30_center_ft", held_out=held, seed=seed,
                               lookahead_s=LOOKAHEAD)),
                os.path.join(EXP, "folds", FOLD_EXP + (f"_s{seed}" if seed else ""), f"{held}.pkl"),
            )
            lodo[held].append(ev)
            del net
            torch.cuda.empty_cache()

    # mean across seeds
    ev_mean = {}
    for ds in DS_KEYS:
        evs = lodo[ds]
        out = {}
        for k in evs[0]:
            r0 = np.mean([e[k]["r0"] for e in evs], 0)
            r1 = np.mean([e[k]["r1"] for e in evs], 0)
            d = dict(evs[0][k]); d["r0"] = r0.astype(np.float32); d["r1"] = r1.astype(np.float32)
            out[k] = d
        ev_mean[ds] = out
    per, mean = score_lodo(ev_mean)

    # all5 external
    ext = {}
    nets = []
    for seed in seeds:
        # train all5: init from all5 pub, FT on all 5 datasets
        init = model_path(PUB_ALL5[seed])
        net = load_net(init)
        opt = torch.optim.AdamW(net.parameters(), lr=LR, weight_decay=5e-4)
        rng = np.random.default_rng(CFG["seed"] + 999 + seed)
        pool = []
        for ds in DS_KEYS:
            for _, r in rows_of(ds).iterrows():
                pool.append(load_trial("eaP_n5_t60", r["key"]))
        pool += load_external(["Fuglsang2020"], "eaP_n5_t60", log)
        for tr in pool:
            tr["_S"] = _targets(tr)
        for step in range(STEPS):
            trial_stream_loss(net, pool[int(rng.integers(0, len(pool)))], opt)
            if (step + 1) % 100 == 0:
                log(f"  all5 seed{seed} step {step + 1}/{STEPS}")
        torch.save(net.state_dict(), os.path.join(CKPT_DIR, f"all5_seed{seed}.pt"))
        nets.append(net)

    mu = float(np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev_mean[ds].values()
    ]).mean())
    sd = float(np.concatenate([
        (e["r0"] - e["r1"]) * np.where(e["att"] == 0, 1.0, -1.0)
        for ds in DS_KEYS for e in ev_mean[ds].values()
    ]).std())
    E.CONF.update(prep="eaP_n5_t60", targets=list(D["targets"]))
    for ds in TEST_NAMES:
        EVs = [ext_evidence(n, ds) for n in nets]
        # mean across seeds
        keys = EVs[0].keys()
        EV = {}
        for k in keys:
            r0 = np.mean([e[k]["r0"] for e in EVs], 0)
            r1 = np.mean([e[k]["r1"] for e in EVs], 0)
            d = dict(EVs[0][k]); d["r0"] = r0.astype(np.float32); d["r1"] = r1.astype(np.float32)
            EV[k] = d
        r = E.score(EV, mu, sd * SCALE, P_STAY, LOOKAHEAD)
        ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT")}
        log(f"EXT {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")

    ok = (mean["H"] > 0.80 and mean["Acc"] > 70 and
          all(ext[ds]["H"] > 0.60 and ext[ds]["Acc"] > 60 for ds in TEST_NAMES))
    payload = dict(
        method="hp30_center_ft", run_id=RUN_ID, seeds=seeds,
        lodo=dict(per=per, mean=mean), ext=ext,
        verdict="PASS" if ok else "FAIL", elapsed_s=time.time() - t0,
        inherited="train+infer causal τ=30 HP on logits (linear_hp30)",
    )
    with open(OUT_JSON, "w") as fh:
        json.dump(payload, fh, indent=2)
    with open(OUT_MD, "w") as fh:
        fh.write(
            "# results_hp30_center_ft\n\n"
            f"LODO mean H={mean['H']:.4f} Acc={mean['Acc']:.2f}\n\n"
            f"AASD {ext['AASD']['H']:.4f}/{ext['AASD']['Acc']:.2f}  "
            f"Carta {ext['Carta2026']['H']:.4f}/{ext['Carta2026']['Acc']:.2f}\n\n"
            f"Verdict: {payload['verdict']}\n"
        )
    log(f"VERDICT {payload['verdict']}")
    return 0 if ok else 1


def Path_walk_ckpts():
    if not os.path.isdir(CKPT_DIR):
        return []
    return [os.path.join(CKPT_DIR, f) for f in os.listdir(CKPT_DIR) if f.endswith(".pt")]


if __name__ == "__main__":
    raise SystemExit(main())
