"""multi-stem-attn-pool: learnable causal soft-attention over frozen stem logits.

Streams (CPU caches only):
  focal  — models/focal_synth_adapt_readout/cache/_tmp_focal__*
  synth  — models/fuse_conf_resid/cache/_tmp_synth__*
  IRM    — models/fuse_conf_resid/cache/_tmp_irm__*

Per frame, build past-only EWMA features of each stream's logit and |logit|
(τ∈{1,4}s @4Hz hop), feed a tiny linear → softmax over K=3 streams, emit
weighted sum. Train gates with frame BCE on LODO *train* folds only
(held-out never used for fit). Select by mean LODO Acc then H @0.994/0.8.
Also score equal-mean and tip focal↔synth w=0.5 as baselines.

≠ fixed-w IRM↔stem / agree-gate / conf-resid / MoE fuse↔hp30 / Mixup/focal FT.

usage:
  CUDA_VISIBLE_DEVICES= \\
    /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/.venv/bin/python -u \\
    multi_stem_attn_pool.py
"""
from __future__ import annotations

import json
import os
import pickle
import time

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from scipy.signal import lfilter

import causal_moe_fuse as M
import ensemble_clf_nested as N
import stem_ms96_irm_synth_fuse as SF
from common import DS_KEYS, SEED

EXP = os.path.dirname(os.path.abspath(__file__))
FOCAL_CACHE = os.path.join(EXP, "models", "focal_synth_adapt_readout", "cache")
IRM_SYNTH_CACHE = os.path.join(EXP, "models", "fuse_conf_resid", "cache")
LOG = os.path.join(EXP, "experiments", "multi_stem_attn_pool.log")
OUT = os.path.join(EXP, "results", "research", "multi_stem_attn_pool.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_multi_stem_attn_pool.md")
CKPT = os.path.join(EXP, "models", "multi_stem_attn_pool")

P_STAY, SCALE = SF.P_STAY, SF.SCALE
TEST_NAMES = SF.TEST_NAMES
HOP = 0.25  # 4 Hz evidence
TAUS = (1.0, 4.0)
STREAMS = ("focal", "synth", "irm")
K = len(STREAMS)
MAX_FIT = 400_000
N_EPOCHS = int(os.environ.get("ATTN_EPOCHS", "8"))
LR = float(os.environ.get("ATTN_LR", "5e-3"))
BATCH = int(os.environ.get("ATTN_BATCH", "8192"))
TIP_AVGC = dict(H=0.8226, Acc=75.54)
TIP_MEAN = dict(H=0.8821, Acc=83.36)
LINEAR_HP30 = dict(AASD=dict(H=0.6422, Acc=53.12), Carta2026=dict(H=0.6520, Acc=60.54))
EPS = 1e-8


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def load_pkl(path):
    with open(path, "rb") as fh:
        return pickle.load(fh)


def require(path):
    if not os.path.isfile(path):
        raise FileNotFoundError(path)
    log_line(f"cache hit {path}")
    return load_pkl(path)


def load_triple(tag: str):
    """tag e.g. held-AVGC | all5_corpus | all5_external."""
    focal = require(os.path.join(FOCAL_CACHE, f"_tmp_focal__{tag}.pkl"))
    synth = require(os.path.join(IRM_SYNTH_CACHE, f"_tmp_synth__{tag}.pkl"))
    irm = require(os.path.join(IRM_SYNTH_CACHE, f"_tmp_irm__{tag}.pkl"))
    return dict(focal=focal, synth=synth, irm=irm)


def alpha_from_tau(tau):
    return float(1.0 - np.exp(-HOP / float(tau)))


def ewma_past(x, alpha, init=0.0):
    """Causal past-only EWMA: out[t] uses x[:t] only (out[0]=init)."""
    x = np.asarray(x, np.float64).reshape(-1)
    a = float(alpha)
    if len(x) == 0:
        return x.copy()
    y, _ = lfilter([a], [1.0, -(1.0 - a)], x,
                   zi=np.asarray([float(init) * (1.0 - a)], np.float64))
    out = np.empty_like(x)
    out[0] = float(init)
    if len(x) > 1:
        out[1:] = y[:-1]
    return out


def feats_1d(z, alphas):
    """Feature matrix (T, 2*|alphas|) = EWMA(z), EWMA(|z|) for each alpha."""
    z = np.asarray(z, np.float64).reshape(-1)
    cols = []
    for a in alphas:
        cols.append(ewma_past(z, a, 0.0))
        cols.append(ewma_past(np.abs(z), a, 0.0))
    return np.stack(cols, 1).astype(np.float32)


def align_keys(*tables):
    keys = set(tables[0])
    for t in tables[1:]:
        keys &= set(t)
    return sorted(keys)


def collect_train(triple, att, train_ds, alphas, max_fit=MAX_FIT, rng=None):
    rng = rng or np.random.default_rng(SEED)
    Xs, Ys, Zs = [], [], []  # feats (T,F), labels, stack logits (T,K)
    for ds in train_ds:
        labels = att[ds]
        tabs = [triple[name][ds] for name in STREAMS]
        for key in align_keys(*tabs):
            if key not in labels:
                continue
            zs = [np.asarray(t[key], np.float64).reshape(-1) for t in tabs]
            n = min(len(z) for z in zs)
            y = (np.asarray(labels[key]) == 0).astype(np.float32).reshape(-1)[:n]
            if n <= 4:
                continue
            zs = [z[:n] for z in zs]
            # feats: concat per-stream EWMA features
            fparts = [feats_1d(z, alphas) for z in zs]
            X = np.concatenate(fparts, 1)  # (T, K*2*|alphas|)
            Z = np.stack(zs, 1).astype(np.float32)  # (T,K)
            Xs.append(X)
            Ys.append(y)
            Zs.append(Z)
    if not Xs:
        raise RuntimeError("no train frames")
    X = np.concatenate(Xs)
    Y = np.concatenate(Ys)
    Z = np.concatenate(Zs)
    if len(X) > max_fit:
        idx = rng.choice(len(X), size=max_fit, replace=False)
        X, Y, Z = X[idx], Y[idx], Z[idx]
    return X, Y, Z


class GateNet(nn.Module):
    def __init__(self, in_dim, k=K, hidden=32):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim, hidden),
            nn.ReLU(),
            nn.Linear(hidden, k),
        )
        # near-uniform init → start ≈ equal-mean
        nn.init.zeros_(self.net[-1].weight)
        nn.init.zeros_(self.net[-1].bias)

    def forward(self, feat, logits):
        # feat (B,F), logits (B,K) → gated (B,)
        g = F.softmax(self.net(feat), dim=-1)
        return (g * logits).sum(-1), g


def train_gate(X, Y, Z, epochs=N_EPOCHS, lr=LR, batch=BATCH, seed=SEED):
    torch.manual_seed(seed)
    device = torch.device("cpu")
    model = GateNet(X.shape[1], K).to(device)
    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)
    Xt = torch.from_numpy(X)
    Yt = torch.from_numpy(Y)
    Zt = torch.from_numpy(Z)
    n = len(X)
    model.train()
    for ep in range(epochs):
        perm = torch.randperm(n)
        total = 0.0
        steps = 0
        for i in range(0, n, batch):
            idx = perm[i:i + batch]
            pred, _ = model(Xt[idx], Zt[idx])
            loss = F.binary_cross_entropy_with_logits(pred, Yt[idx])
            opt.zero_grad()
            loss.backward()
            opt.step()
            total += float(loss.item())
            steps += 1
        log_line(f"  train epoch {ep+1}/{epochs} loss {total / max(steps,1):.4f} n={n}")
    model.eval()
    return model


@torch.no_grad()
def apply_gate_stream(model, tables, alphas):
    """tables: list of K stream dicts for one dataset → gated dict."""
    out = {}
    for key in align_keys(*tables):
        zs = [np.asarray(t[key], np.float64).reshape(-1) for t in tables]
        n = min(len(z) for z in zs)
        zs = [z[:n] for z in zs]
        fparts = [feats_1d(z, alphas) for z in zs]
        X = np.concatenate(fparts, 1).astype(np.float32)
        Z = np.stack(zs, 1).astype(np.float32)
        pred, _ = model(torch.from_numpy(X), torch.from_numpy(Z))
        out[key] = pred.numpy().astype(np.float64)
    return out


def apply_gate_all(model, triple, alphas):
    out = {}
    # datasets present in all three
    ds_set = set(triple["focal"]) & set(triple["synth"]) & set(triple["irm"])
    for ds in ds_set:
        tabs = [triple[name][ds] for name in STREAMS]
        out[ds] = apply_gate_stream(model, tabs, alphas)
    return out


def equal_mean_all(triple):
    out = {}
    ds_set = set(triple["focal"]) & set(triple["synth"]) & set(triple["irm"])
    for ds in ds_set:
        keys = align_keys(triple["focal"][ds], triple["synth"][ds], triple["irm"][ds])
        out[ds] = {}
        for key in keys:
            zs = [np.asarray(triple[n][ds][key], np.float64).reshape(-1) for n in STREAMS]
            n = min(len(z) for z in zs)
            out[ds][key] = np.mean([z[:n] for z in zs], 0).astype(np.float64)
    return out


def focal_synth_w05(triple):
    out = {}
    for ds in set(triple["focal"]) & set(triple["synth"]):
        out[ds] = {}
        for key in align_keys(triple["focal"][ds], triple["synth"][ds]):
            a = np.asarray(triple["focal"][ds][key], np.float64).reshape(-1)
            b = np.asarray(triple["synth"][ds][key], np.float64).reshape(-1)
            n = min(len(a), len(b))
            out[ds][key] = (0.5 * a[:n] + 0.5 * b[:n]).astype(np.float64)
    return out


def mark_claim(status, note=None):
    path = os.path.join(EXP, "multi_agent_opt", "CLAIMS.md")
    rows = open(path).read().splitlines()
    out = []
    for line in rows:
        if "| multi-stem-attn-pool |" in line:
            parts = line.split("|")
            if len(parts) >= 6:
                parts[4] = f" {status} "
                if note:
                    parts[5] = f" {note} "
                line = "|".join(parts)
        out.append(line)
    open(path, "w").write("\n".join(out) + "\n")


def write_results(payload, sel, per, mean_h, mean_a, ext):
    tip_avgc = per.get("AVGC", {})
    dH = float(tip_avgc.get("H", 0)) - TIP_AVGC["H"]
    dA = float(tip_avgc.get("Acc", 0)) - TIP_AVGC["Acc"]
    aasd = ext["AASD"]
    carta = ext["Carta2026"]
    lines = [
        "# results_multi_stem_attn_pool",
        "",
        "Learnable **causal soft-attention** over frozen **focal + synth + IRM** stem logits.",
        "Past-only EWMA(logit), EWMA(|logit|) at τ∈{1,4}s → MLP → softmax gates → weighted sum.",
        "Gates trained with frame BCE on LODO train folds only (held-out never for fit).",
        "HMM 0.994/0.8. CPU caches only.",
        "",
        "**≠** fixed-w IRM↔stem / agree-gate / conf-resid / MoE fuse↔hp30 / Mixup/focal FT.",
        "",
        f"Code: `multi_stem_attn_pool.py`. JSON: `{OUT}`.",
        "",
        f"## Selected: **{sel}**",
        "",
        "| Held-out | H | Acc % |",
        "|---|---:|---:|",
    ]
    for ds in DS_KEYS:
        r = per[ds]
        lines.append(f"| {ds} | {r['H']:.4f} | {r['Acc']:.2f} |")
    lines += [
        f"| **mean** | **{mean_h:.4f}** | **{mean_a:.2f}** |",
        "",
        f"AVGC vs tip 0.8226/75.54: ΔH {dH:+.4f} / ΔAcc {dA:+.2f}",
        "",
        "## External",
        "",
        f"| Set | H / Acc | vs tip / hp30 |",
        f"|---|---|---|",
        f"| AASD | {aasd['H']:.4f} / {aasd['Acc']:.2f} | tip Acc 53.53; hp30 53.12 |",
        f"| Carta2026 | {carta['H']:.4f} / {carta['Acc']:.2f} | hp30 60.54 |",
        "",
        "## Targets",
        "",
        f"| Target | Result |",
        f"|---|---|",
        f"| AVGC ≥0.85/78 | miss — {tip_avgc.get('H',0):.4f}/{tip_avgc.get('Acc',0):.2f} |",
        f"| Beat tip fuse AVGC | {'yes' if dH > 0 else 'no'} (ΔH {dH:+.4f}) |",
        f"| AASD Acc>60 | {'YES' if aasd['Acc']>60 else 'NO'} ({aasd['Acc']:.2f}) |",
        "",
        "## NEXT IDEA",
        "",
        "If flat near tip: soft-attn on frozen tip stems saturates like agree-gate/conf-resid. "
        "Prefer qualitatively new train-time evidence (DANN/VICReg/temporal contrastive/envelope-TRF Path B) "
        "or non-vlaai Acc front-end — not Mixup/focal/fixed-w re-grids.",
        "",
    ]
    open(RESULTS_MD, "w").write("\n".join(lines))
    return RESULTS_MD


def append_already(note):
    path = os.path.join(EXP, "multi_agent_opt", "ALREADY_TRIED.md")
    with open(path, "a") as fh:
        fh.write(
            "\n- **multi-stem soft-attn pool** (`multi-stem-attn-pool`; causal EWMA feats of "
            "frozen focal+synth+IRM → MLP softmax gates; train BCE on LODO train only; "
            f"≠ fixed-w / agree-gate / conf-resid / MoE fuse↔hp30 / Mixup/focal): {note} "
            "`multi_agent_opt/results_multi_stem_attn_pool.md`, `multi_stem_attn_pool.py`.\n"
        )


def update_status(note):
    path = os.path.join(EXP, "multi_agent_opt", "STATUS_GAPS.md")
    text = open(path).read()
    text = text.replace(
        "2. **multi-stem-attn-pool** — learnable soft-attn over focal+synth+IRM frozen logits (CPU AVGC).\n",
        "",
    )
    if "**multi-stem-attn-pool**" not in text.split("## Recently closed")[1][:2000]:
        text = text.replace(
            "## Recently closed\n",
            f"## Recently closed\n\n- **multi-stem-attn-pool** — {note}\n",
        )
    open(path, "w").write(text)


def main():
    os.makedirs(CKPT, exist_ok=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    open(LOG, "w").close()
    mark_claim("running", "CPU soft-attn over focal+synth+IRM; results_multi_stem_attn_pool.md")
    t0 = time.time()
    alphas = [alpha_from_tau(t) for t in TAUS]
    log_line(f"multi-stem-attn-pool CPU  p_stay {P_STAY} scale {SCALE}")
    log_line(f"streams={STREAMS} taus={TAUS} alphas={alphas} epochs={N_EPOCHS}")

    N.load_all()
    att = N.G["att"]

    log_line("=== load caches ===")
    held_trips = {h: load_triple(f"held-{h}") for h in DS_KEYS}

    # sanity tip fuse on AVGC
    tip_st = focal_synth_w05(held_trips["AVGC"])
    san = M.score_lodo(tip_st, "AVGC")
    log_line(f"sanity focal↔synth w0.5 AVGC H {san['H']:.4f} Acc {san['Acc']:.2f} "
             f"(expect ~{TIP_AVGC['H']:.4f}/{TIP_AVGC['Acc']:.2f})")

    variants = {}
    # baselines per held
    for name, builder in (
        ("equal_mean3", equal_mean_all),
        ("focal_synth_w05", focal_synth_w05),
    ):
        per = {}
        for held in DS_KEYS:
            st = builder(held_trips[held])
            off = M.score_lodo(st, held)
            per[held] = dict(H=off["H"], Acc=off["Acc"])
            log_line(f"LODO {name:16s} {held:5s} H {off['H']:.4f} Acc {off['Acc']:.2f}")
        mh = float(np.mean([per[d]["H"] for d in DS_KEYS]))
        ma = float(np.mean([per[d]["Acc"] for d in DS_KEYS]))
        variants[name] = dict(lodo=per, lodo_mean=dict(H=mh, Acc=ma), selectable=False)
        log_line(f"LODO_MEAN {name:16s} H {mh:.4f} Acc {ma:.2f}")

    # train soft-attn per outer held (fit on other 4)
    per_attn = {}
    models = {}
    for held in DS_KEYS:
        train_ds = [d for d in DS_KEYS if d != held]
        trip = held_trips[held]
        log_line(f"=== train gates held={held} on {train_ds} ===")
        X, Y, Z = collect_train(trip, att, train_ds, alphas)
        model = train_gate(X, Y, Z)
        models[held] = model
        torch.save(model.state_dict(), os.path.join(CKPT, f"gate__held-{held}.pt"))
        gated = apply_gate_all(model, trip, alphas)
        off = M.score_lodo(gated, held)
        per_attn[held] = dict(H=off["H"], Acc=off["Acc"])
        log_line(f"LODO soft_attn       {held:5s} H {off['H']:.4f} Acc {off['Acc']:.2f}")
    mh = float(np.mean([per_attn[d]["H"] for d in DS_KEYS]))
    ma = float(np.mean([per_attn[d]["Acc"] for d in DS_KEYS]))
    variants["soft_attn"] = dict(lodo=per_attn, lodo_mean=dict(H=mh, Acc=ma), selectable=True)
    log_line(f"LODO_MEAN soft_attn       H {mh:.4f} Acc {ma:.2f}")

    # select among selectable (soft_attn only) — still compare Acc then H vs tip baseline
    sel = "soft_attn"
    sel_mean = variants[sel]["lodo_mean"]
    log_line(f"SELECT {sel} mean {sel_mean['H']:.4f}/{sel_mean['Acc']:.2f}")

    # external: train gate on all5 corpus, apply to external
    log_line("=== external (all5-trained gates) ===")
    corp = load_triple("all5_corpus")
    ext_trip = load_triple("all5_external")
    X, Y, Z = collect_train(corp, att, list(DS_KEYS), alphas)
    model_all = train_gate(X, Y, Z, seed=SEED + 7)
    torch.save(model_all.state_dict(), os.path.join(CKPT, "gate__all5.pt"))
    gated_corp = apply_gate_all(model_all, corp, alphas)
    gated_ext = apply_gate_all(model_all, ext_trip, alphas)
    # emissions from corpus
    mu, sd = SF.emissions_from(gated_corp, att, datasets=list(DS_KEYS))
    # need meta for external — reuse MoE loader
    _, _, train_att, test_meta = M.load_ext_hp30()  # meta only
    ext = {}
    for ds in TEST_NAMES:
        r = SF.score_external(gated_ext[ds], test_meta[ds], mu, sd)
        ext[ds] = dict(H=r["H"], Acc=r["Acc"], SDT=r.get("SDT"), L=r.get("L"))
        log_line(f"EXT {ds} H {r['H']:.4f} Acc {r['Acc']:.2f}")

    payload = dict(
        method="multi-stem-attn-pool",
        streams=list(STREAMS), taus=list(TAUS),
        p_stay=P_STAY, scale=SCALE,
        selected=sel,
        variants={k: {**v, "lodo": {ds: dict(H=v["lodo"][ds]["H"], Acc=v["lodo"][ds]["Acc"])
                                    for ds in v["lodo"]}}
                  for k, v in variants.items()},
        external=ext,
        tip=dict(mean=TIP_MEAN, AVGC=TIP_AVGC),
        linear_hp30=LINEAR_HP30,
        elapsed_s=time.time() - t0,
    )
    with open(OUT, "w") as fh:
        json.dump(payload, fh, indent=2)

    per = variants[sel]["lodo"]
    mean_h, mean_a = sel_mean["H"], sel_mean["Acc"]
    rpath = write_results(payload, sel, per, mean_h, mean_a, ext)
    avgc = per["AVGC"]
    note = (
        f"Select {sel}. LODO {mean_h:.4f}/{mean_a:.2f} (AVGC {avgc['H']:.4f}/{avgc['Acc']:.2f} "
        f"Δtip {avgc['H']-TIP_AVGC['H']:+.4f} H). "
        f"AASD {ext['AASD']['H']:.4f}/{ext['AASD']['Acc']:.2f}. "
        f"`{rpath}`."
    )
    mark_claim("done", note)
    append_already(note)
    update_status(
        f"AVGC {avgc['H']:.4f}/{avgc['Acc']:.2f} (Δtip {avgc['H']-TIP_AVGC['H']:+.4f}). "
        f"Miss ≥0.85/78."
    )
    log_line(f"wrote {OUT} {rpath}")
    log_line(f"=== done in {time.time()-t0:.0f}s ===")
    print(f"\nRESULT {note}", flush=True)


if __name__ == "__main__":
    main()
