"""Causal logit transformer, third round.

Same official metric as ensemble_clf_transformer_v2.py: H is the harmonic mean of
subject-mean accuracy and switch score L from score.score_dataset. Leave-one-dataset-out.
Inner selection (nested mode) uses the acc74 rule on datasets that are neither the
held-out dataset nor the scored inner dataset. Emissions are fit without the scored
dataset. Test labels are read only inside the scorer.

What is new, relative to stem_ms96 (nested family mean H 0.8677, Acc 82.42):
  gate_local   the 24-d multi-scale logit features are 6 groups x 4 classifier seeds.
               Each seed is projected and passed through a causal depthwise conv, then
               the four seeds are mixed by attention at the same time step. A zero-init
               dilated conv stem and local causal attention (relative bias, 6.25 s)
               sit in front of a zero-init head. The linear skip still starts as the
               mean of the four raw logits. Loss matches stem_ms96.
  gate_robust  the same network with label smoothing, cross-window mixup, per-channel
               gain, mild group-DRO, a small CORAL penalty across training datasets,
               and a lighter temporal-smoothness penalty.

Train seed 20270101.

usage:
  SMOKE=1 STEPS=30 FOLD=AVGC python ensemble_clf_transformer_v3.py
  MODE=screen STEPS=1000 python ensemble_clf_transformer_v3.py
  MODE=nested ONLY=gate_local STEPS=1000 python ensemble_clf_transformer_v3.py
"""
import copy
import math
import os
import time

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import stem_ms96_image_split as IMG
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
CKPT = os.path.join(EXP, "models", "logit_tf_v3")
STEM_CKPT = os.path.join(EXP, "models", "logit_tf_v2")
SCREEN_OUT = os.path.join(EXP, "results", "research", "transformer_v3_screen.csv")
SCREEN_LOG = os.path.join(EXP, "experiments", "transformer_v3_screen.log")
NEST_OUT = os.path.join(EXP, "results", "research", "ensemble_clf_transformer_v3.csv")
NEST_INNER = os.path.join(EXP, "results", "research", "ensemble_clf_transformer_v3_inner.csv")
NEST_LOG = os.path.join(EXP, "experiments", "transformer_v3.log")

# name, family, window, kind, d_model, nlayers, lr, dropout, weight_decay, robust
CONFIGS = (
    ("gate_local", "ms", 96, "gate", 64, 2, 8e-4, 0.1, 1e-4, False),
    ("gate_robust", "ms", 96, "gate", 64, 2, 8e-4, 0.15, 3e-4, True),
)
N_SEEDS = 4
N_GROUPS = 6
TV_ROBUST = 0.02
SMOOTH_EPS = 0.02
CORAL_W = 0.005


def log_line(path, text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "a") as fh:
        fh.write(text + "\n")


def wanted_configs():
    only = os.environ.get("ONLY", "").strip()
    if not only:
        return list(CONFIGS)
    names = {n.strip() for n in only.split(",") if n.strip()}
    cfgs = [c for c in CONFIGS if c[0] in names]
    missing = names - {c[0] for c in cfgs}
    if missing:
        raise SystemExit(f"unknown ONLY names: {sorted(missing)}")
    return cfgs


class LocalCausalBlock(nn.Module):
    """Norm-first local causal attention plus an FFN. Output projections start at zero."""

    def __init__(self, d, nhead, span, dropout):
        super().__init__()
        if d % nhead != 0:
            raise ValueError(f"d_model {d} not divisible by nhead {nhead}")
        self.nhead = nhead
        self.span = span
        self.dh = d // nhead
        self.norm1 = nn.LayerNorm(d)
        self.qkv = nn.Linear(d, 3 * d)
        self.out = nn.Linear(d, d)
        self.rel = nn.Parameter(torch.zeros(nhead, span))
        self.drop = nn.Dropout(dropout)
        self.norm2 = nn.LayerNorm(d)
        self.ff = nn.Sequential(
            nn.Linear(d, 2 * d), nn.GELU(), nn.Dropout(dropout), nn.Linear(2 * d, d))
        nn.init.zeros_(self.out.weight)
        nn.init.zeros_(self.out.bias)
        nn.init.zeros_(self.ff[-1].weight)
        nn.init.zeros_(self.ff[-1].bias)

    def forward(self, x):
        b, t, d = x.shape
        h = self.norm1(x)
        qkv = self.qkv(h).view(b, t, 3, self.nhead, self.dh).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]
        pad = self.span - 1
        k_pad = F.pad(k, (0, 0, pad, 0))
        v_pad = F.pad(v, (0, 0, pad, 0))
        k_u = k_pad.unfold(2, self.span, 1)
        v_u = v_pad.unfold(2, self.span, 1)
        score = (q.unsqueeze(-1) * k_u).sum(-2) / math.sqrt(self.dh)
        score = score + self.rel.view(1, self.nhead, 1, self.span)
        slots = torch.arange(self.span, device=x.device)
        times = torch.arange(t, device=x.device)
        valid = slots.view(1, self.span) >= (self.span - 1 - times).view(t, 1)
        score = score.masked_fill(~valid.view(1, 1, t, self.span), torch.finfo(score.dtype).min)
        weight = self.drop(torch.softmax(score, dim=-1))
        ctx = (weight.unsqueeze(-2) * v_u).sum(-1)
        ctx = ctx.permute(0, 2, 1, 3).contiguous().view(b, t, d)
        x = x + self.drop(self.out(ctx))
        x = x + self.drop(self.ff(self.norm2(x)))
        return x


def causal_pool(h, k):
    total = torch.cumsum(h, dim=1)
    shifted = F.pad(total, (0, 0, k, 0))[:, : h.size(1)]
    count = torch.arange(1, h.size(1) + 1, device=h.device).clamp(max=k).view(1, -1, 1)
    return (total - shifted) / count.to(h.dtype)


class SeedGateLocal(nn.Module):
    """Per-seed causal conv, same-time seed attention, dilated stem, local causal attention."""

    def __init__(self, c_in, window, d_model=64, nhead=4, nlayers=2, dropout=0.1, span=25):
        super().__init__()
        if c_in != N_SEEDS * N_GROUPS:
            raise ValueError(f"expected {N_SEEDS * N_GROUPS} channels, got {c_in}")
        self.window = window
        self.d_model = d_model
        self.dw_pad = 8
        self.seed_proj = nn.Linear(N_GROUPS, d_model)
        self.dw = nn.Conv1d(d_model, d_model, self.dw_pad + 1, groups=d_model)
        self.q = nn.Linear(d_model, d_model)
        self.k = nn.Linear(d_model, d_model)
        self.v = nn.Linear(d_model, d_model)
        self.seed_out = nn.Linear(d_model, d_model)
        self.blocks = nn.ModuleList([
            V.DilatedResidual(d_model, k=5, dilation=dil, dropout=dropout)
            for dil in (1, 2, 4, 8)
        ])
        self.local = nn.ModuleList([
            LocalCausalBlock(d_model, nhead, span, dropout) for _ in range(nlayers)
        ])
        self.head = nn.Linear(d_model * 3, 1)
        self.skip = nn.Linear(c_in, 1, bias=False)
        nn.init.zeros_(self.dw.weight)
        nn.init.zeros_(self.dw.bias)
        nn.init.zeros_(self.seed_out.weight)
        nn.init.zeros_(self.seed_out.bias)
        nn.init.zeros_(self.head.weight)
        nn.init.zeros_(self.head.bias)
        nn.init.zeros_(self.skip.weight)
        with torch.no_grad():
            self.skip.weight[0, :N_SEEDS] = 1.0 / N_SEEDS

    def forward(self, x, return_h=False):
        raw = x
        b, t, _ = x.shape
        seeds = x.contiguous().view(b, t, N_GROUPS, N_SEEDS).permute(0, 3, 1, 2)
        h = self.seed_proj(seeds)
        flat = h.reshape(b * N_SEEDS, t, self.d_model)
        flat = flat + self.dw(F.pad(flat.transpose(1, 2), (self.dw_pad, 0))).transpose(1, 2)
        h = flat.view(b, N_SEEDS, t, self.d_model).permute(0, 2, 1, 3)
        mix = torch.matmul(torch.softmax(torch.matmul(self.q(h), self.k(h).transpose(-1, -2))
                                          / math.sqrt(self.d_model), dim=-1), self.v(h))
        fused = h.mean(dim=2) + self.seed_out(mix.mean(dim=2))
        for block in self.blocks:
            fused = block(fused)
        for layer in self.local:
            fused = layer(fused)
        pooled = torch.cat([fused, causal_pool(fused, 4), causal_pool(fused, 16)], dim=-1)
        logit = self.head(pooled).squeeze(-1) + self.skip(raw).squeeze(-1)
        if return_h:
            return logit, fused
        return logit


def build_model(cfg, c_in):
    _name, _family, window, kind, d_model, nlayers, _lr, dropout, _wd, _robust = cfg
    if kind != "gate":
        raise ValueError(kind)
    return SeedGateLocal(c_in, window, d_model=d_model, nlayers=nlayers, dropout=dropout)


def coral_loss(h, n_g):
    z = h.mean(1)
    covs = []
    index = torch.arange(z.size(0), device=z.device)
    for gi in range(n_g):
        zg = z[index % n_g == gi]
        zg = zg - zg.mean(0, keepdim=True)
        covs.append(zg.transpose(0, 1) @ zg / max(zg.size(0) - 1, 1))
    loss = z.new_zeros(())
    pairs = 0
    for i in range(n_g):
        for j in range(i + 1, n_g):
            loss = loss + (covs[i] - covs[j]).pow(2).mean()
            pairs += 1
    return loss / pairs


def robust_augment(x):
    x = V.augment(x)
    gain = torch.empty(x.size(0), 1, x.size(2), device=x.device).uniform_(math.log(0.75), math.log(1.25)).exp()
    return x * gain


def train_model(cfg, datasets, feat, device, steps):
    name, family, window, _kind, _d_model, _nlayers, lr, _dropout, wd, robust = cfg
    groups, y_all = V.trial_groups(feat[family], datasets)
    mu, sd = V.norm_stats(feat[family], datasets)
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    c_in = groups[0]["trials"][0][0].shape[1]
    torch.manual_seed(V.TRAIN_SEED)
    if device.type == "cuda":
        torch.cuda.manual_seed_all(V.TRAIN_SEED)
    model = build_model(cfg, c_in).to(device)
    ema = copy.deepcopy(model).eval()
    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)
    rng = np.random.default_rng(V.TRAIN_SEED)
    n_g = len(groups)
    dro = torch.zeros(n_g, device=device)
    model.train()
    t0 = time.time()
    last = None
    for step in range(steps):
        V.set_lr(opt, step, steps, lr)
        xb, yb, mb = V.sample_batch(rng, groups, window, V.BATCH, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        m = torch.from_numpy(mb).to(device, non_blocking=True)
        if robust:
            x = robust_augment(x)
            if torch.rand((), device=device) < 0.5:
                lam = float(rng.beta(0.4, 0.4))
                perm = torch.randperm(x.size(0), device=device)
                x = lam * x + (1.0 - lam) * x[perm]
                y = lam * y + (1.0 - lam) * y[perm]
                m = m * m[perm]
            y_hat = y * (1.0 - SMOOTH_EPS) + 0.5 * SMOOTH_EPS
        else:
            x = V.augment(x)
            y_hat = y
        opt.zero_grad(set_to_none=True)
        if robust:
            logit, hidden = model(x, return_h=True)
        else:
            logit = model(x)
            hidden = None
        sw = V.switch_weights(y, m)
        bce = F.binary_cross_entropy_with_logits(logit, y_hat, pos_weight=pos_weight, reduction="none")
        tv_w = TV_ROBUST if robust else 0.05
        if robust:
            per = (bce * sw * m).sum(1) / (sw * m).sum(1).clamp_min(1.0)
            grouped = []
            rows = torch.arange(per.size(0), device=device)
            for gi in range(n_g):
                grouped.append(per[rows % n_g == gi].mean())
            grouped = torch.stack(grouped)
            q = torch.softmax((dro - dro.mean()) / 0.1, dim=0)
            loss = (q * grouped).sum() + CORAL_W * coral_loss(hidden, n_g)
            dro.mul_(0.9).add_(grouped.detach(), alpha=0.1)
        else:
            denom = (sw * m).sum().clamp_min(1.0)
            loss = (bce * sw * m).sum() / denom
        dv = (logit[:, 1:] - logit[:, :-1]).abs()
        m2 = m[:, 1:] * m[:, :-1]
        loss = loss + tv_w * (dv * m2).sum() / m2.sum().clamp_min(1.0)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss at step {step} for {name}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        V.ema_update(model, ema, 0.995)
        last = float(loss.item())
    ema.eval()
    acc = V.train_frame_acc(ema, groups, window, mu, sd, device)
    print(f"  trained {name} on {'+'.join(datasets)}  loss {last:.3f}  frame-acc {acc:.3f}  "
          f"({time.time() - t0:.0f}s)", flush=True)
    return ema, mu, sd


def assert_causal(device):
    torch.manual_seed(0)
    model = SeedGateLocal(N_SEEDS * N_GROUPS, 32, d_model=32, nlayers=1).to(device).eval()
    x = torch.randn(2, 32, N_SEEDS * N_GROUPS, device=device)
    with torch.no_grad():
        base = model(x)
        late = x.clone()
        late[:, -1] = torch.randn(2, N_SEEDS * N_GROUPS, device=device)
        y_late = model(late)
        mid = x.clone()
        mid[:, 10] = torch.randn(2, N_SEEDS * N_GROUPS, device=device)
        y_mid = model(mid)
    if not torch.allclose(base[:, :-1], y_late[:, :-1], atol=1e-5):
        raise RuntimeError("seed-gate readout sees the future")
    if not torch.allclose(base[:, :10], y_mid[:, :10], atol=1e-5):
        raise RuntimeError("seed-gate readout is not causal at mid-window")
    print("causality check passed", flush=True)


def load_stem_streams(tag, feat, device):
    path = os.path.join(STEM_CKPT, f"stem_ms96__{tag}.pt")
    if not os.path.exists(path):
        raise RuntimeError(f"missing stem checkpoint {path}")
    blob = torch.load(path, map_location="cpu", weights_only=False)
    cfg = blob["cfg"]
    c_in = feat["ms"][DS_KEYS[0]][next(iter(feat["ms"][DS_KEYS[0]]))].shape[1]
    model = V.LogitTransformer(c_in, cfg[2], d_model=cfg[3], nlayers=cfg[4], stem=bool(cfg[5])).to(device)
    model.load_state_dict(blob["state"])
    model.eval()
    streams = V.infer_all(model, feat["ms"], blob["mu"], blob["sd"], cfg[2], device)
    del model
    if device.type == "cuda":
        torch.cuda.empty_cache()
    return streams


def select_on_val(streams, val_keys, mu, sd):
    rows = []
    for p in V.P_STAY:
        for scale in V.SCALES:
            merged = []
            for ds, keys in val_keys.items():
                merged.append(IMG.evaluate_keys(ds, streams[ds], mu, sd * scale, p, keys))
            rows.append(dict(p_stay=p, scale=scale,
                             H=float(np.mean([v["H"] for v in merged])),
                             Acc=float(np.mean([v["Acc"] for v in merged])),
                             SDT=float(np.mean([v["SDT"] for v in merged])),
                             L=float(np.mean([v["L"] for v in merged])),
                             acc1=float(np.mean([v["acc1"] for v in merged]))))
    best, reached = IMG.pick_setting(rows)
    return best, reached


def load_streams(path, feat, device, family="ms"):
    blob = torch.load(path, map_location="cpu", weights_only=False)
    cfg = blob["cfg"]
    window, d_model, nlayers, stem = cfg[2], cfg[3], cfg[4], bool(cfg[5])
    c_in = feat[family][DS_KEYS[0]][next(iter(feat[family][DS_KEYS[0]]))].shape[1]
    model = V.LogitTransformer(c_in, window, d_model=d_model, nlayers=nlayers, stem=stem).to(device)
    model.load_state_dict(blob["state"])
    model.eval()
    streams = V.infer_all(model, feat[family], blob["mu"], blob["sd"], window, device)
    del model
    if device.type == "cuda":
        torch.cuda.empty_cache()
    return streams


def run_screen(device, feat, log):
    steps = int(os.environ.get("STEPS", "1000"))
    only = os.environ.get("FOLD", "").strip()
    helds = [h.strip() for h in only.split(",") if h.strip()] or list(DS_KEYS)
    unknown = [h for h in helds if h not in DS_KEYS]
    if unknown:
        raise SystemExit(f"unknown FOLD {unknown}")
    cfgs = wanted_configs()
    rows = []
    for held in helds:
        _val_subj, train_keys, val_keys = IMG.competition_split(held)
        IMG.assert_matches_saved_split(held, _val_subj)
        train_feat = IMG.filter_feat(feat, train_keys)
        fold_streams = []
        for cfg in cfgs:
            t0 = time.time()
            model, mu, sd = train_model(cfg, list(train_keys), train_feat, device, steps)
            if not os.environ.get("SMOKE"):
                path = os.path.join(CKPT, f"{cfg[0]}__screen-{held}.pt")
                torch.save({"state": model.state_dict(), "mu": mu, "sd": sd, "cfg": cfg,
                            "steps": steps, "seed": V.TRAIN_SEED}, path)
            streams = V.infer_all(model, feat[cfg[1]], mu, sd, cfg[2], device)
            del model
            if device.type == "cuda":
                torch.cuda.empty_cache()
            mu_e, sd_e = IMG.emis_on(streams, train_keys)
            best, reached = select_on_val(streams, val_keys, mu_e, sd_e)
            off = IMG.official_test(streams, held, mu_e, sd_e, best.p_stay, best.scale)
            rec = dict(protocol="image_screen", model=cfg[0], ds=held, p_stay=float(best.p_stay),
                       scale=float(best.scale), H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"],
                       acc1=off["acc1"], val_H=float(best.H), val_Acc=float(best.Acc),
                       val_reached_74=bool(reached), seconds=time.time() - t0, scorer="official")
            rows.append(rec)
            log_line(log, f"screen {cfg[0]} {held} p {best.p_stay} sc {best.scale}  "
                          f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f} "
                          f"valH {best.H:.4f} valAcc {best.Acc:.2f} ({rec['seconds']:.0f}s)")
            fold_streams.append(streams)
            if os.environ.get("ENS_IMAGE_STEM") == "1":
                stem_path = os.path.join(EXP, "models", "stem_ms96_image_split", f"stem_ms96__held-{held}.pt")
                mix = V.average_streams([streams, load_streams(stem_path, feat, device, cfg[1])])
                mu_m, sd_m = IMG.emis_on(mix, train_keys)
                best_m, reached_m = select_on_val(mix, val_keys, mu_m, sd_m)
                off_m = IMG.official_test(mix, held, mu_m, sd_m, best_m.p_stay, best_m.scale)
                rows.append(dict(protocol="image_screen", model=f"ens_{cfg[0]}_stem", ds=held,
                                 p_stay=float(best_m.p_stay), scale=float(best_m.scale),
                                 H=off_m["H"], Acc=off_m["Acc"], SDT=off_m["SDT"], L=off_m["L"],
                                 acc1=off_m["acc1"], val_H=float(best_m.H), val_Acc=float(best_m.Acc),
                                 val_reached_74=bool(reached_m), seconds=0.0, scorer="official"))
                log_line(log, f"screen ens_{cfg[0]}_stem {held} p {best_m.p_stay} sc {best_m.scale}  "
                              f"H {off_m['H']:.4f} Acc {off_m['Acc']:.2f} SDT {off_m['SDT']:.2f} L {off_m['L']:.3f}")
            if os.environ.get("SMOKE"):
                return rows
        if len(fold_streams) >= 2:
            avg = V.average_streams(fold_streams)
            mu_e, sd_e = IMG.emis_on(avg, train_keys)
            best, reached = select_on_val(avg, val_keys, mu_e, sd_e)
            off = IMG.official_test(avg, held, mu_e, sd_e, best.p_stay, best.scale)
            rows.append(dict(protocol="image_screen", model="ens_screen", ds=held, p_stay=float(best.p_stay),
                             scale=float(best.scale), H=off["H"], Acc=off["Acc"], SDT=off["SDT"], L=off["L"],
                             acc1=off["acc1"], val_H=float(best.H), val_Acc=float(best.Acc),
                             val_reached_74=bool(reached), seconds=0.0, scorer="official"))
            log_line(log, f"screen ens_screen {held} p {best.p_stay} sc {best.scale}  "
                          f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f}")
    df = pd.DataFrame(rows)
    if not os.environ.get("SMOKE"):
        os.makedirs(os.path.dirname(SCREEN_OUT), exist_ok=True)
        df.to_csv(SCREEN_OUT, index=False)
        log_line(log, "screen means:")
        for model, sub in df.groupby("model"):
            m = sub[["H", "Acc", "SDT", "L"]].mean()
            log_line(log, f"  {model:16s}  H {m.H:.4f}  Acc {m.Acc:.2f}  SDT {m.SDT:.2f}  L {m.L:.3f}")
        log_line(log, f"wrote {SCREEN_OUT}")
    return rows


def run_nested(device, feat, log):
    steps = int(os.environ.get("STEPS", "1000"))
    cfgs = wanted_configs()
    ens_stem = os.environ.get("ENS_STEM", "1") != "0"
    cache = {}
    stem_cache = {}

    def cached_stem(tag):
        if tag not in stem_cache:
            stem_cache[tag] = load_stem_streams(tag, feat, device)
        return stem_cache[tag]
    rows = []
    excluded_pairs = []
    for i, a in enumerate(DS_KEYS):
        for b in DS_KEYS[i + 1:]:
            excluded_pairs.append((a, b))

    def score_name(name):
        for outer in DS_KEYS:
            for inner_ds in DS_KEYS:
                if inner_ds == outer:
                    continue
                train_ds = [d for d in DS_KEYS if d not in (outer, inner_ds)]
                streams = cache[(name, outer, inner_ds)]
                mu, sd = N.emis({d: streams[d] for d in train_ds}, train_ds)
                for p in V.P_STAY:
                    for scale in V.SCALES:
                        r = N.evaluate(inner_ds, streams[inner_ds], mu, sd * scale, p)
                        rows.append(dict(outer=outer, inner=inner_ds, model=name, p_stay=p, scale=scale, **r))
        sub = pd.DataFrame([r for r in rows if r["model"] == name])
        grouped_one = sub.groupby(["outer", "p_stay", "scale"])[["H", "Acc", "SDT"]].mean().reset_index()
        picks, n_ok = V.select_rule(grouped_one.assign(model=name), V.THR)
        log_line(log, f"inner {name}: {n_ok}/5 Acc>=74  H {picks.H.mean():.4f} Acc {picks.Acc.mean():.2f}")

    for cfg in cfgs:
        name, family, window = cfg[0], cfg[1], cfg[2]
        for a, b in excluded_pairs:
            train_ds = [d for d in DS_KEYS if d not in (a, b)]
            model, mu, sd = train_model(cfg, train_ds, feat, device, steps)
            path = os.path.join(CKPT, f"{name}__{V.triplet_key((a, b))}.pt")
            torch.save({"state": model.state_dict(), "mu": mu, "sd": sd, "cfg": cfg, "train": train_ds,
                        "steps": steps}, path)
            streams = V.infer_all(model, feat[family], mu, sd, window, device)
            cache[(name, a, b)] = streams
            cache[(name, b, a)] = streams
            del model
            if device.type == "cuda":
                torch.cuda.empty_cache()
        score_name(name)

    names = [c[0] for c in cfgs]
    extra = []
    if len(names) >= 2:
        for a, b in excluded_pairs:
            avg = V.average_streams([cache[(n, a, b)] for n in names])
            cache[("ens_new", a, b)] = avg
            cache[("ens_new", b, a)] = avg
        extra.append("ens_new")
        score_name("ens_new")
    if ens_stem:
        for a, b in excluded_pairs:
            stem = cached_stem(V.triplet_key((a, b)))
            for name in names:
                avg = V.average_streams([cache[(name, a, b)], stem])
                tag = f"ens_{name}_stem"
                cache[(tag, a, b)] = avg
                cache[(tag, b, a)] = avg
            if len(names) >= 2:
                avg = V.average_streams([cache[(n, a, b)] for n in names] + [stem])
                cache[("ens_all_stem", a, b)] = avg
                cache[("ens_all_stem", b, a)] = avg
        for name in names:
            extra.append(f"ens_{name}_stem")
            score_name(f"ens_{name}_stem")
        if len(names) >= 2:
            extra.append("ens_all_stem")
            score_name("ens_all_stem")

    model_names = names + extra
    inner = pd.DataFrame(rows)
    os.makedirs(os.path.dirname(NEST_INNER), exist_ok=True)
    inner.to_csv(NEST_INNER, index=False)
    log_line(log, f"wrote {NEST_INNER}")
    grouped = inner.groupby(["outer", "model", "p_stay", "scale"])[["H", "Acc", "SDT", "L", "acc1"]].mean().reset_index()
    refit = {}

    def streams_for(name, outer):
        key = (name, outer)
        if key in refit:
            return refit[key]
        if name == "ens_new":
            refit[key] = V.average_streams([streams_for(n, outer) for n in names])
            return refit[key]
        if name == "ens_all_stem":
            refit[key] = V.average_streams([streams_for(n, outer) for n in names]
                                           + [cached_stem(f"held-{outer}")])
            return refit[key]
        if name.startswith("ens_") and name.endswith("_stem"):
            base = name[len("ens_"):-len("_stem")]
            refit[key] = V.average_streams([
                streams_for(base, outer),
                cached_stem(f"held-{outer}"),
            ])
            return refit[key]
        cfg = next(c for c in cfgs if c[0] == name)
        train_ds = [d for d in DS_KEYS if d != outer]
        model, mu, sd = train_model(cfg, train_ds, feat, device, steps)
        path = os.path.join(CKPT, f"{name}__held-{outer}.pt")
        torch.save({"state": model.state_dict(), "mu": mu, "sd": sd, "cfg": cfg, "train": train_ds,
                    "steps": steps}, path)
        refit[key] = V.infer_all(model, feat[cfg[1]], mu, sd, cfg[2], device)
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
        return refit[key]

    out_rows = []

    def emit(role, picks):
        for _, b in picks.iterrows():
            streams = streams_for(b.model, b.outer)
            off = V.score_pick(streams, b.outer, b.p_stay, b.scale)
            out_rows.append(dict(role=role, ds=b.outer, model=b.model, p_stay=float(b.p_stay),
                                 scale=float(b.scale), H=off["H"], Acc=off["Acc"], SDT=off["SDT"],
                                 L=off["L"], acc1=off["acc1"], inner_H=float(b.H), inner_Acc=float(b.Acc),
                                 scorer="official"))
            log_line(log, f"{role} {b.outer} {b.model} p {b.p_stay} sc {b.scale}  "
                          f"H {off['H']:.4f} Acc {off['Acc']:.2f} SDT {off['SDT']:.2f} L {off['L']:.3f}")

    picks, n_ok = V.select_rule(grouped, V.THR)
    log_line(log, f"select acc74: {n_ok}/5 folds reached inner Acc>={V.THR:g}  "
                  f"inner H {picks.H.mean():.4f} Acc {picks.Acc.mean():.2f}")
    for _, b in picks.iterrows():
        log_line(log, f"  {b.outer} {b.model} p {b.p_stay} sc {b.scale}  inner H {b.H:.4f} Acc {b.Acc:.2f}")
    emit("acc74", picks)
    for name in model_names:
        sub = grouped[grouped.model == name]
        fp, n_family = V.select_rule(sub, V.THR)
        log_line(log, f"family {name}: {n_family}/5 inner Acc>=74  inner H {fp.H.mean():.4f} Acc {fp.Acc.mean():.2f}")
        emit(f"family_{name}", fp)

    df = pd.DataFrame(out_rows)
    df.to_csv(NEST_OUT, index=False)
    log_line(log, "means:")
    for role, sub in df.groupby("role"):
        m = sub[["H", "Acc", "SDT", "L", "acc1"]].mean()
        log_line(log, f"  {role:28s}  H {m.H:.4f}  Acc {m.Acc:.2f}  SDT {m.SDT:.2f}  L {m.L:.3f}  acc1 {m.acc1:.3f}")
    prim = df[df.role == "acc74"]
    h, acc = float(prim.H.mean()), float(prim.Acc.mean())
    log_line(log, f"PRIMARY acc74  H {h:.4f}  Acc {acc:.2f}")
    best_role, best_h, best_acc = "acc74", h, acc
    for role, sub in df.groupby("role"):
        hh, aa = float(sub.H.mean()), float(sub.Acc.mean())
        if (hh > best_h) or (math.isclose(hh, best_h) and aa > best_acc):
            best_role, best_h, best_acc = role, hh, aa
    log_line(log, f"BEST role {best_role}  H {best_h:.4f}  Acc {best_acc:.2f}")
    log_line(log, f"goal H>0.9 and Acc>85: {bool(best_h > 0.9 and best_acc > 85)}")
    log_line(log, f"wrote {NEST_OUT}")
    return df


def main():
    mode = os.environ.get("MODE", "screen").strip().lower()
    log = NEST_LOG if mode == "nested" else SCREEN_LOG
    if os.environ.get("SMOKE"):
        open(log, "a").close()
    else:
        os.makedirs(os.path.dirname(log), exist_ok=True)
        open(log, "w").close()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    steps = int(os.environ.get("STEPS", "1000"))
    log_line(log, f"device {device}  mode {mode}  steps {steps}  batch {V.BATCH}")
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    N.load_all()
    checks = N.check_fast_matches_official()
    base_h = float(np.mean([c[1]["H"] for c in checks]))
    base_acc = float(np.mean([c[1]["Acc"] for c in checks]))
    log_line(log, f"reproduced lda4 @ 0.998  H {base_h:.4f}  Acc {base_acc:.2f}")
    if abs(base_h - 0.7965) > 5e-4 or abs(base_acc - 73.01) > 0.02:
        raise RuntimeError("baseline reproduction does not match the published LDA numbers")
    assert_causal(device)
    feat = V.build_features()
    os.makedirs(CKPT, exist_ok=True)
    if mode == "screen":
        run_screen(device, feat, log)
    elif mode == "nested":
        run_nested(device, feat, log)
    else:
        raise SystemExit(f"unknown MODE {mode}")


if __name__ == "__main__":
    main()
