"""RoPE (rotary) Q/K fine-tune of stem_ms96 (replace absolute pos).

Removes absolute `self.pos` embeddings from LogitTransformer and applies
rotary embeddings to projected Q/K inside a custom causal TransformerEncoder
(2 layers, d=64, window 96, ms features, dilated stem kept).

NOT ALiBi. NOT v3/v4 local relative-bias span 6.25s. NOT env_xattn / TFDirect.

Init from models/stem_ms96_swa/{held-*,all5}.pt (drop `pos`; remap
TransformerEncoderLayer MHA → RoPE qkv/out_proj; restore causal mask).
EMA 0.995 eval. Score LODO 5 + AASD + Carta at HMM p_stay=0.994 scale=0.8;
also report linear_hp30 on logits.

Early abort: if after EARLY_STEPS the train loss is still ~chance (≈0.69), stop.

usage:
  STEPS=1000 BATCH=512 python stem_ms96_rope.py
"""
import copy
import json
import math
import os
import pickle
import time

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import ensemble_clf_nested as N
import ensemble_clf_transformer_v2 as V
import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS

EXP = os.path.dirname(os.path.abspath(__file__))
SWA_DIR = os.path.join(EXP, "models", "stem_ms96_swa")
STEM_DIR = os.path.join(EXP, "models", "logit_tf_v2")
CKPT_DIR = os.path.join(EXP, "models", "stem_ms96_rope")
LOG = os.path.join(EXP, "experiments", "stem_ms96_rope.log")
OUT = os.path.join(EXP, "results", "research", "stem_ms96_rope.json")
EVID = "/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research"
TEST_NAMES = ("AASD", "Carta2026")

STEPS = int(os.environ.get("STEPS", "1000"))
BATCH = int(os.environ.get("BATCH", "512"))
EARLY_STEPS = int(os.environ.get("EARLY_STEPS", "80"))
EARLY_ABORT_LOSS = float(os.environ.get("EARLY_ABORT_LOSS", "0.65"))
EMA_DECAY = 0.995
LR = 8e-4
P_STAY = 0.994
SCALE = 0.8
CFG = ("stem_ms96_rope", "ms", 96, 64, 2, True, LR)
WINDOW = 96
NHEAD = 4
ROPE_BASE = 10000.0


def log_line(text):
    print(text, flush=True)
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as fh:
        fh.write(text + "\n")


def rotate_half(x):
    x1, x2 = x.chunk(2, dim=-1)
    return torch.cat((-x2, x1), dim=-1)


def build_rope_cache(seq_len, head_dim, base=ROPE_BASE, device=None, dtype=None):
    """cos/sin tables shaped (1, 1, T, Dh) for broadcast over (B, H, T, Dh)."""
    if head_dim % 2 != 0:
        raise ValueError(f"RoPE head_dim must be even, got {head_dim}")
    inv_freq = 1.0 / (base ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
    t = torch.arange(seq_len, dtype=torch.float32)
    freqs = torch.outer(t, inv_freq)  # (T, Dh/2)
    emb = torch.cat((freqs, freqs), dim=-1)  # (T, Dh)
    cos = emb.cos()[None, None, :, :]
    sin = emb.sin()[None, None, :, :]
    if device is not None:
        cos = cos.to(device=device)
        sin = sin.to(device=device)
    if dtype is not None:
        cos = cos.to(dtype=dtype)
        sin = sin.to(dtype=dtype)
    return cos, sin


def apply_rope(x, cos, sin):
    """x, cos, sin: (..., T, Dh)."""
    t = x.size(-2)
    return x * cos[..., :t, :] + rotate_half(x) * sin[..., :t, :]


class RopeEncoderLayer(nn.Module):
    """Pre-norm causal attention with RoPE on Q/K + GELU FFN (matches stem TF layer)."""

    def __init__(self, d_model, nhead, dim_feedforward, dropout=0.1):
        super().__init__()
        if d_model % nhead != 0:
            raise ValueError("d_model must be divisible by nhead")
        self.d_model = d_model
        self.nhead = nhead
        self.head_dim = d_model // nhead
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.qkv = nn.Linear(d_model, 3 * d_model)
        self.out_proj = nn.Linear(d_model, d_model)
        self.linear1 = nn.Linear(d_model, dim_feedforward)
        self.linear2 = nn.Linear(dim_feedforward, d_model)
        self.dropout = nn.Dropout(dropout)
        self.attn_drop = nn.Dropout(dropout)

    def forward(self, x, cos, sin, causal_mask):
        # causal_mask: (T, T) with 0 allowed / -inf blocked (additive)
        b, t, _ = x.shape
        h = self.norm1(x)
        qkv = self.qkv(h).view(b, t, 3, self.nhead, self.head_dim)
        qkv = qkv.permute(2, 0, 3, 1, 4)  # (3, B, H, T, Dh)
        q, k, v = qkv[0], qkv[1], qkv[2]
        q = apply_rope(q, cos, sin)
        k = apply_rope(k, cos, sin)
        scale = self.head_dim ** -0.5
        attn = torch.matmul(q, k.transpose(-2, -1)) * scale
        attn = attn + causal_mask[:t, :t].to(dtype=attn.dtype, device=attn.device)
        attn = self.attn_drop(torch.softmax(attn, dim=-1))
        out = torch.matmul(attn, v)  # (B, H, T, Dh)
        out = out.transpose(1, 2).contiguous().view(b, t, self.d_model)
        x = x + self.dropout(self.out_proj(out))
        h = self.norm2(x)
        h = self.linear2(self.dropout(F.gelu(self.linear1(h))))
        return x + self.dropout(h)


class LogitTransformerRoPE(nn.Module):
    """stem_ms96 LogitTransformer with absolute pos removed and RoPE on attn Q/K."""

    def __init__(self, c_in, window, d_model=64, nhead=4, nlayers=2, dropout=0.1, stem=True):
        super().__init__()
        if not stem:
            raise ValueError("RoPE script expects stem=True")
        self.window = window
        self.nhead = nhead
        self.d_model = d_model
        self.head_dim = d_model // nhead
        self.dw_pad = 8
        self.dw = nn.Conv1d(c_in, c_in, self.dw_pad + 1, groups=c_in)
        nn.init.zeros_(self.dw.weight)
        nn.init.zeros_(self.dw.bias)
        self.proj = nn.Linear(c_in, d_model)
        self.blocks = nn.ModuleList([
            V.DilatedResidual(d_model, k=5, dilation=d, dropout=dropout)
            for d in (1, 2, 4, 8)
        ])
        # NO absolute self.pos — RoPE replaces it
        self.layers = nn.ModuleList([
            RopeEncoderLayer(d_model, nhead, d_model * 2, dropout=dropout)
            for _ in range(nlayers)
        ])
        self.head = nn.Linear(d_model, 1)
        self.skip = nn.Linear(c_in, 1, bias=False)
        mask = nn.Transformer.generate_square_subsequent_mask(window)
        self.register_buffer("mask", mask)
        cos, sin = build_rope_cache(window, self.head_dim)
        self.register_buffer("rope_cos", cos)
        self.register_buffer("rope_sin", sin)
        nn.init.zeros_(self.head.weight)
        nn.init.zeros_(self.head.bias)
        nn.init.zeros_(self.skip.weight)
        with torch.no_grad():
            self.skip.weight[0, : min(4, c_in)] = 1.0 / 4.0

    def forward(self, x):
        raw = x
        x = x + self.dw(F.pad(x.transpose(1, 2), (self.dw_pad, 0))).transpose(1, 2)
        h = self.proj(x)
        for block in self.blocks:
            h = block(h)
        t = h.size(1)
        cos = self.rope_cos[..., :t, :].to(dtype=h.dtype)
        sin = self.rope_sin[..., :t, :].to(dtype=h.dtype)
        for layer in self.layers:
            h = layer(h, cos, sin, self.mask)
        return self.head(h).squeeze(-1) + self.skip(raw).squeeze(-1)


def remap_swa_state(state):
    """Map stem SWA / nn.TransformerEncoder keys onto RoPE module names.

    Drops absolute `pos`. Remaps enc.layers.i.self_attn.{in_proj,out_proj}
    and enc.layers.i.{linear*,norm*} → layers.i.*.
    """
    out = {}
    for k, v in state.items():
        if k == "pos":
            continue
        if k.startswith("enc.layers."):
            # enc.layers.{i}.self_attn.in_proj_weight → layers.{i}.qkv.weight
            rest = k[len("enc.layers."):]
            parts = rest.split(".", 1)
            if len(parts) != 2:
                continue
            idx, tail = parts
            if tail == "self_attn.in_proj_weight":
                out[f"layers.{idx}.qkv.weight"] = v
            elif tail == "self_attn.in_proj_bias":
                out[f"layers.{idx}.qkv.bias"] = v
            elif tail == "self_attn.out_proj.weight":
                out[f"layers.{idx}.out_proj.weight"] = v
            elif tail == "self_attn.out_proj.bias":
                out[f"layers.{idx}.out_proj.bias"] = v
            elif tail.startswith("linear1.") or tail.startswith("linear2.") \
                    or tail.startswith("norm1.") or tail.startswith("norm2."):
                out[f"layers.{idx}.{tail}"] = v
            # drop unused attn dropout buffers / nested tensors
            continue
        if k in ("mask",) or k.startswith("enc."):
            continue
        out[k] = v
    return out


def assert_causal_rope(device):
    torch.manual_seed(0)
    model = LogitTransformerRoPE(8, 16, d_model=32, nhead=4, nlayers=1).to(device).eval()
    x = torch.randn(2, 16, 8, device=device)
    x2 = x.clone()
    x2[:, -1] = torch.randn(2, 8, device=device)
    with torch.no_grad():
        y1 = model(x)
        y2 = model(x2)
    if not torch.allclose(y1[:, :-1], y2[:, :-1], atol=1e-5):
        raise RuntimeError("RoPE transformer readout is not causal")
    log_line("RoPE causality check passed")


def resolve_init(held):
    """Prefer SWA held-*/all5; fall back to published stem_ms96."""
    if held == "all5":
        swa = os.path.join(SWA_DIR, "all5.pt")
        if os.path.isfile(swa):
            return swa
        raise FileNotFoundError(swa)
    swa = os.path.join(SWA_DIR, f"held-{held}.pt")
    if os.path.isfile(swa):
        return swa
    stem = os.path.join(STEM_DIR, f"stem_ms96__held-{held}.pt")
    if os.path.isfile(stem):
        log_line(f"  SWA missing for {held}; falling back to {stem}")
        return stem
    raise FileNotFoundError(f"no init for {held}: tried {swa} and {stem}")


def load_rope_from_swa(path, device):
    blob = torch.load(path, map_location="cpu", weights_only=False)
    name, family, window, d_model, nlayers, stem, _lr = blob["cfg"]
    if not stem or int(window) != WINDOW or int(d_model) != 64 or int(nlayers) != 2:
        raise RuntimeError(f"unexpected stem cfg {blob['cfg']} from {path}")
    c_in = int(np.asarray(blob["mu"]).shape[0])
    net = LogitTransformerRoPE(
        c_in, WINDOW, d_model=d_model, nhead=NHEAD, nlayers=nlayers, stem=True)
    good_mask = net.mask.detach().clone()
    good_cos = net.rope_cos.detach().clone()
    good_sin = net.rope_sin.detach().clone()
    remapped = remap_swa_state(blob["state"])
    missing, unexpected = net.load_state_dict(remapped, strict=False)
    miss_ok = {m for m in missing if not m.startswith(("rope_cos", "rope_sin", "mask"))}
    # qkv/out_proj should load; anything else is a warning
    if miss_ok:
        log_line(f"  warn missing keys after RoPE remap: {sorted(miss_ok)}")
    if unexpected:
        log_line(f"  warn unexpected keys: {sorted(unexpected)}")
    net.mask.copy_(good_mask)
    net.rope_cos.copy_(good_cos)
    net.rope_sin.copy_(good_sin)
    if torch.isnan(net.mask).any():
        raise RuntimeError(f"causal mask still has nan after restore from {path}")
    net = net.to(device)
    mu = np.asarray(blob["mu"], np.float32)
    sd = np.asarray(blob["sd"], np.float32)
    return net, mu, sd, list(blob["train"])


def weighted_bce(logit, y, m, pos_weight):
    sw = V.switch_weights(y, m)
    bce = F.binary_cross_entropy_with_logits(logit, y, pos_weight=pos_weight, reduction="none")
    denom = (sw * m).sum().clamp_min(1.0)
    loss = (bce * sw * m).sum() / denom
    dv = (logit[:, 1:] - logit[:, :-1]).abs()
    m2 = m[:, 1:] * m[:, :-1]
    loss = loss + 0.05 * (dv * m2).sum() / m2.sum().clamp_min(1.0)
    return loss


def set_lr(opt, step, total, base):
    warm = min(40, max(total // 10, 1))
    if step < warm:
        lr = base * (step + 1) / warm
    else:
        p = (step - warm) / max(total - warm, 1)
        lr = 1e-5 + 0.5 * (base - 1e-5) * (1.0 + math.cos(math.pi * p))
    for group in opt.param_groups:
        group["lr"] = lr
    return lr


class ChanceLikeAbort(RuntimeError):
    pass


def train_rope(model, groups, y_all, mu, sd, device, steps, batch, tag, early_abort=False):
    pos = float(y_all.mean())
    pos_weight = torch.tensor([(1.0 - pos) / max(pos, 1e-3)], device=device)
    opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
    rng = np.random.default_rng(V.TRAIN_SEED)
    ema = copy.deepcopy(model).eval()
    model.train()
    t0 = time.time()
    last = None
    early_losses = []
    for step in range(steps):
        set_lr(opt, step, steps, LR)
        xb, yb, mb = V.sample_batch(rng, groups, WINDOW, batch, mu, sd)
        x = torch.from_numpy(xb).to(device, non_blocking=True)
        y = torch.from_numpy(yb).to(device, non_blocking=True)
        m = torch.from_numpy(mb).to(device, non_blocking=True)
        x = V.augment(x)
        opt.zero_grad(set_to_none=True)
        logit = model(x)
        loss = weighted_bce(logit, y, m, pos_weight)
        if not torch.isfinite(loss):
            raise RuntimeError(f"non-finite loss at step {step} for {tag}")
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
        V.ema_update(model, ema, EMA_DECAY)
        last = float(loss.item())
        if step < EARLY_STEPS:
            early_losses.append(last)
        if early_abort and step + 1 == EARLY_STEPS:
            mean_early = float(np.mean(early_losses[-40:]))
            log_line(f"    {tag} early@{EARLY_STEPS} mean_loss {mean_early:.4f} "
                     f"(abort if >= {EARLY_ABORT_LOSS})")
            if mean_early >= EARLY_ABORT_LOSS:
                raise ChanceLikeAbort(
                    f"chance-like loss {mean_early:.4f} after {EARLY_STEPS} steps")
        if step == 0 or (step + 1) % 200 == 0 or step + 1 == steps:
            log_line(f"    {tag} step {step + 1}/{steps} loss {last:.4f}")
    ema.eval()
    acc = V.train_frame_acc(ema, groups, WINDOW, mu, sd, device)
    log_line(f"  trained {tag} loss {last:.4f} frame-acc {acc:.3f} ({time.time() - t0:.0f}s)")
    return ema, dict(loss=last, frame_acc=acc)


def fit_with_oom(init_path, groups, y_all, device, steps, batch, tag,
                 early_abort=False, halved=False):
    model, mu, sd, _train = load_rope_from_swa(init_path, device)
    try:
        ema, stats = train_rope(
            model, groups, y_all, mu, sd, device, steps, batch, tag,
            early_abort=early_abort)
        return ema, mu, sd, batch, stats
    except torch.cuda.OutOfMemoryError:
        if halved:
            raise
        log_line(f"  OOM on {tag} at batch {batch}; halving once")
        del model
        if device.type == "cuda":
            torch.cuda.empty_cache()
        return fit_with_oom(
            init_path, groups, y_all, device, steps, batch // 2, tag,
            early_abort=early_abort, halved=True)


def hp30_stream(stream):
    out = {}
    for key, d in stream.items():
        out[key] = V4.highpass(np.asarray(d, np.float64)[:, None]).reshape(-1).astype(np.float64)
    return out


def score_lodo(model, feat, mu, sd, held, device):
    streams = V.infer_all(model, feat, mu, sd, WINDOW, device)
    ev = N.to_ev(streams)
    off = N.official(held, ev, P_STAY, SCALE)
    hp = {ds: hp30_stream(streams[ds]) for ds in streams}
    ev_hp = N.to_ev(hp)
    off_hp = N.official(held, ev_hp, P_STAY, SCALE)
    log_line(f"LODO {held} p {P_STAY} sc {SCALE}  H {off['H']:.4f} Acc {off['Acc']:.2f} "
             f"L {off['L']:.3f} SDT {off['SDT']:.2f}")
    log_line(f"LODO_hp30 {held} p {P_STAY} sc {SCALE}  H {off_hp['H']:.4f} Acc {off_hp['Acc']:.2f}")
    return dict(rope=off, hp30=off_hp), streams


def score_external(model, feat_ext, meta, mu_in, sd_in, corpus_streams, corpus_att, device):
    parts = []
    for ds, table in corpus_streams.items():
        labels = corpus_att[ds]
        for key, d in table.items():
            parts.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    mu_e = float(np.concatenate(parts).mean())
    sd_e = float(np.concatenate(parts).std())
    log_line(f"  external emissions mu {mu_e:.4f} sd {sd_e:.4f} scale {SCALE}")

    hp_corpus = {ds: hp30_stream(corpus_streams[ds]) for ds in corpus_streams}
    parts_hp = []
    for ds, table in hp_corpus.items():
        labels = corpus_att[ds]
        for key, d in table.items():
            parts_hp.append(np.asarray(d, np.float64) * np.where(labels[key] == 0, 1.0, -1.0))
    mu_hp = float(np.concatenate(parts_hp).mean())
    sd_hp = float(np.concatenate(parts_hp).std())
    log_line(f"  external hp30 emissions mu {mu_hp:.4f} sd {sd_hp:.4f} scale {SCALE}")

    rows = {}
    for ds in TEST_NAMES:
        stream = V.infer_all(model, {ds: feat_ext[ds]}, mu_in, sd_in, WINDOW, device)[ds]
        ev = {}
        for key, d in stream.items():
            meta_k = meta[ds][key]
            d = np.asarray(d, np.float64)
            ev[key] = dict(r0=d, r1=np.zeros(len(d)), att=meta_k["att"], t_end=meta_k["t_end"],
                           att128=meta_k["att128"], n128=meta_k["n128"], subject=meta_k["subject"])
        r = E.score(ev, mu_e, sd_e * SCALE, P_STAY, 0.75)
        stream_hp = hp30_stream(stream)
        ev_hp = {}
        for key, d in stream_hp.items():
            meta_k = meta[ds][key]
            d = np.asarray(d, np.float64)
            ev_hp[key] = dict(r0=d, r1=np.zeros(len(d)), att=meta_k["att"], t_end=meta_k["t_end"],
                              att128=meta_k["att128"], n128=meta_k["n128"], subject=meta_k["subject"])
        r_hp = E.score(ev_hp, mu_hp, sd_hp * SCALE, P_STAY, 0.75)
        rows[ds] = dict(rope=r, hp30=r_hp)
        log_line(f"EXT {ds} p {P_STAY} sc {SCALE}  H {r['H']:.4f} Acc {r['Acc']:.2f} "
                 f"L {r['L']:.3f} SDT {r['SDT']:.2f}")
        log_line(f"EXT_hp30 {ds} p {P_STAY} sc {SCALE}  H {r_hp['H']:.4f} Acc {r_hp['Acc']:.2f}")
    return rows


def load_external():
    E.CONF.update(prep="eaP_n5_t60", targets=["env", "onset", "dom"])
    logits, meta = {}, {}
    for ds in TEST_NAMES:
        evs = []
        for i in range(4):
            path = os.path.join(EVID, f"ext_clf_{ds}_w10b3sw2P_evidence_s{i}.pkl")
            with open(path, "rb") as fh:
                evs.append(pickle.load(fh))
        logits[ds], meta[ds] = {}, {}
        for key in evs[0]:
            cols = [(ev[key]["r0"] - ev[key]["r1"]).astype(np.float32) for ev in evs]
            e0 = evs[0][key]
            logits[ds][key] = np.stack(cols, 1)
            meta[ds][key] = dict(
                att=np.asarray(e0["att"]), t_end=np.asarray(e0["t_end"], np.float64),
                att128=np.asarray(e0["att128"]), n128=int(e0["n128"]), subject=int(e0["subject"]))
        log_line(f"  external {ds}: {len(logits[ds])} trials")
    return logits, meta


def main():
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    open(LOG, "w").close()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    log_line(f"device {device} steps {STEPS} batch {BATCH} EMA {EMA_DECAY} "
             f"RoPE base {ROPE_BASE} early_abort@{EARLY_STEPS}>={EARLY_ABORT_LOSS}")
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    assert_causal_rope(device)
    N.load_all()
    feat_all = V.build_features()["ms"]
    os.makedirs(CKPT_DIR, exist_ok=True)

    per = {}
    for i, held in enumerate(DS_KEYS):
        path = resolve_init(held)
        train_ds = [d for d in DS_KEYS if d != held]
        groups, y_all = V.trial_groups(feat_all, train_ds)
        log_line(f"fold held-out={held} init {path}")
        try:
            ema, mu, sd, used, stats = fit_with_oom(
                path, groups, y_all, device, STEPS, BATCH, f"held-{held}",
                early_abort=(i == 0))
        except ChanceLikeAbort as exc:
            log_line(f"ABANDONED: {exc}")
            out = dict(abandoned=True, reason=str(exc), held=held)
            os.makedirs(os.path.dirname(OUT), exist_ok=True)
            with open(OUT, "w") as fh:
                json.dump(out, fh, indent=1, default=float)
            raise SystemExit(2)
        torch.save({"state": ema.state_dict(), "mu": mu, "sd": sd, "init": path,
                    "train": train_ds, "steps": STEPS, "batch": used,
                    "stats": stats, "cfg": CFG, "rope_base": ROPE_BASE},
                   os.path.join(CKPT_DIR, f"held-{held}.pt"))
        scores, _streams = score_lodo(ema, feat_all, mu, sd, held, device)
        per[held] = dict(scores=scores, stats=stats, batch=used)
        del ema
        if device.type == "cuda":
            torch.cuda.empty_cache()

    all_path = resolve_init("all5")
    log_line(f"all-5 RoPE fine-tune for AASD and Carta, init {all_path}")
    groups, y_all = V.trial_groups(feat_all, list(DS_KEYS))
    ema, mu, sd, used, stats = fit_with_oom(
        all_path, groups, y_all, device, STEPS, BATCH, "all5", early_abort=False)
    torch.save({"state": ema.state_dict(), "mu": mu, "sd": sd, "init": all_path,
                "train": list(DS_KEYS), "steps": STEPS, "batch": used,
                "stats": stats, "cfg": CFG, "rope_base": ROPE_BASE},
               os.path.join(CKPT_DIR, "all5.pt"))
    corpus_streams = V.infer_all(ema, feat_all, mu, sd, WINDOW, device)
    ext_logits, ext_meta = load_external()
    ext_feat = V4.build_features_from(ext_logits)["ms"]
    ext = score_external(ema, ext_feat, ext_meta, mu, sd, corpus_streams, N.G["att"], device)

    mean_h = float(np.mean([per[d]["scores"]["rope"]["H"] for d in DS_KEYS]))
    mean_acc = float(np.mean([per[d]["scores"]["rope"]["Acc"] for d in DS_KEYS]))
    log_line(f"LODO mean H {mean_h:.4f} Acc {mean_acc:.2f}")

    out = dict(
        p_stay=P_STAY, scale=SCALE, steps=STEPS, batch=BATCH,
        rope_base=ROPE_BASE,
        lodo={d: per[d]["scores"] for d in DS_KEYS},
        lodo_stats={d: per[d]["stats"] for d in DS_KEYS},
        lodo_mean=dict(H=mean_h, Acc=mean_acc),
        external=ext,
        all5_stats=stats,
    )
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log_line(f"wrote {OUT}")


if __name__ == "__main__":
    main()
