"""Refine soft_hp(τ∈{30,35,40}) ↔ tip at low w — LODO-pass with best Carta so far.

multitau found soft_hp35 w=0.15 → LODO 0.8517/77.60 Carta 0.6422 (LODO pass,
Carta short by ~0.008). Dense w∈[0.08,0.22] + p_stay grid to close Carta gap
without dropping LODO H≤0.85.

Distinct from soft_hp10 refine (short-τ near-miss) and τ=30-only stem_hp screens.
"""
from __future__ import annotations

import json
import os
import time
from pathlib import Path

import numpy as np

import ensemble_clf_transformer_v4 as V4
import eval_external as E
from common import DS_KEYS
from common_exp import EXP
from dual_hp30_sw2_bmms import (
    LOOKAHEAD,
    TEST_NAMES,
    as_sym,
    blend_ev,
    load_ext_mean4,
    score_lodo,
)
from accfuse_hp30_hselect import align_hp, load_accfuse
from stem_hp30_accfuse import (
    TIP_ALPHA,
    blend_signed,
    emis_from,
    ev_to_signed,
    load_stem_ext,
    load_stem_honest,
)
from multitau_stem_hp_accfuse import soft_hp

RUN_ID = "soft_hp35_tip_refine_20261108"
RUN_DIR = os.path.join(EXP, "experiments", "multi_agent_opt", "runs", RUN_ID)
OUT_JSON = os.path.join(EXP, "results", "research", "soft_hp35_tip_refine.json")
RESULTS_MD = os.path.join(EXP, "multi_agent_opt", "results_soft_hp35_tip_refine.md")
WS = [round(x, 2) for x in np.arange(0.08, 0.231, 0.02)]
TAUS = (30, 35, 40)
P_CANDS = [
    (0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 0.9), (0.998, 1.0),
    (0.997, 0.8), (0.995, 0.8), (0.999, 0.8),
]


def log(m):
    print(f"{time.strftime('%F %T')} {m}", flush=True)
    os.makedirs(RUN_DIR, exist_ok=True)
    with open(os.path.join(RUN_DIR, "run.log"), "a") as fh:
        fh.write(m + "\n")


def main():
    t0 = time.time()
    os.makedirs(RUN_DIR, exist_ok=True)
    open(os.path.join(RUN_DIR, "run.log"), "w").close()
    log("=== soft_hp35 tip refine ===")

    af = load_accfuse()
    _, lin_hp, _ = V4._mean_and_hp_from_folds()
    lin_al = align_hp(af, lin_hp)
    tip_ev = {ds: blend_ev(af[ds], lin_al[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    stem_s = load_stem_honest()
    tmpl = af

    sw2e = load_ext_mean4("w10b3sw2P")
    bmmse = load_ext_mean4("w10bmms3enhP")
    af_ext = {ds: blend_ev(sw2e[ds], bmmse[ds], 0.5) for ds in TEST_NAMES}
    train_logits, _ = V4.load_train()
    test_logits, _ = V4.load_test()
    base = {}
    base.update(train_logits)
    base.update(test_logits)
    streams = V4.linear_streams(base, "hp30")
    hp_ext = {}
    for ds in TEST_NAMES:
        hp_ext[ds] = {}
        for k, d in streams[ds].items():
            if k not in af_ext[ds]:
                continue
            n = min(len(d), len(af_ext[ds][k]["r0"]))
            hp_ext[ds][k] = as_sym(af_ext[ds][k], np.asarray(d, float)[:n])
    tip_ext = {ds: blend_ev(af_ext[ds], hp_ext[ds], TIP_ALPHA) for ds in TEST_NAMES}
    tip_ext_s = ev_to_signed(tip_ext)
    stem_ext_s = load_stem_ext()

    rows = []
    for tau in TAUS:
        vname = f"soft_hp{tau}"
        stem_v = soft_hp(stem_s, tau)
        stem_ext_v = soft_hp(stem_ext_s, tau)
        log(f"=== {vname} ===")
        for w in WS:
            ev = blend_signed(tip_s, stem_v, w, tmpl)
            pgrid = []
            for p, sc in P_CANDS:
                per, mH, mA = score_lodo(ev, p_stay=p, scale=sc)
                pgrid.append(dict(p=p, sc=sc, H=mH, Acc=mA))
            ok75 = [r for r in pgrid if r["Acc"] > 75]
            best_p = sorted(ok75 or pgrid, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
            mu, sd = emis_from(ev)
            ext_ev = blend_signed(tip_ext_s, stem_ext_v, w, tip_ext)
            ext, ext996 = {}, {}
            for ds in TEST_NAMES:
                r = E.score(ext_ev[ds], mu, sd * best_p["sc"], best_p["p"], LOOKAHEAD)
                ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
                r2 = E.score(ext_ev[ds], mu, sd * 0.8, 0.996, LOOKAHEAD)
                ext996[ds] = {k: float(r2[k]) for k in ("H", "Acc", "L", "SDT") if k in r2}
            lodo996 = next(r for r in pgrid if r["p"] == 0.996 and r["sc"] == 0.8)
            pass_new = (
                best_p["H"] > 0.85 and best_p["Acc"] > 75
                and (ext["Carta2026"]["H"] >= 0.65 or ext["AASD"]["H"] >= 0.65)
            )
            pass996 = (
                lodo996["H"] > 0.85 and lodo996["Acc"] > 75
                and (ext996["Carta2026"]["H"] >= 0.65 or ext996["AASD"]["H"] >= 0.65)
            )
            row = dict(
                variant=vname, w=w, lodo=best_p, lodo996=lodo996,
                external=ext, external_996=ext996,
                pass_new=pass_new, pass_at_996=pass996,
            )
            rows.append(row)
            log(
                f"  w={w:.2f} LODO {best_p['H']:.4f}/{best_p['Acc']:.2f}@p{best_p['p']}/{best_p['sc']} "
                f"Carta {ext['Carta2026']['H']:.4f} AASD {ext['AASD']['H']:.4f} "
                f"@996 C {ext996['Carta2026']['H']:.4f} "
                f"lodo996 {lodo996['H']:.4f}/{lodo996['Acc']:.2f} pass={pass_new or pass996}"
            )

    passers = [r for r in rows if r["pass_new"] or r["pass_at_996"]]
    lp = [r for r in rows if r["lodo"]["H"] > 0.85 and r["lodo"]["Acc"] > 75]
    if passers:
        chosen = sorted(passers, key=lambda r: (r["lodo"]["H"], r["lodo"]["Acc"]), reverse=True)[0]
    elif lp:
        chosen = sorted(lp, key=lambda r: (r["lodo"]["H"], r["lodo"]["Acc"]), reverse=True)[0]
        bc = sorted(lp, key=lambda r: r["external"]["Carta2026"]["H"], reverse=True)[0]
        log(f"NO PASS; LODO-pick {chosen['variant']} w={chosen['w']}; "
            f"best_carta {bc['variant']} w={bc['w']} "
            f"Carta={bc['external']['Carta2026']['H']:.4f} "
            f"LODO {bc['lodo']['H']:.4f}/{bc['lodo']['Acc']:.2f}")
    else:
        chosen = sorted(rows, key=lambda r: (r["lodo"]["H"], r["lodo"]["Acc"]), reverse=True)[0]

    out = dict(method="soft_hp35_tip_refine", rows=rows, chosen=chosen,
               any_pass=bool(passers), elapsed_s=time.time() - t0)
    with open(OUT_JSON, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    lines = ["# results_soft_hp35_tip_refine\n\n", f"any_pass={bool(passers)}\n\n",
             "| var | w | LODO | Carta | AASD | pass |\n",
             "|---|---:|---:|---:|---:|:---:|\n"]
    for r in rows:
        lines.append(
            f"| {r['variant']} | {r['w']:.2f} | {r['lodo']['H']:.4f}/{r['lodo']['Acc']:.2f} | "
            f"{r['external']['Carta2026']['H']:.4f} | {r['external']['AASD']['H']:.4f} | "
            f"{r['pass_new'] or r['pass_at_996']} |\n"
        )
    Path(RESULTS_MD).write_text("".join(lines))
    log(f"DONE any_pass={bool(passers)} elapsed={time.time()-t0:.1f}s")


if __name__ == "__main__":
    main()
