"""Feature-level ensemble: AccFuse↔hp30 tip ⊕ EEG-demean SW2 soft_hp50 (no stem).

Diversity at emission level of two independently debiased front-ends.
≠ tip↔stem soft-w / AC / σ / |Δ|. Uses published tip + cached demean evidence
if present; otherwise AccFuse tip alone soft bank (abort if demean missing).

CPU-only.
"""
from __future__ import annotations

import json
import os
import pickle
from pathlib import Path

import numpy as np

import ensemble_clf_transformer_v4 as V4
import eval_external as E
import score as S
from common import DS_KEYS
from common_exp import EXP
from dual_hp30_sw2_bmms import LOOKAHEAD, TEST_NAMES, as_sym, blend_ev, load_ext_mean4, score_lodo
from accfuse_hp30_hselect import align_hp, load_accfuse
from stem_hp30_accfuse import TIP_ALPHA, emis_from, ev_to_signed

RUN = "fe_ensemble_tip_demean"
OUT = os.path.join(EXP, "results", "research", f"{RUN}.json")
MD = os.path.join(EXP, "multi_agent_opt", f"results_{RUN}.md")
DEMEAN_EXP = "direct_clf__eaP_n5_t60_w10b3_eegdemean_t30_s0"
WS = (0.1, 0.2, 0.3, 0.4, 0.5)
GRID = [(0.994, 0.8), (0.996, 0.8), (0.998, 0.8), (0.998, 1.0)]


def log(m):
    print(m, flush=True)


def load_demean_lodo():
    try:
        ev, _ = S.load_folds(DEMEAN_EXP, "causal")
        return ev
    except Exception as e:
        log(f"demean folds missing: {e}")
        return None


def soft_hp_signed(streams, tau=50.0):
    out = {}
    for ds in streams:
        out[ds] = {}
        for k, d in streams[ds].items():
            d = np.asarray(d, float)
            h = V4.highpass(d[:, None], tau=tau)[:, 0]
            out[ds][k] = 0.5 * d + 0.5 * h
    return out


def main():
    af = load_accfuse()
    _, lin_hp, _ = V4._mean_and_hp_from_folds()
    tip_ev = {ds: blend_ev(af[ds], align_hp(af, lin_hp)[ds], TIP_ALPHA) for ds in DS_KEYS}
    tip_s = ev_to_signed(tip_ev)
    dem = load_demean_lodo()
    if dem is None:
        log("ABORT: demean evidence wiped with failing ckpts")
        Path(MD).write_text("# results_fe_ensemble_tip_demean\n\nABORT: demean folds deleted.\n")
        return 1
    dem_s = ev_to_signed(dem)
    dem_soft = soft_hp_signed(dem_s, 50)

    results = []
    for w in WS:
        blended = {}
        for ds in DS_KEYS:
            blended[ds] = {}
            for k in tip_s[ds]:
                if k not in dem_soft[ds]:
                    continue
                a, b = tip_s[ds][k], dem_soft[ds][k]
                n = min(len(a), len(b))
                s = (1 - w) * a[:n] + w * b[:n]
                blended[ds][k] = as_sym(tip_ev[ds][k], s)
        for p, sc in GRID:
            per, mH, mA = score_lodo(blended, p_stay=p, scale=sc)
            results.append(dict(w=w, p=p, sc=sc, H=mH, Acc=mA))
            log(f"  w={w} p={p} sc={sc} H={mH:.4f} Acc={mA:.2f}")

    ok = [r for r in results if r["H"] > 0.85 and r["Acc"] > 75]
    mild = [r for r in results if r["p"] == 0.996 and r["sc"] == 0.8]
    best = sorted(ok or mild or results, key=lambda r: (r["H"], r["Acc"]), reverse=True)[0]
    log(f"SELECTED {best}")

    # external: need demean ext — may be deleted; skip if missing
    dem_ext_paths = {
        ds: os.path.join(EXP, "results", "research", f"ext_clf_{ds}_w10b3_ptSW2_eegdemean_t30_evidence_s0.pkl")
        for ds in TEST_NAMES
    }
    if not all(os.path.exists(p) for p in dem_ext_paths.values()):
        log("demean ext missing; score tip-only ext for reference")
        ext = {}
    else:
        sw2e = load_ext_mean4("w10b3sw2P")
        bmmse = load_ext_mean4("w10bmms3enhP")
        af_ext = {ds: blend_ev(sw2e[ds], bmmse[ds], 0.5) for ds in TEST_NAMES}
        train_logits, _ = V4.load_train()
        test_logits, _ = V4.load_test()
        base = {}
        base.update(train_logits)
        base.update(test_logits)
        streams = V4.linear_streams(base, "hp30")
        tip_ext = {}
        for ds in TEST_NAMES:
            tip_ext[ds] = {}
            for k, d in streams[ds].items():
                if k not in af_ext[ds]:
                    continue
                n = min(len(d), len(af_ext[ds][k]["r0"]))
                tip_ext[ds][k] = as_sym(af_ext[ds][k],
                    (1 - TIP_ALPHA) * (np.asarray(af_ext[ds][k]["r0"]) - np.asarray(af_ext[ds][k]["r1"]))[:n]
                    + TIP_ALPHA * np.asarray(d, float)[:n])
        # rebuild proper tip_ext via blend
        tip_ext = {ds: blend_ev(af_ext[ds], {
            k: as_sym(af_ext[ds][k], np.asarray(streams[ds][k], float)[:len(af_ext[ds][k]["r0"])])
            for k in af_ext[ds] if k in streams[ds]
        }, TIP_ALPHA) for ds in TEST_NAMES}
        tip_ext_s = ev_to_signed(tip_ext)
        dem_ext_s = {}
        for ds, path in dem_ext_paths.items():
            with open(path, "rb") as fh:
                EV = pickle.load(fh)
            dem_ext_s[ds] = {}
            for k, e in EV.items():
                d = np.asarray(e["r0"], float) - np.asarray(e["r1"], float)
                h = V4.highpass(d[:, None], tau=50)[:, 0]
                dem_ext_s[ds][k] = 0.5 * d + 0.5 * h
        w = best["w"]
        # LODO emis from blended
        # rebuild blended at best w
        blended = {}
        for ds in DS_KEYS:
            blended[ds] = {}
            for k in tip_s[ds]:
                if k not in dem_soft[ds]:
                    continue
                a, b = tip_s[ds][k], dem_soft[ds][k]
                n = min(len(a), len(b))
                s = (1 - w) * a[:n] + w * b[:n]
                blended[ds][k] = as_sym(tip_ev[ds][k], s)
        mu, sd = emis_from(blended)
        ext = {}
        for ds in TEST_NAMES:
            ev = {}
            for k in tip_ext_s[ds]:
                if k not in dem_ext_s[ds]:
                    continue
                a, b = tip_ext_s[ds][k], dem_ext_s[ds][k]
                n = min(len(a), len(b))
                s = (1 - w) * a[:n] + w * b[:n]
                ev[k] = as_sym(tip_ext[ds][k], s)
            r = E.score(ev, mu, sd * best["sc"], best["p"], LOOKAHEAD)
            ext[ds] = {k: float(r[k]) for k in ("H", "Acc", "L", "SDT") if k in r}
            log(f"  EXT {ds} H={ext[ds]['H']:.4f} Acc={ext[ds]['Acc']:.2f}")

    gate = bool(ok) and ext and ext.get("AASD", {}).get("H", 0) >= 0.65 and ext.get("Carta2026", {}).get("H", 0) >= 0.65
    out = dict(selected=best, grid=results, external=ext, pass_joint=gate)
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    Path(MD).write_text(
        f"# results_fe_ensemble_tip_demean\n\nselected {best}; ext {ext}; pass={gate}\n"
    )
    log(f"DONE pass={gate}")
    return 0 if gate else 1


if __name__ == "__main__":
    raise SystemExit(main())
