#!/usr/bin/env python3
"""
causal_eeg_nested_lodo.py

Nested-LODO redesign of the causal EEG decision layer.

Scope:
- Uses the existing focal<->synthetic fused evidence caches (w=0.5).
- Jointly selects HMM persistence, emission scale, posterior EMA tau, and
  decision threshold inside INNER LODO.
- Selects configurations using a predeclared balanced H/Accuracy objective
  plus a Pareto-frontier report.
- Evaluates each selected configuration once on the OUTER held-out dataset.
- Does not use AASD/Carta labels for selection.

Important limitation:
The supplied cache is already fused evidence. This script cannot learn a new
focal/synthetic fusion weight unless the upstream project exposes the separate
unfused focal and synthetic streams. It redesigns the complete decision layer
that is observable in causal_ds_temp.py.

Run from the same project directory/environment as causal_ds_temp.py:
    CUDA_VISIBLE_DEVICES= OMP_NUM_THREADS=8 python -u causal_eeg_nested_lodo.py

The script intentionally reuses project-specific official scorer and cache
helpers; it is not a replacement for the project's EEG feature/model training.
"""

from __future__ import annotations

import itertools
import json
import os
import pickle
import time
from typing import Any

import numpy as np

import ensemble_clf_nested as N
import score as OfficialScore
from common import CACHE as EDA_CACHE, DS_KEYS, FS as FS128
from common_exp import rows_of
from e1_switch import expand, hmm_filter, synth_trial
from scorer import aggregate, fast_score_subject

# External real-switch evaluation helpers from the original project.
TEST_NAMES = ("AASD", "Carta2026")

# --------------------------- Configuration ---------------------------------

EXP = os.path.dirname(os.path.abspath(__file__))
SRC_CACHE = os.path.join(
    EXP, "models", "focal_synth_adapt_readout", "cache"
)
OUT_DIR = os.path.join(EXP, "results", "research")
OUT_JSON = os.path.join(OUT_DIR, "causal_eeg_nested_lodo.json")
OUT_MD = os.path.join(OUT_DIR, "causal_eeg_nested_lodo.md")
LOG_PATH = os.path.join(EXP, "experiments", "causal_eeg_nested_lodo.log")

# Balanced objective: H is already on its official scale; Acc is converted
# from percentage to [0, 1]. lambda=0.5 gives equal weight after scaling.
LAMBDA_H = 0.5

# Coarse grid to keep nested evaluation tractable. Refine only after observing
# inner-fold behavior; do not tune from outer-fold results.
P_STAY_GRID = (0.98, 0.985, 0.99, 0.994, 0.997, 0.999)
EMISSION_SCALE_GRID = (0.5, 0.65, 0.8, 1.0, 1.25, 1.5)
TAU_GRID = (0.0, 0.25, 0.5, 1.0, 2.0, 4.0)
THRESHOLD_GRID = (0.35, 0.40, 0.45, 0.50, 0.55, 0.60, 0.65)

# The original project code uses the official scorer's seed and dataset index.
EPS = 1e-12


def log(message: str) -> None:
    print(message, flush=True)
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
    with open(LOG_PATH, "a", encoding="utf-8") as f:
        f.write(message + "\n")


def load_pickle(path: str) -> Any:
    with open(path, "rb") as f:
        return pickle.load(f)


def load_fused_caches() -> dict[str, dict[str, np.ndarray]]:
    """Load the existing frozen focal-synthetic w=0.5 evidence caches."""
    result = {}
    for held in DS_KEYS:
        path = os.path.join(
            SRC_CACHE, f"focal_synth_w05__held-{held}.pkl"
        )
        if not os.path.isfile(path):
            raise FileNotFoundError(
                f"Missing cache: {path}\n"
                "Run the upstream focal_synth_adapt_readout/fusion pipeline "
                "first, or update SRC_CACHE/cache naming for this project."
            )
        result[held] = load_pickle(path)
        log(f"cache hit: {path}")
    return result


def smooth_posterior(posterior: np.ndarray, tau_s: float,
                     hop_s: float = 0.25) -> np.ndarray:
    """Causal exponential moving average; tau=0 is identity."""
    p = np.asarray(posterior, dtype=np.float64).reshape(-1)
    if tau_s <= 0 or len(p) < 2:
        return p.copy()
    alpha = 1.0 - np.exp(-float(hop_s) / float(tau_s))
    out = np.empty_like(p)
    out[0] = p[0]
    for i in range(1, len(p)):
        out[i] = alpha * p[i] + (1.0 - alpha) * out[i - 1]
    return out


def causal_hmm_posterior(
    evidence: np.ndarray, mu: float, sd: float, p_stay: float
) -> np.ndarray:
    """Causal two-state HMM posterior using the project's HMM filter."""
    d = np.asarray(evidence, dtype=np.float64).reshape(-1)
    sd = max(float(sd), EPS)
    ll0 = -0.5 * ((d - float(mu)) / sd) ** 2
    ll1 = -0.5 * ((d + float(mu)) / sd) ** 2
    return hmm_filter(ll0, ll1, float(p_stay))


def make_predictions(
    streams: dict[str, dict[str, np.ndarray]],
    target: str,
    train_ds: list[str],
    config: dict[str, float],
) -> dict[str, Any]:
    """
    Score one target dataset with emission parameters fitted only on train_ds.
    The target labels are used only by the official evaluation scorer after
    predictions have been produced.
    """
    mu, sd = N.emis({ds: streams[ds] for ds in train_ds}, train_ds)
    sd_eff = max(float(sd) * float(config["emission_scale"]), EPS)

    rng = np.random.default_rng(
        OfficialScore.SEED + DS_KEYS.index(target)
    )
    per_subject: dict[str, list[tuple[np.ndarray, np.ndarray]]] = {}

    for _, row in rows_of(target).iterrows():
        key = row["key"]
        total = int(row["n"])
        label_path = os.path.join(EDA_CACHE, "cnd", f"{key}.npz")
        if not os.path.isfile(label_path):
            raise FileNotFoundError(f"Missing label cache: {label_path}")

        labels_blob = np.load(label_path)
        attention = labels_blob["att_col"][:total].astype(int)

        d = np.asarray(streams[target][key], dtype=np.float64).reshape(-1)
        t_end = np.asarray(N.G["tend"][target][key], dtype=np.float64)

        # Same synthetic switch schedule as the supplied baseline evaluator.
        switches, _d_syn, _ = synth_trial(key, total, d, t_end, rng)

        posterior = causal_hmm_posterior(
            d, mu, sd_eff, float(config["p_stay"])
        )
        posterior = smooth_posterior(posterior, float(config["tau_s"]))
        decision = (posterior > float(config["threshold"])).astype(np.int8)

        guess = int(rng.integers(0, 2))
        predicted = expand(
            decision, t_end, N.G["lookahead"], total, guess
        )
        truth = np.where(switches, 1 - attention, attention)

        per_subject.setdefault(row["subj_uid"], []).append(
            (predicted, truth)
        )

    subject_scores = []
    for trials in per_subject.values():
        predicted = np.concatenate([x[0] for x in trials])
        truth = np.concatenate([x[1] for x in trials])
        trial_ids = np.concatenate([
            np.full(len(pred), trial_id, dtype=np.int64)
            for trial_id, (pred, _truth) in enumerate(trials)
        ])
        subject_scores.append(
            fast_score_subject(predicted, truth, trial_ids, fs=FS128)
        )

    metrics = aggregate(subject_scores)
    return {
        "H": float(metrics["H"]),
        "Acc": float(metrics["Acc"]),
        "L": float(metrics["L"]),
        "SDT": float(metrics["SDT"]),
        "nSwitches": int(metrics["nSwitches"]),
        "nUndetected": int(metrics["nUndetected"]),
        "miss_rate": float(
            metrics["nUndetected"] / max(metrics["nSwitches"], 1)
        ),
        "emis_mu": float(mu),
        "emis_sd": float(sd_eff),
    }



def score_external_dataset(
    stream: dict[str, np.ndarray],
    meta: dict[str, Any],
    mu: float,
    sd: float,
    config: dict[str, float],
) -> dict[str, Any]:
    """Evaluate one real-switch external set after all parameters are frozen.

    External labels (meta[*]["att128"]) are read only to compute final metrics;
    this function is never called during inner/outer parameter selection.
    """
    rng = np.random.default_rng(OfficialScore.SEED)
    per_subject: dict[str, list[tuple[np.ndarray, np.ndarray]]] = {}
    for key in sorted(stream):
        item = meta[key]
        evidence = np.asarray(stream[key], dtype=np.float64).reshape(-1)
        posterior = causal_hmm_posterior(
            evidence, mu, max(float(sd) * float(config["emission_scale"]), EPS),
            float(config["p_stay"]),
        )
        posterior = smooth_posterior(posterior, float(config["tau_s"]))
        decision = (posterior > float(config["threshold"])).astype(np.int8)
        predicted = expand(
            decision, np.asarray(item["t_end"], dtype=np.float64),
            0.75, int(item["n128"]), int(rng.integers(0, 2)),
        )
        truth = np.asarray(item["att128"], dtype=np.int8)
        per_subject.setdefault(item["subject"], []).append((predicted, truth))

    subject_scores = []
    for trials in per_subject.values():
        predicted = np.concatenate([x[0] for x in trials])
        truth = np.concatenate([x[1] for x in trials])
        trial_ids = np.concatenate([
            np.full(len(pred), trial_id, dtype=np.int64)
            for trial_id, (pred, _truth) in enumerate(trials)
        ])
        subject_scores.append(
            fast_score_subject(predicted, truth, trial_ids, fs=FS128)
        )
    metrics = aggregate(subject_scores)
    n_switches = int(metrics["nSwitches"])
    n_undetected = int(metrics["nUndetected"])
    return {
        "H": float(metrics["H"]),
        "Acc": float(metrics["Acc"]),
        "L": float(metrics["L"]),
        "SDT": float(metrics["SDT"]),
        "nSwitches": n_switches,
        "nUndetected": n_undetected,
        "miss_rate": float(n_undetected / max(n_switches, 1)),
    }


def evaluate_external(
    candidates: list[dict[str, float]],
) -> dict[str, Any]:
    """Select on all five corpus datasets by LODO, then score AASD/Carta once."""
    log("=== external AASD/Carta2026 evaluation (labels never used for selection) ===")
    import stem_ms96_irm_synth_fuse as SF
    import ensemble_clf_transformer_v4 as V4

    corp_path = os.path.join(SRC_CACHE, "focal_synth_w05__all5_corpus.pkl")
    ext_path = os.path.join(SRC_CACHE, "focal_synth_w05__all5_external.pkl")
    for path in (corp_path, ext_path):
        if not os.path.isfile(path):
            raise FileNotFoundError(
                f"Required external-evaluation cache is missing: {path}"
            )
    log(f"loading 5-corpus evidence: {corp_path}")
    corp = load_pickle(corp_path)
    log(f"loading external test evidence: {ext_path}")
    ext_streams = load_pickle(ext_path)
    _ext_logits, ext_meta = SF.load_external()

    # These are mandatory final test sets, not optional validation datasets.
    # Their labels are accessed only by score_external_dataset after the
    # deployment configuration has been selected and frozen on corpus LODO.
    missing_streams = [name for name in TEST_NAMES if name not in ext_streams]
    missing_meta = [name for name in TEST_NAMES if name not in ext_meta]
    if missing_streams or missing_meta:
        raise KeyError(
            "Missing required external test data: "
            f"streams={missing_streams}, metadata={missing_meta}. "
            f"Expected both {TEST_NAMES}."
        )

    # Select one deployment configuration using inner LODO over all five
    # corpus datasets. AASD/Carta are never passed to select_config().
    corpus_datasets = list(V4.CORPUS)
    log(f"selecting frozen external config on corpus only: {corpus_datasets}")
    best_config, selection = select_config(corp, corpus_datasets, candidates)
    att = N.G["att"]
    mu, sd = SF.emissions_from(corp, att, datasets=corpus_datasets)
    log(
        "frozen external config: "
        + json.dumps(best_config, sort_keys=True)
        + f"; corpus emission mu={float(mu):.6g}, sd={float(sd):.6g}"
    )

    external_scores = {}
    for dataset in TEST_NAMES:
        log(f"scoring untouched external test set: {dataset}")
        score = score_external_dataset(
            ext_streams[dataset], ext_meta[dataset], mu, sd, best_config
        )
        external_scores[dataset] = score
        log(
            f"EXTERNAL {dataset}: H={score['H']:.4f} Acc={score['Acc']:.2f} "
            f"L={score['L']:.3f} SDT={score['SDT']:.2f} "
            f"miss_rate={score['miss_rate']:.3f}"
        )

    return {
        "test_datasets": list(TEST_NAMES),
        "selection_protocol": (
            "deployment config selected by inner LODO over all five corpus "
            "datasets only; AASD and Carta2026 labels used exclusively for "
            "the final, post-selection score"
        ),
        "selection_config": best_config,
        "corpus_datasets": corpus_datasets,
        "corpus_inner_selection": selection,
        "emission_mu": float(mu),
        "emission_sd": float(sd),
        "datasets": external_scores,
    }

def objective(score: dict[str, float]) -> float:
    """Balanced scalar objective with H and Acc placed on [0, 1] scales."""
    h = float(score["H"])
    acc01 = float(score["Acc"]) / 100.0
    return LAMBDA_H * h + (1.0 - LAMBDA_H) * acc01


def config_grid() -> list[dict[str, float]]:
    return [
        {
            "p_stay": float(p),
            "emission_scale": float(s),
            "tau_s": float(tau),
            "threshold": float(thr),
        }
        for p, s, tau, thr in itertools.product(
            P_STAY_GRID,
            EMISSION_SCALE_GRID,
            TAU_GRID,
            THRESHOLD_GRID,
        )
    ]


def evaluate_config_on_inner_lodo(
    streams: dict[str, dict[str, np.ndarray]],
    inner_datasets: list[str],
    config: dict[str, float],
) -> dict[str, Any]:
    """Evaluate a fixed config with inner LODO over the outer-training set."""
    folds = []
    for inner_held in inner_datasets:
        inner_train = [d for d in inner_datasets if d != inner_held]
        if not inner_train:
            continue
        score = make_predictions(streams, inner_held, inner_train, config)
        folds.append({"held": inner_held, **score})

    if not folds:
        raise RuntimeError("No valid inner LODO folds.")

    mean_h = float(np.mean([x["H"] for x in folds]))
    mean_acc = float(np.mean([x["Acc"] for x in folds]))
    mean_j = float(np.mean([objective(x) for x in folds]))
    return {
        "config": config,
        "folds": folds,
        "mean_H": mean_h,
        "mean_Acc": mean_acc,
        "mean_objective": mean_j,
    }


def pareto_front(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """Return non-dominated configurations in (H, Acc), both maximized."""
    front = []
    for candidate in rows:
        dominated = False
        for other in rows:
            if other is candidate:
                continue
            no_worse = (
                other["mean_H"] >= candidate["mean_H"]
                and other["mean_Acc"] >= candidate["mean_Acc"]
            )
            strictly_better = (
                other["mean_H"] > candidate["mean_H"]
                or other["mean_Acc"] > candidate["mean_Acc"]
            )
            if no_worse and strictly_better:
                dominated = True
                break
        if not dominated:
            front.append(candidate)
    return sorted(
        front,
        key=lambda x: (
            x["mean_objective"], x["mean_H"], x["mean_Acc"]
        ),
        reverse=True,
    )


def select_config(
    streams: dict[str, dict[str, np.ndarray]],
    outer_train: list[str],
    candidates: list[dict[str, float]],
) -> tuple[dict[str, float], dict[str, Any]]:
    """Select only from inner-fold scores; outer-held is not passed here."""
    rows = []
    for i, cfg in enumerate(candidates, start=1):
        row = evaluate_config_on_inner_lodo(streams, outer_train, cfg)
        rows.append(row)
        if i % 25 == 0:
            log(
                f"  inner configs {i}/{len(candidates)}; "
                f"last H={row['mean_H']:.4f} "
                f"Acc={row['mean_Acc']:.2f} "
                f"J={row['mean_objective']:.5f}"
            )

    front = pareto_front(rows)
    # Primary selection: balanced objective; tie-break on H, then Acc.
    best = max(
        rows,
        key=lambda x: (
            x["mean_objective"], x["mean_H"], x["mean_Acc"]
        ),
    )
    return best["config"], {
        "best_inner": best,
        "pareto_front": front,
        "n_configs": len(rows),
        "selection_objective": (
            "0.5 * H + 0.5 * (Acc / 100); ties by H then Acc"
        ),
    }


def main() -> None:
    os.makedirs(OUT_DIR, exist_ok=True)
    os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
    with open(LOG_PATH, "w", encoding="utf-8"):
        pass

    started = time.time()
    log("=== causal EEG nested LODO decision-layer redesign ===")
    log(f"datasets={list(DS_KEYS)}")
    log(f"lambda_H={LAMBDA_H}")
    log(
        "WARNING: this optimizes the existing fused evidence decision layer; "
        "it does not retrain upstream EEG models or learn focal/synth fusion."
    )

    # Load project metadata and labels using the existing project API.
    N.load_all()
    fused_by_outer = load_fused_caches()
    candidates = config_grid()
    log(f"candidate configs: {len(candidates)}")

    results: dict[str, Any] = {
        "method": "nested LODO joint HMM/EMA/threshold selection",
        "scope": "frozen focal-synthetic w=0.5 evidence; decision layer only",
        "objective": {
            "lambda_H": LAMBDA_H,
            "formula": "J = lambda_H * H + (1-lambda_H) * Acc/100",
            "selection": "maximize inner-LODO mean J; tie-break H then Acc",
        },
        "grids": {
            "p_stay": list(P_STAY_GRID),
            "emission_scale": list(EMISSION_SCALE_GRID),
            "tau_s": list(TAU_GRID),
            "threshold": list(THRESHOLD_GRID),
        },
        "outer_folds": {},
    }

    outer_means = []
    for outer_held in DS_KEYS:
        t0 = time.time()
        log(f"\n=== OUTER held={outer_held} ===")
        outer_train = [d for d in DS_KEYS if d != outer_held]
        streams = fused_by_outer[outer_held]

        best_config, selection = select_config(
            streams, outer_train, candidates
        )

        # Final outer evaluation. The outer-held labels are not involved in
        # selecting best_config; they are only consumed by this scorer.
        outer_score = make_predictions(
            streams, outer_held, outer_train, best_config
        )

        row = {
            "outer_train_datasets": outer_train,
            "selected_config": best_config,
            "inner_selection": selection,
            "outer_score": outer_score,
            "elapsed_seconds": float(time.time() - t0),
        }
        results["outer_folds"][outer_held] = row
        outer_means.append(outer_score)
        log(
            f"OUTER {outer_held}: H={outer_score['H']:.4f} "
            f"Acc={outer_score['Acc']:.2f} "
            f"tau={best_config['tau_s']:.2f} "
            f"p_stay={best_config['p_stay']:.4f} "
            f"scale={best_config['emission_scale']:.3f} "
            f"threshold={best_config['threshold']:.2f}"
        )

    # Mandatory final external test stage, matching the original workflow:
    # AASD and Carta2026 are evaluated only after LODO and config freezing.
    try:
        log("\n=== POST-LODO EXTERNAL TESTS: AASD + Carta2026 ===")
        results["external_evaluation"] = evaluate_external(candidates)
    except Exception as exc:
        # Keep LODO outputs even if external data/dependencies are unavailable.
        results["external_evaluation_error"] = repr(exc)
        log(
            "ERROR: required external AASD/Carta2026 evaluation did not complete: "
            f"{exc!r}. Check focal_synth_w05__all5_external.pkl, "
            "focal_synth_w05__all5_corpus.pkl, SF.load_external(), and test metadata."
        )

    summary = {
        "mean_H": float(np.mean([x["H"] for x in outer_means])),
        "mean_Acc": float(np.mean([x["Acc"] for x in outer_means])),
        "mean_L": float(np.mean([x["L"] for x in outer_means])),
        "mean_SDT": float(np.mean([x["SDT"] for x in outer_means])),
        "mean_miss_rate": float(
            np.mean([x["miss_rate"] for x in outer_means])
        ),
    }
    results["summary"] = summary
    results["elapsed_seconds"] = float(time.time() - started)

    with open(OUT_JSON, "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, default=float)

    md = [
        "# Causal EEG nested LODO results",
        "",
        "**Scope:** joint decision-layer selection on frozen focal-synthetic "
        "w=0.5 evidence. This does not retrain upstream models.",
        "",
        f"**Objective:** `J = {LAMBDA_H} * H + "
        f"{1-LAMBDA_H} * (Acc / 100)`; selection uses inner LODO only.",
        "",
        "| Outer held | H | Acc (%) | L | SDT (s) | Miss rate | p_stay | scale | tau (s) | threshold |",
        "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
    ]
    for held in DS_KEYS:
        fold = results["outer_folds"][held]
        s = fold["outer_score"]
        c = fold["selected_config"]
        md.append(
            f"| {held} | {s['H']:.4f} | {s['Acc']:.2f} | "
            f"{s['L']:.3f} | {s['SDT']:.2f} | {s['miss_rate']:.3f} | "
            f"{c['p_stay']:.4f} | {c['emission_scale']:.3f} | "
            f"{c['tau_s']:.2f} | {c['threshold']:.2f} |"
        )
    md += [
        "",
        "## Mean outer-fold metrics",
        "",
        f"- H: **{summary['mean_H']:.4f}**",
        f"- Accuracy: **{summary['mean_Acc']:.2f}%**",
        f"- L: {summary['mean_L']:.3f}",
        f"- SDT: {summary['mean_SDT']:.2f} s",
        f"- Miss rate: {summary['mean_miss_rate']:.3f}",
        "",
        "## External real-switch evaluation (AASD / Carta2026)",
        "",
    ]
    if "external_evaluation" in results:
        ext = results["external_evaluation"]
        md += [
            f"- Selection: {ext['selection_protocol']}.",
            f"- Frozen external config: `{json.dumps(ext['selection_config'], sort_keys=True)}`",
            "",
            "| External test | H | Acc (%) | L | SDT (s) | Switches | Undetected | Miss rate |",
            "|---|---:|---:|---:|---:|---:|---:|---:|",
        ]
        for dataset in TEST_NAMES:
            e = ext["datasets"][dataset]
            md.append(
                f"| {dataset} | {e['H']:.4f} | {e['Acc']:.2f} | {e['L']:.3f} | "
                f"{e['SDT']:.2f} | {e['nSwitches']} | {e['nUndetected']} | {e['miss_rate']:.3f} |"
            )
    elif "external_evaluation_error" in results:
        md.append(f"External evaluation not completed: `{results['external_evaluation_error']}`")
    md += [
        "",
        "## Leakage and scope notes",
        "",
        "- Each outer-held dataset is excluded from hyperparameter selection.",
        "- HMM emission statistics are estimated from outer-training datasets.",
        "- Candidate parameters are selected by inner LODO only.",
        "- Confirm that upstream evidence caches were generated without leakage.",
        "- The current cache is already fused at w=0.5; focal/synthetic fusion "
        "weight learning requires separate upstream streams.",
        "- These LODO scores use the project's synthetic-switch evaluation "
        "protocol; they are not a substitute for real-switch external evaluation.",
    ]
    with open(OUT_MD, "w", encoding="utf-8") as f:
        f.write("\n".join(md) + "\n")

    log(f"\nSUMMARY {json.dumps(summary)}")
    log(f"wrote {OUT_JSON}")
    log(f"wrote {OUT_MD}")
    log(f"elapsed={results['elapsed_seconds']:.1f}s")


if __name__ == "__main__":
    main()
