# -*- coding: utf-8 -*-
"""Robust selective-cascade ensemble for academic early warning (grade < 5).

Experiment C — robust V4 nonlinear residual specialist
------------------------------------------------------
1. Balanced RF + Balanced ExtraTrees + HistGradientBoosting are unchanged.
2. Historical expert probabilities are generated out-of-fold, preferably by
   strictly temporal expanding cross-fitting.
3. Routing is learned from batch-relative committee rank + disagreement.
4. Easy negatives soft-exit: they bypass the specialist but retain the
   committee score and remain eligible for global warning selection.
5. Routed cases use top-k raw features + expert probabilities + committee
   uncertainty in a compact nonlinear residual MLP anchored to committee logit.
6. The MLP output head is zero-initialized; residuals are asymmetric/bounded,
   magnitude-regularized, temporally early-stopped, and class-weight tempering
   is selected on temporal validation.
7. Specialist authority is validation-selected (including zero), support-capped,
   and guarded against material AP/AUC degradation versus the committee.
8. Strong-positive consensus cases cannot be downgraded below committee score
   and remain warnings at decision time.

Target: target_grade10 < 5.
"""
from __future__ import annotations

import argparse
import copy
import math
import pickle
import random
import warnings
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Dict, Sequence

import numpy as np
import pandas as pd
import torch
import torch.nn.functional as F
from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.feature_selection import f_classif
from sklearn.metrics import average_precision_score, confusion_matrix, precision_recall_fscore_support, roc_auc_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from torch import nn

TARGET, FAILURE_BOUNDARY = "target_grade10", 5.0
BASE_NAMES = ["random_forest_balanced", "extra_trees_balanced", "hist_gradient_boosting"]
UNCERTAINTY_NAMES = [
    "committee_mean", "committee_min", "committee_max", "committee_std",
    "committee_spread", "committee_entropy", "committee_vote_fraction",
]
EXCLUDE = {
    "dataset_split", "student_faculty_key", "student_faculty_name", "IDSV", "target_NHHK",
    "target_semester_ordinal", "is_summer_target", "IDKQDK", "IDToHoc", "IDMH",
    "target_IDCongDiem", "target_grade10", "target_grade4", "target_status", "base_row_id",
    "course_cluster_label",
}
IDS = ["IDSV", "IDMH", "IDKQDK", "IDToHoc", "target_NHHK", "target_semester_ordinal",
       "student_faculty_key", "student_faculty_name"]


@dataclass
class RoutingPolicy:
    negative_exit_quantile: float
    negative_max_spread: float
    negative_enabled: bool
    negative_relaxation: str
    max_negative_exit_failure_fraction: float
    max_negative_exit_failure_rate: float
    negative_exit_rate_train: float
    negative_exit_failure_rate_train: float
    negative_exit_failure_fraction_train: float
    routed_rate_train: float
    routed_prevalence_train: float
    routed_enrichment_train: float
    positive_rank_quantile: float
    positive_max_spread: float
    positive_preserve_min_precision: float
    positive_preserve_enabled: bool
    positive_relaxation: str
    positive_preserve_rate_train: float
    positive_preserve_precision_train: float
    positive_preserve_recall_train: float


@dataclass
class FoldRecord:
    fold: int
    y: np.ndarray
    score: np.ndarray
    committee_score: np.ndarray
    prevalence: float
    easy_negative: np.ndarray
    strong_positive: np.ndarray
    hard_negative_guard: bool
    path: Path


@dataclass
class ThresholdPolicy:
    threshold: float
    beta: float
    development_ap: float
    development_auc: float
    development_prevalence: float
    flag_rate: float
    warning_ratio: float
    precision: float
    recall: float
    f1: float
    fbeta: float
    constraints_relaxed: str


@dataclass
class CrossFitResult:
    mode: str
    probabilities: np.ndarray
    covered_mask: np.ndarray
    block_ids: np.ndarray
    source: str
    report_rows: list[dict]
    warning: str


class ResidualMLP(nn.Module):
    """Compact nonlinear residual model with a zero-initialized scalar output head."""

    def __init__(self, input_dim: int, hidden_dims=(32, 16), dropout=0.20):
        super().__init__()
        dims = [int(input_dim), *[int(h) for h in hidden_dims]]
        layers = []
        for din, dout in zip(dims[:-1], dims[1:]):
            layers.extend([nn.Linear(din, dout), nn.ReLU(), nn.Dropout(float(dropout))])
        self.backbone = nn.Sequential(*layers)
        last_dim = dims[-1]
        self.output = nn.Linear(last_dim, 1)
        nn.init.zeros_(self.output.weight)
        nn.init.zeros_(self.output.bias)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        h = self.backbone(x) if len(self.backbone) else x
        return self.output(h).squeeze(1)


class RawFeatureSelector:
    """Training-only standardized ANOVA/F-score raw-feature filter."""

    def __init__(self, max_features=48):
        self.max_features = int(max_features)
        self.scaler_: StandardScaler | None = None
        self.selected_indices_: np.ndarray | None = None
        self.scores_: np.ndarray | None = None
        self.feature_names_: list[str] = []
        self.selected_feature_names_: list[str] = []
        self.report_: list[dict] = []

    def fit(self, X, y, feature_names):
        X = clean_X(X)
        y = np.asarray(y, int).reshape(-1)
        self.feature_names_ = list(feature_names)
        if X.shape[1] != len(self.feature_names_):
            raise ValueError("Raw feature names do not match raw feature matrix")
        if len(X) != len(y) or len(X) < 2 or np.unique(y).size < 2:
            raise ValueError("Feature selection requires >=2 rows and both classes")
        self.scaler_ = StandardScaler().fit(X)
        Xs = self.scaler_.transform(X)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            scores, _ = f_classif(Xs, y)
        scores = np.asarray(scores, float)
        scores = np.nan_to_num(scores, nan=-np.inf, posinf=np.finfo(float).max, neginf=-np.inf)
        order = np.lexsort((np.arange(len(scores)), -scores))
        k = min(max(1, self.max_features), X.shape[1])
        self.selected_indices_ = np.sort(order[:k])
        self.scores_ = scores
        self.selected_feature_names_ = [self.feature_names_[i] for i in self.selected_indices_]
        rank_lookup = {int(idx): rank + 1 for rank, idx in enumerate(order)}
        self.report_ = [{
            "raw_feature_index": i,
            "feature": name,
            "f_score": float(scores[i]) if np.isfinite(scores[i]) else np.nan,
            "rank": rank_lookup[i],
            "selected": bool(i in set(self.selected_indices_.tolist())),
        } for i, name in enumerate(self.feature_names_)]
        return self

    def transform(self, X):
        if self.selected_indices_ is None:
            raise RuntimeError("RawFeatureSelector is not fitted")
        return clean_X(X)[:, self.selected_indices_]


def seed_all(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)


def resolve_device(device):
    if device == "auto":
        return torch.device("cuda" if torch.cuda.is_available() else "cpu")
    return torch.device(device)


def clean_X(X):
    X = X.to_numpy() if hasattr(X, "to_numpy") else X
    X = np.asarray(X, dtype=np.float64)
    if X.ndim != 2:
        raise ValueError(f"X must be 2D, got {X.shape}")
    return np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)


def clip_probability(p):
    return np.clip(np.asarray(p, float).reshape(-1), 1e-7, 1 - 1e-7)


def safe_ap(y, score):
    y = np.asarray(y, int)
    return float("nan") if np.unique(y).size < 2 else float(average_precision_score(y, score))


def safe_auc(y, score):
    y = np.asarray(y, int)
    return float("nan") if np.unique(y).size < 2 else float(roc_auc_score(y, score))


def effective_sample_weights(y, beta=0.9999):
    y = np.asarray(y, int).reshape(-1)
    n_failure, n_normal = int((y == 1).sum()), int((y == 0).sum())
    if not 0 <= beta < 1:
        raise ValueError(f"effective_beta must satisfy 0 <= beta < 1, got {beta}")
    if n_failure == 0 or n_normal == 0:
        wf = wn = 1.0
    else:
        raw_f = (1 - beta) / (1 - beta ** float(n_failure))
        raw_n = (1 - beta) / (1 - beta ** float(n_normal))
        z = (n_failure * raw_f + n_normal * raw_n) / (n_failure + n_normal)
        wf, wn = raw_f / z, raw_n / z
    w = np.where(y == 1, wf, wn).astype(np.float32)
    return w, {
        "beta": float(beta), "n_failure": n_failure, "n_normal": n_normal,
        "failure_weight": float(wf), "normal_weight": float(wn),
        "effective_ratio": float(wf / max(wn, 1e-12)),
    }


def sample_weights_from_stats(y, stats):
    y = np.asarray(y, int).reshape(-1)
    return np.where(y == 1, stats["failure_weight"], stats["normal_weight"]).astype(np.float32)


def tempered_sample_weights(y, effective_stats, gamma=1.0):
    """Temper effective-number class weights and renormalize mean sample weight to one."""
    y = np.asarray(y, int).reshape(-1)
    gamma = float(gamma)
    if not 0.0 <= gamma <= 1.0:
        raise ValueError(f"weight gamma must be in [0,1], got {gamma}")
    wf0 = max(float(effective_stats["failure_weight"]), 1e-12)
    wn0 = max(float(effective_stats["normal_weight"]), 1e-12)
    wf_raw, wn_raw = wf0 ** gamma, wn0 ** gamma
    nf, nn = int((y == 1).sum()), int((y == 0).sum())
    z = (nf * wf_raw + nn * wn_raw) / max(nf + nn, 1)
    wf, wn = wf_raw / max(z, 1e-12), wn_raw / max(z, 1e-12)
    w = np.where(y == 1, wf, wn).astype(np.float32)
    return w, {
        "gamma": gamma,
        "beta": float(effective_stats.get("beta", np.nan)),
        "n_failure": nf,
        "n_normal": nn,
        "failure_weight": float(wf),
        "normal_weight": float(wn),
        "tempered_ratio": float(wf / max(wn, 1e-12)),
        "base_effective_ratio": float(effective_stats.get("effective_ratio", np.nan)),
    }


def committee_statistics(expert_probabilities):
    p = np.asarray(expert_probabilities, float)
    if p.ndim != 2 or p.shape[1] != len(BASE_NAMES):
        raise ValueError(f"Expected expert probability matrix (N,{len(BASE_NAMES)}), got {p.shape}")
    p = np.clip(p, 1e-7, 1 - 1e-7)
    mean = p.mean(axis=1)
    pmin, pmax = p.min(axis=1), p.max(axis=1)
    std = p.std(axis=1)
    spread = pmax - pmin
    q = np.clip(mean, 1e-7, 1 - 1e-7)
    entropy = -(q * np.log(q) + (1 - q) * np.log(1 - q))
    vote = (p >= 0.5).mean(axis=1)
    return {
        "committee_mean": mean, "committee_min": pmin, "committee_max": pmax,
        "committee_std": std, "committee_spread": spread,
        "committee_entropy": entropy, "committee_vote_fraction": vote,
    }


def batch_percentile_rank(score):
    """Return ascending percentile ranks in (0,1], stable under monotonic score shifts."""
    score = np.asarray(score, float).reshape(-1)
    if not len(score):
        return np.array([], dtype=float)
    return pd.Series(score).rank(method="average", pct=True).to_numpy(float)


def ranks_within_blocks(score, block_ids):
    score = np.asarray(score, float).reshape(-1)
    blocks = np.asarray(block_ids)
    if len(score) != len(blocks):
        raise ValueError("score and block_ids must have equal lengths")
    out = np.empty(len(score), float)
    for block in pd.unique(blocks):
        idx = np.where(blocks == block)[0]
        out[idx] = batch_percentile_rank(score[idx])
    return out


def temporal_crossfit_plan(groups, min_train_rows=50):
    groups = np.asarray(groups)
    valid = ~pd.isna(groups)
    unique = np.unique(groups[valid])
    unique = np.sort(unique)
    plans = []
    for block in unique[1:]:
        train_idx = np.where(valid & (groups < block))[0]
        val_idx = np.where(valid & (groups == block))[0]
        if len(train_idx) >= int(min_train_rows) and len(val_idx):
            plans.append((train_idx, val_idx, block))
    return plans


def extract_temporal_groups(frame):
    if "target_semester_ordinal" in frame.columns:
        s = pd.to_numeric(frame["target_semester_ordinal"], errors="coerce")
        if s.notna().sum() >= 2 and s.nunique(dropna=True) >= 2:
            return s.to_numpy(float), "target_semester_ordinal"
    if "target_NHHK" in frame.columns:
        raw = frame["target_NHHK"]
        numeric = pd.to_numeric(raw, errors="coerce")
        if numeric.notna().sum() >= 2 and numeric.nunique(dropna=True) >= 2:
            return numeric.to_numpy(float), "target_NHHK_numeric"
        text = raw.astype("string")
        vals = sorted(v for v in text.dropna().unique().tolist())
        if len(vals) >= 2:
            mapping = {v: i for i, v in enumerate(vals)}
            return text.map(mapping).to_numpy(float), "target_NHHK_ordered"
    return np.full(len(frame), np.nan), "unavailable"


def asymmetric_bounded_delta_numpy(raw, positive_cap=2.0, negative_cap=0.75):
    raw = np.asarray(raw, float)
    t = np.tanh(raw)
    return np.where(raw >= 0, float(positive_cap) * t, float(negative_cap) * t)


def asymmetric_bounded_delta_torch(raw, positive_cap=2.0, negative_cap=0.75):
    t = torch.tanh(raw)
    return torch.where(raw >= 0, float(positive_cap) * t, float(negative_cap) * t)


def apply_residual_correction(committee_score, delta, authority):
    q = clip_probability(committee_score)
    delta = np.asarray(delta, float).reshape(-1)
    if len(q) != len(delta):
        raise ValueError("committee_score and delta must have equal lengths")
    z = np.log(q / (1 - q)) + float(authority) * delta
    return clip_probability(1.0 / (1.0 + np.exp(-np.clip(z, -50, 50))))


def specialist_loss_components_numpy(logits, y, weights, delta, residual_penalty):
    logits = np.asarray(logits, float)
    y = np.asarray(y, float)
    weights = np.asarray(weights, float)
    delta = np.asarray(delta, float)
    # stable BCE-with-logits
    per = np.maximum(logits, 0) - logits * y + np.log1p(np.exp(-np.abs(logits)))
    bce = float(np.mean(per * weights))
    penalty = float(residual_penalty) * float(np.mean(delta ** 2))
    return bce, penalty, bce + penalty


def make_base_models(seed, n_estimators, hist_max_iter):
    return {
        BASE_NAMES[0]: RandomForestClassifier(
            n_estimators=int(n_estimators), min_samples_leaf=4, max_features="sqrt",
            class_weight="balanced_subsample", bootstrap=True, n_jobs=-1, random_state=int(seed)),
        BASE_NAMES[1]: ExtraTreesClassifier(
            n_estimators=int(n_estimators), min_samples_leaf=3, max_features=0.70,
            class_weight="balanced", n_jobs=-1, random_state=int(seed) + 1),
        BASE_NAMES[2]: HistGradientBoostingClassifier(
            learning_rate=0.05, max_iter=int(hist_max_iter), max_leaf_nodes=31,
            min_samples_leaf=20, l2_regularization=1.0, early_stopping=True,
            random_state=int(seed) + 2),
    }


def positive_probability(model, X):
    proba, classes = np.asarray(model.predict_proba(X), float), np.asarray(model.classes_)
    idx = np.where(classes == 1)[0]
    if len(idx) != 1:
        raise RuntimeError("Positive class 1 is missing")
    return clip_probability(proba[:, int(idx[0])])


def predict_experts(models, X):
    X = clean_X(X)
    return np.column_stack([positive_probability(models[n], X) for n in BASE_NAMES])


def generate_crossfit_predictions(
    X, y, temporal_groups, temporal_source, *, seed, n_estimators, hist_max_iter,
    crossfit_folds=5, min_train_rows=100, min_oof_coverage=0.40, verbose=True,
):
    X, y = clean_X(X), np.asarray(y, int).reshape(-1)
    n = len(y)
    p = np.full((n, len(BASE_NAMES)), np.nan, float)
    blocks = np.full(n, "", dtype=object)
    report = []
    warning = ""

    plans = temporal_crossfit_plan(temporal_groups, min_train_rows=min_train_rows)
    temporal_ok = len(plans) >= 2
    if temporal_ok:
        for j, (tr, va, block) in enumerate(plans, 1):
            if np.unique(y[tr]).size < 2:
                continue
            models = make_base_models(seed + 1000 * j, n_estimators, hist_max_iter)
            for model in models.values():
                model.fit(X[tr], y[tr])
            p[va] = predict_experts(models, X[va])
            blocks[va] = str(block)
            report.append({
                "row_type": "block", "mode": "temporal_expanding", "block": str(block),
                "train_rows": len(tr), "validation_rows": len(va),
                "train_failures": int(y[tr].sum()), "validation_failures": int(y[va].sum()),
                "validation_prevalence": float(y[va].mean()),
                "train_max_group": float(np.nanmax(np.asarray(temporal_groups)[tr])),
                "validation_group": float(block),
            })
        covered = np.isfinite(p).all(axis=1)
        coverage = float(covered.mean())
        if covered.sum() >= 20 and np.unique(y[covered]).size == 2 and coverage >= min_oof_coverage and len(pd.unique(blocks[covered])) >= 2:
            mode = "temporal_expanding"
        else:
            temporal_ok = False
            warning = (
                f"Temporal cross-fit insufficient (coverage={coverage:.1%}, blocks={len(pd.unique(blocks[covered]))}); "
                "falling back to stratified OOF."
            )

    if not temporal_ok:
        if not warning:
            warning = "Temporal metadata/blocks unavailable; falling back to stratified OOF."
        p[:] = np.nan
        blocks[:] = ""
        report = []
        min_class = int(np.bincount(y, minlength=2).min())
        n_splits = min(int(crossfit_folds), min_class)
        if n_splits < 2:
            raise ValueError("Not enough examples in both classes for cross-fitting")
        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=int(seed))
        for j, (tr, va) in enumerate(skf.split(X, y), 1):
            models = make_base_models(seed + 1000 * j, n_estimators, hist_max_iter)
            for model in models.values():
                model.fit(X[tr], y[tr])
            p[va] = predict_experts(models, X[va])
            blocks[va] = f"skf_{j}"
            report.append({
                "row_type": "block", "mode": "stratified_oof", "block": f"skf_{j}",
                "train_rows": len(tr), "validation_rows": len(va),
                "train_failures": int(y[tr].sum()), "validation_failures": int(y[va].sum()),
                "validation_prevalence": float(y[va].mean()),
                "train_max_group": np.nan, "validation_group": np.nan,
            })
        covered = np.isfinite(p).all(axis=1)
        mode = "stratified_oof"

    idx = np.where(covered)[0]
    q = committee_statistics(p[idx])["committee_mean"]
    report.insert(0, {
        "row_type": "overall", "mode": mode, "block": "ALL",
        "train_rows": n, "validation_rows": len(idx), "train_failures": int(y.sum()),
        "validation_failures": int(y[idx].sum()), "validation_prevalence": float(y[idx].mean()),
        "train_max_group": np.nan, "validation_group": np.nan,
        "oof_coverage": float(covered.mean()), "temporal_source": temporal_source,
        "committee_ap": safe_ap(y[idx], q), "committee_auc": safe_auc(y[idx], q),
    })
    for i, name in enumerate(BASE_NAMES):
        report.append({
            "row_type": "expert_metric", "mode": mode, "block": "ALL", "expert": name,
            "oof_ap": safe_ap(y[idx], p[idx, i]), "oof_auc": safe_auc(y[idx], p[idx, i]),
        })
    if verbose:
        print(f"[Cross-fit] mode={mode} | source={temporal_source} | OOF={len(idx):,}/{n:,} ({100*covered.mean():.2f}%) | failures={int(y[idx].sum()):,}")
        if warning:
            print(f"[Cross-fit WARNING] {warning}")
        for row in report:
            if row.get("row_type") == "block":
                print(f"  block={row['block']} | train={row['train_rows']:,} | OOF={row['validation_rows']:,} | prevalence={100*row['validation_prevalence']:.2f}%")
        print(f"  committee OOF AP/AUC={safe_ap(y[idx], q):.4f}/{safe_auc(y[idx], q):.4f}")
        for i, name in enumerate(BASE_NAMES):
            print(f"  {name:24s} OOF AP={safe_ap(y[idx], p[idx,i]):.4f} | AUC={safe_auc(y[idx], p[idx,i]):.4f}")
    return CrossFitResult(mode, p, covered, blocks, temporal_source, report, warning)


def _block_metrics(y, mask, blocks, min_block_failures=5, min_block_rows=30):
    rows = []
    y = np.asarray(y, int)
    blocks = np.asarray(blocks)
    for block in pd.unique(blocks):
        idx = blocks == block
        n, failures = int(idx.sum()), int(y[idx].sum())
        selected = idx & mask
        k, sf = int(selected.sum()), int(y[selected].sum())
        rows.append({
            "block": str(block), "rows": n, "failures": failures,
            "selected": k, "selected_failures": sf,
            "selected_rate": k / max(n, 1),
            "selected_failure_rate": sf / max(k, 1),
            "selected_failure_fraction": sf / max(failures, 1),
            "supported": bool(n >= min_block_rows and failures >= min_block_failures),
            "precision": sf / max(k, 1),
        })
    return rows


def fit_routing_policy(
    y, expert_probabilities, block_ids,
    max_negative_exit_failure_fraction=0.05,
    max_negative_exit_failure_rate=0.01,
    positive_preserve_min_precision=0.60,
    min_route_count=5,
    min_block_failures=5,
):
    y = np.asarray(y, int).reshape(-1)
    p = np.asarray(expert_probabilities, float)
    blocks = np.asarray(block_ids)
    stats = committee_statistics(p)
    q, spread = stats["committee_mean"], stats["committee_spread"]
    rank = ranks_within_blocks(q, blocks)
    n, total_fail = len(y), int(y.sum())
    min_count = max(1, min(int(min_route_count), n))

    neg_quantiles = np.unique(np.r_[np.linspace(.05, .50, 10), np.linspace(.55, .85, 7)])
    spread_grid = np.unique(np.quantile(spread, np.linspace(.20, 1.0, 17)))
    tiers = {"strict_all_blocks": [], "pooled_plus_block_failure_fraction": [], "pooled_only": []}
    for rq in neg_quantiles:
        for sd in spread_grid:
            mask = (rank <= rq) & (spread <= sd)
            count = int(mask.sum())
            if count < min_count:
                continue
            ef = int(y[mask].sum())
            frac, rate = ef / max(total_fail, 1), ef / max(count, 1)
            if frac > max_negative_exit_failure_fraction + 1e-12 or rate > max_negative_exit_failure_rate + 1e-12:
                continue
            bm = _block_metrics(y, mask, blocks, min_block_failures=min_block_failures)
            supported = [r for r in bm if r["supported"]]
            all_both = all(r["selected_failure_fraction"] <= max_negative_exit_failure_fraction + 1e-12 and
                           r["selected_failure_rate"] <= max_negative_exit_failure_rate + 1e-12 for r in supported)
            all_frac = all(r["selected_failure_fraction"] <= max_negative_exit_failure_fraction + 1e-12 for r in supported)
            cand = (count, -frac, -rate, float(rq), float(sd), mask, bm)
            tiers["pooled_only"].append(cand)
            if all_frac:
                tiers["pooled_plus_block_failure_fraction"].append(cand)
            if all_both:
                tiers["strict_all_blocks"].append(cand)
    chosen_tier = next((t for t in ["strict_all_blocks", "pooled_plus_block_failure_fraction", "pooled_only"] if tiers[t]), "disabled")
    if chosen_tier == "disabled":
        easy = np.zeros(n, bool); neg_q = -1.0; neg_d = 0.0; neg_enabled = False; neg_blocks = _block_metrics(y, easy, blocks, min_block_failures)
    else:
        *_, neg_q, neg_d, easy, neg_blocks = max(tiers[chosen_tier], key=lambda z: z[:3])
        neg_enabled = True

    exited, exited_fail = int(easy.sum()), int(y[easy].sum())
    routed = ~easy
    overall_prev = float(y.mean()) if n else 0.0
    routed_prev = float(y[routed].mean()) if routed.any() else 0.0

    # Positive preservation: rank-aware high-risk low-disagreement region.
    pos_quantiles = np.unique(np.r_[np.linspace(.70, .95, 6), .97, .98, .99])
    pos_spread = np.unique(np.quantile(spread[routed] if routed.any() else spread, np.linspace(.20, 1.0, 17)))
    strict_pos, pooled_pos = [], []
    for rq in pos_quantiles:
        for sd in pos_spread:
            mask = routed & (rank >= rq) & (spread <= sd)
            count = int(mask.sum())
            if count < min_count:
                continue
            precision = float(y[mask].mean())
            if precision + 1e-12 < positive_preserve_min_precision:
                continue
            bm = _block_metrics(y, mask, blocks, min_block_failures=min_block_failures)
            supported_selected = [r for r in bm if r["supported"] and r["selected"] >= min_count]
            all_prec = all(r["precision"] + 1e-12 >= positive_preserve_min_precision for r in supported_selected)
            cand = (count, precision, float(rq), float(sd), mask, bm)
            pooled_pos.append(cand)
            if all_prec:
                strict_pos.append(cand)
    if strict_pos:
        pos_relax = "strict_all_supported_blocks"; pool = strict_pos
    elif pooled_pos:
        pos_relax = "pooled_precision_only"; pool = pooled_pos
    else:
        pos_relax = "disabled"; pool = []
    if pool:
        _, pos_prec, pos_q, pos_d, strong, pos_blocks = max(pool, key=lambda z: (z[0], z[1]))
        pos_enabled = True
    else:
        pos_prec, pos_q, pos_d = 0.0, 2.0, 0.0
        strong = np.zeros(n, bool); pos_blocks = _block_metrics(y, strong, blocks, min_block_failures)
        pos_enabled = False

    pos_count = int(strong.sum())
    policy = RoutingPolicy(
        negative_exit_quantile=float(neg_q), negative_max_spread=float(neg_d), negative_enabled=bool(neg_enabled),
        negative_relaxation=chosen_tier,
        max_negative_exit_failure_fraction=float(max_negative_exit_failure_fraction),
        max_negative_exit_failure_rate=float(max_negative_exit_failure_rate),
        negative_exit_rate_train=exited / max(n, 1),
        negative_exit_failure_rate_train=exited_fail / max(exited, 1),
        negative_exit_failure_fraction_train=exited_fail / max(total_fail, 1),
        routed_rate_train=float(routed.mean()) if n else 0.0,
        routed_prevalence_train=routed_prev,
        routed_enrichment_train=routed_prev / max(overall_prev, 1e-12),
        positive_rank_quantile=float(pos_q), positive_max_spread=float(pos_d),
        positive_preserve_min_precision=float(positive_preserve_min_precision),
        positive_preserve_enabled=bool(pos_enabled), positive_relaxation=pos_relax,
        positive_preserve_rate_train=pos_count / max(n, 1),
        positive_preserve_precision_train=float(pos_prec),
        positive_preserve_recall_train=float(y[strong].sum() / max(total_fail, 1)),
    )
    block_diag = []
    for row in neg_blocks:
        block_diag.append({"route_region": "easy_negative", **row})
        # The specialist region is exactly the complement of the negative exit.
        block_diag.append({
            "route_region": "specialist_routed", "block": row["block"],
            "rows": row["rows"], "failures": row["failures"],
            "selected": row["rows"] - row["selected"],
            "selected_failures": row["failures"] - row["selected_failures"],
            "selected_rate": 1.0 - row["selected_rate"],
            "selected_failure_rate": (row["failures"] - row["selected_failures"]) / max(row["rows"] - row["selected"], 1),
            "selected_failure_fraction": (row["failures"] - row["selected_failures"]) / max(row["failures"], 1),
            "supported": row["supported"],
            "precision": (row["failures"] - row["selected_failures"]) / max(row["rows"] - row["selected"], 1),
        })
    for row in pos_blocks:
        block_diag.append({"route_region": "strong_positive", **row})
    return policy, block_diag


def apply_routing_policy(expert_probabilities, policy: RoutingPolicy, block_ids=None):
    p = np.asarray(expert_probabilities, float)
    stats = committee_statistics(p)
    q, spread = stats["committee_mean"], stats["committee_spread"]
    rank = batch_percentile_rank(q) if block_ids is None else ranks_within_blocks(q, block_ids)
    easy = ((rank <= policy.negative_exit_quantile) & (spread <= policy.negative_max_spread)) if policy.negative_enabled else np.zeros(len(q), bool)
    routed = ~easy
    strong = (routed & (rank >= policy.positive_rank_quantile) & (spread <= policy.positive_max_spread)) if policy.positive_preserve_enabled else np.zeros(len(q), bool)
    return {"committee_rank": rank, "easy_negative": easy, "routed": routed, "strong_positive": strong}


def print_routing_policy(p: RoutingPolicy):
    print("\n[Rank-based routing policy learned from cross-fitted historical rows]")
    if p.negative_enabled:
        print(
            f"  Easy-negative exit : rank <= {p.negative_exit_quantile:.3f} AND spread <= {p.negative_max_spread:.6f} | mode={p.negative_relaxation}\n"
            f"                       exit={100*p.negative_exit_rate_train:.2f}% | exit failure rate={100*p.negative_exit_failure_rate_train:.3f}% | "
            f"share of failures exited={100*p.negative_exit_failure_fraction_train:.2f}%"
        )
    else:
        print("  Easy-negative exit : DISABLED (no pooled-safe candidate found)")
    print(f"  Specialist routing : {100*p.routed_rate_train:.2f}% | prevalence={100*p.routed_prevalence_train:.2f}% | enrichment={p.routed_enrichment_train:.2f}x")
    if p.positive_preserve_enabled:
        print(
            f"  Positive preserve  : rank >= {p.positive_rank_quantile:.3f} AND spread <= {p.positive_max_spread:.6f} | mode={p.positive_relaxation}\n"
            f"                       coverage={100*p.positive_preserve_rate_train:.2f}% | precision={100*p.positive_preserve_precision_train:.2f}% | "
            f"failure recall={100*p.positive_preserve_recall_train:.2f}%"
        )
    else:
        print("  Positive preserve  : DISABLED (minimum precision not met)")


def choose_specialist_validation(y, block_ids, crossfit_mode, seed=42, min_val_rows=30):
    y = np.asarray(y, int).reshape(-1)
    blocks = np.asarray(block_ids)
    n = len(y)
    if crossfit_mode == "temporal_expanding" and n >= 2:
        unique = list(pd.unique(blocks))
        # Crossfit blocks are chronological strings of numeric values; sort numerically when possible.
        try:
            unique = sorted(unique, key=float)
        except Exception:
            unique = sorted(unique, key=str)
        val_blocks = []
        for block in reversed(unique):
            val_blocks.append(block)
            val = np.isin(blocks, val_blocks)
            tr = ~val
            if val.sum() >= min_val_rows and tr.sum() >= min_val_rows and np.unique(y[val]).size == 2 and np.unique(y[tr]).size == 2:
                return np.where(tr)[0], np.where(val)[0], "temporal_latest_blocks", ",".join(map(str, reversed(val_blocks)))
    if n < 4 or np.unique(y).size < 2:
        return np.arange(n), np.array([], dtype=int), "unavailable", ""
    tr, va = train_test_split(np.arange(n), test_size=.20, stratify=y, random_state=int(seed))
    return np.asarray(tr), np.asarray(va), "stratified_fallback", ""


def build_specialist_features(X_raw, expert_probabilities, selector: RawFeatureSelector):
    p = np.asarray(expert_probabilities, float)
    Xsel = selector.transform(X_raw)
    stats = committee_statistics(p)
    U = np.column_stack([stats[n] for n in UNCERTAINTY_NAMES])
    Z = np.column_stack([Xsel, p, U])
    names = list(selector.selected_feature_names_) + [f"expert_probability_{n}" for n in BASE_NAMES] + list(UNCERTAINTY_NAMES)
    return Z, names


def max_fbeta_score(y, score, beta=2.0):
    y = np.asarray(y, int); score = np.asarray(score, float)
    if not len(y) or np.unique(y).size < 2:
        return float("nan")
    thresholds = np.unique(score)
    if len(thresholds) > 300:
        thresholds = np.unique(np.quantile(thresholds, np.linspace(.01, .99, 300)))
    best = 0.0
    b2 = beta * beta
    for t in thresholds:
        pred = score >= t
        tp = np.sum((pred == 1) & (y == 1)); fp = np.sum((pred == 1) & (y == 0)); fn = np.sum((pred == 0) & (y == 1))
        p = tp / max(tp + fp, 1); r = tp / max(tp + fn, 1)
        den = b2 * p + r
        f = 0.0 if den <= 0 else (1 + b2) * p * r / den
        best = max(best, f)
    return float(best)


class NonlinearBoundedResidualSpecialist:
    """Compact nonlinear residual specialist selected on forward temporal validation."""

    def __init__(self, max_epochs=300, lr=1e-3, weight_decay=1e-3, effective_beta=.9999,
                 weight_gamma_grid=(0.0,.5,1.0), hidden_dims=(32,16), dropout=.20,
                 positive_cap=2.0, negative_cap=.75, residual_penalty=.05,
                 patience=30, min_improvement=1e-4, device="auto", seed=42, verbose=True):
        self.max_epochs=int(max_epochs); self.lr=float(lr); self.weight_decay=float(weight_decay)
        self.effective_beta=float(effective_beta); self.weight_gamma_grid=sorted(set(float(x) for x in weight_gamma_grid))
        self.hidden_dims=tuple(int(h) for h in hidden_dims); self.dropout=float(dropout)
        self.positive_cap=float(positive_cap); self.negative_cap=float(negative_cap); self.residual_penalty=float(residual_penalty)
        self.patience=int(patience); self.min_improvement=float(min_improvement); self.device=resolve_device(device); self.seed=int(seed); self.verbose=bool(verbose)
        self.scaler_: StandardScaler|None=None; self.model_: ResidualMLP|None=None; self.history_:list[dict]=[]
        self.candidate_summaries_:list[dict]=[]; self.selected_gamma_:float=0.0; self.selected_weight_stats_:dict={}
        self.best_epoch_=0; self.best_val_bce_=float("nan"); self.best_val_ap_=float("nan"); self.best_val_auc_=float("nan"); self.best_val_f2_=float("nan")
        self.n_parameters_=0; self.refit_rows_=0; self.refit_failures_=0

    def _new_model(self, input_dim):
        model=ResidualMLP(input_dim,self.hidden_dims,self.dropout).to(self.device)
        self.n_parameters_=sum(p.numel() for p in model.parameters() if p.requires_grad)
        return model

    def _delta(self, raw):
        return asymmetric_bounded_delta_torch(raw,self.positive_cap,self.negative_cap)

    @staticmethod
    def _metric_key(ap, auc, f2, bce):
        def finite(v, bad=-np.inf): return float(v) if np.isfinite(v) else bad
        return (finite(ap), finite(auc), finite(f2), -finite(bce, bad=np.inf))

    def _evaluate_tensor(self, model, z_t, q_t, y_t, w_t):
        model.eval()
        with torch.no_grad():
            raw=model(z_t); delta=self._delta(raw); base=torch.logit(q_t.clamp(1e-6,1-1e-6)); logits=base+delta
            bce=F.binary_cross_entropy_with_logits(logits,y_t,weight=w_t); prob=torch.sigmoid(logits)
        return float(bce.cpu()),prob.cpu().numpy(),delta.cpu().numpy(),raw.cpu().numpy()

    def fit_select(self,Z_train,q_train,y_train,Z_val,q_val,y_val,beta_for_f2=2.0):
        Z_train,Z_val=clean_X(Z_train),clean_X(Z_val); q_train,q_val=clip_probability(q_train),clip_probability(q_val)
        y_train,y_val=np.asarray(y_train,int),np.asarray(y_val,int)
        self.scaler_=StandardScaler().fit(Z_train)
        ztr=torch.from_numpy(self.scaler_.transform(Z_train).astype(np.float32)).to(self.device); zva=torch.from_numpy(self.scaler_.transform(Z_val).astype(np.float32)).to(self.device)
        qtr=torch.from_numpy(q_train.astype(np.float32)).to(self.device); qva=torch.from_numpy(q_val.astype(np.float32)).to(self.device)
        ytr=torch.from_numpy(y_train.astype(np.float32)).to(self.device); yva=torch.from_numpy(y_val.astype(np.float32)).to(self.device)
        _,eff_stats=effective_sample_weights(y_train,self.effective_beta)
        all_history=[]; candidate_states=[]; self.candidate_summaries_=[]
        for gi,gamma in enumerate(self.weight_gamma_grid):
            seed_all(self.seed)  # identical initialization/dropout stream for fair gamma comparison
            wtr_np,wstats=tempered_sample_weights(y_train,eff_stats,gamma); wva_np=sample_weights_from_stats(y_val,wstats)
            wtr=torch.from_numpy(wtr_np).to(self.device); wva=torch.from_numpy(wva_np).to(self.device)
            model=self._new_model(Z_train.shape[1]); opt=torch.optim.AdamW(model.parameters(),lr=self.lr,weight_decay=self.weight_decay)
            base_tr=torch.logit(qtr.clamp(1e-6,1-1e-6)); best_state=copy.deepcopy(model.state_dict()); best_row=None; best_key=None; bad=0
            gamma_history=[]
            for epoch in range(1,self.max_epochs+1):
                model.train(); raw=model(ztr); delta=self._delta(raw); logits=base_tr+delta
                bce=F.binary_cross_entropy_with_logits(logits,ytr,weight=wtr); penalty=self.residual_penalty*torch.mean(delta**2); total=bce+penalty
                opt.zero_grad(); total.backward(); opt.step()
                vbce,vprob,vdelta,_=self._evaluate_tensor(model,zva,qva,yva,wva)
                vap=safe_ap(y_val,vprob); vauc=safe_auc(y_val,vprob); vf2=max_fbeta_score(y_val,vprob,beta_for_f2)
                row={"phase":"selection","gamma":gamma,"epoch":epoch,"train_tempered_bce":float(bce.detach().cpu()),"residual_penalty_term":float(penalty.detach().cpu()),"total_loss":float(total.detach().cpu()),"val_tempered_bce":vbce,"val_ap":vap,"val_auc":vauc,"val_best_f2":vf2,"delta_mean":float(delta.detach().mean().cpu()),"delta_std":float(delta.detach().std(unbiased=False).cpu()),"delta_min":float(delta.detach().min().cpu()),"delta_max":float(delta.detach().max().cpu()),"delta_increase_fraction":float((delta.detach()>0).float().mean().cpu()),"delta_decrease_fraction":float((delta.detach()<0).float().mean().cpu())}
                gamma_history.append(row); key=self._metric_key(vap,vauc,vf2,vbce)
                improved = best_key is None or key[0] > best_key[0] + self.min_improvement or (abs(key[0]-best_key[0]) <= self.min_improvement and key[1:] > best_key[1:])
                if improved:
                    best_key=key; best_row=row.copy(); best_state=copy.deepcopy(model.state_dict()); bad=0
                else: bad+=1
                if self.verbose and (epoch==1 or epoch%max(1,self.max_epochs//5)==0):
                    print(f"  gamma={gamma:.2f} epoch {epoch:4d}/{self.max_epochs} | BCE={row['train_tempered_bce']:.6f}+pen={row['residual_penalty_term']:.6f} | val AP/AUC/F2={vap:.4f}/{vauc:.4f}/{vf2:.4f} BCE={vbce:.6f} | delta={row['delta_mean']:+.3f}±{row['delta_std']:.3f} range=[{row['delta_min']:+.3f},{row['delta_max']:+.3f}]")
                if bad>=self.patience: break
            assert best_row is not None
            all_history.extend(gamma_history); candidate_states.append((gamma,best_row,best_state,wstats))
            self.candidate_summaries_.append({"gamma":gamma,"best_epoch":int(best_row['epoch']),"val_ap":best_row['val_ap'],"val_auc":best_row['val_auc'],"val_best_f2":best_row['val_best_f2'],"val_tempered_bce":best_row['val_tempered_bce'],"failure_weight":wstats['failure_weight'],"normal_weight":wstats['normal_weight'],"tempered_ratio":wstats['tempered_ratio'],"base_effective_ratio":wstats['base_effective_ratio']})
        def ckey(item):
            gamma,row,_,_=item; return (*self._metric_key(row['val_ap'],row['val_auc'],row['val_best_f2'],row['val_tempered_bce']),-gamma)
        gamma,brow,bstate,wstats=max(candidate_states,key=ckey)
        self.selected_gamma_=float(gamma); self.selected_weight_stats_=dict(wstats); self.best_epoch_=int(brow['epoch']); self.best_val_bce_=float(brow['val_tempered_bce']); self.best_val_ap_=float(brow['val_ap']); self.best_val_auc_=float(brow['val_auc']); self.best_val_f2_=float(brow['val_best_f2'])
        self.model_=self._new_model(Z_train.shape[1]); self.model_.load_state_dict(bstate); self.history_=all_history
        return self

    def predict_delta(self,Z):
        if self.scaler_ is None or self.model_ is None: raise RuntimeError("Specialist not fitted")
        z=torch.from_numpy(self.scaler_.transform(clean_X(Z)).astype(np.float32)).to(self.device); self.model_.eval()
        with torch.no_grad(): raw=self.model_(z); delta=self._delta(raw)
        return delta.cpu().numpy(),raw.cpu().numpy()

    def refit_all(self,Z,q,y,epochs):
        seed_all(self.seed); Z=clean_X(Z); q=clip_probability(q); y=np.asarray(y,int); self.refit_rows_=len(y); self.refit_failures_=int(y.sum())
        self.scaler_=StandardScaler().fit(Z); z=torch.from_numpy(self.scaler_.transform(Z).astype(np.float32)).to(self.device); qt=torch.from_numpy(q.astype(np.float32)).to(self.device); yt=torch.from_numpy(y.astype(np.float32)).to(self.device)
        _,eff_stats=effective_sample_weights(y,self.effective_beta); w_np,wstats=tempered_sample_weights(y,eff_stats,self.selected_gamma_); self.selected_weight_stats_=dict(wstats); wt=torch.from_numpy(w_np).to(self.device)
        self.model_=self._new_model(Z.shape[1]); opt=torch.optim.AdamW(self.model_.parameters(),lr=self.lr,weight_decay=self.weight_decay); base=torch.logit(qt.clamp(1e-6,1-1e-6)); ref=[]
        for epoch in range(1,int(epochs)+1):
            self.model_.train(); raw=self.model_(z); delta=self._delta(raw); logits=base+delta; bce=F.binary_cross_entropy_with_logits(logits,yt,weight=wt); penalty=self.residual_penalty*torch.mean(delta**2); total=bce+penalty
            opt.zero_grad(); total.backward(); opt.step()
            if epoch==1 or epoch==int(epochs) or epoch%max(1,int(epochs)//5)==0:
                self.model_.eval()
                with torch.no_grad(): prob=torch.sigmoid(base+self._delta(self.model_(z))).cpu().numpy(); d=self._delta(self.model_(z)).cpu().numpy()
                row={"phase":"refit_all","gamma":self.selected_gamma_,"epoch":epoch,"train_tempered_bce":float(bce.detach().cpu()),"residual_penalty_term":float(penalty.detach().cpu()),"total_loss":float(total.detach().cpu()),"val_tempered_bce":np.nan,"val_ap":safe_ap(y,prob),"val_auc":safe_auc(y,prob),"val_best_f2":max_fbeta_score(y,prob,2.0),"delta_mean":float(np.mean(d)),"delta_std":float(np.std(d)),"delta_min":float(np.min(d)),"delta_max":float(np.max(d)),"delta_increase_fraction":float(np.mean(d>0)),"delta_decrease_fraction":float(np.mean(d<0))}; ref.append(row)
                if self.verbose: print(f"  specialist refit {epoch:4d}/{epochs} | gamma={self.selected_gamma_:.2f} BCE={row['train_tempered_bce']:.6f}+pen={row['residual_penalty_term']:.6f} | AP/AUC={row['val_ap']:.4f}/{row['val_auc']:.4f} | delta={row['delta_mean']:+.3f}±{row['delta_std']:.3f}")
        self.history_.extend(ref); return self


def authority_ceiling(n_selection):
    n = int(n_selection)
    if n < 500: return .25
    if n < 1000: return .50
    if n < 2000: return .75
    return 1.0


def select_authority_safe(y_val,q_val,delta_val,grid,n_selection_rows,beta=2.0,ap_tolerance=.005,auc_tolerance=.005):
    ceiling=authority_ceiling(n_selection_rows); committee_ap=safe_ap(y_val,q_val); committee_auc=safe_auc(y_val,q_val); candidates=[]
    allowed=sorted(set(float(x) for x in grid if float(x)<=ceiling+1e-12)|{0.0})
    for a in allowed:
        score=apply_residual_correction(q_val,delta_val,a); ap=safe_ap(y_val,score); auc=safe_auc(y_val,score); f2=max_fbeta_score(y_val,score,beta)
        safe=(not np.isfinite(committee_ap) or ap>=committee_ap-float(ap_tolerance)-1e-12) and (not np.isfinite(committee_auc) or auc>=committee_auc-float(auc_tolerance)-1e-12)
        candidates.append({"authority":a,"ap":ap,"auc":auc,"best_f2":f2,"support_ceiling":ceiling,"committee_ap":committee_ap,"committee_auc":committee_auc,"ap_delta_vs_committee":ap-committee_ap,"auc_delta_vs_committee":auc-committee_auc,"ranking_safe":bool(safe)})
    safe_rows=[r for r in candidates if r['ranking_safe']]
    if not safe_rows:
        return 0.0,candidates,float(ceiling),"fallback_to_committee_no_ranking_safe_candidate"
    def key(r):
        vals=[r['ap'],r['best_f2'],r['auc']]; vals=[-np.inf if not np.isfinite(v) else v for v in vals]; return (*vals,-r['authority'])
    chosen=max(safe_rows,key=key); reason="committee_selected" if chosen['authority']==0 else "best_ranking_safe_candidate"
    return float(chosen['authority']),candidates,float(ceiling),reason


def combine_cascade_scores(committee_score, specialist_probability, easy_negative, strong_positive):
    q = clip_probability(committee_score)
    sp = np.asarray(specialist_probability, float).reshape(-1)
    easy = np.asarray(easy_negative, bool); strong = np.asarray(strong_positive, bool)
    out = q.copy(); routed = ~easy; usable = routed & np.isfinite(sp); out[usable] = sp[usable]
    protected = strong & usable; out[protected] = np.maximum(out[protected], q[protected])
    return clip_probability(out)


class SelectiveCascadeEnsemble:
    def __init__(self, random_state=42, n_estimators=500, hist_max_iter=250,
                 max_negative_exit_failure_fraction=.05, max_negative_exit_failure_rate=.01,
                 positive_preserve_min_precision=.60, min_route_count=5, routing_min_block_failures=5,
                 crossfit_folds=5, crossfit_min_train_rows=100, min_oof_coverage=.40,
                 specialist_top_raw_features=48, specialist_epochs=300, specialist_lr=1e-3,
                 specialist_weight_decay=1e-3, effective_beta=.9999, weight_gamma_grid=(0,.5,1.0),
                 specialist_hidden_dims=(32,16), specialist_dropout=.20, specialist_positive_cap=2.0,
                 specialist_negative_cap=.75, residual_penalty=.05, specialist_patience=30,
                 specialist_min_improvement=1e-4, authority_grid=(0,.25,.5,.75,1.0),
                 authority_ap_tolerance=.005, authority_auc_tolerance=.005,
                 hard_negative_guard=False, device="auto", verbose=True):
        self.seed=int(random_state); self.n_estimators=int(n_estimators); self.hist_max_iter=int(hist_max_iter)
        self.max_negative_exit_failure_fraction=float(max_negative_exit_failure_fraction); self.max_negative_exit_failure_rate=float(max_negative_exit_failure_rate)
        self.positive_preserve_min_precision=float(positive_preserve_min_precision); self.min_route_count=int(min_route_count); self.routing_min_block_failures=int(routing_min_block_failures)
        self.crossfit_folds=int(crossfit_folds); self.crossfit_min_train_rows=int(crossfit_min_train_rows); self.min_oof_coverage=float(min_oof_coverage)
        self.specialist_top_raw_features=int(specialist_top_raw_features); self.specialist_epochs=int(specialist_epochs); self.specialist_lr=float(specialist_lr); self.specialist_weight_decay=float(specialist_weight_decay)
        self.effective_beta=float(effective_beta); self.weight_gamma_grid=list(weight_gamma_grid); self.specialist_hidden_dims=tuple(int(h) for h in specialist_hidden_dims); self.specialist_dropout=float(specialist_dropout)
        self.specialist_positive_cap=float(specialist_positive_cap); self.specialist_negative_cap=float(specialist_negative_cap); self.residual_penalty=float(residual_penalty)
        self.specialist_patience=int(specialist_patience); self.specialist_min_improvement=float(specialist_min_improvement); self.authority_grid=list(authority_grid); self.authority_ap_tolerance=float(authority_ap_tolerance); self.authority_auc_tolerance=float(authority_auc_tolerance)
        self.hard_negative_guard=bool(hard_negative_guard); self.device_arg=device; self.verbose=bool(verbose)
        self.base_models_: Dict[str, object] = {}; self.base_report_: list[dict]=[]; self.crossfit_report_: list[dict]=[]
        self.routing_policy_: RoutingPolicy|None=None; self.routing_block_diagnostics_: list[dict]=[]; self.route_train_diagnostics_: dict={}
        self.selector_: RawFeatureSelector|None=None; self.selection_feature_selection_report_: list[dict]=[]; self.feature_selection_report_: list[dict]=[]; self.specialist_: NonlinearBoundedResidualSpecialist|None=None
        self.specialist_feature_names_: list[str]=[]; self.specialist_summary_: dict={}; self.weight_gamma_candidates_: list[dict]=[]; self.authority_candidates_: list[dict]=[]; self.authority_=0.0; self.authority_reason_=""
        self.feature_names_: list[str]=[]; self.crossfit_mode_=""; self.temporal_source_=""; self.is_fitted_=False

    def expert_probabilities(self, X):
        if not self.base_models_: raise RuntimeError("Production experts not fitted")
        return predict_experts(self.base_models_, X)

    def fit(self, X, y, train_frame, feature_names):
        seed_all(self.seed); X=clean_X(X); y=np.asarray(y,int); self.feature_names_=list(feature_names)
        groups, source = extract_temporal_groups(train_frame); self.temporal_source_=source
        cf = generate_crossfit_predictions(X,y,groups,source,seed=self.seed,n_estimators=self.n_estimators,hist_max_iter=self.hist_max_iter,crossfit_folds=self.crossfit_folds,min_train_rows=self.crossfit_min_train_rows,min_oof_coverage=self.min_oof_coverage,verbose=self.verbose)
        self.crossfit_mode_=cf.mode; self.crossfit_report_=cf.report_rows
        idx=np.where(cf.covered_mask)[0]; p_oof=cf.probabilities[idx]; y_oof=y[idx]; X_oof=X[idx]; blocks=cf.block_ids[idx]
        q_oof=committee_statistics(p_oof)["committee_mean"]
        self.base_report_=[{"model":name,"oof_ap":safe_ap(y_oof,p_oof[:,i]),"oof_auc":safe_auc(y_oof,p_oof[:,i]),"crossfit_mode":cf.mode} for i,name in enumerate(BASE_NAMES)]
        self.base_report_.append({"model":"committee_mean","oof_ap":safe_ap(y_oof,q_oof),"oof_auc":safe_auc(y_oof,q_oof),"crossfit_mode":cf.mode})

        self.routing_policy_, self.routing_block_diagnostics_ = fit_routing_policy(y_oof,p_oof,blocks,self.max_negative_exit_failure_fraction,self.max_negative_exit_failure_rate,self.positive_preserve_min_precision,self.min_route_count,self.routing_min_block_failures)
        masks=apply_routing_policy(p_oof,self.routing_policy_,blocks); routed=masks["routed"]
        if self.verbose:
            print_routing_policy(self.routing_policy_)
            print("[Per-block routing diagnostics]")
            for row in self.routing_block_diagnostics_:
                if row["route_region"] in {"easy_negative", "specialist_routed"}:
                    print(
                        f"  {row['route_region']:18s} block={row['block']} | "
                        f"selected={row['selected']:,}/{row['rows']:,} ({100*row['selected_rate']:.2f}%) | "
                        f"failure prevalence={100*row['selected_failure_rate']:.2f}% | "
                        f"failure share={100*row['selected_failure_fraction']:.2f}%"
                    )
        routed_idx=np.where(routed)[0]; yr=y_oof[routed]; Xr=X_oof[routed]; pr=p_oof[routed]; qr=q_oof[routed]; br=blocks[routed]
        overall_prev=float(y_oof.mean()); routed_prev=float(yr.mean()) if len(yr) else 0.0
        if self.verbose:
            print(f"[Specialist population] rows={len(yr):,}/{len(y_oof):,} ({100*len(yr)/max(len(y_oof),1):.2f}%) | failures={int(yr.sum()):,} | prevalence={100*routed_prev:.2f}% | enrichment={routed_prev/max(overall_prev,1e-12):.2f}x")

        self.authority_=0.0; self.specialist_=None; self.selector_=None
        if len(yr) >= max(20,self.min_route_count*2) and np.unique(yr).size==2:
            tr_rel, va_rel, val_mode, val_blocks = choose_specialist_validation(yr,br,cf.mode,self.seed)
            if len(va_rel) and np.unique(yr[tr_rel]).size==2 and np.unique(yr[va_rel]).size==2:
                selection_selector=RawFeatureSelector(self.specialist_top_raw_features).fit(Xr[tr_rel],yr[tr_rel],self.feature_names_)
                self.selection_feature_selection_report_=selection_selector.report_
                Zsel,names_sel=build_specialist_features(Xr,pr,selection_selector)
                if self.verbose:
                    top=sorted(selection_selector.report_,key=lambda z:z["rank"])[:min(12,len(selection_selector.report_))]
                    print(f"[Feature selection — selection train only] selected={len(selection_selector.selected_indices_):,}/{X.shape[1]:,} raw -> specialist input={Zsel.shape[1]:,}")
                    print("  selected raw features:", ", ".join(selection_selector.selected_feature_names_))
                    print("  top F-scores:", ", ".join(f"{r['feature']}({r['f_score']:.2f})" for r in top if r['selected']))
                    print(f"[Specialist temporal validation] mode={val_mode} | train={len(tr_rel):,} ({100*yr[tr_rel].mean():.2f}% failures) | val={len(va_rel):,} ({100*yr[va_rel].mean():.2f}% failures) | val blocks={val_blocks or 'n/a'}")
                spec=NonlinearBoundedResidualSpecialist(self.specialist_epochs,self.specialist_lr,self.specialist_weight_decay,self.effective_beta,self.weight_gamma_grid,self.specialist_hidden_dims,self.specialist_dropout,self.specialist_positive_cap,self.specialist_negative_cap,self.residual_penalty,self.specialist_patience,self.specialist_min_improvement,self.device_arg,self.seed,self.verbose)
                if self.verbose:
                    probe=ResidualMLP(Zsel.shape[1],self.specialist_hidden_dims,self.specialist_dropout)
                    nparam=sum(p.numel() for p in probe.parameters() if p.requires_grad)
                    print(f"[Specialist architecture] input={Zsel.shape[1]} -> {' -> '.join(map(str,self.specialist_hidden_dims))} -> 1 | dropout={self.specialist_dropout:.2f} | params={nparam:,} | selection rows/param={len(tr_rel)/max(nparam,1):.3f} | selection failures/param={int(yr[tr_rel].sum())/max(nparam,1):.4f}")
                spec.fit_select(Zsel[tr_rel],qr[tr_rel],yr[tr_rel],Zsel[va_rel],qr[va_rel],yr[va_rel],2.0)
                self.weight_gamma_candidates_=spec.candidate_summaries_
                if self.verbose:
                    print("[Failure-weight tempering candidates]")
                    for row in self.weight_gamma_candidates_: print(f"  gamma={row['gamma']:.2f} | epoch={row['best_epoch']:3d} | AP/AUC/F2={row['val_ap']:.4f}/{row['val_auc']:.4f}/{row['val_best_f2']:.4f} | BCE={row['val_tempered_bce']:.6f} | weights={row['failure_weight']:.3f}/{row['normal_weight']:.3f} ratio={row['tempered_ratio']:.2f}")
                    print(f"  SELECTED gamma={spec.selected_gamma_:.2f}")
                delta_val,_=spec.predict_delta(Zsel[va_rel]); self.authority_,self.authority_candidates_,ceiling,self.authority_reason_=select_authority_safe(yr[va_rel],qr[va_rel],delta_val,self.authority_grid,len(tr_rel),2.0,self.authority_ap_tolerance,self.authority_auc_tolerance)
                if self.verbose:
                    print(f"[Specialist authority] support ceiling={ceiling:.2f} | AP tolerance={self.authority_ap_tolerance:.4f} | AUC tolerance={self.authority_auc_tolerance:.4f}")
                    for row in self.authority_candidates_: print(f"  authority={row['authority']:.2f} | AP={row['ap']:.4f} ({row['ap_delta_vs_committee']:+.4f}) | AUC={row['auc']:.4f} ({row['auc_delta_vs_committee']:+.4f}) | bestF2={row['best_f2']:.4f} | safe={row['ranking_safe']}")
                    print(f"  SELECTED authority={self.authority_:.2f} | reason={self.authority_reason_}")
                best_epoch=max(1,spec.best_epoch_)
                # Final feature selector is refit on all routed OOF rows only after temporal model selection.
                self.selector_=RawFeatureSelector(self.specialist_top_raw_features).fit(Xr,yr,self.feature_names_); self.feature_selection_report_=self.selector_.report_
                Zr,names=build_specialist_features(Xr,pr,self.selector_); self.specialist_feature_names_=names
                overlap=len(set(selection_selector.selected_feature_names_)&set(self.selector_.selected_feature_names_))
                spec.refit_all(Zr,qr,yr,best_epoch); self.specialist_=spec
                rows_per_param=len(yr)/max(spec.n_parameters_,1); fails_per_param=int(yr.sum())/max(spec.n_parameters_,1)
                if self.verbose:
                    print(f"[Final specialist refit] input={Zr.shape[1]} -> {' -> '.join(map(str,self.specialist_hidden_dims))} -> 1 | params={spec.n_parameters_:,} | rows={len(yr):,} ({rows_per_param:.3f}/param) | failures={int(yr.sum()):,} ({fails_per_param:.4f}/param) | feature overlap selection/final={overlap}/{len(self.selector_.selected_feature_names_)}")
                    print("  final selected raw features:", ", ".join(self.selector_.selected_feature_names_))
                self.specialist_summary_={"enabled":True,"type":"compact_nonlinear_bounded_residual_mlp","crossfit_mode":cf.mode,"validation_mode":val_mode,"validation_blocks":val_blocks,"routed_rows":len(yr),"routed_failures":int(yr.sum()),"selection_train_rows":len(tr_rel),"selection_val_rows":len(va_rel),"selected_raw_features":len(self.selector_.selected_indices_),"specialist_input_dim":Zr.shape[1],"hidden_dims":"x".join(map(str,self.specialist_hidden_dims)),"dropout":self.specialist_dropout,"trainable_parameters":spec.n_parameters_,"rows_per_parameter":rows_per_param,"failures_per_parameter":fails_per_param,"selected_epoch":best_epoch,"selected_weight_gamma":spec.selected_gamma_,"selected_failure_weight":spec.selected_weight_stats_.get('failure_weight',np.nan),"selected_normal_weight":spec.selected_weight_stats_.get('normal_weight',np.nan),"selected_tempered_ratio":spec.selected_weight_stats_.get('tempered_ratio',np.nan),"best_val_tempered_bce":spec.best_val_bce_,"best_val_ap":spec.best_val_ap_,"best_val_auc":spec.best_val_auc_,"best_val_f2":spec.best_val_f2_,"authority":self.authority_,"authority_reason":self.authority_reason_,"authority_support_ceiling":ceiling,"authority_ap_tolerance":self.authority_ap_tolerance,"authority_auc_tolerance":self.authority_auc_tolerance,"positive_cap":self.specialist_positive_cap,"negative_cap":self.specialist_negative_cap,"residual_penalty":self.residual_penalty,"selection_final_feature_overlap":overlap}
            else:
                self.specialist_summary_={"enabled":False,"reason":"temporal/stratified specialist validation unavailable","authority":0.0}
        else:
            self.specialist_summary_={"enabled":False,"reason":"routed population too small or single-class","authority":0.0}
        if not self.specialist_summary_.get("enabled") and self.verbose: print(f"[Specialist] DISABLED -> committee authority 100% ({self.specialist_summary_.get('reason')})")

        # Cross-fitted historical diagnostics with final refit specialist.
        c_oof=self._components_from_known(X_oof,p_oof,block_ids=blocks); self.route_train_diagnostics_=routing_diagnostics(y_oof,c_oof,self.specialist_positive_cap,self.specialist_negative_cap,self.authority_)
        if self.verbose: print_routing_diagnostics("CROSS-FITTED HISTORICAL",self.route_train_diagnostics_)

        # Production experts use ALL historical rows.
        self.base_models_=make_base_models(self.seed,self.n_estimators,self.hist_max_iter)
        if self.verbose: print(f"[Production experts] fitting all historical rows: {len(X):,}")
        for model in self.base_models_.values(): model.fit(X,y)
        self.crossfit_report_.append({
            "row_type": "production_fit", "mode": cf.mode, "block": "FULL_HISTORY",
            "train_rows": len(X), "train_failures": int(y.sum()),
            "validation_rows": 0, "validation_failures": 0, "validation_prevalence": np.nan,
            "temporal_source": source,
        })
        self.is_fitted_=True; return self

    def _components_from_known(self,X,p,block_ids=None):
        p=np.asarray(p,float); stats=committee_statistics(p); q=stats["committee_mean"]
        masks=apply_routing_policy(p,self.routing_policy_,block_ids); routed=masks["routed"]; strong=masks["strong_positive"]
        sp=np.full(len(q),np.nan); delta=np.full(len(q),np.nan); raw_delta=np.full(len(q),np.nan)
        if self.specialist_ is not None and self.selector_ is not None and routed.any():
            Z,_=build_specialist_features(clean_X(X)[routed],p[routed],self.selector_); d,raw=self.specialist_.predict_delta(Z)
            pred=apply_residual_correction(q[routed],d,self.authority_); sp[routed]=pred; delta[routed]=d; raw_delta[routed]=raw
        else:
            sp[routed]=q[routed]; delta[routed]=0.; raw_delta[routed]=0.
        sp_full=np.full(len(q),np.nan); valid_delta=routed & np.isfinite(delta); sp_full[valid_delta]=apply_residual_correction(q[valid_delta],delta[valid_delta],1.0); sp_full[~routed]=q[~routed]
        final=combine_cascade_scores(q,sp,masks["easy_negative"],strong)
        full_final=combine_cascade_scores(q,sp_full,masks["easy_negative"],strong)
        return {"expert_probabilities":p,**stats,**masks,"specialist_probability":sp,"specialist_probability_full_authority":sp_full,"residual_delta":delta,"raw_residual":raw_delta,"specialist_authority":np.full(len(q),self.authority_),"final_probability":final,"full_authority_counterfactual":full_final}

    def predict_components(self,X):
        if not self.is_fitted_: raise RuntimeError("Cascade not fitted")
        X=clean_X(X); return self._components_from_known(X,self.expert_probabilities(X))

    def predict_proba(self,X): return self.predict_components(X)["final_probability"]

    def save(self,path):
        specialist_payload=None
        if self.specialist_ is not None and self.specialist_.model_ is not None:
            specialist_payload={"type":"compact_nonlinear_bounded_residual_mlp","scaler":self.specialist_.scaler_,"state_dict":{k:v.detach().cpu() for k,v in self.specialist_.model_.state_dict().items()},"input_dim":len(self.specialist_feature_names_),"hidden_dims":self.specialist_hidden_dims,"dropout":self.specialist_dropout,"trainable_parameters":self.specialist_.n_parameters_,"selected_weight_gamma":self.specialist_.selected_gamma_,"authority":self.authority_,"authority_reason":self.authority_reason_,"positive_cap":self.specialist_positive_cap,"negative_cap":self.specialist_negative_cap,"residual_penalty":self.residual_penalty}
        payload={"target":TARGET,"failure_boundary":FAILURE_BOUNDARY,"feature_names":self.feature_names_,"base_model_names":BASE_NAMES,"base_models":self.base_models_,"crossfit_mode":self.crossfit_mode_,"temporal_source":self.temporal_source_,"routing_policy":asdict(self.routing_policy_) if self.routing_policy_ else None,"raw_feature_selector":self.selector_,"specialist_feature_names":self.specialist_feature_names_,"specialist":specialist_payload,"specialist_summary":self.specialist_summary_,"hard_negative_guard":self.hard_negative_guard}
        with Path(path).open("wb") as f: pickle.dump(payload,f)


def routing_diagnostics(y,components,positive_cap=2.0,negative_cap=.75,authority=1.0):
    y=np.asarray(y,int); n=len(y); total_fail=int(y.sum()); easy=np.asarray(components["easy_negative"],bool); routed=np.asarray(components["routed"],bool); strong=np.asarray(components["strong_positive"],bool)
    q=np.asarray(components["committee_mean"],float); final=np.asarray(components["final_probability"],float); sp=np.asarray(components["specialist_probability"],float); sp_full=np.asarray(components.get("specialist_probability_full_authority",sp),float); delta=np.asarray(components["residual_delta"],float)
    exit_count=int(easy.sum()); exit_fail=int(y[easy].sum()); routed_count=int(routed.sum()); routed_fail=int(y[routed].sum()); strong_count=int(strong.sum()); strong_fail=int(y[strong].sum())
    valid=routed & np.isfinite(sp); fd=delta[routed & np.isfinite(delta)]; protection=strong & valid & (sp<q)
    sat=np.zeros(len(fd),bool)
    if len(fd): sat=(fd>=.98*positive_cap)|(fd<=-.98*negative_cap)
    return {"n":n,"failures":total_fail,"prevalence":float(y.mean()) if n else 0.,"easy_negative_count":exit_count,"easy_negative_rate":exit_count/max(n,1),"easy_negative_failures":exit_fail,"easy_negative_failure_rate":exit_fail/max(exit_count,1),"easy_negative_failure_fraction_of_all_failures":exit_fail/max(total_fail,1),"routed_count":routed_count,"routing_rate":routed_count/max(n,1),"routed_failures":routed_fail,"routed_failure_prevalence":routed_fail/max(routed_count,1),"routed_enrichment":(routed_fail/max(routed_count,1))/max(float(y.mean()) if n else 0.,1e-12),"strong_positive_count":strong_count,"strong_positive_rate":strong_count/max(n,1),"strong_positive_precision":strong_fail/max(strong_count,1),"strong_positive_recall":strong_fail/max(total_fail,1),"positive_score_protection_count":int(protection.sum()),"committee_ap":safe_ap(y,q),"committee_auc":safe_auc(y,q),"final_ap":safe_ap(y,final),"final_auc":safe_auc(y,final),"routed_committee_ap":safe_ap(y[routed],q[routed]) if routed.any() else np.nan,"routed_committee_auc":safe_auc(y[routed],q[routed]) if routed.any() else np.nan,"routed_specialist_ap":safe_ap(y[valid],sp[valid]) if valid.any() else np.nan,"routed_specialist_auc":safe_auc(y[valid],sp[valid]) if valid.any() else np.nan,"routed_full_authority_ap":safe_ap(y[valid],sp_full[valid]) if valid.any() else np.nan,"routed_full_authority_auc":safe_auc(y[valid],sp_full[valid]) if valid.any() else np.nan,"residual_delta_mean":float(np.mean(fd)) if len(fd) else np.nan,"residual_delta_std":float(np.std(fd)) if len(fd) else np.nan,"residual_delta_min":float(np.min(fd)) if len(fd) else np.nan,"residual_delta_max":float(np.max(fd)) if len(fd) else np.nan,"residual_increase_fraction":float(np.mean(fd>0)) if len(fd) else np.nan,"residual_decrease_fraction":float(np.mean(fd<0)) if len(fd) else np.nan,"residual_cap_saturation_fraction":float(np.mean(sat)) if len(fd) else np.nan,"specialist_authority":float(authority)}


def print_routing_diagnostics(title,d):
    print(f"\n[{title} routing diagnostics]")
    print(f"  Easy-negative exit : {d['easy_negative_count']:,}/{d['n']:,} ({100*d['easy_negative_rate']:.2f}%) | failures inside={d['easy_negative_failures']:,}/{d['failures']:,} ({100*d['easy_negative_failure_fraction_of_all_failures']:.2f}% of failures) | exit failure rate={100*d['easy_negative_failure_rate']:.3f}%")
    print(f"  Specialist routed  : {d['routed_count']:,}/{d['n']:,} ({100*d['routing_rate']:.2f}%) | failures={d['routed_failures']:,} | prevalence={100*d['routed_failure_prevalence']:.2f}% | enrichment={d['routed_enrichment']:.2f}x")
    print(f"  Strong positives   : {d['strong_positive_count']:,} ({100*d['strong_positive_rate']:.2f}%) | precision={100*d['strong_positive_precision']:.2f}% | recall={100*d['strong_positive_recall']:.2f}% | score protection={d['positive_score_protection_count']:,}")
    print(f"  Whole-set ranking  : committee AP/AUC={d['committee_ap']:.4f}/{d['committee_auc']:.4f} -> cascade AP/AUC={d['final_ap']:.4f}/{d['final_auc']:.4f}")
    print(f"  Routed-only ranking: committee AP/AUC={d['routed_committee_ap']:.4f}/{d['routed_committee_auc']:.4f} -> selected-authority AP/AUC={d['routed_specialist_ap']:.4f}/{d['routed_specialist_auc']:.4f} | full-authority counterfactual={d['routed_full_authority_ap']:.4f}/{d['routed_full_authority_auc']:.4f}")
    print(f"  Bounded residual   : authority={d['specialist_authority']:.2f} | mean={d['residual_delta_mean']:+.4f} std={d['residual_delta_std']:.4f} min={d['residual_delta_min']:+.4f} max={d['residual_delta_max']:+.4f} | increase={100*d['residual_increase_fraction']:.1f}% decrease={100*d['residual_decrease_fraction']:.1f}% | cap saturation={100*d['residual_cap_saturation_fraction']:.2f}%")


def parse_folds(spec): return sorted({int(x.strip()) for x in spec.split(",") if x.strip()})


def prepare_features(train,test,columns):
    Xtr,Xte=train.loc[:,columns].copy(),test.reindex(columns=columns).copy()
    cat=[c for c in columns if pd.api.types.is_object_dtype(Xtr[c]) or isinstance(Xtr[c].dtype,pd.CategoricalDtype) or pd.api.types.is_string_dtype(Xtr[c])]; num=[c for c in columns if c not in cat]
    for c in num: Xtr[c],Xte[c]=pd.to_numeric(Xtr[c],errors="coerce"),pd.to_numeric(Xte[c],errors="coerce")
    if cat:
        try: enc=OneHotEncoder(handle_unknown="ignore",sparse_output=False,dtype=float)
        except TypeError: enc=OneHotEncoder(handle_unknown="ignore",sparse=False,dtype=float)
        A=enc.fit_transform(Xtr[cat].astype("string").fillna("__MISSING__")); B=enc.transform(Xte[cat].astype("string").fillna("__MISSING__")); names=enc.get_feature_names_out(cat).tolist()
        Xtr=pd.concat([Xtr[num].reset_index(drop=True),pd.DataFrame(A,columns=names)],axis=1); Xte=pd.concat([Xte[num].reset_index(drop=True),pd.DataFrame(B,columns=names)],axis=1)
    else: Xtr,Xte=Xtr[num].reset_index(drop=True),Xte[num].reset_index(drop=True)
    means=Xtr.mean(axis=0,numeric_only=True); Xtr=Xtr.fillna(means).fillna(0).astype(float); Xte=Xte.fillna(means).fillna(0).astype(float)
    if list(Xtr.columns)!=list(Xte.columns): raise RuntimeError("Train/test feature mismatch")
    return Xtr,Xte,list(Xtr.columns)


def load_fold(splits_dir,faculty,fold):
    root=Path(splits_dir)/f"fold_{fold}"; trp=root/f"{faculty}_train.csv"; tep=root/f"{faculty}_test.csv"
    if not trp.exists() or not tep.exists(): return None
    train,test=pd.read_csv(trp,low_memory=False),pd.read_csv(tep,low_memory=False)
    if TARGET not in train or TARGET not in test: raise KeyError(f"{TARGET} missing in fold {fold}")
    train[TARGET]=pd.to_numeric(train[TARGET],errors="coerce"); test[TARGET]=pd.to_numeric(test[TARGET],errors="coerce")
    train=train.dropna(subset=[TARGET]).reset_index(drop=True); test=test.dropna(subset=[TARGET]).reset_index(drop=True)
    if len(train)<50:return None
    cols=[c for c in train.columns if c not in EXCLUDE]; Xtr,Xte,names=prepare_features(train,test,cols)
    return Xtr,Xte,train[TARGET].to_numpy(float),test[TARGET].to_numpy(float),train,test,names


def binary_metrics(y,pred):
    tn,fp,fn,tp=confusion_matrix(y,pred,labels=[0,1]).ravel(); p,r,f1,_=precision_recall_fscore_support(y,pred,average="binary",zero_division=0); n=tn+fp+fn+tp
    return {"tn":int(tn),"fp":int(fp),"fn":int(fn),"tp":int(tp),"accuracy":float((tn+tp)/n),"precision":float(p),"recall":float(r),"specificity":float(tn/(tn+fp)) if tn+fp else np.nan,"f1":float(f1),"flag_rate":float((tp+fp)/n)}


def report(y,score,pred,beta):
    m=binary_metrics(np.asarray(y,int),np.asarray(pred,int)); b2=beta*beta; den=b2*m["precision"]+m["recall"]
    m.update(ap=safe_ap(y,score),auc=safe_auc(y,score),prevalence=float(np.mean(y)),n=len(y),fbeta=0. if den<=0 else float((1+b2)*m["precision"]*m["recall"]/den)); return m


def apply_decision_rule(score,threshold,easy_negative=None,strong_positive=None,hard_negative_guard=False):
    pred=(np.asarray(score,float).reshape(-1)>=threshold).astype(int)
    if hard_negative_guard and easy_negative is not None: pred[np.asarray(easy_negative,bool)]=0
    if strong_positive is not None: pred[np.asarray(strong_positive,bool)]=1
    return pred


def tune_threshold(y_failure,score,easy_negative=None,strong_positive=None,beta=2.,min_precision=.15,max_warning_ratio=2.5,min_flags=5,hard_negative_guard=False):
    y=np.asarray(y_failure,int); score=np.asarray(score,float); easy=np.zeros(len(y),bool) if easy_negative is None else np.asarray(easy_negative,bool); strong=np.zeros(len(y),bool) if strong_positive is None else np.asarray(strong_positive,bool)
    prev=float(y.mean()); unique=np.unique(score); thresholds=unique if len(unique)<=1000 else np.unique(np.quantile(unique,np.linspace(.001,.999,1000))); c=[]
    for t in thresholds: c.append((float(t),report(y,score,apply_decision_rule(score,t,easy,strong,hard_negative_guard),beta)))
    max_rate=min(.99,max_warning_ratio*max(prev,1e-8)); enough=[x for x in c if x[1]["flag_rate"]*len(y)>=min_flags]; strict=[x for x in enough if x[1]["precision"]>=min_precision and x[1]["flag_rate"]<=max_rate]; ratio=[x for x in enough if x[1]["flag_rate"]<=max_rate]
    if strict: eligible,relaxed=strict,"none"
    elif ratio: eligible,relaxed=ratio,"minimum precision"
    elif enough: eligible,relaxed=enough,"minimum precision and maximum warning ratio"
    else: eligible,relaxed=c,"all constraints"
    t,m=max(eligible,key=lambda x:(x[1]["fbeta"],x[1]["recall"],x[1]["precision"],-x[1]["flag_rate"])); return ThresholdPolicy(t,beta,m["ap"],m["auc"],prev,m["flag_rate"],m["flag_rate"]/max(prev,1e-8),m["precision"],m["recall"],m["f1"],m["fbeta"],relaxed)


def warn_by_rate(score,rate,easy_negative=None,strong_positive=None,hard_negative_guard=False):
    score=np.asarray(score,float); n=len(score); easy=np.zeros(n,bool) if easy_negative is None else np.asarray(easy_negative,bool); strong=np.zeros(n,bool) if strong_positive is None else np.asarray(strong_positive,bool); out=np.zeros(n,int); out[strong]=1
    if not n or rate<=0:return out
    target=min(n,max(1,int(round(rate*n)))); need=max(0,target-int(out.sum()));
    if need==0:return out
    blocked=strong | (easy if hard_negative_guard else False); cand=np.where(~blocked)[0]
    if not len(cand):return out
    need=min(need,len(cand)); local=np.argpartition(score[cand],-need)[-need:]; out[cand[local]]=1; return out


def save_results(path,frame,c):
    cols=[x for x in IDS if x in frame.columns]+[TARGET]; out=frame.loc[:,list(dict.fromkeys(cols))].copy(); out["actual_failure_lt5"]=(frame[TARGET].to_numpy(float)<FAILURE_BOUNDARY).astype(np.int8)
    for i,n in enumerate(BASE_NAMES):out[f"expert_probability_{n}"]=c["expert_probabilities"][:,i]
    for n in UNCERTAINTY_NAMES:out[n]=c[n]
    out["committee_rank"]=c["committee_rank"]; out["easy_negative_exit"]=c["easy_negative"].astype(np.int8); out["routed_to_specialist"]=c["routed"].astype(np.int8); out["strong_positive_preserved"]=c["strong_positive"].astype(np.int8)
    out["specialist_probability_lt5"]=c["specialist_probability"]; out["specialist_probability_full_authority_lt5"]=c["specialist_probability_full_authority"]; out["specialist_residual_delta_logit"]=c["residual_delta"]; out["specialist_raw_residual"]=c["raw_residual"]; out["specialist_authority"]=c["specialist_authority"]; out["committee_probability_lt5"]=c["committee_mean"]; out["failure_probability_lt5"]=c["final_probability"]; out.to_csv(path,index=False)


def train_one(splits_dir,faculty,fold,args,outdir):
    data=load_fold(splits_dir,faculty,fold)
    if data is None: print(f"Fold {fold}: skipped"); return None
    X,Xtest,grades,test_grades,train_frame,test_frame,names=data; y=(grades<FAILURE_BOUNDARY).astype(int); yt=(test_grades<FAILURE_BOUNDARY).astype(int)
    print("\n"+"#"*118); print(f"FOLD {fold} | historical_train={len(X):,} | test={len(Xtest):,} | raw/prepared features={X.shape[1]:,}"); print(f"Failure prevalence: train={100*y.mean():.2f}% | test={100*yt.mean():.2f}%"); print("#"*118)
    model=SelectiveCascadeEnsemble(random_state=args.seed,n_estimators=args.n_estimators,hist_max_iter=args.hist_max_iter,max_negative_exit_failure_fraction=args.max_negative_exit_failure_fraction,max_negative_exit_failure_rate=args.max_negative_exit_failure_rate,positive_preserve_min_precision=args.positive_preserve_min_precision,min_route_count=args.min_route_count,routing_min_block_failures=args.routing_min_block_failures,crossfit_folds=args.crossfit_folds,crossfit_min_train_rows=args.crossfit_min_train_rows,min_oof_coverage=args.min_oof_coverage,specialist_top_raw_features=args.specialist_top_raw_features,specialist_epochs=args.specialist_epochs,specialist_lr=args.specialist_lr,specialist_weight_decay=args.specialist_weight_decay,effective_beta=args.effective_beta,weight_gamma_grid=args.weight_gamma_grid,specialist_hidden_dims=args.specialist_hidden_dims,specialist_dropout=args.specialist_dropout,specialist_positive_cap=args.specialist_positive_cap,specialist_negative_cap=args.specialist_negative_cap,residual_penalty=args.residual_penalty,specialist_patience=args.specialist_patience,specialist_min_improvement=args.specialist_min_improvement,authority_grid=args.authority_grid,authority_ap_tolerance=args.authority_ap_tolerance,authority_auc_tolerance=args.authority_auc_tolerance,hard_negative_guard=args.hard_negative_guard,device=args.device,verbose=True).fit(X,y,train_frame,names)
    c=model.predict_components(Xtest); diag=routing_diagnostics(yt,c,args.specialist_positive_cap,args.specialist_negative_cap,model.authority_)
    print("\n[Temporal test expert ranking]")
    for i,n in enumerate(BASE_NAMES):print(f"  {n:24s} AP={safe_ap(yt,c['expert_probabilities'][:,i]):.4f} | AUC={safe_auc(yt,c['expert_probabilities'][:,i]):.4f}")
    print_routing_diagnostics("TEMPORAL TEST",diag)
    tag=f"{faculty}_fold_{fold}"; model.save(outdir/f"{tag}_classifier.pkl"); pd.DataFrame(model.base_report_).to_csv(outdir/f"{tag}_base_models.csv",index=False); pd.DataFrame(model.crossfit_report_).to_csv(outdir/f"{tag}_crossfit_report.csv",index=False); pd.DataFrame([asdict(model.routing_policy_)]).to_csv(outdir/f"{tag}_routing_policy.csv",index=False); pd.DataFrame(model.routing_block_diagnostics_).to_csv(outdir/f"{tag}_routing_block_diagnostics.csv",index=False); pd.DataFrame([model.route_train_diagnostics_]).to_csv(outdir/f"{tag}_route_train_diagnostics.csv",index=False); pd.DataFrame([diag]).to_csv(outdir/f"{tag}_test_routing_diagnostics.csv",index=False); pd.DataFrame(model.feature_selection_report_).to_csv(outdir/f"{tag}_feature_selection.csv",index=False); pd.DataFrame(model.selection_feature_selection_report_).to_csv(outdir/f"{tag}_selection_feature_selection.csv",index=False); pd.DataFrame(model.weight_gamma_candidates_).to_csv(outdir/f"{tag}_weight_gamma_candidates.csv",index=False); pd.DataFrame(model.authority_candidates_).to_csv(outdir/f"{tag}_authority_candidates.csv",index=False); pd.DataFrame([model.specialist_summary_]).to_csv(outdir/f"{tag}_specialist_summary.csv",index=False)
    hist=model.specialist_.history_ if model.specialist_ is not None else []; pd.DataFrame(hist).to_csv(outdir/f"{tag}_specialist_training.csv",index=False)
    path=outdir/f"{tag}_test_results.csv"; save_results(path,test_frame,c)
    return FoldRecord(fold,yt,c["final_probability"],c["committee_mean"],float(yt.mean()),c["easy_negative"],c["strong_positive"],args.hard_negative_guard,path)


def print_cm(title,m):
    print(f"\n  Confusion matrix — {title}"); print(f"  {'Actual / Predicted':<22}{'Normal >=5':>16}{'Failure <5':>16}"); print(f"  {'Normal >=5':<22}{m['tn']:>16,d}{m['fp']:>16,d}"); print(f"  {'Failure <5':<22}{m['fn']:>16,d}{m['tp']:>16,d}"); print(f"  Normal correctly rejected: {100*m['specificity']:.2f}%"); print(f"  Failures successfully detected: {100*m['recall']:.2f}%")


def evaluate(records:Sequence[FoldRecord],policy,prevalence_map,fallback_prevalence,title,confusion=False):
    print(f"\n### {title} — ROBUST V4 SELECTIVE CASCADE [beta={policy.beta:g}]"); print(f"  {'fold':>6} {'test':>7} {'base%':>7} | {'AP':>7} {'AUC':>7} | {'ACC@t':>7} {'P@t':>7} {'R@t':>7} {'F1@t':>7} {'Fβ@t':>7} {'rate@t':>8} | {'ACC@r':>7} {'P@r':>7} {'R@r':>7} {'F1@r':>7} {'Fβ@r':>7} {'rate@r':>8}")
    rows=[]; ys=[]; scores=[]; fixeds=[]; drifts=[]; cfs=[]; cdr=[]
    for r in records:
        rate=float(np.clip(policy.warning_ratio*prevalence_map.get(r.fold-1,fallback_prevalence),1e-4,.99)); wf=apply_decision_rule(r.score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard); wd=warn_by_rate(r.score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard); mf,md=report(r.y,r.score,wf,policy.beta),report(r.y,r.score,wd,policy.beta)
        cf=apply_decision_rule(r.committee_score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard); cd=warn_by_rate(r.committee_score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard); mcf,mcd=report(r.y,r.committee_score,cf,policy.beta),report(r.y,r.committee_score,cd,policy.beta)
        fixed_without_positive=apply_decision_rule(r.score,policy.threshold,r.easy_negative,None,r.hard_negative_guard)
        drift_without_positive=warn_by_rate(r.score,rate,r.easy_negative,None,r.hard_negative_guard)
        pos_fixed_added=int(np.sum((wf==1)&(fixed_without_positive==0)&r.strong_positive))
        pos_drift_added=int(np.sum((wd==1)&(drift_without_positive==0)&r.strong_positive))
        print(f"  {r.fold:>6} {len(r.y):>7} {100*r.prevalence:>6.2f}% | {mf['ap']:>7.4f} {mf['auc']:>7.4f} | {mf['accuracy']:>7.4f} {mf['precision']:>7.4f} {mf['recall']:>7.4f} {mf['f1']:>7.4f} {mf['fbeta']:>7.4f} {100*mf['flag_rate']:>7.2f}% | {md['accuracy']:>7.4f} {md['precision']:>7.4f} {md['recall']:>7.4f} {md['f1']:>7.4f} {md['fbeta']:>7.4f} {100*md['flag_rate']:>7.2f}%")
        print(f"           committee counterfactual: AP/AUC={mcf['ap']:.4f}/{mcf['auc']:.4f} | fixed F2={mcf['fbeta']:.4f} R={mcf['recall']:.4f} | drift F2={mcd['fbeta']:.4f} R={mcd['recall']:.4f}")
        print(f"           positive decision protection activated: fixed={pos_fixed_added:,} | drift={pos_drift_added:,}")
        if confusion: print_cm(f"fold {r.fold} — fixed threshold",mf); print_cm(f"fold {r.fold} — drift-aware",md)
        rows.append({"fold":r.fold,"n":len(r.y),"prevalence":r.prevalence,"drift_rate":rate,"easy_negative_rate":float(r.easy_negative.mean()),"strong_positive_rate":float(r.strong_positive.mean()),"positive_decision_protection_fixed":pos_fixed_added,"positive_decision_protection_drift":pos_drift_added,**{f"fixed_{k}":v for k,v in mf.items()},**{f"drift_{k}":v for k,v in md.items()},**{f"committee_fixed_{k}":v for k,v in mcf.items()},**{f"committee_drift_{k}":v for k,v in mcd.items()}}); ys.append(r.y);scores.append(r.score);fixeds.append(wf);drifts.append(wd);cfs.append(cf);cdr.append(cd)
    y,score,wf,wd=np.concatenate(ys),np.concatenate(scores),np.concatenate(fixeds),np.concatenate(drifts); mf,md=report(y,score,wf,policy.beta),report(y,score,wd,policy.beta)
    cscore=np.concatenate([r.committee_score for r in records]); mcf=report(y,cscore,np.concatenate(cfs),policy.beta); mcd=report(y,cscore,np.concatenate(cdr),policy.beta)
    print("-"*154); print(f"  {'POOLED':>6} {len(y):>7} {100*y.mean():>6.2f}% | {mf['ap']:>7.4f} {mf['auc']:>7.4f} | {mf['accuracy']:>7.4f} {mf['precision']:>7.4f} {mf['recall']:>7.4f} {mf['f1']:>7.4f} {mf['fbeta']:>7.4f} {100*mf['flag_rate']:>7.2f}% | {md['accuracy']:>7.4f} {md['precision']:>7.4f} {md['recall']:>7.4f} {md['f1']:>7.4f} {md['fbeta']:>7.4f} {100*md['flag_rate']:>7.2f}%"); print(f"  POOLED committee counterfactual: AP/AUC={mcf['ap']:.4f}/{mcf['auc']:.4f} | fixed F2={mcf['fbeta']:.4f} R={mcf['recall']:.4f} | drift F2={mcd['fbeta']:.4f} R={mcd['recall']:.4f}")
    if confusion: print_cm("POOLED — fixed development threshold",mf); print_cm("POOLED — drift-aware",md)
    return rows,{**{f"fixed_{k}":v for k,v in mf.items()},**{f"drift_{k}":v for k,v in md.items()},**{f"committee_fixed_{k}":v for k,v in mcf.items()},**{f"committee_drift_{k}":v for k,v in mcd.items()}}


def add_policy_columns(r,policy,prevalence_map,fallback):
    frame=pd.read_csv(r.path,low_memory=False); rate=float(np.clip(policy.warning_ratio*prevalence_map.get(r.fold-1,fallback),1e-4,.99)); frame["fixed_threshold"]=policy.threshold; frame["warning_fixed"]=apply_decision_rule(r.score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["drift_rate"]=rate; frame["warning_drift"]=warn_by_rate(r.score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["committee_warning_fixed_counterfactual"]=apply_decision_rule(r.committee_score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["committee_warning_drift_counterfactual"]=warn_by_rate(r.committee_score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame.to_csv(r.path,index=False)


def parse_float_grid(s):
    vals=[float(x.strip()) for x in str(s).split(",") if x.strip()]
    return sorted(set(vals))


def parse_hidden_dims(s):
    vals=tuple(int(x.strip()) for x in str(s).split(",") if x.strip())
    if not vals or any(v<1 for v in vals): raise argparse.ArgumentTypeError("hidden dims must be positive comma-separated integers")
    return vals

def parse_authority_grid(s):
    vals=[float(x.strip()) for x in str(s).split(",") if x.strip()];
    if not vals:return [0,.25,.5,.75,1.0]
    return sorted(set(vals))


def parse_args():
    p=argparse.ArgumentParser(description="Robust V4 cross-fitted selective cascade with compact nonlinear residual MLP")
    p.add_argument("--faculty",default="khoa_cong_nghe_thong_tin"); p.add_argument("--splits_dir",required=True); p.add_argument("--dev_folds",default="1,2,3,4,5"); p.add_argument("--holdout_folds",default="6,7"); p.add_argument("--models_dir",default="results/train_expert_v4_robust"); p.add_argument("--seed",type=int,default=42); p.add_argument("--n_estimators",type=int,default=500); p.add_argument("--hist_max_iter",type=int,default=250)
    p.add_argument("--crossfit_folds",type=int,default=5); p.add_argument("--crossfit_min_train_rows",type=int,default=100); p.add_argument("--min_oof_coverage",type=float,default=.40)
    p.add_argument("--max_negative_exit_failure_fraction",type=float,default=.05); p.add_argument("--max_negative_exit_failure_rate",type=float,default=.01); p.add_argument("--positive_preserve_min_precision",type=float,default=.60); p.add_argument("--min_route_count",type=int,default=5); p.add_argument("--routing_min_block_failures",type=int,default=5); p.add_argument("--hard_negative_guard",action="store_true",help="Legacy ablation: force easy-negative exits to normal at decision time")
    p.add_argument("--specialist_top_raw_features",type=int,default=48); p.add_argument("--specialist_epochs",type=int,default=300); p.add_argument("--specialist_lr",type=float,default=1e-3); p.add_argument("--specialist_weight_decay",type=float,default=1e-3); p.add_argument("--effective_beta",type=float,default=.9999); p.add_argument("--weight_gamma_grid",type=parse_float_grid,default=parse_float_grid("0,0.5,1.0")); p.add_argument("--specialist_hidden_dims",type=parse_hidden_dims,default=parse_hidden_dims("32,16")); p.add_argument("--specialist_dropout",type=float,default=.20); p.add_argument("--specialist_positive_cap",type=float,default=2.0); p.add_argument("--specialist_negative_cap",type=float,default=.75); p.add_argument("--residual_penalty",type=float,default=.05); p.add_argument("--specialist_patience",type=int,default=30); p.add_argument("--specialist_min_improvement",type=float,default=1e-4); p.add_argument("--authority_grid",type=parse_authority_grid,default=parse_authority_grid("0,0.25,0.5,0.75,1.0")); p.add_argument("--authority_ap_tolerance",type=float,default=.005); p.add_argument("--authority_auc_tolerance",type=float,default=.005); p.add_argument("--device",default="auto",choices=["auto","cpu","cuda"])
    p.add_argument("--threshold_beta",type=float,default=2.0); p.add_argument("--min_precision",type=float,default=.15); p.add_argument("--max_warning_ratio",type=float,default=2.5); p.add_argument("--min_flags",type=int,default=5); return p.parse_args()


def validate_args(a):
    for n in ["max_negative_exit_failure_fraction","max_negative_exit_failure_rate","positive_preserve_min_precision","min_oof_coverage","min_precision"]:
        if not 0<=getattr(a,n)<=1:raise ValueError(f"{n} must be in [0,1]")
    if a.crossfit_folds<2 or a.crossfit_min_train_rows<2 or a.min_route_count<1 or a.routing_min_block_failures<1:raise ValueError("Invalid crossfit/routing arguments")
    if a.specialist_top_raw_features<1 or a.specialist_epochs<1 or a.specialist_lr<=0 or a.specialist_weight_decay<0 or not 0<=a.specialist_dropout<1 or a.specialist_positive_cap<=0 or a.specialist_negative_cap<=0 or a.residual_penalty<0 or a.specialist_patience<1:raise ValueError("Invalid specialist arguments")
    if any(g<0 or g>1 for g in a.weight_gamma_grid) or not a.weight_gamma_grid: raise ValueError("weight_gamma_grid values must be in [0,1]")
    if a.authority_ap_tolerance<0 or a.authority_auc_tolerance<0: raise ValueError("authority tolerances must be nonnegative")
    if not 0<=a.effective_beta<1:raise ValueError("effective_beta must satisfy 0<=beta<1")
    if any(x<0 or x>1 for x in a.authority_grid):raise ValueError("authority_grid values must be in [0,1]")
    if a.threshold_beta<=0 or a.max_warning_ratio<=0 or a.min_flags<1:raise ValueError("Invalid threshold arguments")
    if a.device=="cuda" and not torch.cuda.is_available():raise SystemExit("[ERROR] CUDA requested but unavailable")


def main():
    a=parse_args(); validate_args(a); seed_all(a.seed); splits=Path(a.splits_dir).expanduser().resolve(); out=Path(a.models_dir).expanduser().resolve()
    if not splits.exists():raise SystemExit(f"[ERROR] Missing splits directory: {splits}")
    out.mkdir(parents=True,exist_ok=True); dev,hold=parse_folds(a.dev_folds),parse_folds(a.holdout_folds)
    if set(dev)&set(hold):raise ValueError("Development and holdout folds overlap")
    print("="*118); print("ROBUST V4 SELECTIVE CASCADE — CROSS-FITTED RANK ROUTING + COMPACT NONLINEAR RESIDUAL MLP"); print(f"Faculty: {a.faculty} | target: grade<5 | dev={dev} | holdout={hold}"); print("Experts unchanged: Balanced RF + Balanced ExtraTrees + HistGB | committee=simple mean"); print(f"Cross-fit: temporal expanding preferred; stratified {a.crossfit_folds}-fold fallback | min OOF coverage={100*a.min_oof_coverage:.0f}%"); print(f"Routing: rank-based soft negative exit (hard guard={a.hard_negative_guard}) | positive preserve >= {100*a.positive_preserve_min_precision:.1f}% precision"); print(f"Specialist: top-{a.specialist_top_raw_features} raw + 3 expert + 7 uncertainty | MLP {'->'.join(map(str,a.specialist_hidden_dims))}->1 dropout={a.specialist_dropout:g} | bounded residual [-{a.specialist_negative_cap:g},+{a.specialist_positive_cap:g}] | penalty={a.residual_penalty:g}"); print(f"Weight gamma grid={a.weight_gamma_grid} | temporal AP checkpointing | effective BCE beta={a.effective_beta:.4f}"); print(f"Authority grid={a.authority_grid} with support ceiling + ranking guard (AP/AUC tol={a.authority_ap_tolerance:g}/{a.authority_auc_tolerance:g})"); print(f"Operational objective: F{a.threshold_beta:g} | outputs: {out}"); print("="*118)
    print("\n>>> DEVELOPMENT FOLDS"); dr=[r for f in dev if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
    if not dr:raise SystemExit("[ERROR] No development folds trained")
    yd=np.concatenate([r.y for r in dr]); sd=np.concatenate([r.score for r in dr]); nd=np.concatenate([r.easy_negative for r in dr]); pdp=np.concatenate([r.strong_positive for r in dr]); pol=tune_threshold(yd,sd,nd,pdp,a.threshold_beta,a.min_precision,a.max_warning_ratio,a.min_flags,a.hard_negative_guard)
    print("\n>>> POOLED DEVELOPMENT THRESHOLD"); print(f"AP={pol.development_ap:.4f} | AUC={pol.development_auc:.4f} | threshold={pol.threshold:.6f} | flag={100*pol.flag_rate:.2f}% | ratio={pol.warning_ratio:.3f} | P={pol.precision:.4f} R={pol.recall:.4f} F1={pol.f1:.4f} F{pol.beta:g}={pol.fbeta:.4f}")
    prev={r.fold:r.prevalence for r in dr}; fallback=float(yd.mean()); drows,dpool=evaluate(dr,pol,prev,fallback,"DEVELOPMENT")
    hr=[];hrows=[];hpool={}
    if hold:
        print("\n"+"!"*118); print("HOLDOUT: labels do not tune cross-fitting, routing, feature selection, specialist, authority, threshold, or warning budget."); print("!"*118); hr=[r for f in hold if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
        for r in hr:prev[r.fold]=r.prevalence
        if hr:hrows,hpool=evaluate(hr,pol,prev,fallback,"HOLDOUT (test)",True)
    for r in dr+hr:add_policy_columns(r,pol,prev,fallback)
    pd.DataFrame([asdict(pol)]).to_csv(out/f"{a.faculty}_threshold_policy.csv",index=False); rows=[{"split":"development",**x} for x in drows]+[{"split":"holdout",**x} for x in hrows]+[{"split":"development","fold":"POOLED",**dpool}];
    if hpool:rows.append({"split":"holdout","fold":"POOLED",**hpool})
    pd.DataFrame(rows).to_csv(out/f"{a.faculty}_warning_metrics.csv",index=False); print("\n"+"="*118); print(f"Frozen threshold: {pol.threshold:.6f} | warning ratio: {pol.warning_ratio:.3f}"); print(f"Completed {len(dr)+len(hr)} fold(s) | outputs: {out}"); print("="*118)


if __name__ == "__main__":
    main()