# -*- coding: utf-8 -*-
"""Train V3: correction-opportunity selective cascade for academic early warning (grade < 5).

Experiment C — robust V3 redesign
---------------------------------
1. Balanced RF + Balanced ExtraTrees + HistGradientBoosting are unchanged.
2. Historical expert probabilities are generated out-of-fold, preferably by
   strictly temporal expanding cross-fitting.
3. Routing is learned from batch-relative committee rank + disagreement.
4. Easy negatives *soft exit*: they bypass the specialist but retain the
   committee score and remain eligible for global warning selection.
5. Routed cases use a low-dimensional, zero-initialized linear residual
   specialist anchored to the committee logit.
6. Residuals are asymmetric/bounded, magnitude-regularized, temporally
   early-stopped, and given validation-selected authority (including 0).
7. Strong-positive consensus cases cannot be downgraded below the committee
   score and remain warnings at decision time.

Target: target_grade10 < 5.
"""
from __future__ import annotations

import argparse
import copy
import math
import pickle
import random
import warnings
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Dict, Sequence

import numpy as np
import pandas as pd
import torch
import torch.nn.functional as F
from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.feature_selection import f_classif
from sklearn.metrics import average_precision_score, confusion_matrix, precision_recall_fscore_support, roc_auc_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from torch import nn

TARGET, FAILURE_BOUNDARY = "target_grade10", 5.0
BASE_NAMES = ["random_forest_balanced", "extra_trees_balanced", "hist_gradient_boosting"]
UNCERTAINTY_NAMES = [
    "committee_mean", "committee_min", "committee_max", "committee_std",
    "committee_spread", "committee_entropy", "committee_vote_fraction",
]
EXCLUDE = {
    "dataset_split", "student_faculty_key", "student_faculty_name", "IDSV", "target_NHHK",
    "target_semester_ordinal", "is_summer_target", "IDKQDK", "IDToHoc", "IDMH",
    "target_IDCongDiem", "target_grade10", "target_grade4", "target_status", "base_row_id",
    "course_cluster_label",
}
IDS = ["IDSV", "IDMH", "IDKQDK", "IDToHoc", "target_NHHK", "target_semester_ordinal",
       "student_faculty_key", "student_faculty_name"]


@dataclass
class RoutingPolicy:
    negative_exit_quantile: float
    negative_max_spread: float
    negative_enabled: bool
    negative_relaxation: str
    max_negative_exit_failure_fraction: float
    max_negative_exit_failure_rate: float
    negative_exit_rate_train: float
    negative_exit_failure_rate_train: float
    negative_exit_failure_fraction_train: float
    routed_rate_train: float
    routed_prevalence_train: float
    routed_enrichment_train: float
    positive_rank_quantile: float
    positive_max_spread: float
    positive_preserve_min_precision: float
    positive_preserve_enabled: bool
    positive_relaxation: str
    positive_preserve_rate_train: float
    positive_preserve_precision_train: float
    positive_preserve_recall_train: float


@dataclass
class FoldRecord:
    fold: int
    y: np.ndarray
    score: np.ndarray
    committee_score: np.ndarray
    prevalence: float
    easy_negative: np.ndarray
    strong_positive: np.ndarray
    hard_negative_guard: bool
    path: Path


@dataclass
class ThresholdPolicy:
    threshold: float
    beta: float
    development_ap: float
    development_auc: float
    development_prevalence: float
    flag_rate: float
    warning_ratio: float
    precision: float
    recall: float
    f1: float
    fbeta: float
    constraints_relaxed: str


@dataclass
class CrossFitResult:
    mode: str
    probabilities: np.ndarray
    covered_mask: np.ndarray
    block_ids: np.ndarray
    source: str
    report_rows: list[dict]
    warning: str


class ResidualCorrectionLayer(nn.Module):
    """Zero-initialized scalar residual in committee-logit space."""

    def __init__(self, input_dim: int):
        super().__init__()
        self.linear = nn.Linear(input_dim, 1)
        nn.init.zeros_(self.linear.weight)
        nn.init.zeros_(self.linear.bias)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.linear(x).squeeze(1)


class RawFeatureSelector:
    """Training-only standardized ANOVA/F-score raw-feature filter."""

    def __init__(self, max_features=48):
        self.max_features = int(max_features)
        self.scaler_: StandardScaler | None = None
        self.selected_indices_: np.ndarray | None = None
        self.scores_: np.ndarray | None = None
        self.feature_names_: list[str] = []
        self.selected_feature_names_: list[str] = []
        self.report_: list[dict] = []

    def fit(self, X, y, feature_names):
        X = clean_X(X)
        y = np.asarray(y, int).reshape(-1)
        self.feature_names_ = list(feature_names)
        if X.shape[1] != len(self.feature_names_):
            raise ValueError("Raw feature names do not match raw feature matrix")
        if len(X) != len(y) or len(X) < 2 or np.unique(y).size < 2:
            raise ValueError("Feature selection requires >=2 rows and both classes")
        self.scaler_ = StandardScaler().fit(X)
        Xs = self.scaler_.transform(X)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            scores, _ = f_classif(Xs, y)
        scores = np.asarray(scores, float)
        scores = np.nan_to_num(scores, nan=-np.inf, posinf=np.finfo(float).max, neginf=-np.inf)
        order = np.lexsort((np.arange(len(scores)), -scores))
        k = min(max(1, self.max_features), X.shape[1])
        self.selected_indices_ = np.sort(order[:k])
        self.scores_ = scores
        self.selected_feature_names_ = [self.feature_names_[i] for i in self.selected_indices_]
        rank_lookup = {int(idx): rank + 1 for rank, idx in enumerate(order)}
        self.report_ = [{
            "raw_feature_index": i,
            "feature": name,
            "f_score": float(scores[i]) if np.isfinite(scores[i]) else np.nan,
            "rank": rank_lookup[i],
            "selected": bool(i in set(self.selected_indices_.tolist())),
        } for i, name in enumerate(self.feature_names_)]
        return self

    def transform(self, X):
        if self.selected_indices_ is None:
            raise RuntimeError("RawFeatureSelector is not fitted")
        return clean_X(X)[:, self.selected_indices_]


def seed_all(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)


def resolve_device(device):
    if device == "auto":
        return torch.device("cuda" if torch.cuda.is_available() else "cpu")
    return torch.device(device)


def clean_X(X):
    X = X.to_numpy() if hasattr(X, "to_numpy") else X
    X = np.asarray(X, dtype=np.float64)
    if X.ndim != 2:
        raise ValueError(f"X must be 2D, got {X.shape}")
    return np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)


def clip_probability(p):
    return np.clip(np.asarray(p, float).reshape(-1), 1e-7, 1 - 1e-7)


def safe_ap(y, score):
    y = np.asarray(y, int)
    return float("nan") if np.unique(y).size < 2 else float(average_precision_score(y, score))


def safe_auc(y, score):
    y = np.asarray(y, int)
    return float("nan") if np.unique(y).size < 2 else float(roc_auc_score(y, score))


def effective_sample_weights(y, beta=0.9999):
    y = np.asarray(y, int).reshape(-1)
    n_failure, n_normal = int((y == 1).sum()), int((y == 0).sum())
    if not 0 <= beta < 1:
        raise ValueError(f"effective_beta must satisfy 0 <= beta < 1, got {beta}")
    if n_failure == 0 or n_normal == 0:
        wf = wn = 1.0
    else:
        raw_f = (1 - beta) / (1 - beta ** float(n_failure))
        raw_n = (1 - beta) / (1 - beta ** float(n_normal))
        z = (n_failure * raw_f + n_normal * raw_n) / (n_failure + n_normal)
        wf, wn = raw_f / z, raw_n / z
    w = np.where(y == 1, wf, wn).astype(np.float32)
    return w, {
        "beta": float(beta), "n_failure": n_failure, "n_normal": n_normal,
        "failure_weight": float(wf), "normal_weight": float(wn),
        "effective_ratio": float(wf / max(wn, 1e-12)),
    }


def sample_weights_from_stats(y, stats):
    y = np.asarray(y, int).reshape(-1)
    return np.where(y == 1, stats["failure_weight"], stats["normal_weight"]).astype(np.float32)


def committee_statistics(expert_probabilities):
    p = np.asarray(expert_probabilities, float)
    if p.ndim != 2 or p.shape[1] != len(BASE_NAMES):
        raise ValueError(f"Expected expert probability matrix (N,{len(BASE_NAMES)}), got {p.shape}")
    p = np.clip(p, 1e-7, 1 - 1e-7)
    mean = p.mean(axis=1)
    pmin, pmax = p.min(axis=1), p.max(axis=1)
    std = p.std(axis=1)
    spread = pmax - pmin
    q = np.clip(mean, 1e-7, 1 - 1e-7)
    entropy = -(q * np.log(q) + (1 - q) * np.log(1 - q))
    vote = (p >= 0.5).mean(axis=1)
    return {
        "committee_mean": mean, "committee_min": pmin, "committee_max": pmax,
        "committee_std": std, "committee_spread": spread,
        "committee_entropy": entropy, "committee_vote_fraction": vote,
    }


def batch_percentile_rank(score):
    """Return ascending percentile ranks in (0,1], stable under monotonic score shifts."""
    score = np.asarray(score, float).reshape(-1)
    if not len(score):
        return np.array([], dtype=float)
    return pd.Series(score).rank(method="average", pct=True).to_numpy(float)


def ranks_within_blocks(score, block_ids):
    score = np.asarray(score, float).reshape(-1)
    blocks = np.asarray(block_ids)
    if len(score) != len(blocks):
        raise ValueError("score and block_ids must have equal lengths")
    out = np.empty(len(score), float)
    for block in pd.unique(blocks):
        idx = np.where(blocks == block)[0]
        out[idx] = batch_percentile_rank(score[idx])
    return out


def temporal_crossfit_plan(groups, min_train_rows=50):
    groups = np.asarray(groups)
    valid = ~pd.isna(groups)
    unique = np.unique(groups[valid])
    unique = np.sort(unique)
    plans = []
    for block in unique[1:]:
        train_idx = np.where(valid & (groups < block))[0]
        val_idx = np.where(valid & (groups == block))[0]
        if len(train_idx) >= int(min_train_rows) and len(val_idx):
            plans.append((train_idx, val_idx, block))
    return plans


def extract_temporal_groups(frame):
    if "target_semester_ordinal" in frame.columns:
        s = pd.to_numeric(frame["target_semester_ordinal"], errors="coerce")
        if s.notna().sum() >= 2 and s.nunique(dropna=True) >= 2:
            return s.to_numpy(float), "target_semester_ordinal"
    if "target_NHHK" in frame.columns:
        raw = frame["target_NHHK"]
        numeric = pd.to_numeric(raw, errors="coerce")
        if numeric.notna().sum() >= 2 and numeric.nunique(dropna=True) >= 2:
            return numeric.to_numpy(float), "target_NHHK_numeric"
        text = raw.astype("string")
        vals = sorted(v for v in text.dropna().unique().tolist())
        if len(vals) >= 2:
            mapping = {v: i for i, v in enumerate(vals)}
            return text.map(mapping).to_numpy(float), "target_NHHK_ordered"
    return np.full(len(frame), np.nan), "unavailable"


def asymmetric_bounded_delta_numpy(raw, positive_cap=2.0, negative_cap=0.75):
    raw = np.asarray(raw, float)
    t = np.tanh(raw)
    return np.where(raw >= 0, float(positive_cap) * t, float(negative_cap) * t)


def asymmetric_bounded_delta_torch(raw, positive_cap=2.0, negative_cap=0.75):
    t = torch.tanh(raw)
    return torch.where(raw >= 0, float(positive_cap) * t, float(negative_cap) * t)


def apply_residual_correction(committee_score, delta, authority):
    q = clip_probability(committee_score)
    delta = np.asarray(delta, float).reshape(-1)
    if len(q) != len(delta):
        raise ValueError("committee_score and delta must have equal lengths")
    z = np.log(q / (1 - q)) + float(authority) * delta
    return clip_probability(1.0 / (1.0 + np.exp(-np.clip(z, -50, 50))))


def specialist_loss_components_numpy(logits, y, weights, delta, residual_penalty):
    logits = np.asarray(logits, float)
    y = np.asarray(y, float)
    weights = np.asarray(weights, float)
    delta = np.asarray(delta, float)
    # stable BCE-with-logits
    per = np.maximum(logits, 0) - logits * y + np.log1p(np.exp(-np.abs(logits)))
    bce = float(np.mean(per * weights))
    penalty = float(residual_penalty) * float(np.mean(delta ** 2))
    return bce, penalty, bce + penalty


def make_base_models(seed, n_estimators, hist_max_iter):
    return {
        BASE_NAMES[0]: RandomForestClassifier(
            n_estimators=int(n_estimators), min_samples_leaf=4, max_features="sqrt",
            class_weight="balanced_subsample", bootstrap=True, n_jobs=-1, random_state=int(seed)),
        BASE_NAMES[1]: ExtraTreesClassifier(
            n_estimators=int(n_estimators), min_samples_leaf=3, max_features=0.70,
            class_weight="balanced", n_jobs=-1, random_state=int(seed) + 1),
        BASE_NAMES[2]: HistGradientBoostingClassifier(
            learning_rate=0.05, max_iter=int(hist_max_iter), max_leaf_nodes=31,
            min_samples_leaf=20, l2_regularization=1.0, early_stopping=True,
            random_state=int(seed) + 2),
    }


def positive_probability(model, X):
    proba, classes = np.asarray(model.predict_proba(X), float), np.asarray(model.classes_)
    idx = np.where(classes == 1)[0]
    if len(idx) != 1:
        raise RuntimeError("Positive class 1 is missing")
    return clip_probability(proba[:, int(idx[0])])


def predict_experts(models, X):
    X = clean_X(X)
    return np.column_stack([positive_probability(models[n], X) for n in BASE_NAMES])


def generate_crossfit_predictions(
    X, y, temporal_groups, temporal_source, *, seed, n_estimators, hist_max_iter,
    crossfit_folds=5, min_train_rows=100, min_oof_coverage=0.40, verbose=True,
):
    X, y = clean_X(X), np.asarray(y, int).reshape(-1)
    n = len(y)
    p = np.full((n, len(BASE_NAMES)), np.nan, float)
    blocks = np.full(n, "", dtype=object)
    report = []
    warning = ""

    plans = temporal_crossfit_plan(temporal_groups, min_train_rows=min_train_rows)
    temporal_ok = len(plans) >= 2
    if temporal_ok:
        for j, (tr, va, block) in enumerate(plans, 1):
            if np.unique(y[tr]).size < 2:
                continue
            models = make_base_models(seed + 1000 * j, n_estimators, hist_max_iter)
            for model in models.values():
                model.fit(X[tr], y[tr])
            p[va] = predict_experts(models, X[va])
            blocks[va] = str(block)
            report.append({
                "row_type": "block", "mode": "temporal_expanding", "block": str(block),
                "train_rows": len(tr), "validation_rows": len(va),
                "train_failures": int(y[tr].sum()), "validation_failures": int(y[va].sum()),
                "validation_prevalence": float(y[va].mean()),
                "train_max_group": float(np.nanmax(np.asarray(temporal_groups)[tr])),
                "validation_group": float(block),
            })
        covered = np.isfinite(p).all(axis=1)
        coverage = float(covered.mean())
        if covered.sum() >= 20 and np.unique(y[covered]).size == 2 and coverage >= min_oof_coverage and len(pd.unique(blocks[covered])) >= 2:
            mode = "temporal_expanding"
        else:
            temporal_ok = False
            warning = (
                f"Temporal cross-fit insufficient (coverage={coverage:.1%}, blocks={len(pd.unique(blocks[covered]))}); "
                "falling back to stratified OOF."
            )

    if not temporal_ok:
        if not warning:
            warning = "Temporal metadata/blocks unavailable; falling back to stratified OOF."
        p[:] = np.nan
        blocks[:] = ""
        report = []
        min_class = int(np.bincount(y, minlength=2).min())
        n_splits = min(int(crossfit_folds), min_class)
        if n_splits < 2:
            raise ValueError("Not enough examples in both classes for cross-fitting")
        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=int(seed))
        for j, (tr, va) in enumerate(skf.split(X, y), 1):
            models = make_base_models(seed + 1000 * j, n_estimators, hist_max_iter)
            for model in models.values():
                model.fit(X[tr], y[tr])
            p[va] = predict_experts(models, X[va])
            blocks[va] = f"skf_{j}"
            report.append({
                "row_type": "block", "mode": "stratified_oof", "block": f"skf_{j}",
                "train_rows": len(tr), "validation_rows": len(va),
                "train_failures": int(y[tr].sum()), "validation_failures": int(y[va].sum()),
                "validation_prevalence": float(y[va].mean()),
                "train_max_group": np.nan, "validation_group": np.nan,
            })
        covered = np.isfinite(p).all(axis=1)
        mode = "stratified_oof"

    idx = np.where(covered)[0]
    q = committee_statistics(p[idx])["committee_mean"]
    report.insert(0, {
        "row_type": "overall", "mode": mode, "block": "ALL",
        "train_rows": n, "validation_rows": len(idx), "train_failures": int(y.sum()),
        "validation_failures": int(y[idx].sum()), "validation_prevalence": float(y[idx].mean()),
        "train_max_group": np.nan, "validation_group": np.nan,
        "oof_coverage": float(covered.mean()), "temporal_source": temporal_source,
        "committee_ap": safe_ap(y[idx], q), "committee_auc": safe_auc(y[idx], q),
    })
    for i, name in enumerate(BASE_NAMES):
        report.append({
            "row_type": "expert_metric", "mode": mode, "block": "ALL", "expert": name,
            "oof_ap": safe_ap(y[idx], p[idx, i]), "oof_auc": safe_auc(y[idx], p[idx, i]),
        })
    if verbose:
        print(f"[Cross-fit] mode={mode} | source={temporal_source} | OOF={len(idx):,}/{n:,} ({100*covered.mean():.2f}%) | failures={int(y[idx].sum()):,}")
        if warning:
            print(f"[Cross-fit WARNING] {warning}")
        for row in report:
            if row.get("row_type") == "block":
                print(f"  block={row['block']} | train={row['train_rows']:,} | OOF={row['validation_rows']:,} | prevalence={100*row['validation_prevalence']:.2f}%")
        print(f"  committee OOF AP/AUC={safe_ap(y[idx], q):.4f}/{safe_auc(y[idx], q):.4f}")
        for i, name in enumerate(BASE_NAMES):
            print(f"  {name:24s} OOF AP={safe_ap(y[idx], p[idx,i]):.4f} | AUC={safe_auc(y[idx], p[idx,i]):.4f}")
    return CrossFitResult(mode, p, covered, blocks, temporal_source, report, warning)



def max_fbeta_score(y, score, beta=2.0):
    y=np.asarray(y,int);score=np.asarray(score,float)
    if not len(y) or np.unique(y).size<2:return float("nan")
    thresholds=np.unique(score)
    if len(thresholds)>300:thresholds=np.unique(np.quantile(thresholds,np.linspace(.01,.99,300)))
    best=0.0;b2=beta*beta
    for t in thresholds:
        pred=score>=t;tp=np.sum((pred==1)&(y==1));fp=np.sum((pred==1)&(y==0));fn=np.sum((pred==0)&(y==1));prec=tp/max(tp+fp,1);rec=tp/max(tp+fn,1);den=b2*prec+rec;f=0.0 if den<=0 else (1+b2)*prec*rec/den;best=max(best,f)
    return float(best)

# ============================================================================
# TRAIN V3 — SPECIALIZED CORRECTION-OPPORTUNITY ROUTING
# ============================================================================

@dataclass
class CorrectionRoutingPolicy:
    negative_bypass_quantile: float
    positive_bypass_quantile: float
    negative_max_spread: float
    positive_max_spread: float
    route_rate_train: float
    route_failure_prevalence_train: float
    route_failure_enrichment_train: float
    route_error_capture_train: float
    route_error_enrichment_train: float
    positive_under_capture_train: float
    negative_over_capture_train: float
    selection_score: float
    route_min_rate: float
    route_max_rate: float
    median_block_error_enrichment: float = float("nan")
    min_block_error_enrichment: float = float("nan")
    supported_block_win_fraction: float = float("nan")
    negative_bypass_rate_train: float = 0.0
    positive_bypass_rate_train: float = 0.0
    source: str = "correction_opportunity"


def committee_log_loss_each(y, q):
    y = np.asarray(y, int).reshape(-1)
    q = clip_probability(q)
    return -(y * np.log(q) + (1 - y) * np.log(1 - q))


def correction_opportunity_metrics(y, q, routed):
    y = np.asarray(y, int).reshape(-1)
    q = clip_probability(q)
    routed = np.asarray(routed, bool).reshape(-1)
    n = len(y)
    if len(q) != n or len(routed) != n:
        raise ValueError("y, q, routed must have equal length")
    route_rate = float(routed.mean()) if n else 0.0
    loss = committee_log_loss_each(y, q)
    total_loss = float(loss.sum())
    route_loss = float(loss[routed].sum())
    error_capture = route_loss / max(total_loss, 1e-12)
    error_enrichment = error_capture / max(route_rate, 1e-12)
    pos_under = y * (1.0 - q)
    neg_over = (1 - y) * q
    pos_capture = float(pos_under[routed].sum()) / max(float(pos_under.sum()), 1e-12)
    neg_capture = float(neg_over[routed].sum()) / max(float(neg_over.sum()), 1e-12)
    pos_enrichment = pos_capture / max(route_rate, 1e-12)
    neg_enrichment = neg_capture / max(route_rate, 1e-12)
    overall_prev = float(y.mean()) if n else 0.0
    route_prev = float(y[routed].mean()) if routed.any() else 0.0
    return {
        "route_rate": route_rate,
        "route_rows": int(routed.sum()),
        "route_failures": int(y[routed].sum()),
        "route_failure_prevalence": route_prev,
        "route_failure_enrichment": route_prev / max(overall_prev, 1e-12),
        "error_capture": error_capture,
        "error_enrichment": error_enrichment,
        "positive_under_capture": pos_capture,
        "positive_under_enrichment": pos_enrichment,
        "negative_over_capture": neg_capture,
        "negative_over_enrichment": neg_enrichment,
    }


def _sorted_blocks(blocks):
    vals = list(pd.unique(np.asarray(blocks)))
    try:
        return sorted(vals, key=float)
    except Exception:
        return sorted(vals, key=str)


def _route_block_metrics(y, q, routed, blocks, min_rows=100, min_failures=5):
    rows = []
    for block in _sorted_blocks(blocks):
        idx = np.asarray(blocks) == block
        m = correction_opportunity_metrics(np.asarray(y)[idx], np.asarray(q)[idx], np.asarray(routed)[idx])
        rows.append({
            "block": str(block), "rows": int(idx.sum()), "failures": int(np.asarray(y)[idx].sum()),
            "supported": bool(idx.sum() >= min_rows and np.asarray(y)[idx].sum() >= min_failures), **m,
        })
    return rows


def _routing_masks_from_params(expert_probabilities, neg_q, pos_q, neg_spread, pos_spread, block_ids=None):
    p = np.asarray(expert_probabilities, float)
    stats = committee_statistics(p)
    q, spread = stats["committee_mean"], stats["committee_spread"]
    rank = batch_percentile_rank(q) if block_ids is None else ranks_within_blocks(q, block_ids)
    negative = (rank <= float(neg_q)) & (spread <= float(neg_spread))
    positive = (rank >= float(pos_q)) & (spread <= float(pos_spread))
    positive &= ~negative
    routed = ~(negative | positive)
    return {
        "committee_rank": rank,
        "negative_bypass": negative,
        "positive_bypass": positive,
        "routed": routed,
    }


def generate_correction_route_candidates(
    y, expert_probabilities, block_ids, route_min_rate=.25, route_max_rate=.65,
    target_route_rate=.45, top_k=4, min_block_rows=100, min_block_failures=5,
):
    y = np.asarray(y, int).reshape(-1)
    p = np.asarray(expert_probabilities, float)
    blocks = np.asarray(block_ids)
    stats = committee_statistics(p)
    q, spread = stats["committee_mean"], stats["committee_spread"]
    neg_grid = [0.30, 0.40, 0.50, 0.60, 0.70]
    pos_grid = [0.90, 0.95, 0.98]
    spread_grid = np.unique(np.quantile(spread, [0.25, 0.50, 0.75, 1.00]))
    candidates = []
    for nq in neg_grid:
        for pq in pos_grid:
            if nq >= pq:
                continue
            for ns in spread_grid:
                for ps in spread_grid:
                    masks = _routing_masks_from_params(p, nq, pq, ns, ps, blocks)
                    routed = masks["routed"]
                    m = correction_opportunity_metrics(y, q, routed)
                    if m["route_rate"] < route_min_rate - 1e-12 or m["route_rate"] > route_max_rate + 1e-12:
                        continue
                    br = _route_block_metrics(y, q, routed, blocks, min_block_rows, min_block_failures)
                    supported = [r for r in br if r["supported"] and r["route_rate"] > 0]
                    enrich = [r["error_enrichment"] for r in supported if np.isfinite(r["error_enrichment"])]
                    median_enrich = float(np.median(enrich)) if enrich else m["error_enrichment"]
                    min_enrich = float(np.min(enrich)) if enrich else m["error_enrichment"]
                    win_fraction = float(np.mean(np.asarray(enrich) > 1.0)) if enrich else 0.0
                    route_stability = float(np.mean([route_min_rate*.60 <= r["route_rate"] <= min(.85, route_max_rate*1.25) for r in supported])) if supported else 0.0
                    target_penalty = abs(m["route_rate"] - target_route_rate) / max(route_max_rate - route_min_rate, 1e-6)
                    score = (
                        .40 * m["error_enrichment"] + .30 * median_enrich
                        + .10 * m["positive_under_enrichment"] + .05 * m["negative_over_enrichment"]
                        + .10 * m["error_capture"] + .05 * win_fraction
                        + .05 * route_stability - .15 * target_penalty
                    )
                    policy = CorrectionRoutingPolicy(
                        negative_bypass_quantile=float(nq), positive_bypass_quantile=float(pq),
                        negative_max_spread=float(ns), positive_max_spread=float(ps),
                        route_rate_train=m["route_rate"], route_failure_prevalence_train=m["route_failure_prevalence"],
                        route_failure_enrichment_train=m["route_failure_enrichment"], route_error_capture_train=m["error_capture"],
                        route_error_enrichment_train=m["error_enrichment"], positive_under_capture_train=m["positive_under_capture"],
                        negative_over_capture_train=m["negative_over_capture"], selection_score=float(score),
                        route_min_rate=float(route_min_rate), route_max_rate=float(route_max_rate),
                        median_block_error_enrichment=median_enrich, min_block_error_enrichment=min_enrich,
                        supported_block_win_fraction=win_fraction,
                        negative_bypass_rate_train=float(masks["negative_bypass"].mean()),
                        positive_bypass_rate_train=float(masks["positive_bypass"].mean()),
                    )
                    candidates.append((policy, br))
    if not candidates:
        # Safe fallback: middle half of rank routed, no disagreement gate.
        nq, pq, s = .25, .75, float(np.max(spread) + 1e-9)
        masks = _routing_masks_from_params(p, nq, pq, s, s, blocks)
        m = correction_opportunity_metrics(y, q, masks["routed"])
        br = _route_block_metrics(y, q, masks["routed"], blocks, min_block_rows, min_block_failures)
        policy = CorrectionRoutingPolicy(nq,pq,s,s,m["route_rate"],m["route_failure_prevalence"],m["route_failure_enrichment"],m["error_capture"],m["error_enrichment"],m["positive_under_capture"],m["negative_over_capture"],0.0,route_min_rate,route_max_rate,source="fallback_middle_half")
        candidates = [(policy, br)]
    candidates.sort(key=lambda x: (x[0].selection_score, x[0].route_error_enrichment_train, x[0].route_error_capture_train), reverse=True)
    return candidates[:max(1, int(top_k))]


def apply_correction_routing_policy(expert_probabilities, policy: CorrectionRoutingPolicy, block_ids=None):
    masks = _routing_masks_from_params(
        expert_probabilities, policy.negative_bypass_quantile, policy.positive_bypass_quantile,
        policy.negative_max_spread, policy.positive_max_spread, block_ids,
    )
    # Compatibility aliases: negative bypass is a soft exit, strong_positive is never forced.
    masks["easy_negative"] = masks["negative_bypass"]
    masks["strong_positive"] = np.zeros(len(masks["routed"]), bool)
    return masks


def print_correction_routing_policy(p: CorrectionRoutingPolicy):
    print("\n[Three-way correction-opportunity routing]")
    print(f"  Negative bypass : rank <= {p.negative_bypass_quantile:.2f} AND spread <= {p.negative_max_spread:.6f} | soft committee bypass")
    print(f"  Positive bypass : rank >= {p.positive_bypass_quantile:.2f} AND spread <= {p.positive_max_spread:.6f} | soft committee bypass")
    print(f"  Specialist route: {100*p.route_rate_train:.2f}% | failure prevalence={100*p.route_failure_prevalence_train:.2f}% | failure enrichment={p.route_failure_enrichment_train:.2f}x")
    print(f"  Correction opportunity: error capture={100*p.route_error_capture_train:.2f}% | error enrichment={p.route_error_enrichment_train:.2f}x | median-block enrichment={p.median_block_error_enrichment:.2f}x")
    print(f"  Error types captured: positive underprediction={100*p.positive_under_capture_train:.2f}% | negative overprediction={100*p.negative_over_capture_train:.2f}%")
    print(f"  Bypass rates: negative={100*p.negative_bypass_rate_train:.2f}% | positive={100*p.positive_bypass_rate_train:.2f}% | selection score={p.selection_score:.4f}")


def choose_multiblock_validation(y, block_ids, crossfit_mode, min_val_rows=5000, min_val_failures=150, min_val_blocks=2, seed=42):
    y = np.asarray(y, int).reshape(-1)
    blocks = np.asarray(block_ids)
    if crossfit_mode == "temporal_expanding" and len(y) >= 4:
        unique = _sorted_blocks(blocks)
        chosen = []
        for block in reversed(unique):
            idx = blocks == block
            # Tiny temporal fragments can join a window, but never satisfy block count by themselves.
            chosen.append(block)
            va = np.isin(blocks, chosen)
            tr = ~va
            supported_blocks = [b for b in chosen if np.sum(blocks == b) >= 30 and np.sum(y[blocks == b]) >= 2]
            if (va.sum() >= min_val_rows and y[va].sum() >= min_val_failures and len(supported_blocks) >= min_val_blocks
                    and tr.sum() >= 100 and np.unique(y[tr]).size == 2 and np.unique(y[va]).size == 2):
                return np.where(tr)[0], np.where(va)[0], {
                    "mode":"temporal_recent_multiblock", "validation_blocks":list(reversed(chosen)),
                    "supported_validation_blocks":list(reversed(supported_blocks)), "validation_rows":int(va.sum()),
                    "validation_failures":int(y[va].sum()),
                }
    if len(y) < 10 or np.unique(y).size < 2:
        return np.arange(len(y)), np.array([], dtype=int), {"mode":"unavailable","validation_blocks":[]}
    tr, va = train_test_split(np.arange(len(y)), test_size=.20, stratify=y, random_state=int(seed))
    return np.asarray(tr), np.asarray(va), {"mode":"stratified_fallback","validation_blocks":[]}


def build_history_windows(train_indices, y, block_ids, min_rows=5000, min_failures=150, recent_counts=(4,6)):
    train_indices = np.asarray(train_indices, int)
    y = np.asarray(y, int); blocks = np.asarray(block_ids)
    rows = []
    if len(train_indices) and np.unique(y[train_indices]).size == 2:
        rows.append({"name":"all_history", "indices":train_indices.copy(), "block_count":len(pd.unique(blocks[train_indices]))})
    unique = _sorted_blocks(blocks[train_indices]) if len(train_indices) else []
    for count in recent_counts:
        if len(unique) < count:
            continue
        use = unique[-int(count):]
        idx = train_indices[np.isin(blocks[train_indices], use)]
        if len(idx) >= min_rows and int(y[idx].sum()) >= min_failures and np.unique(y[idx]).size == 2:
            rows.append({"name":f"last_{int(count)}_blocks", "indices":idx, "block_count":int(count)})
    # Keep distinct index sets only.
    dedup = []
    seen = set()
    for row in rows:
        key = tuple(row["indices"].tolist())
        if key not in seen:
            seen.add(key); dedup.append(row)
    return dedup


def build_contextual_specialist_features(X_raw, expert_probabilities, selector: RawFeatureSelector, block_ids=None, committee_rank=None):
    p = np.asarray(expert_probabilities, float)
    Xsel = selector.transform(X_raw)
    stats = committee_statistics(p)
    U = np.column_stack([stats[n] for n in UNCERTAINTY_NAMES])
    if committee_rank is None:
        rank = batch_percentile_rank(stats["committee_mean"]) if block_ids is None else ranks_within_blocks(stats["committee_mean"], block_ids)
    else:
        rank = np.asarray(committee_rank, float).reshape(-1)
    spread = stats["committee_spread"]
    entropy = stats["committee_entropy"]
    interactions = np.column_stack([Xsel * rank[:,None], Xsel * spread[:,None], Xsel * entropy[:,None]])
    Z = np.column_stack([Xsel, p, U, interactions])
    raw_names = list(selector.selected_feature_names_)
    names = raw_names + [f"expert_probability_{n}" for n in BASE_NAMES] + list(UNCERTAINTY_NAMES)
    names += [f"{n} x_committee_rank" for n in raw_names]
    names += [f"{n} x_spread" for n in raw_names]
    names += [f"{n} x_entropy" for n in raw_names]
    return Z, names


class ContextualResidualLayer(nn.Module):
    def __init__(self, input_dim):
        super().__init__()
        self.linear = nn.Linear(int(input_dim), 1)
        nn.init.zeros_(self.linear.weight); nn.init.zeros_(self.linear.bias)
    def forward(self, x):
        return self.linear(x).squeeze(1)


def tempered_sample_weights(y, effective_stats, gamma=1.0):
    y = np.asarray(y, int).reshape(-1); gamma=float(gamma)
    if not 0 <= gamma <= 1: raise ValueError("weight gamma must be in [0,1]")
    wf0=float(effective_stats["failure_weight"]); wn0=float(effective_stats["normal_weight"])
    wf,wn=wf0**gamma,wn0**gamma
    n1,n0=int((y==1).sum()),int((y==0).sum())
    norm=(n1*wf+n0*wn)/max(len(y),1); wf/=max(norm,1e-12); wn/=max(norm,1e-12)
    return np.where(y==1,wf,wn).astype(np.float32), {"gamma":gamma,"failure_weight":wf,"normal_weight":wn,"tempered_ratio":wf/max(wn,1e-12),"base_effective_ratio":wf0/max(wn0,1e-12)}


def make_ranking_pairs(y, q, max_pairs=8192, seed=42):
    y=np.asarray(y,int); q=np.asarray(q,float); pos=np.where(y==1)[0]; neg=np.where(y==0)[0]
    if not len(pos) or not len(neg): return np.array([],int),np.array([],int)
    rng=np.random.default_rng(int(seed)); k=max(1,int(max_pairs)); half=k//2
    hard_pos=pos[np.argsort(q[pos])[:max(1,len(pos)//2)]]; hard_neg=neg[np.argsort(-q[neg])[:max(1,len(neg)//2)]]
    pi=np.r_[rng.choice(hard_pos,half,replace=True),rng.choice(pos,k-half,replace=True)]
    ni=np.r_[rng.choice(hard_neg,half,replace=True),rng.choice(neg,k-half,replace=True)]
    return pi.astype(int),ni.astype(int)


def pairwise_ranking_loss(logits, y, pairs):
    pi,ni=pairs
    if len(pi)==0: return logits.sum()*0.0
    pi=pi.to(logits.device) if isinstance(pi,torch.Tensor) else torch.as_tensor(pi,dtype=torch.long,device=logits.device)
    ni=ni.to(logits.device) if isinstance(ni,torch.Tensor) else torch.as_tensor(ni,dtype=torch.long,device=logits.device)
    return F.softplus(-(logits[pi]-logits[ni])).mean()


def block_center_penalty(delta, block_codes):
    if delta.numel()==0: return delta.sum()*0.0
    codes=block_codes.to(delta.device) if isinstance(block_codes,torch.Tensor) else torch.as_tensor(block_codes,dtype=torch.long,device=delta.device)
    vals=[]
    for c in torch.unique(codes):
        m=codes==c
        if torch.any(m): vals.append(delta[m].mean()**2)
    return torch.stack(vals).mean() if vals else delta.sum()*0.0


def _supported_validation_block_uplifts(y, q, score, blocks, beta=2.0, min_rows=30, min_failures=2):
    rows=[]
    y=np.asarray(y,int); q=np.asarray(q,float); score=np.asarray(score,float); blocks=np.asarray(blocks)
    for b in _sorted_blocks(blocks):
        idx=blocks==b; n=int(idx.sum()); fails=int(y[idx].sum())
        if n<min_rows or fails<min_failures or fails>=n: continue
        cap,cau=safe_ap(y[idx],q[idx]),safe_auc(y[idx],q[idx]); sap,sau=safe_ap(y[idx],score[idx]),safe_auc(y[idx],score[idx])
        rows.append({"block":str(b),"rows":n,"failures":fails,"committee_ap":cap,"specialist_ap":sap,"delta_ap":sap-cap,"committee_auc":cau,"specialist_auc":sau,"delta_auc":sau-cau,"committee_f2":max_fbeta_score(y[idx],q[idx],beta),"specialist_f2":max_fbeta_score(y[idx],score[idx],beta)})
    return rows


def temporal_uplift_summary(y,q,score,blocks,beta=2.0):
    rows=_supported_validation_block_uplifts(y,q,score,blocks,beta)
    aps=np.array([r["delta_ap"] for r in rows],float); aucs=np.array([r["delta_auc"] for r in rows],float)
    f2=np.array([r["specialist_f2"]-r["committee_f2"] for r in rows],float)
    return {
        "per_block":rows,"supported_blocks":len(rows),
        "median_ap_uplift":float(np.median(aps)) if len(aps) else -np.inf,
        "mean_ap_uplift":float(np.mean(aps)) if len(aps) else -np.inf,
        "worst_ap_uplift":float(np.min(aps)) if len(aps) else -np.inf,
        "ap_win_fraction":float(np.mean(aps>1e-6)) if len(aps) else 0.0,
        "median_auc_uplift":float(np.median(aucs)) if len(aucs) else -np.inf,
        "worst_auc_uplift":float(np.min(aucs)) if len(aucs) else -np.inf,
        "median_f2_uplift":float(np.median(f2)) if len(f2) else -np.inf,
        "pooled_ap":safe_ap(y,score),"pooled_auc":safe_auc(y,score),"pooled_f2":max_fbeta_score(y,score,beta),
    }


def select_authority_multiblock(y, q, delta, blocks, grid, min_median_ap_uplift=.001, min_win_fraction=.60, worst_ap_drop=.005, worst_auc_drop=.005, beta=2.0):
    y=np.asarray(y,int);q=clip_probability(q);delta=np.asarray(delta,float);blocks=np.asarray(blocks)
    candidates=[]
    for a in sorted(set(float(x) for x in grid)|{0.0}):
        score=apply_residual_correction(q,delta,a); s=temporal_uplift_summary(y,q,score,blocks,beta)
        eligible=(a==0.0) or (s["supported_blocks"]>0 and s["median_ap_uplift"]>=min_median_ap_uplift-1e-12 and s["ap_win_fraction"]>=min_win_fraction-1e-12 and s["worst_ap_uplift"]>=-worst_ap_drop-1e-12 and s["worst_auc_uplift"]>=-worst_auc_drop-1e-12)
        candidates.append({"authority":a,"eligible":bool(eligible),**{k:v for k,v in s.items() if k!="per_block"}})
    eligible=[r for r in candidates if r["eligible"] and r["authority"]>0]
    if not eligible: return 0.0,candidates,"committee_fallback_no_consistent_temporal_uplift"
    chosen=max(eligible,key=lambda r:(r["median_ap_uplift"],r["ap_win_fraction"],r["mean_ap_uplift"],r["median_f2_uplift"],-r["authority"]))
    return float(chosen["authority"]),candidates,"consistent_multiblock_temporal_uplift"


class ContextualResidualSpecialist:
    def __init__(self,max_epochs=200,lr=5e-3,weight_decay=1e-3,effective_beta=.9999,weight_gamma_grid=(0,.5,1),positive_cap=2.,negative_cap=.75,residual_penalty=.05,ranking_penalty=.20,center_penalty=.02,rank_pairs=8192,patience=20,min_improvement=1e-4,device="auto",seed=42,verbose=True):
        self.max_epochs=int(max_epochs);self.lr=float(lr);self.weight_decay=float(weight_decay);self.effective_beta=float(effective_beta);self.weight_gamma_grid=list(weight_gamma_grid);self.positive_cap=float(positive_cap);self.negative_cap=float(negative_cap);self.residual_penalty=float(residual_penalty);self.ranking_penalty=float(ranking_penalty);self.center_penalty_weight=float(center_penalty);self.rank_pairs=int(rank_pairs);self.patience=int(patience);self.min_improvement=float(min_improvement);self.device=resolve_device(device);self.seed=int(seed);self.verbose=bool(verbose)
        self.scaler_=None;self.model_=None;self.history_=[];self.candidate_summaries_=[];self.selected_gamma_=0.;self.best_epoch_=1;self.selected_weight_stats_={}
    def _delta(self,raw):return asymmetric_bounded_delta_torch(raw,self.positive_cap,self.negative_cap)
    def fit_select(self,Ztr,qtr,ytr,btr,Zva,qva,yva,bva,full_y,full_q,full_blocks,full_routed_mask,beta=2.0):
        Ztr,Zva=clean_X(Ztr),clean_X(Zva);qtr,qva=clip_probability(qtr),clip_probability(qva);ytr,yva=np.asarray(ytr,int),np.asarray(yva,int);btr=np.asarray(btr);bva=np.asarray(bva)
        self.scaler_=StandardScaler().fit(Ztr); a=torch.from_numpy(self.scaler_.transform(Ztr).astype(np.float32)).to(self.device); v=torch.from_numpy(self.scaler_.transform(Zva).astype(np.float32)).to(self.device)
        qt=torch.from_numpy(qtr.astype(np.float32)).to(self.device); yt=torch.from_numpy(ytr.astype(np.float32)).to(self.device); base=torch.logit(qt.clamp(1e-6,1-1e-6))
        _,eff=effective_sample_weights(ytr,self.effective_beta); pair_np=make_ranking_pairs(ytr,qtr,self.rank_pairs,self.seed); pair_t=(torch.as_tensor(pair_np[0],dtype=torch.long,device=self.device),torch.as_tensor(pair_np[1],dtype=torch.long,device=self.device))
        _,codes=np.unique(btr,return_inverse=True); codes_t=torch.as_tensor(codes,dtype=torch.long,device=self.device)
        all_candidates=[]; self.history_=[]; self.candidate_summaries_=[]
        for gamma in self.weight_gamma_grid:
            seed_all(self.seed); w_np,wstats=tempered_sample_weights(ytr,eff,gamma); wt=torch.from_numpy(w_np).to(self.device); model=ContextualResidualLayer(Ztr.shape[1]).to(self.device);opt=torch.optim.AdamW(model.parameters(),lr=self.lr,weight_decay=self.weight_decay)
            best_state=copy.deepcopy(model.state_dict());best_key=(-np.inf,-np.inf,-np.inf);best_epoch=1;best_summary=None;bad=0
            for epoch in range(1,self.max_epochs+1):
                model.train();raw=model(a);d=self._delta(raw);logits=base+d;bce=F.binary_cross_entropy_with_logits(logits,yt,weight=wt);rankloss=pairwise_ranking_loss(logits,yt,pair_t);rpen=self.residual_penalty*torch.mean(d**2);cpen=self.center_penalty_weight*block_center_penalty(d,codes_t);loss=bce+self.ranking_penalty*rankloss+rpen+cpen
                opt.zero_grad();loss.backward();opt.step()
                model.eval()
                with torch.no_grad(): rv=model(v); dv=self._delta(rv).cpu().numpy()
                score_full=np.asarray(full_q,float).copy(); score_full[np.asarray(full_routed_mask,bool)] = apply_residual_correction(qva,dv,1.0)
                summ=temporal_uplift_summary(full_y,full_q,score_full,full_blocks,beta); key=(summ["median_ap_uplift"],summ["ap_win_fraction"],summ["mean_ap_uplift"])
                row={"phase":"selection","gamma":float(gamma),"epoch":epoch,"train_bce":float(bce.detach().cpu()),"pairwise_ranking_loss":float(rankloss.detach().cpu()),"residual_penalty_term":float(rpen.detach().cpu()),"center_penalty_term":float(cpen.detach().cpu()),"total_loss":float(loss.detach().cpu()),"val_median_ap_uplift":summ["median_ap_uplift"],"val_ap_win_fraction":summ["ap_win_fraction"],"val_worst_ap_uplift":summ["worst_ap_uplift"],"val_median_auc_uplift":summ["median_auc_uplift"],"val_pooled_ap":summ["pooled_ap"],"delta_mean":float(d.detach().mean().cpu()),"delta_std":float(d.detach().std(unbiased=False).cpu()),"delta_min":float(d.detach().min().cpu()),"delta_max":float(d.detach().max().cpu())};self.history_.append(row)
                if best_summary is None or key > tuple(x+self.min_improvement for x in best_key): best_key=key;best_state=copy.deepcopy(model.state_dict());best_epoch=epoch;best_summary=summ;bad=0
                else: bad+=1
                if self.verbose and (epoch==1 or epoch%max(1,self.max_epochs//4)==0): print(f"    gamma={gamma:.2f} epoch={epoch:3d} | BCE={row['train_bce']:.5f} rank={row['pairwise_ranking_loss']:.5f} residual={row['residual_penalty_term']:.5f} center={row['center_penalty_term']:.5f} | median ΔAP={summ['median_ap_uplift']:+.4f} wins={100*summ['ap_win_fraction']:.0f}% worst ΔAP={summ['worst_ap_uplift']:+.4f}")
                if bad>=self.patience: break
            model.load_state_dict(best_state); all_candidates.append((float(gamma),best_epoch,best_summary,best_state,wstats))
            self.candidate_summaries_.append({"gamma":float(gamma),"best_epoch":best_epoch,"median_ap_uplift":best_summary["median_ap_uplift"],"ap_win_fraction":best_summary["ap_win_fraction"],"worst_ap_uplift":best_summary["worst_ap_uplift"],"median_auc_uplift":best_summary["median_auc_uplift"],"pooled_ap":best_summary["pooled_ap"],"failure_weight":wstats["failure_weight"],"normal_weight":wstats["normal_weight"],"tempered_ratio":wstats["tempered_ratio"]})
        chosen=max(all_candidates,key=lambda x:(x[2]["median_ap_uplift"],x[2]["ap_win_fraction"],x[2]["mean_ap_uplift"],-x[0])); self.selected_gamma_,self.best_epoch_,_,state,self.selected_weight_stats_=chosen; self.model_=ContextualResidualLayer(Ztr.shape[1]).to(self.device);self.model_.load_state_dict(state);return self
    def predict_delta(self,Z):
        if self.model_ is None:return np.zeros(len(Z)),np.zeros(len(Z))
        z=torch.from_numpy(self.scaler_.transform(clean_X(Z)).astype(np.float32)).to(self.device);self.model_.eval()
        with torch.no_grad():raw=self.model_(z);d=self._delta(raw)
        return d.cpu().numpy(),raw.cpu().numpy()
    def refit_all(self,Z,q,y,blocks,epochs):
        seed_all(self.seed);Z=clean_X(Z);q=clip_probability(q);y=np.asarray(y,int);blocks=np.asarray(blocks);self.scaler_=StandardScaler().fit(Z);z=torch.from_numpy(self.scaler_.transform(Z).astype(np.float32)).to(self.device);qt=torch.from_numpy(q.astype(np.float32)).to(self.device);yt=torch.from_numpy(y.astype(np.float32)).to(self.device);base=torch.logit(qt.clamp(1e-6,1-1e-6));_,eff=effective_sample_weights(y,self.effective_beta);w_np,wstats=tempered_sample_weights(y,eff,self.selected_gamma_);self.selected_weight_stats_=wstats;wt=torch.from_numpy(w_np).to(self.device);pairs=make_ranking_pairs(y,q,self.rank_pairs,self.seed);pairs=(torch.as_tensor(pairs[0],dtype=torch.long,device=self.device),torch.as_tensor(pairs[1],dtype=torch.long,device=self.device));_,codes=np.unique(blocks,return_inverse=True);codes=torch.as_tensor(codes,dtype=torch.long,device=self.device)
        self.model_=ContextualResidualLayer(Z.shape[1]).to(self.device);opt=torch.optim.AdamW(self.model_.parameters(),lr=self.lr,weight_decay=self.weight_decay)
        for epoch in range(1,max(1,int(epochs))+1):
            self.model_.train();raw=self.model_(z);d=self._delta(raw);logits=base+d;bce=F.binary_cross_entropy_with_logits(logits,yt,weight=wt);rankloss=pairwise_ranking_loss(logits,yt,pairs);rpen=self.residual_penalty*torch.mean(d**2);cpen=self.center_penalty_weight*block_center_penalty(d,codes);loss=bce+self.ranking_penalty*rankloss+rpen+cpen;opt.zero_grad();loss.backward();opt.step();self.history_.append({"phase":"refit_all","gamma":self.selected_gamma_,"epoch":epoch,"train_bce":float(bce.detach().cpu()),"pairwise_ranking_loss":float(rankloss.detach().cpu()),"residual_penalty_term":float(rpen.detach().cpu()),"center_penalty_term":float(cpen.detach().cpu()),"total_loss":float(loss.detach().cpu()),"delta_mean":float(d.detach().mean().cpu()),"delta_std":float(d.detach().std(unbiased=False).cpu())})
        return self
    def coefficient_report(self,names):
        if self.model_ is None:return []
        w=self.model_.linear.weight.detach().cpu().numpy().reshape(-1); rows=[]
        for i,(n,c) in enumerate(zip(names,w)):rows.append({"feature":n,"coefficient":float(c),"abs_coefficient":float(abs(c)),"rank_abs":0})
        order=np.argsort(-np.abs(w));ranks={int(idx):r+1 for r,idx in enumerate(order)}
        for i,row in enumerate(rows):row["rank_abs"]=ranks[i]
        return rows


def novelty_subgroup_diagnostics(train_frame,test_frame,y,components):
    if "IDSV" not in train_frame or "IDSV" not in test_frame or "IDMH" not in train_frame or "IDMH" not in test_frame:return []
    seen_students=set(train_frame["IDSV"].dropna().astype(str));seen_courses=set(train_frame["IDMH"].dropna().astype(str));ts=test_frame["IDSV"].astype(str).isin(seen_students).to_numpy();tc=test_frame["IDMH"].astype(str).isin(seen_courses).to_numpy();y=np.asarray(y,int);r=np.asarray(components["routed"],bool);q=np.asarray(components["committee_mean"],float);f=np.asarray(components["final_probability"],float)
    defs=[("Returning student + seen course",ts&tc),("Returning student + new course",ts&~tc),("New student + seen course",~ts&tc),("New student + new course",~ts&~tc)];rows=[]
    for name,m in defs:
        if not m.any():continue
        rows.append({"subgroup":name,"rows":int(m.sum()),"failures":int(y[m].sum()),"failure_rate":float(y[m].mean()),"route_rate":float(r[m].mean()),"committee_ap":safe_ap(y[m],q[m]),"final_ap":safe_ap(y[m],f[m]),"delta_ap":safe_ap(y[m],f[m])-safe_ap(y[m],q[m]),"committee_auc":safe_auc(y[m],q[m]),"final_auc":safe_auc(y[m],f[m])})
    return rows


class SelectiveCascadeEnsemble:
    def __init__(self,random_state=42,n_estimators=500,hist_max_iter=250,crossfit_folds=5,crossfit_min_train_rows=100,min_oof_coverage=.40,specialist_top_raw_features=48,specialist_epochs=120,specialist_lr=5e-3,specialist_weight_decay=1e-3,effective_beta=.9999,weight_gamma_grid=(0,.5,1),specialist_positive_cap=2.,specialist_negative_cap=.75,residual_penalty=.05,ranking_penalty=.20,center_penalty=.02,rank_pairs=8192,specialist_patience=15,specialist_min_improvement=1e-4,authority_grid=(0,.25,.5,.75,1),route_min_rate=.25,route_max_rate=.65,target_route_rate=.45,route_top_k=3,validation_min_rows=5000,validation_min_failures=150,validation_min_blocks=2,history_min_rows=5000,history_min_failures=150,history_recent_counts=(4,6),min_median_ap_uplift=.001,min_ap_win_fraction=.60,worst_ap_drop=.005,worst_auc_drop=.005,device="auto",verbose=True):
        self.seed=int(random_state);self.n_estimators=int(n_estimators);self.hist_max_iter=int(hist_max_iter);self.crossfit_folds=int(crossfit_folds);self.crossfit_min_train_rows=int(crossfit_min_train_rows);self.min_oof_coverage=float(min_oof_coverage);self.specialist_top_raw_features=int(specialist_top_raw_features);self.specialist_epochs=int(specialist_epochs);self.specialist_lr=float(specialist_lr);self.specialist_weight_decay=float(specialist_weight_decay);self.effective_beta=float(effective_beta);self.weight_gamma_grid=list(weight_gamma_grid);self.specialist_positive_cap=float(specialist_positive_cap);self.specialist_negative_cap=float(specialist_negative_cap);self.residual_penalty=float(residual_penalty);self.ranking_penalty=float(ranking_penalty);self.center_penalty=float(center_penalty);self.rank_pairs=int(rank_pairs);self.specialist_patience=int(specialist_patience);self.specialist_min_improvement=float(specialist_min_improvement);self.authority_grid=list(authority_grid);self.route_min_rate=float(route_min_rate);self.route_max_rate=float(route_max_rate);self.target_route_rate=float(target_route_rate);self.route_top_k=int(route_top_k);self.validation_min_rows=int(validation_min_rows);self.validation_min_failures=int(validation_min_failures);self.validation_min_blocks=int(validation_min_blocks);self.history_min_rows=int(history_min_rows);self.history_min_failures=int(history_min_failures);self.history_recent_counts=tuple(history_recent_counts);self.min_median_ap_uplift=float(min_median_ap_uplift);self.min_ap_win_fraction=float(min_ap_win_fraction);self.worst_ap_drop=float(worst_ap_drop);self.worst_auc_drop=float(worst_auc_drop);self.device_arg=device;self.verbose=bool(verbose);self.hard_negative_guard=False
        self.base_models_={};self.base_report_=[];self.crossfit_report_=[];self.routing_policy_=None;self.routing_block_diagnostics_=[];self.routing_candidates_=[];self.joint_selection_candidates_=[];self.selector_=None;self.feature_selection_report_=[];self.specialist_=None;self.specialist_feature_names_=[];self.specialist_summary_={};self.authority_candidates_=[];self.authority_=0.;self.authority_reason_="";self.coefficient_report_=[];self.temporal_uplift_blocks_=[];self.route_train_diagnostics_={};self.feature_names_=[];self.is_fitted_=False;self.crossfit_mode_="";self.temporal_source_=""
    def expert_probabilities(self,X):return predict_experts(self.base_models_,X)
    def fit(self,X,y,train_frame,feature_names):
        seed_all(self.seed);X=clean_X(X);y=np.asarray(y,int);self.feature_names_=list(feature_names);groups,source=extract_temporal_groups(train_frame);self.temporal_source_=source;cf=generate_crossfit_predictions(X,y,groups,source,seed=self.seed,n_estimators=self.n_estimators,hist_max_iter=self.hist_max_iter,crossfit_folds=self.crossfit_folds,min_train_rows=self.crossfit_min_train_rows,min_oof_coverage=self.min_oof_coverage,verbose=self.verbose);self.crossfit_mode_=cf.mode;self.crossfit_report_=cf.report_rows;idx=np.where(cf.covered_mask)[0];Xo=X[idx];yo=y[idx];po=cf.probabilities[idx];bo=cf.block_ids[idx];qo=committee_statistics(po)["committee_mean"]
        self.base_report_=[{"model":n,"oof_ap":safe_ap(yo,po[:,i]),"oof_auc":safe_auc(yo,po[:,i]),"crossfit_mode":cf.mode} for i,n in enumerate(BASE_NAMES)]+[{"model":"committee_mean","oof_ap":safe_ap(yo,qo),"oof_auc":safe_auc(yo,qo),"crossfit_mode":cf.mode}]
        meta_tr,meta_va,vinfo=choose_multiblock_validation(yo,bo,cf.mode,self.validation_min_rows,self.validation_min_failures,self.validation_min_blocks,self.seed)
        if self.verbose:print(f"[Meta-selection temporal window] mode={vinfo['mode']} | train={len(meta_tr):,} | val={len(meta_va):,} failures={int(yo[meta_va].sum()) if len(meta_va) else 0:,} | blocks={','.join(map(str,vinfo.get('validation_blocks',[]))) or 'n/a'}")
        route_cands=generate_correction_route_candidates(yo[meta_tr],po[meta_tr],bo[meta_tr],self.route_min_rate,self.route_max_rate,self.target_route_rate,self.route_top_k)
        self.routing_candidates_=[{**asdict(p),"candidate_rank":i+1} for i,(p,_) in enumerate(route_cands)]
        best=None
        for ridx,(policy,train_blockdiag) in enumerate(route_cands,1):
            masks_all=apply_correction_routing_policy(po,policy,bo); routed_all=masks_all["routed"]; train_route_idx=meta_tr[routed_all[meta_tr]]; val_route_idx=meta_va[routed_all[meta_va]]
            if self.verbose:print(f"\n[Route candidate {ridx}/{len(route_cands)}] route={100*policy.route_rate_train:.1f}% error enrichment={policy.route_error_enrichment_train:.2f}x capture={100*policy.route_error_capture_train:.1f}% low/high bypass={100*policy.negative_bypass_rate_train:.1f}%/{100*policy.positive_bypass_rate_train:.1f}%")
            min_specialist_rows=max(100,min(self.history_min_rows,1000))
            if len(train_route_idx)<min_specialist_rows or len(val_route_idx)<30 or np.unique(yo[train_route_idx]).size<2 or np.unique(yo[val_route_idx]).size<2:continue
            windows=build_history_windows(train_route_idx,yo,bo,self.history_min_rows,self.history_min_failures,self.history_recent_counts)
            for win in windows:
                widx=win["indices"]; selector=RawFeatureSelector(self.specialist_top_raw_features).fit(Xo[widx],yo[widx],self.feature_names_); rank_all=ranks_within_blocks(qo,bo); Zw,names=build_contextual_specialist_features(Xo[widx],po[widx],selector,bo[widx],rank_all[widx]); Zv,_=build_contextual_specialist_features(Xo[val_route_idx],po[val_route_idx],selector,bo[val_route_idx],rank_all[val_route_idx])
                spec=ContextualResidualSpecialist(self.specialist_epochs,self.specialist_lr,self.specialist_weight_decay,self.effective_beta,self.weight_gamma_grid,self.specialist_positive_cap,self.specialist_negative_cap,self.residual_penalty,self.ranking_penalty,self.center_penalty,self.rank_pairs,self.specialist_patience,self.specialist_min_improvement,self.device_arg,self.seed,self.verbose)
                if self.verbose:print(f"  [Specialist candidate] window={win['name']} rows={len(widx):,} failures={int(yo[widx].sum()):,} | contextual input={Zw.shape[1]}")
                full_y=yo[meta_va];full_q=qo[meta_va];full_b=bo[meta_va];full_routed=routed_all[meta_va]
                spec.fit_select(Zw,qo[widx],yo[widx],bo[widx],Zv,qo[val_route_idx],yo[val_route_idx],bo[val_route_idx],full_y,full_q,full_b,full_routed,2.0)
                dv,_=spec.predict_delta(Zv);delta_full=np.zeros(len(meta_va),float);delta_full[full_routed]=dv
                authority,arows,reason=select_authority_multiblock(full_y,full_q,delta_full,full_b,self.authority_grid,self.min_median_ap_uplift,self.min_ap_win_fraction,self.worst_ap_drop,self.worst_auc_drop,2.0)
                chosen_row=next(r for r in arows if abs(r["authority"]-authority)<1e-12); result={"route_candidate":ridx,"window":win["name"],"route_rate":policy.route_rate_train,"route_error_enrichment":policy.route_error_enrichment_train,"route_error_capture":policy.route_error_capture_train,"gamma":spec.selected_gamma_,"epoch":spec.best_epoch_,"authority":authority,"authority_reason":reason,"median_ap_uplift":chosen_row["median_ap_uplift"],"mean_ap_uplift":chosen_row["mean_ap_uplift"],"worst_ap_uplift":chosen_row["worst_ap_uplift"],"ap_win_fraction":chosen_row["ap_win_fraction"],"median_auc_uplift":chosen_row["median_auc_uplift"],"median_f2_uplift":chosen_row["median_f2_uplift"]};self.joint_selection_candidates_.append(result)
                selected_val_score=apply_residual_correction(full_q,delta_full,authority)
                perblock_selected=_supported_validation_block_uplifts(full_y,full_q,selected_val_score,full_b,2.0)
                key=(authority>0,chosen_row["median_ap_uplift"],chosen_row["ap_win_fraction"],chosen_row["mean_ap_uplift"],policy.route_error_enrichment_train,policy.selection_score)
                if best is None or key>best[0]:best=(key,policy,train_blockdiag,win,spec,authority,arows,reason,selector,names,routed_all,perblock_selected)
        if best is None:
            policy,train_blockdiag=route_cands[0];self.routing_policy_=policy;self.routing_block_diagnostics_=train_blockdiag;self.authority_=0.;self.authority_reason_="committee_fallback_no_viable_specialist";self.specialist_summary_={"enabled":False,"reason":"no viable specialist candidate","authority":0.0}
        else:
            _,policy,train_blockdiag,win,spec,authority,arows,reason,_,_,routed_all,perblock_selected=best;self.temporal_uplift_blocks_=perblock_selected;self.routing_policy_=policy;self.routing_block_diagnostics_=train_blockdiag;self.authority_=authority;self.authority_candidates_=arows;self.authority_reason_=reason
            # Refit using same window semantics but now include historical meta-validation blocks.
            all_route_idx=np.where(routed_all)[0]; final_windows=build_history_windows(all_route_idx,yo,bo,self.history_min_rows,self.history_min_failures,self.history_recent_counts);fwin=next((w for w in final_windows if w["name"]==win["name"]),final_windows[0]);fidx=fwin["indices"];self.selector_=RawFeatureSelector(self.specialist_top_raw_features).fit(Xo[fidx],yo[fidx],self.feature_names_);self.feature_selection_report_=self.selector_.report_;rank_all=ranks_within_blocks(qo,bo);Zf,names=build_contextual_specialist_features(Xo[fidx],po[fidx],self.selector_,bo[fidx],rank_all[fidx]);spec.refit_all(Zf,qo[fidx],yo[fidx],bo[fidx],spec.best_epoch_);self.specialist_=spec;self.specialist_feature_names_=names;self.coefficient_report_=spec.coefficient_report(names);self.specialist_summary_={"enabled":True,"type":"contextual_interaction_linear_residual","route_rate":float(routed_all.mean()),"route_error_enrichment":policy.route_error_enrichment_train,"route_error_capture":policy.route_error_capture_train,"selected_window":win["name"],"final_window":fwin["name"],"final_training_rows":len(fidx),"final_training_failures":int(yo[fidx].sum()),"specialist_input_dim":len(names),"selected_gamma":spec.selected_gamma_,"selected_epoch":spec.best_epoch_,"authority":authority,"authority_reason":reason,"ranking_penalty":self.ranking_penalty,"residual_penalty":self.residual_penalty,"center_penalty":self.center_penalty}
        selected_masks_full=apply_correction_routing_policy(po,self.routing_policy_,bo);self.routing_block_diagnostics_=_route_block_metrics(yo,qo,selected_masks_full["routed"],bo,100,5)
        if self.verbose:
            print_correction_routing_policy(self.routing_policy_)
            print("[Selected route — per-block correction opportunity]")
            for rr in self.routing_block_diagnostics_:
                if rr["rows"]>=30:
                    print(f"  block={rr['block']:>6s} rows={rr['rows']:>6,} route={100*rr['route_rate']:>6.1f}% | error capture={100*rr['error_capture']:>6.1f}% enrich={rr['error_enrichment']:.2f}x | +under={100*rr['positive_under_capture']:>6.1f}% -over={100*rr['negative_over_capture']:>6.1f}%")
            print("[Joint route / specialist selection]")
            for r in self.joint_selection_candidates_:print(f"  route#{r['route_candidate']} window={r['window']:>13s} gamma={r['gamma']:.2f} epoch={r['epoch']:3d} authority={r['authority']:.2f} | median ΔAP={r['median_ap_uplift']:+.4f} wins={100*r['ap_win_fraction']:.0f}% worst ΔAP={r['worst_ap_uplift']:+.4f} | route error enrich={r['route_error_enrichment']:.2f}x")
            print(f"  SELECTED: authority={self.authority_:.2f} reason={self.authority_reason_} | window={self.specialist_summary_.get('selected_window','n/a')}")
            if self.temporal_uplift_blocks_:
                print("[Selected specialist — validation uplift by temporal block]")
                for ur in self.temporal_uplift_blocks_:print(f"  block={ur['block']:>6s} rows={ur['rows']:>6,} fail={ur['failures']:>5,} | ΔAP={ur['delta_ap']:+.4f} ΔAUC={ur['delta_auc']:+.4f} ΔF2={ur['specialist_f2']-ur['committee_f2']:+.4f}")
            if self.coefficient_report_:
                top=sorted(self.coefficient_report_,key=lambda r:r['rank_abs'])[:16];print("[Top contextual specialist coefficients]");
                for r in top:print(f"  {r['coefficient']:+.5f}  {r['feature']}")
                inter=[r for r in sorted(self.coefficient_report_,key=lambda r:r['rank_abs']) if " x_" in r["feature"]][:12]
                if inter:
                    print("[Top interaction coefficients]")
                    for r in inter:print(f"  {r['coefficient']:+.5f}  {r['feature']}")
        c_oof=self._components_from_known(Xo,po,bo);self.route_train_diagnostics_=routing_diagnostics(yo,c_oof,self.specialist_positive_cap,self.specialist_negative_cap,self.authority_);print_routing_diagnostics("CROSS-FITTED HISTORICAL",self.route_train_diagnostics_) if self.verbose else None
        self.base_models_=make_base_models(self.seed,self.n_estimators,self.hist_max_iter);print(f"[Production experts] fitting all historical rows: {len(X):,}") if self.verbose else None
        for model in self.base_models_.values():model.fit(X,y)
        self.is_fitted_=True;return self
    def _components_from_known(self,X,p,block_ids=None):
        X=clean_X(X);p=np.asarray(p,float);stats=committee_statistics(p);q=stats["committee_mean"];m=apply_correction_routing_policy(p,self.routing_policy_,block_ids);r=m["routed"];rank=m["committee_rank"];sp=np.full(len(q),np.nan);dlt=np.zeros(len(q));raw=np.zeros(len(q))
        if self.specialist_ is not None and self.selector_ is not None and r.any():
            Z,_=build_contextual_specialist_features(X[r],p[r],self.selector_,None,rank[r]);d,rr=self.specialist_.predict_delta(Z);sp[r]=apply_residual_correction(q[r],d,self.authority_);dlt[r]=d;raw[r]=rr
        else:sp[r]=q[r]
        final=q.copy();final[r]=sp[r];return {"expert_probabilities":p,**stats,**m,"specialist_probability":sp,"residual_delta":dlt,"raw_residual":raw,"specialist_authority":np.full(len(q),self.authority_),"final_probability":clip_probability(final)}
    def predict_components(self,X):
        if not self.is_fitted_:raise RuntimeError("Cascade not fitted")
        X=clean_X(X);return self._components_from_known(X,self.expert_probabilities(X))
    def predict_proba(self,X):return self.predict_components(X)["final_probability"]
    def save(self,path):
        payload={"target":TARGET,"failure_boundary":FAILURE_BOUNDARY,"feature_names":self.feature_names_,"base_model_names":BASE_NAMES,"base_models":self.base_models_,"routing_policy":asdict(self.routing_policy_) if self.routing_policy_ else None,"raw_feature_selector":self.selector_,"specialist_feature_names":self.specialist_feature_names_,"specialist_summary":self.specialist_summary_,"authority":self.authority_,"coefficient_report":self.coefficient_report_}
        with Path(path).open("wb") as f:pickle.dump(payload,f)


def routing_diagnostics(y,components,positive_cap=2.0,negative_cap=.75,authority=1.0):
    y=np.asarray(y,int);q=np.asarray(components["committee_mean"],float);final=np.asarray(components["final_probability"],float);r=np.asarray(components["routed"],bool);neg=np.asarray(components["negative_bypass"],bool);pos=np.asarray(components["positive_bypass"],bool);d=np.asarray(components["residual_delta"],float);m=correction_opportunity_metrics(y,q,r);fd=d[r];loss=committee_log_loss_each(y,q);return {"n":len(y),"failures":int(y.sum()),"prevalence":float(y.mean()),"negative_bypass_count":int(neg.sum()),"negative_bypass_rate":float(neg.mean()),"positive_bypass_count":int(pos.sum()),"positive_bypass_rate":float(pos.mean()),"routed_count":int(r.sum()),"routing_rate":float(r.mean()),"routed_failures":int(y[r].sum()),"routed_failure_prevalence":m["route_failure_prevalence"],"routed_enrichment":m["route_failure_enrichment"],"route_error_capture":m["error_capture"],"route_error_enrichment":m["error_enrichment"],"positive_under_capture":m["positive_under_capture"],"negative_over_capture":m["negative_over_capture"],"committee_ap":safe_ap(y,q),"committee_auc":safe_auc(y,q),"final_ap":safe_ap(y,final),"final_auc":safe_auc(y,final),"routed_committee_ap":safe_ap(y[r],q[r]) if r.any() else np.nan,"routed_committee_auc":safe_auc(y[r],q[r]) if r.any() else np.nan,"routed_specialist_ap":safe_ap(y[r],final[r]) if r.any() else np.nan,"routed_specialist_auc":safe_auc(y[r],final[r]) if r.any() else np.nan,"residual_delta_mean":float(fd.mean()) if len(fd) else np.nan,"residual_delta_std":float(fd.std()) if len(fd) else np.nan,"residual_delta_min":float(fd.min()) if len(fd) else np.nan,"residual_delta_max":float(fd.max()) if len(fd) else np.nan,"residual_increase_fraction":float(np.mean(fd>0)) if len(fd) else np.nan,"residual_decrease_fraction":float(np.mean(fd<0)) if len(fd) else np.nan,"specialist_authority":float(authority),"committee_total_logloss":float(loss.sum())}


def print_routing_diagnostics(title,d):
    print(f"\n[{title} routing diagnostics]")
    print(f"  Soft bypass       : negative={d['negative_bypass_count']:,} ({100*d['negative_bypass_rate']:.2f}%) | positive={d['positive_bypass_count']:,} ({100*d['positive_bypass_rate']:.2f}%)")
    print(f"  Specialist routed : {d['routed_count']:,}/{d['n']:,} ({100*d['routing_rate']:.2f}%) | failures={d['routed_failures']:,} | prevalence={100*d['routed_failure_prevalence']:.2f}% | failure enrichment={d['routed_enrichment']:.2f}x")
    print(f"  Error specialization: committee-error capture={100*d['route_error_capture']:.2f}% | error enrichment={d['route_error_enrichment']:.2f}x | positive-under capture={100*d['positive_under_capture']:.2f}% | negative-over capture={100*d['negative_over_capture']:.2f}%")
    print(f"  Whole-set ranking : committee AP/AUC={d['committee_ap']:.4f}/{d['committee_auc']:.4f} -> cascade AP/AUC={d['final_ap']:.4f}/{d['final_auc']:.4f}")
    print(f"  Routed ranking    : committee AP/AUC={d['routed_committee_ap']:.4f}/{d['routed_committee_auc']:.4f} -> specialist AP/AUC={d['routed_specialist_ap']:.4f}/{d['routed_specialist_auc']:.4f}")
    print(f"  Residual          : authority={d['specialist_authority']:.2f} | mean={d['residual_delta_mean']:+.4f} std={d['residual_delta_std']:.4f} range=[{d['residual_delta_min']:+.4f},{d['residual_delta_max']:+.4f}] | up/down={100*d['residual_increase_fraction']:.1f}%/{100*d['residual_decrease_fraction']:.1f}%")

def parse_folds(spec): return sorted({int(x.strip()) for x in spec.split(",") if x.strip()})


def prepare_features(train,test,columns):
    Xtr,Xte=train.loc[:,columns].copy(),test.reindex(columns=columns).copy()
    cat=[c for c in columns if pd.api.types.is_object_dtype(Xtr[c]) or isinstance(Xtr[c].dtype,pd.CategoricalDtype) or pd.api.types.is_string_dtype(Xtr[c])]; num=[c for c in columns if c not in cat]
    for c in num: Xtr[c],Xte[c]=pd.to_numeric(Xtr[c],errors="coerce"),pd.to_numeric(Xte[c],errors="coerce")
    if cat:
        try: enc=OneHotEncoder(handle_unknown="ignore",sparse_output=False,dtype=float)
        except TypeError: enc=OneHotEncoder(handle_unknown="ignore",sparse=False,dtype=float)
        A=enc.fit_transform(Xtr[cat].astype("string").fillna("__MISSING__")); B=enc.transform(Xte[cat].astype("string").fillna("__MISSING__")); names=enc.get_feature_names_out(cat).tolist()
        Xtr=pd.concat([Xtr[num].reset_index(drop=True),pd.DataFrame(A,columns=names)],axis=1); Xte=pd.concat([Xte[num].reset_index(drop=True),pd.DataFrame(B,columns=names)],axis=1)
    else: Xtr,Xte=Xtr[num].reset_index(drop=True),Xte[num].reset_index(drop=True)
    means=Xtr.mean(axis=0,numeric_only=True); Xtr=Xtr.fillna(means).fillna(0).astype(float); Xte=Xte.fillna(means).fillna(0).astype(float)
    if list(Xtr.columns)!=list(Xte.columns): raise RuntimeError("Train/test feature mismatch")
    return Xtr,Xte,list(Xtr.columns)


def load_fold(splits_dir,faculty,fold):
    root=Path(splits_dir)/f"fold_{fold}"; trp=root/f"{faculty}_train.csv"; tep=root/f"{faculty}_test.csv"
    if not trp.exists() or not tep.exists(): return None
    train,test=pd.read_csv(trp,low_memory=False),pd.read_csv(tep,low_memory=False)
    if TARGET not in train or TARGET not in test: raise KeyError(f"{TARGET} missing in fold {fold}")
    train[TARGET]=pd.to_numeric(train[TARGET],errors="coerce"); test[TARGET]=pd.to_numeric(test[TARGET],errors="coerce")
    train=train.dropna(subset=[TARGET]).reset_index(drop=True); test=test.dropna(subset=[TARGET]).reset_index(drop=True)
    if len(train)<50:return None
    cols=[c for c in train.columns if c not in EXCLUDE]; Xtr,Xte,names=prepare_features(train,test,cols)
    return Xtr,Xte,train[TARGET].to_numpy(float),test[TARGET].to_numpy(float),train,test,names


def binary_metrics(y,pred):
    tn,fp,fn,tp=confusion_matrix(y,pred,labels=[0,1]).ravel(); p,r,f1,_=precision_recall_fscore_support(y,pred,average="binary",zero_division=0); n=tn+fp+fn+tp
    return {"tn":int(tn),"fp":int(fp),"fn":int(fn),"tp":int(tp),"accuracy":float((tn+tp)/n),"precision":float(p),"recall":float(r),"specificity":float(tn/(tn+fp)) if tn+fp else np.nan,"f1":float(f1),"flag_rate":float((tp+fp)/n)}


def report(y,score,pred,beta):
    m=binary_metrics(np.asarray(y,int),np.asarray(pred,int)); b2=beta*beta; den=b2*m["precision"]+m["recall"]
    m.update(ap=safe_ap(y,score),auc=safe_auc(y,score),prevalence=float(np.mean(y)),n=len(y),fbeta=0. if den<=0 else float((1+b2)*m["precision"]*m["recall"]/den)); return m


def apply_decision_rule(score,threshold,easy_negative=None,strong_positive=None,hard_negative_guard=False):
    pred=(np.asarray(score,float).reshape(-1)>=threshold).astype(int)
    if hard_negative_guard and easy_negative is not None: pred[np.asarray(easy_negative,bool)]=0
    if strong_positive is not None: pred[np.asarray(strong_positive,bool)]=1
    return pred


def tune_threshold(y_failure,score,easy_negative=None,strong_positive=None,beta=2.,min_precision=.15,max_warning_ratio=2.5,min_flags=5,hard_negative_guard=False):
    y=np.asarray(y_failure,int); score=np.asarray(score,float); easy=np.zeros(len(y),bool) if easy_negative is None else np.asarray(easy_negative,bool); strong=np.zeros(len(y),bool) if strong_positive is None else np.asarray(strong_positive,bool)
    prev=float(y.mean()); unique=np.unique(score); thresholds=unique if len(unique)<=1000 else np.unique(np.quantile(unique,np.linspace(.001,.999,1000))); c=[]
    for t in thresholds: c.append((float(t),report(y,score,apply_decision_rule(score,t,easy,strong,hard_negative_guard),beta)))
    max_rate=min(.99,max_warning_ratio*max(prev,1e-8)); enough=[x for x in c if x[1]["flag_rate"]*len(y)>=min_flags]; strict=[x for x in enough if x[1]["precision"]>=min_precision and x[1]["flag_rate"]<=max_rate]; ratio=[x for x in enough if x[1]["flag_rate"]<=max_rate]
    if strict: eligible,relaxed=strict,"none"
    elif ratio: eligible,relaxed=ratio,"minimum precision"
    elif enough: eligible,relaxed=enough,"minimum precision and maximum warning ratio"
    else: eligible,relaxed=c,"all constraints"
    t,m=max(eligible,key=lambda x:(x[1]["fbeta"],x[1]["recall"],x[1]["precision"],-x[1]["flag_rate"])); return ThresholdPolicy(t,beta,m["ap"],m["auc"],prev,m["flag_rate"],m["flag_rate"]/max(prev,1e-8),m["precision"],m["recall"],m["f1"],m["fbeta"],relaxed)


def warn_by_rate(score,rate,easy_negative=None,strong_positive=None,hard_negative_guard=False):
    score=np.asarray(score,float); n=len(score); easy=np.zeros(n,bool) if easy_negative is None else np.asarray(easy_negative,bool); strong=np.zeros(n,bool) if strong_positive is None else np.asarray(strong_positive,bool); out=np.zeros(n,int); out[strong]=1
    if not n or rate<=0:return out
    target=min(n,max(1,int(round(rate*n)))); need=max(0,target-int(out.sum()));
    if need==0:return out
    blocked=strong | (easy if hard_negative_guard else False); cand=np.where(~blocked)[0]
    if not len(cand):return out
    need=min(need,len(cand)); local=np.argpartition(score[cand],-need)[-need:]; out[cand[local]]=1; return out


def save_results(path,frame,c):
    cols=[x for x in IDS if x in frame.columns]+[TARGET]; out=frame.loc[:,list(dict.fromkeys(cols))].copy(); out["actual_failure_lt5"]=(frame[TARGET].to_numpy(float)<FAILURE_BOUNDARY).astype(np.int8)
    for i,n in enumerate(BASE_NAMES):out[f"expert_probability_{n}"]=c["expert_probabilities"][:,i]
    for n in UNCERTAINTY_NAMES:out[n]=c[n]
    out["committee_rank"]=c["committee_rank"]; out["easy_negative_exit"]=c["easy_negative"].astype(np.int8); out["routed_to_specialist"]=c["routed"].astype(np.int8); out["strong_positive_preserved"]=c["strong_positive"].astype(np.int8)
    out["specialist_probability_lt5"]=c["specialist_probability"]; out["specialist_residual_delta_logit"]=c["residual_delta"]; out["specialist_raw_residual"]=c["raw_residual"]; out["specialist_authority"]=c["specialist_authority"]; out["committee_probability_lt5"]=c["committee_mean"]; out["failure_probability_lt5"]=c["final_probability"]; out.to_csv(path,index=False)


def train_one(splits_dir,faculty,fold,args,outdir):
    data=load_fold(splits_dir,faculty,fold)
    if data is None: print(f"Fold {fold}: skipped"); return None
    X,Xtest,grades,test_grades,train_frame,test_frame,names=data; y=(grades<FAILURE_BOUNDARY).astype(int); yt=(test_grades<FAILURE_BOUNDARY).astype(int)
    print("\n"+"#"*118); print(f"FOLD {fold} | historical_train={len(X):,} | test={len(Xtest):,} | raw/prepared features={X.shape[1]:,}"); print(f"Failure prevalence: train={100*y.mean():.2f}% | test={100*yt.mean():.2f}%"); print("#"*118)
    model=SelectiveCascadeEnsemble(random_state=args.seed,n_estimators=args.n_estimators,hist_max_iter=args.hist_max_iter,max_negative_exit_failure_fraction=args.max_negative_exit_failure_fraction,max_negative_exit_failure_rate=args.max_negative_exit_failure_rate,positive_preserve_min_precision=args.positive_preserve_min_precision,min_route_count=args.min_route_count,routing_min_block_failures=args.routing_min_block_failures,crossfit_folds=args.crossfit_folds,crossfit_min_train_rows=args.crossfit_min_train_rows,min_oof_coverage=args.min_oof_coverage,specialist_top_raw_features=args.specialist_top_raw_features,specialist_epochs=args.specialist_epochs,specialist_lr=args.specialist_lr,specialist_weight_decay=args.specialist_weight_decay,effective_beta=args.effective_beta,specialist_positive_cap=args.specialist_positive_cap,specialist_negative_cap=args.specialist_negative_cap,residual_penalty=args.residual_penalty,specialist_patience=args.specialist_patience,specialist_min_improvement=args.specialist_min_improvement,authority_grid=args.authority_grid,hard_negative_guard=args.hard_negative_guard,device=args.device,verbose=True).fit(X,y,train_frame,names)
    c=model.predict_components(Xtest); diag=routing_diagnostics(yt,c,args.specialist_positive_cap,args.specialist_negative_cap,model.authority_)
    print("\n[Temporal test expert ranking]")
    for i,n in enumerate(BASE_NAMES):print(f"  {n:24s} AP={safe_ap(yt,c['expert_probabilities'][:,i]):.4f} | AUC={safe_auc(yt,c['expert_probabilities'][:,i]):.4f}")
    print_routing_diagnostics("TEMPORAL TEST",diag)
    tag=f"{faculty}_fold_{fold}"; model.save(outdir/f"{tag}_classifier.pkl"); pd.DataFrame(model.base_report_).to_csv(outdir/f"{tag}_base_models.csv",index=False); pd.DataFrame(model.crossfit_report_).to_csv(outdir/f"{tag}_crossfit_report.csv",index=False); pd.DataFrame([asdict(model.routing_policy_)]).to_csv(outdir/f"{tag}_routing_policy.csv",index=False); pd.DataFrame(model.routing_block_diagnostics_).to_csv(outdir/f"{tag}_routing_block_diagnostics.csv",index=False); pd.DataFrame([model.route_train_diagnostics_]).to_csv(outdir/f"{tag}_route_train_diagnostics.csv",index=False); pd.DataFrame([diag]).to_csv(outdir/f"{tag}_test_routing_diagnostics.csv",index=False); pd.DataFrame(model.feature_selection_report_).to_csv(outdir/f"{tag}_feature_selection.csv",index=False); pd.DataFrame(model.authority_candidates_).to_csv(outdir/f"{tag}_authority_candidates.csv",index=False); pd.DataFrame([model.specialist_summary_]).to_csv(outdir/f"{tag}_specialist_summary.csv",index=False)
    hist=model.specialist_.history_ if model.specialist_ is not None else []; pd.DataFrame(hist).to_csv(outdir/f"{tag}_specialist_training.csv",index=False)
    path=outdir/f"{tag}_test_results.csv"; save_results(path,test_frame,c)
    return FoldRecord(fold,yt,c["final_probability"],c["committee_mean"],float(yt.mean()),c["easy_negative"],c["strong_positive"],args.hard_negative_guard,path)


def print_cm(title,m):
    print(f"\n  Confusion matrix — {title}"); print(f"  {'Actual / Predicted':<22}{'Normal >=5':>16}{'Failure <5':>16}"); print(f"  {'Normal >=5':<22}{m['tn']:>16,d}{m['fp']:>16,d}"); print(f"  {'Failure <5':<22}{m['fn']:>16,d}{m['tp']:>16,d}"); print(f"  Normal correctly rejected: {100*m['specificity']:.2f}%"); print(f"  Failures successfully detected: {100*m['recall']:.2f}%")


def evaluate(records:Sequence[FoldRecord],policy,prevalence_map,fallback_prevalence,title,confusion=False):
    print(f"\n### {title} — ROBUST SELECTIVE CASCADE [beta={policy.beta:g}]"); print(f"  {'fold':>6} {'test':>7} {'base%':>7} | {'AP':>7} {'AUC':>7} | {'ACC@t':>7} {'P@t':>7} {'R@t':>7} {'F1@t':>7} {'Fβ@t':>7} {'rate@t':>8} | {'ACC@r':>7} {'P@r':>7} {'R@r':>7} {'F1@r':>7} {'Fβ@r':>7} {'rate@r':>8}")
    rows=[]; ys=[]; scores=[]; fixeds=[]; drifts=[]; cfs=[]; cdr=[]
    for r in records:
        rate=float(np.clip(policy.warning_ratio*prevalence_map.get(r.fold-1,fallback_prevalence),1e-4,.99)); wf=apply_decision_rule(r.score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard); wd=warn_by_rate(r.score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard); mf,md=report(r.y,r.score,wf,policy.beta),report(r.y,r.score,wd,policy.beta)
        cf=apply_decision_rule(r.committee_score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard); cd=warn_by_rate(r.committee_score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard); mcf,mcd=report(r.y,r.committee_score,cf,policy.beta),report(r.y,r.committee_score,cd,policy.beta)
        fixed_without_positive=apply_decision_rule(r.score,policy.threshold,r.easy_negative,None,r.hard_negative_guard)
        drift_without_positive=warn_by_rate(r.score,rate,r.easy_negative,None,r.hard_negative_guard)
        pos_fixed_added=int(np.sum((wf==1)&(fixed_without_positive==0)&r.strong_positive))
        pos_drift_added=int(np.sum((wd==1)&(drift_without_positive==0)&r.strong_positive))
        print(f"  {r.fold:>6} {len(r.y):>7} {100*r.prevalence:>6.2f}% | {mf['ap']:>7.4f} {mf['auc']:>7.4f} | {mf['accuracy']:>7.4f} {mf['precision']:>7.4f} {mf['recall']:>7.4f} {mf['f1']:>7.4f} {mf['fbeta']:>7.4f} {100*mf['flag_rate']:>7.2f}% | {md['accuracy']:>7.4f} {md['precision']:>7.4f} {md['recall']:>7.4f} {md['f1']:>7.4f} {md['fbeta']:>7.4f} {100*md['flag_rate']:>7.2f}%")
        print(f"           committee counterfactual: AP/AUC={mcf['ap']:.4f}/{mcf['auc']:.4f} | fixed F2={mcf['fbeta']:.4f} R={mcf['recall']:.4f} | drift F2={mcd['fbeta']:.4f} R={mcd['recall']:.4f}")
        print(f"           positive decision protection activated: fixed={pos_fixed_added:,} | drift={pos_drift_added:,}")
        if confusion: print_cm(f"fold {r.fold} — fixed threshold",mf); print_cm(f"fold {r.fold} — drift-aware",md)
        rows.append({"fold":r.fold,"n":len(r.y),"prevalence":r.prevalence,"drift_rate":rate,"easy_negative_rate":float(r.easy_negative.mean()),"strong_positive_rate":float(r.strong_positive.mean()),"positive_decision_protection_fixed":pos_fixed_added,"positive_decision_protection_drift":pos_drift_added,**{f"fixed_{k}":v for k,v in mf.items()},**{f"drift_{k}":v for k,v in md.items()},**{f"committee_fixed_{k}":v for k,v in mcf.items()},**{f"committee_drift_{k}":v for k,v in mcd.items()}}); ys.append(r.y);scores.append(r.score);fixeds.append(wf);drifts.append(wd);cfs.append(cf);cdr.append(cd)
    y,score,wf,wd=np.concatenate(ys),np.concatenate(scores),np.concatenate(fixeds),np.concatenate(drifts); mf,md=report(y,score,wf,policy.beta),report(y,score,wd,policy.beta)
    cscore=np.concatenate([r.committee_score for r in records]); mcf=report(y,cscore,np.concatenate(cfs),policy.beta); mcd=report(y,cscore,np.concatenate(cdr),policy.beta)
    print("-"*154); print(f"  {'POOLED':>6} {len(y):>7} {100*y.mean():>6.2f}% | {mf['ap']:>7.4f} {mf['auc']:>7.4f} | {mf['accuracy']:>7.4f} {mf['precision']:>7.4f} {mf['recall']:>7.4f} {mf['f1']:>7.4f} {mf['fbeta']:>7.4f} {100*mf['flag_rate']:>7.2f}% | {md['accuracy']:>7.4f} {md['precision']:>7.4f} {md['recall']:>7.4f} {md['f1']:>7.4f} {md['fbeta']:>7.4f} {100*md['flag_rate']:>7.2f}%"); print(f"  POOLED committee counterfactual: AP/AUC={mcf['ap']:.4f}/{mcf['auc']:.4f} | fixed F2={mcf['fbeta']:.4f} R={mcf['recall']:.4f} | drift F2={mcd['fbeta']:.4f} R={mcd['recall']:.4f}")
    if confusion: print_cm("POOLED — fixed development threshold",mf); print_cm("POOLED — drift-aware",md)
    return rows,{**{f"fixed_{k}":v for k,v in mf.items()},**{f"drift_{k}":v for k,v in md.items()},**{f"committee_fixed_{k}":v for k,v in mcf.items()},**{f"committee_drift_{k}":v for k,v in mcd.items()}}


def add_policy_columns(r,policy,prevalence_map,fallback):
    frame=pd.read_csv(r.path,low_memory=False); rate=float(np.clip(policy.warning_ratio*prevalence_map.get(r.fold-1,fallback),1e-4,.99)); frame["fixed_threshold"]=policy.threshold; frame["warning_fixed"]=apply_decision_rule(r.score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["drift_rate"]=rate; frame["warning_drift"]=warn_by_rate(r.score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["committee_warning_fixed_counterfactual"]=apply_decision_rule(r.committee_score,policy.threshold,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame["committee_warning_drift_counterfactual"]=warn_by_rate(r.committee_score,rate,r.easy_negative,r.strong_positive,r.hard_negative_guard).astype(np.int8); frame.to_csv(r.path,index=False)


def parse_authority_grid(s):
    vals=[float(x.strip()) for x in str(s).split(",") if x.strip()];
    if not vals:return [0,.25,.5,.75,1.0]
    return sorted(set(vals))


def parse_args():
    p=argparse.ArgumentParser(description="Robust negative-exit + positive-preserving selective cascade")
    p.add_argument("--faculty",default="khoa_cong_nghe_thong_tin"); p.add_argument("--splits_dir",required=True); p.add_argument("--dev_folds",default="1,2,3,4,5"); p.add_argument("--holdout_folds",default="6,7"); p.add_argument("--models_dir",default="results/train_expert_v3"); p.add_argument("--seed",type=int,default=42); p.add_argument("--n_estimators",type=int,default=500); p.add_argument("--hist_max_iter",type=int,default=250)
    p.add_argument("--crossfit_folds",type=int,default=5); p.add_argument("--crossfit_min_train_rows",type=int,default=100); p.add_argument("--min_oof_coverage",type=float,default=.40)
    p.add_argument("--max_negative_exit_failure_fraction",type=float,default=.05); p.add_argument("--max_negative_exit_failure_rate",type=float,default=.01); p.add_argument("--positive_preserve_min_precision",type=float,default=.60); p.add_argument("--min_route_count",type=int,default=5); p.add_argument("--routing_min_block_failures",type=int,default=5); p.add_argument("--hard_negative_guard",action="store_true",help="Legacy ablation: force easy-negative exits to normal at decision time")
    p.add_argument("--specialist_top_raw_features",type=int,default=48); p.add_argument("--specialist_epochs",type=int,default=300); p.add_argument("--specialist_lr",type=float,default=1e-2); p.add_argument("--specialist_weight_decay",type=float,default=1e-3); p.add_argument("--effective_beta",type=float,default=.9999); p.add_argument("--specialist_positive_cap",type=float,default=2.0); p.add_argument("--specialist_negative_cap",type=float,default=.75); p.add_argument("--residual_penalty",type=float,default=.05); p.add_argument("--specialist_patience",type=int,default=30); p.add_argument("--specialist_min_improvement",type=float,default=1e-4); p.add_argument("--authority_grid",type=parse_authority_grid,default=parse_authority_grid("0,0.25,0.5,0.75,1.0")); p.add_argument("--device",default="auto",choices=["auto","cpu","cuda"])
    p.add_argument("--threshold_beta",type=float,default=2.0); p.add_argument("--min_precision",type=float,default=.15); p.add_argument("--max_warning_ratio",type=float,default=2.5); p.add_argument("--min_flags",type=int,default=5); return p.parse_args()


def validate_args(a):
    for n in ["max_negative_exit_failure_fraction","max_negative_exit_failure_rate","positive_preserve_min_precision","min_oof_coverage","min_precision"]:
        if not 0<=getattr(a,n)<=1:raise ValueError(f"{n} must be in [0,1]")
    if a.crossfit_folds<2 or a.crossfit_min_train_rows<2 or a.min_route_count<1 or a.routing_min_block_failures<1:raise ValueError("Invalid crossfit/routing arguments")
    if a.specialist_top_raw_features<1 or a.specialist_epochs<1 or a.specialist_lr<=0 or a.specialist_weight_decay<0 or a.specialist_positive_cap<=0 or a.specialist_negative_cap<=0 or a.residual_penalty<0 or a.specialist_patience<1:raise ValueError("Invalid specialist arguments")
    if not 0<=a.effective_beta<1:raise ValueError("effective_beta must satisfy 0<=beta<1")
    if any(x<0 or x>1 for x in a.authority_grid):raise ValueError("authority_grid values must be in [0,1]")
    if a.threshold_beta<=0 or a.max_warning_ratio<=0 or a.min_flags<1:raise ValueError("Invalid threshold arguments")
    if a.device=="cuda" and not torch.cuda.is_available():raise SystemExit("[ERROR] CUDA requested but unavailable")


def main():
    a=parse_args(); validate_args(a); seed_all(a.seed); splits=Path(a.splits_dir).expanduser().resolve(); out=Path(a.models_dir).expanduser().resolve()
    if not splits.exists():raise SystemExit(f"[ERROR] Missing splits directory: {splits}")
    out.mkdir(parents=True,exist_ok=True); dev,hold=parse_folds(a.dev_folds),parse_folds(a.holdout_folds)
    if set(dev)&set(hold):raise ValueError("Development and holdout folds overlap")
    print("="*118); print("ROBUST SELECTIVE CASCADE — CROSS-FITTED RANK ROUTING + BOUNDED RESIDUAL SPECIALIST"); print(f"Faculty: {a.faculty} | target: grade<5 | dev={dev} | holdout={hold}"); print("Experts unchanged: Balanced RF + Balanced ExtraTrees + HistGB | committee=simple mean"); print(f"Cross-fit: temporal expanding preferred; stratified {a.crossfit_folds}-fold fallback | min OOF coverage={100*a.min_oof_coverage:.0f}%"); print(f"Routing: rank-based soft negative exit (hard guard={a.hard_negative_guard}) | positive preserve >= {100*a.positive_preserve_min_precision:.1f}% precision"); print(f"Specialist: top-{a.specialist_top_raw_features} raw ANOVA features + 3 expert + 7 uncertainty | bounded residual [-{a.specialist_negative_cap:g},+{a.specialist_positive_cap:g}] | penalty={a.residual_penalty:g}"); print(f"Authority grid={a.authority_grid} with support ceiling | temporal validation | effective BCE beta={a.effective_beta:.4f}"); print(f"Operational objective: F{a.threshold_beta:g} | outputs: {out}"); print("="*118)
    print("\n>>> DEVELOPMENT FOLDS"); dr=[r for f in dev if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
    if not dr:raise SystemExit("[ERROR] No development folds trained")
    yd=np.concatenate([r.y for r in dr]); sd=np.concatenate([r.score for r in dr]); nd=np.concatenate([r.easy_negative for r in dr]); pdp=np.concatenate([r.strong_positive for r in dr]); pol=tune_threshold(yd,sd,nd,pdp,a.threshold_beta,a.min_precision,a.max_warning_ratio,a.min_flags,a.hard_negative_guard)
    print("\n>>> POOLED DEVELOPMENT THRESHOLD"); print(f"AP={pol.development_ap:.4f} | AUC={pol.development_auc:.4f} | threshold={pol.threshold:.6f} | flag={100*pol.flag_rate:.2f}% | ratio={pol.warning_ratio:.3f} | P={pol.precision:.4f} R={pol.recall:.4f} F1={pol.f1:.4f} F{pol.beta:g}={pol.fbeta:.4f}")
    prev={r.fold:r.prevalence for r in dr}; fallback=float(yd.mean()); drows,dpool=evaluate(dr,pol,prev,fallback,"DEVELOPMENT")
    hr=[];hrows=[];hpool={}
    if hold:
        print("\n"+"!"*118); print("HOLDOUT: labels do not tune cross-fitting, routing, feature selection, specialist, authority, threshold, or warning budget."); print("!"*118); hr=[r for f in hold if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
        for r in hr:prev[r.fold]=r.prevalence
        if hr:hrows,hpool=evaluate(hr,pol,prev,fallback,"HOLDOUT (test)",True)
    for r in dr+hr:add_policy_columns(r,pol,prev,fallback)
    pd.DataFrame([asdict(pol)]).to_csv(out/f"{a.faculty}_threshold_policy.csv",index=False); rows=[{"split":"development",**x} for x in drows]+[{"split":"holdout",**x} for x in hrows]+[{"split":"development","fold":"POOLED",**dpool}];
    if hpool:rows.append({"split":"holdout","fold":"POOLED",**hpool})
    pd.DataFrame(rows).to_csv(out/f"{a.faculty}_warning_metrics.csv",index=False); print("\n"+"="*118); print(f"Frozen threshold: {pol.threshold:.6f} | warning ratio: {pol.warning_ratio:.3f}"); print(f"Completed {len(dr)+len(hr)} fold(s) | outputs: {out}"); print("="*118)



# ============================================================================
# TRAIN V3 CLI / ARTIFACT OVERRIDES
# ============================================================================

def save_results(path,frame,c):
    cols=[x for x in IDS if x in frame.columns]+[TARGET];out=frame.loc[:,list(dict.fromkeys(cols))].copy();out["actual_failure_lt5"]=(frame[TARGET].to_numpy(float)<FAILURE_BOUNDARY).astype(np.int8)
    for i,n in enumerate(BASE_NAMES):out[f"expert_probability_{n}"]=c["expert_probabilities"][:,i]
    for n in UNCERTAINTY_NAMES:out[n]=c[n]
    out["committee_rank"]=c["committee_rank"];out["negative_soft_bypass"]=c["negative_bypass"].astype(np.int8);out["positive_soft_bypass"]=c["positive_bypass"].astype(np.int8);out["routed_to_specialist"]=c["routed"].astype(np.int8);out["specialist_probability_lt5"]=c["specialist_probability"];out["specialist_residual_delta_logit"]=c["residual_delta"];out["specialist_raw_residual"]=c["raw_residual"];out["specialist_authority"]=c["specialist_authority"];out["committee_probability_lt5"]=c["committee_mean"];out["failure_probability_lt5"]=c["final_probability"];out.to_csv(path,index=False)


def train_one(splits_dir,faculty,fold,args,outdir):
    data=load_fold(splits_dir,faculty,fold)
    if data is None:print(f"Fold {fold}: skipped");return None
    X,Xtest,grades,test_grades,train_frame,test_frame,names=data;y=(grades<FAILURE_BOUNDARY).astype(int);yt=(test_grades<FAILURE_BOUNDARY).astype(int)
    print("\n"+"#"*120);print(f"FOLD {fold} | historical_train={len(X):,} | test={len(Xtest):,} | raw/prepared features={X.shape[1]:,}");print(f"Failure prevalence: train={100*y.mean():.2f}% | test={100*yt.mean():.2f}%");print("#"*120)
    model=SelectiveCascadeEnsemble(random_state=args.seed,n_estimators=args.n_estimators,hist_max_iter=args.hist_max_iter,crossfit_folds=args.crossfit_folds,crossfit_min_train_rows=args.crossfit_min_train_rows,min_oof_coverage=args.min_oof_coverage,specialist_top_raw_features=args.specialist_top_raw_features,specialist_epochs=args.specialist_epochs,specialist_lr=args.specialist_lr,specialist_weight_decay=args.specialist_weight_decay,effective_beta=args.effective_beta,weight_gamma_grid=args.weight_gamma_grid,specialist_positive_cap=args.specialist_positive_cap,specialist_negative_cap=args.specialist_negative_cap,residual_penalty=args.residual_penalty,ranking_penalty=args.ranking_penalty,center_penalty=args.center_penalty,rank_pairs=args.rank_pairs,specialist_patience=args.specialist_patience,specialist_min_improvement=args.specialist_min_improvement,authority_grid=args.authority_grid,route_min_rate=args.route_min_rate,route_max_rate=args.route_max_rate,target_route_rate=args.target_route_rate,route_top_k=args.route_top_k,validation_min_rows=args.validation_min_rows,validation_min_failures=args.validation_min_failures,validation_min_blocks=args.validation_min_blocks,history_min_rows=args.history_min_rows,history_min_failures=args.history_min_failures,history_recent_counts=args.history_recent_counts,min_median_ap_uplift=args.min_median_ap_uplift,min_ap_win_fraction=args.min_ap_win_fraction,worst_ap_drop=args.worst_ap_drop,worst_auc_drop=args.worst_auc_drop,device=args.device,verbose=True).fit(X,y,train_frame,names)
    c=model.predict_components(Xtest);diag=routing_diagnostics(yt,c,args.specialist_positive_cap,args.specialist_negative_cap,model.authority_);print("\n[Temporal test expert ranking]")
    for i,n in enumerate(BASE_NAMES):print(f"  {n:24s} AP={safe_ap(yt,c['expert_probabilities'][:,i]):.4f} | AUC={safe_auc(yt,c['expert_probabilities'][:,i]):.4f}")
    print_routing_diagnostics("TEMPORAL TEST",diag);sub=novelty_subgroup_diagnostics(train_frame,test_frame,yt,c)
    if sub:
        print("[Student/course novelty subgroups]")
        for r in sub:print(f"  {r['subgroup']:<34s} rows={r['rows']:>6,} fail={100*r['failure_rate']:>6.2f}% route={100*r['route_rate']:>6.2f}% | committee AP={r['committee_ap']:.4f} -> final AP={r['final_ap']:.4f} (Δ={r['delta_ap']:+.4f})")
    tag=f"{faculty}_fold_{fold}";model.save(outdir/f"{tag}_classifier.pkl");pd.DataFrame(model.base_report_).to_csv(outdir/f"{tag}_base_models.csv",index=False);pd.DataFrame(model.crossfit_report_).to_csv(outdir/f"{tag}_crossfit_report.csv",index=False);pd.DataFrame([asdict(model.routing_policy_)]).to_csv(outdir/f"{tag}_routing_policy.csv",index=False);pd.DataFrame(model.routing_candidates_).to_csv(outdir/f"{tag}_routing_candidates.csv",index=False);pd.DataFrame(model.routing_block_diagnostics_).to_csv(outdir/f"{tag}_routing_block_diagnostics.csv",index=False);pd.DataFrame(model.joint_selection_candidates_).to_csv(outdir/f"{tag}_joint_selection_candidates.csv",index=False);pd.DataFrame(model.feature_selection_report_).to_csv(outdir/f"{tag}_feature_selection.csv",index=False);pd.DataFrame(model.authority_candidates_).to_csv(outdir/f"{tag}_authority_candidates.csv",index=False);pd.DataFrame(model.coefficient_report_).to_csv(outdir/f"{tag}_specialist_coefficients.csv",index=False);pd.DataFrame(model.temporal_uplift_blocks_).to_csv(outdir/f"{tag}_temporal_uplift_blocks.csv",index=False);pd.DataFrame([model.specialist_summary_]).to_csv(outdir/f"{tag}_specialist_summary.csv",index=False);pd.DataFrame([model.route_train_diagnostics_]).to_csv(outdir/f"{tag}_route_train_diagnostics.csv",index=False);pd.DataFrame([diag]).to_csv(outdir/f"{tag}_test_routing_diagnostics.csv",index=False);pd.DataFrame(sub).to_csv(outdir/f"{tag}_novelty_subgroups.csv",index=False);hist=model.specialist_.history_ if model.specialist_ is not None else [];pd.DataFrame(hist).to_csv(outdir/f"{tag}_specialist_training.csv",index=False);path=outdir/f"{tag}_test_results.csv";save_results(path,test_frame,c)
    return FoldRecord(fold,yt,c["final_probability"],c["committee_mean"],float(yt.mean()),c["negative_bypass"],np.zeros(len(yt),bool),False,path)


def _parse_float_grid(s):return sorted(set(float(x.strip()) for x in str(s).split(',') if x.strip()))
def _parse_int_grid(s):return tuple(sorted(set(int(x.strip()) for x in str(s).split(',') if x.strip())))
def parse_args():
    p=argparse.ArgumentParser(description="Train V3: correction-opportunity selective cascade with contextual residual specialist")
    p.add_argument("--faculty",default="khoa_cong_nghe_thong_tin");p.add_argument("--splits_dir",required=True);p.add_argument("--dev_folds",default="1,2,3,4,5");p.add_argument("--holdout_folds",default="6,7");p.add_argument("--models_dir",default="results/train_v3");p.add_argument("--seed",type=int,default=42);p.add_argument("--n_estimators",type=int,default=500);p.add_argument("--hist_max_iter",type=int,default=250)
    p.add_argument("--crossfit_folds",type=int,default=5);p.add_argument("--crossfit_min_train_rows",type=int,default=100);p.add_argument("--min_oof_coverage",type=float,default=.40)
    p.add_argument("--route_min_rate",type=float,default=.25);p.add_argument("--route_max_rate",type=float,default=.65);p.add_argument("--target_route_rate",type=float,default=.45);p.add_argument("--route_top_k",type=int,default=3)
    p.add_argument("--validation_min_rows",type=int,default=5000);p.add_argument("--validation_min_failures",type=int,default=150);p.add_argument("--validation_min_blocks",type=int,default=2);p.add_argument("--history_min_rows",type=int,default=5000);p.add_argument("--history_min_failures",type=int,default=150);p.add_argument("--history_recent_counts",type=_parse_int_grid,default=_parse_int_grid("4,6"))
    p.add_argument("--specialist_top_raw_features",type=int,default=48);p.add_argument("--specialist_epochs",type=int,default=120);p.add_argument("--specialist_lr",type=float,default=5e-3);p.add_argument("--specialist_weight_decay",type=float,default=1e-3);p.add_argument("--effective_beta",type=float,default=.9999);p.add_argument("--weight_gamma_grid",type=_parse_float_grid,default=_parse_float_grid("0,0.5,1"));p.add_argument("--specialist_positive_cap",type=float,default=2.0);p.add_argument("--specialist_negative_cap",type=float,default=.75);p.add_argument("--residual_penalty",type=float,default=.05);p.add_argument("--ranking_penalty",type=float,default=.20);p.add_argument("--center_penalty",type=float,default=.02);p.add_argument("--rank_pairs",type=int,default=8192);p.add_argument("--specialist_patience",type=int,default=15);p.add_argument("--specialist_min_improvement",type=float,default=1e-4);p.add_argument("--authority_grid",type=parse_authority_grid,default=parse_authority_grid("0,0.25,0.5,0.75,1"));p.add_argument("--min_median_ap_uplift",type=float,default=.001);p.add_argument("--min_ap_win_fraction",type=float,default=.60);p.add_argument("--worst_ap_drop",type=float,default=.005);p.add_argument("--worst_auc_drop",type=float,default=.005);p.add_argument("--device",default="auto",choices=["auto","cpu","cuda"])
    p.add_argument("--threshold_beta",type=float,default=2.0);p.add_argument("--min_precision",type=float,default=.15);p.add_argument("--max_warning_ratio",type=float,default=2.5);p.add_argument("--min_flags",type=int,default=5);return p.parse_args()


def validate_args(a):
    if not 0<a.route_min_rate<a.route_max_rate<1:raise ValueError("route rates must satisfy 0 < min < max < 1")
    if not a.route_min_rate<=a.target_route_rate<=a.route_max_rate:raise ValueError("target_route_rate must fall inside route range")
    if not 0<=a.min_ap_win_fraction<=1 or not 0<=a.min_oof_coverage<=1 or not 0<=a.min_precision<=1:raise ValueError("fraction arguments must be in [0,1]")
    if any(g<0 or g>1 for g in a.weight_gamma_grid):raise ValueError("weight gamma values must be in [0,1]")
    if any(x<0 or x>1 for x in a.authority_grid):raise ValueError("authority values must be in [0,1]")
    if a.specialist_epochs<1 or a.specialist_lr<=0 or a.specialist_weight_decay<0 or a.residual_penalty<0 or a.ranking_penalty<0 or a.center_penalty<0:raise ValueError("invalid specialist hyperparameters")
    if a.validation_min_rows<1 or a.validation_min_failures<1 or a.validation_min_blocks<1 or a.route_top_k<1:raise ValueError("invalid selection support")
    if a.device=="cuda" and not torch.cuda.is_available():raise SystemExit("[ERROR] CUDA requested but unavailable")


def main():
    a=parse_args();validate_args(a);seed_all(a.seed);splits=Path(a.splits_dir).expanduser().resolve();out=Path(a.models_dir).expanduser().resolve();
    if not splits.exists():raise SystemExit(f"[ERROR] Missing splits directory: {splits}")
    out.mkdir(parents=True,exist_ok=True);dev,hold=parse_folds(a.dev_folds),parse_folds(a.holdout_folds)
    if set(dev)&set(hold):raise ValueError("Development and holdout folds overlap")
    print("="*120);print("TRAIN V3 — CORRECTION-OPPORTUNITY ROUTING + CONTEXTUAL INTERACTION RESIDUAL SPECIALIST");print(f"Faculty: {a.faculty} | target: grade<5 | dev={dev} | holdout={hold}");print("Experts unchanged: Balanced RF + Balanced ExtraTrees + HistGB | committee=simple mean");print(f"Routing: three-way soft bypass | target route={100*a.target_route_rate:.0f}% admissible={100*a.route_min_rate:.0f}-{100*a.route_max_rate:.0f}% | top candidates={a.route_top_k}");print(f"Specialist: top-{a.specialist_top_raw_features} raw + expert/uncertainty + raw×(rank,spread,entropy) interactions | expected input={4*a.specialist_top_raw_features+10}");print(f"Loss: tempered BCE + {a.ranking_penalty:g} pairwise ranking + {a.residual_penalty:g} residual + {a.center_penalty:g} block-centering | bounded [-{a.specialist_negative_cap:g},+{a.specialist_positive_cap:g}]");print(f"Temporal selection: recent multiblock >= {a.validation_min_rows} rows, >= {a.validation_min_failures} failures, >= {a.validation_min_blocks} blocks | authority grid={a.authority_grid}");print(f"Outputs: {out}");print("="*120)
    print("\n>>> DEVELOPMENT FOLDS");dr=[r for f in dev if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
    if not dr:raise SystemExit("[ERROR] No development folds trained")
    yd=np.concatenate([r.y for r in dr]);sd=np.concatenate([r.score for r in dr]);pol=tune_threshold(yd,sd,None,None,a.threshold_beta,a.min_precision,a.max_warning_ratio,a.min_flags,False);print("\n>>> POOLED DEVELOPMENT THRESHOLD");print(f"AP={pol.development_ap:.4f} | AUC={pol.development_auc:.4f} | threshold={pol.threshold:.6f} | flag={100*pol.flag_rate:.2f}% | ratio={pol.warning_ratio:.3f} | P={pol.precision:.4f} R={pol.recall:.4f} F1={pol.f1:.4f} F{pol.beta:g}={pol.fbeta:.4f}")
    prev={r.fold:r.prevalence for r in dr};fallback=float(yd.mean());drows,dpool=evaluate(dr,pol,prev,fallback,"DEVELOPMENT")
    hr=[];hrows=[];hpool={}
    if hold:
        print("\n"+"!"*120);print("HOLDOUT: labels do not tune routing, specialist window/gamma/epoch/authority, threshold, or warning budget.");print("!"*120);hr=[r for f in hold if (r:=train_one(splits,a.faculty,f,a,out)) is not None]
        for r in hr:prev[r.fold]=r.prevalence
        if hr:hrows,hpool=evaluate(hr,pol,prev,fallback,"HOLDOUT (test)",True)
    for r in dr+hr:add_policy_columns(r,pol,prev,fallback)
    pd.DataFrame([asdict(pol)]).to_csv(out/f"{a.faculty}_threshold_policy.csv",index=False);rows=[{"split":"development",**x} for x in drows]+[{"split":"holdout",**x} for x in hrows]+[{"split":"development","fold":"POOLED",**dpool}];
    if hpool:rows.append({"split":"holdout","fold":"POOLED",**hpool})
    pd.DataFrame(rows).to_csv(out/f"{a.faculty}_warning_metrics.csv",index=False);print("\n"+"="*120);print(f"Frozen threshold: {pol.threshold:.6f} | warning ratio: {pol.warning_ratio:.3f}");print(f"Completed {len(dr)+len(hr)} fold(s) | outputs: {out}");print("="*120)

if __name__ == "__main__":
    main()
