"""Keyed, support-aware evaluation for Suite27 memory-conditioned boosting.

All ranking summaries give each eligible faculty/fold cell equal weight.
One-class cells have undefined AP/AUC and are counted explicitly. Missing scores
remain missing. Operational thresholds are learned only from development folds.
"""
from __future__ import annotations

from collections.abc import Mapping
from pathlib import Path
import json
import math
import re

import numpy as np
import pandas as pd
from sklearn.metrics import average_precision_score, roc_auc_score


BUDGETS = (10, 15, 20, 25)
CORE_MODELS = tuple(f"suite27_{name}" for name in ("stock", "native", "deterministic", "global", "path", "mcgb"))


def _vectors(y, score):
    y = np.asarray(y, dtype=float)
    score = np.asarray(score, dtype=float)
    if y.ndim != 1 or score.ndim != 1 or len(y) != len(score):
        raise ValueError("Labels and scores must be equal-length one-dimensional arrays")
    if not np.isfinite(y).all() or not np.isin(y, (0, 1)).all():
        raise ValueError("Actual labels must be finite binary values 0 or 1")
    if np.isinf(score).any() or ((score < 0) | (score > 1)).any():
        raise ValueError("Scores must be probabilities in [0, 1], or NaN for missing coverage")
    return y.astype(np.int8), score


def _rank_metrics(y, score, sample_weight=None):
    if sample_weight is not None:
        active = np.asarray(sample_weight) > 0
        if not active.any() or len(np.unique(y[active])) < 2:
            return math.nan, math.nan
    elif len(y) == 0 or len(np.unique(y)) < 2:
        return math.nan, math.nan
    return (float(average_precision_score(y, score, sample_weight=sample_weight)),
            float(roc_auc_score(y, score, sample_weight=sample_weight)))


def _confusion(y, predicted):
    predicted = np.asarray(predicted, dtype=bool)
    tp = int(np.sum((y == 1) & predicted))
    fp = int(np.sum((y == 0) & predicted))
    fn = int(np.sum((y == 1) & ~predicted))
    tn = int(np.sum((y == 0) & ~predicted))
    precision = tp / max(tp + fp, 1)
    recall = tp / max(tp + fn, 1)
    return {"tp": tp, "fp": fp, "fn": fn, "tn": tn,
            "precision": precision, "recall": recall,
            "f1": 2 * tp / max(2 * tp + fp + fn, 1),
            "f2": 5 * tp / max(5 * tp + fp + 4 * fn, 1),
            "flagged": tp + fp,
            "flag_rate": (tp + fp) / len(y) if len(y) else math.nan,
            "accuracy": (tp + tn) / len(y) if len(y) else math.nan}


def metrics(y, score):
    """Return ranking, probability-calibration, and top-budget metrics.

    NaN scores are excluded with explicit support counts. A warning budget uses
    round(rate * N), at least one warning when N > 0, and stable input-order
    tie-breaking. This matches the previous Suite reporting convention. AP/AUC
    are undefined for one-class cells; calibration remains defined.
    """
    y, score = _vectors(y, score)
    total = len(y)
    keep = np.isfinite(score)
    y, score = y[keep], score[keep]
    ap, auc = _rank_metrics(y, score)
    n = len(y)
    positives = int(y.sum())
    ece = math.nan
    if n:
        bins = np.minimum((score * 10).astype(int), 9)
        ece = sum(float(np.mean(bins == i)) *
                  abs(float(np.mean(score[bins == i])) - float(np.mean(y[bins == i])))
                  for i in range(10) if np.any(bins == i))
    result = {"total_rows": total, "rows": n, "missing_scores": total - n,
              "positives": positives, "negatives": n - positives,
              "prevalence": positives / n if n else math.nan,
              "coverage": n / total if total else math.nan,
              "ap": ap, "auc": auc,
              "brier": float(np.mean((score - y) ** 2)) if n else math.nan,
              "ece10": ece, "ece": ece,
              "mean_score": float(np.mean(score)) if n else math.nan,
              "calibration_gap": float(np.mean(score) - np.mean(y)) if n else math.nan}
    order = np.argsort(-score, kind="stable")
    for budget in BUDGETS:
        predicted = np.zeros(n, dtype=bool)
        if n:
            predicted[order[:max(1, min(n, round(budget * n / 100)))]] = True
        conf = _confusion(y, predicted)
        for key, value in conf.items():
            result[f"{key}_at_{budget}"] = value if n else (0 if key in {"tp", "fp", "fn", "tn", "flagged"} else math.nan)
        result[f"fbeta_at_{budget}"] = result[f"f2_at_{budget}"]
    return result


def _cell_groups(frame):
    keys = [key for key in ("fold", "faculty") if key in frame.columns]
    if not keys:
        return [((), frame)]
    return frame.groupby(keys if len(keys) > 1 else keys[0], sort=True, dropna=False)


def macro_ap(frame, score_column):
    """Equal-weight mean AP over two-class faculty/fold cells with coverage.

    For validation frames without a fold column the cells are faculties.
    """
    if "actual" not in frame or score_column not in frame:
        raise ValueError(f"macro_ap requires actual and {score_column!r} columns")
    values = [metrics(cell.actual, cell[score_column])["ap"] for _, cell in _cell_groups(frame)]
    values = [value for value in values if np.isfinite(value)]
    return float(np.mean(values)) if values else math.nan


REFERENCE_ALIASES = {
    "suite14": ["suite14_authority_safe_final", "suite14", "final"],
    "suite16": ["suite16_hgb_final", "suite16", "final"],
    "suite17": ["suite17_hgb_final", "suite17_final", "final"],
    "suite18": ["suite18_hgb_final", "suite18_final", "final"],
    "suite20": ["suite20_transformer", "suite20_semester_target_transformer", "transformer", "final"],
    "suite21": ["suite21_guarded_temporal_residual", "suite21_guarded", "final"],
    "suite22": ["suite22_safe_blend", "safe_blend", "final"],
    "suite22_primary": ["suite22_sftr_tabm", "sftr_tabm", "primary"],
    "suite22_relational": ["suite22_relational_ft_tabm", "relational_ft_tabm", "relational"],
    "suite23": ["suite23_faculty_safe_fusion", "faculty_safe_fusion", "final"],
    "suite23_graph": ["suite23_cs_hagt", "cs_hagt"],
    "suite24": ["suite24_evidence_residual", "evidence_residual", "primary"],
    "suite24_posthoc": ["suite24_posthoc_blend", "posthoc_blend"],
    "suite25": ["suite25_sacr", "sacr", "primary"],
    "suite25_blend": ["suite25_committee_blend", "committee_blend"],
    "council": ["suite4_competence_council", "competence_council", "council", "final"],
    "v2": ["failure_probability_lt5", "failure_probability", "stacked_probability", "final"],
    "suite26_mint": ["p_suite26_mint", "suite26_mint", "mint"],
    "suite26_memory": ["p_suite26_memory", "suite26_memory", "memory"],
    "suite26_tree": ["p_suite26_tree", "suite26_tree_only", "suite26_tree", "tree_only"],
    "suite26_original": ["suite26_tair"],
    "suite26_original_memory": ["suite26_tree_memory"],
    "suite26_original_tree": ["suite26_tree"],
    "suite26_original_joint_head": ["suite26_joint_head"],
}
IDENTIFIER_ALIASES = {
    "sample_id": ("sample_id", "target_sample_id"),
    "student_id": ("student_id", "student", "IDSV"),
    "course_id": ("course_id", "course", "IDMH"),
    "semester": ("semester", "target_NHHK", "NHHK", "target_semester"),
    "fold": ("fold", "fold_id"),
    "faculty": ("faculty", "faculty_name"),
    "actual": ("actual", "actual_failure_lt5", "y", "label", "target"),
}
AUDIT_COLUMNS = ("reference", "fold", "faculty", "status", "detail", "path", "column", "keys",
                 "target_rows", "source_rows", "matched_rows", "scored_rows", "missing_rows",
                 "invalid_scores", "extra_reference_rows")


def _canonical_id(value, key):
    if pd.isna(value):
        return None
    value = str(value).strip()
    if not value:
        return None
    # Preserve leading zeros in student/course/sample identifiers.
    if re.fullmatch(r"-?(0|[1-9][0-9]*)\.0+", value):
        value = value.split(".")[0]
    if key == "semester":
        try:
            number = int(value)
            year, term = divmod(number, 10)
            if 1900 <= year <= 9999 and term in (1, 2, 3):
                number = year * 3 + term - 1
            return str(number)
        except ValueError:
            return value
    return value


def _normalize_identifiers(frame):
    """Normalize known aliases and reject disagreement between duplicate aliases."""
    output = frame.copy()
    lower_columns = {}
    for column in frame.columns:
        lower_columns.setdefault(str(column).lower(), []).append(column)
    for key, aliases in IDENTIFIER_ALIASES.items():
        columns = list(dict.fromkeys(column for alias in aliases
                                    for column in lower_columns.get(alias.lower(), [])))
        normalized = []
        for column in columns:
            if key == "actual":
                value = pd.to_numeric(frame[column], errors="coerce")
            else:
                value = frame[column].map(lambda v: _canonical_id(v, key))
            normalized.append(value)
        if normalized:
            base = normalized[0]
            for other in normalized[1:]:
                same = base.eq(other) | (base.isna() & other.isna())
                if not bool(same.all()):
                    raise ValueError(f"Conflicting aliases for {key}: {columns}")
            output[key] = base
    return output


def _reference_candidates(root, fold, faculty):
    root = Path(root)
    fold, faculty = str(fold), str(faculty)
    scoped = [root / f"fold_{fold}" / f"{faculty}_scores.csv",
              root / f"{faculty}_fold_{fold}_scores.csv",
              root / f"fold_{fold}" / f"{faculty}_target_scores.csv",
              root / f"{faculty}_fold_{fold}_target_scores.csv",
              root / f"{faculty}_fold_{fold}_test_results.csv",
              root / f"fold_{fold}" / f"{faculty}_test_results.csv"]
    if root.is_file():
        fixed = (root.name in {f"{faculty}_fold_{fold}_{tail}.csv" for tail in
                             ("scores", "target_scores", "test_results")}
                 or (root.parent.name == f"fold_{fold}" and root.name in
                     {f"{faculty}_{tail}.csv" for tail in ("scores", "target_scores", "test_results")}))
        return [(root, fixed)]
    return [(path, True) for path in scoped] + [(root / "predictions.csv", False),
                                               (root / "scores.csv", False)]


def _align_reference(cell, source, column, fixed_cell):
    left = _normalize_identifiers(cell).reset_index(drop=True)
    right = _normalize_identifiers(source).reset_index(drop=True)
    if "actual" not in right:
        raise ValueError("Reference has no actual labels to verify")
    # A consolidated file may contain multiple folds/faculties. A fixed-cell file
    # is instead verified against its stated scope, never silently relabeled.
    if not fixed_cell:
        for key in ("fold", "faculty"):
            if key in right and key in left:
                right = right[right[key] == left[key].iloc[0]].copy()
    choices = [("sample_id",), ("student_id", "course_id", "semester")]
    fold_proven = fixed_cell or ("fold" in right and "fold" in left)
    if fold_proven:
        choices.append(("student_id", "course_id"))
    keys = None
    for candidate in choices:
        if not all(key in left and key in right for key in candidate):
            continue
        if left[list(candidate)].isna().any().any() or right[list(candidate)].isna().any().any():
            continue
        if left.duplicated(list(candidate)).any() or right.duplicated(list(candidate)).any():
            continue
        keys = list(candidate)
        break
    if keys is None:
        raise ValueError("No sufficiently unique, complete identifiers: need sample_id, "
                         "student/course/semester, or unique student/course in a fixed fold")
    right = right.reset_index(drop=True)
    left_keys = pd.MultiIndex.from_frame(left[keys])
    right_keys = pd.MultiIndex.from_frame(right[keys])
    locations = right_keys.get_indexer(left_keys)
    matched = locations >= 0
    lpos = np.flatnonzero(matched)
    rpos = locations[matched]
    for key in IDENTIFIER_ALIASES:
        if key not in left or key not in right:
            continue
        lv = left[key].iloc[lpos].reset_index(drop=True)
        rv = right[key].iloc[rpos].reset_index(drop=True)
        if key == "actual":
            if not np.isfinite(rv.to_numpy(float)).all() or not rv.isin((0, 1)).all():
                raise ValueError("Invalid actual labels in matched reference rows")
        if lv.isna().any() or rv.isna().any() or not bool(lv.eq(rv).all()):
            raise ValueError(f"Matched reference {key} mismatch; reference cell rejected")
    score = pd.to_numeric(right[column], errors="coerce").to_numpy(float)
    valid = np.isfinite(score) & (score >= 0) & (score <= 1)
    output = np.full(len(cell), np.nan)
    good = valid[rpos]
    output[lpos[good]] = score[rpos[good]]
    scored = int(good.sum())
    return output, {"keys": ",".join(keys), "source_rows": len(right),
                    "matched_rows": int(matched.sum()), "scored_rows": scored,
                    "missing_rows": len(cell) - scored,
                    "invalid_scores": int((~valid[rpos]).sum()),
                    "extra_reference_rows": len(right) - int(matched.sum())}


def add_references(frame, reference_dirs):
    """Add legacy probability columns using validated one-to-one keyed joins.

    ``reference_dirs`` maps report names to directories (or CSV paths). Missing,
    ambiguous, malformed, or conflicting files yield explicit audit rows and NaN
    scores. Every supplied identifier and the actual label are checked after the
    join; matching label arrays are never sufficient alignment evidence.
    """
    if not {"fold", "faculty", "actual"}.issubset(frame.columns):
        raise ValueError("Reference alignment requires fold, faculty, and actual")
    _vectors(frame.actual, np.full(len(frame), np.nan))
    if reference_dirs is None:
        reference_dirs = {}
    if not isinstance(reference_dirs, Mapping):
        reference_dirs = {Path(path).name: path for path in reference_dirs}
    wide = frame.copy()
    audit = []
    for name, root in reference_dirs.items():
        if name in wide.columns:
            raise ValueError(f"Reference {name!r} would overwrite an existing column")
        scores = np.full(len(wide), np.nan)
        for (fold, faculty), positions in wide.groupby(["fold", "faculty"], sort=True).indices.items():
            cell = wide.iloc[positions]
            record = dict.fromkeys(AUDIT_COLUMNS, "")
            record.update(reference=name, fold=fold, faculty=faculty, target_rows=len(cell),
                          source_rows=0, matched_rows=0, scored_rows=0, missing_rows=len(cell),
                          invalid_scores=0, extra_reference_rows=0)
            candidate = next(((p, fixed) for p, fixed in _reference_candidates(root, fold, faculty)
                              if p.is_file()), None) if root else None
            if candidate is None:
                record.update(status="missing", detail="Reference file not found")
                audit.append(record)
                continue
            path, fixed = candidate
            record["path"] = str(path)
            try:
                source = pd.read_csv(path, dtype=object)
                record["source_rows"] = len(source)
                column = next((alias for alias in [name, *REFERENCE_ALIASES.get(name, ["final"])]
                               if alias in source.columns), None)
                if column is None:
                    raise ValueError(f"No probability column for {name}; tried known score aliases")
                record["column"] = column
                aligned, counts = _align_reference(cell, source, column, fixed)
                scores[positions] = aligned
                record.update(counts)
                if counts["scored_rows"] == len(cell):
                    record.update(status="loaded", detail="Verified one-to-one keyed alignment")
                elif counts["scored_rows"]:
                    record.update(status="partial", detail="Unmatched or invalid scores remain NaN")
                else:
                    record.update(status="unmatched", detail="No valid scores on matched identifiers")
            except (ValueError, OSError, KeyError, pd.errors.ParserError) as exc:
                record.update(status="rejected", detail=str(exc))
            audit.append(record)
        wide[name] = scores
    audit_frame = pd.DataFrame(audit, columns=AUDIT_COLUMNS)
    wide.attrs["reference_columns"] = list(dict.fromkeys([
        *frame.attrs.get("reference_columns", []), *reference_dirs]))
    wide.attrs["reference_alignment_audit"] = [*frame.attrs.get("reference_alignment_audit", []),
                                               *audit_frame.to_dict(orient="records")]
    return wide, audit_frame


COMPONENTS = (
    ("suite27_native", "suite27_stock", "native_hard_growth"),
    ("suite27_deterministic", "suite27_native", "deterministic_memory"),
    ("suite27_global", "suite27_deterministic", "learned_root_memory"),
    ("suite27_path", "suite27_global", "ancestor_conditioning"),
    ("suite27_mcgb", "suite27_path", "optional_coadaptation"),
)
PAIR_COLUMNS = ("fold", "faculty", "model_a", "model_b", "total_rows", "rows", "students",
                "available_a", "available_b", "coverage", "ap_a", "ap_b", "delta_ap",
                "auc_a", "auc_b", "delta_auc", "f2_at_20_a", "f2_at_20_b", "delta_f2_at_20",
                "brier_a", "brier_b", "delta_brier", "ece10_a", "ece10_b", "delta_ece10")
BOOT_COLUMNS = ("model_a", "model_b", "split", "statistic", "delta_ap", "ci_low", "ci_high",
                "cells", "excluded_cells", "rows", "students", "requested_replicates",
                "valid_replicates", "invalid_replicates", "seed", "status")
DRAW_COLUMNS = ("model_a", "model_b", "split", "replicate", "delta_ap", "valid_cells",
                "required_cells", "unique_sampled_students")
THRESHOLD_COLUMNS = ("faculty", "model", "status", "threshold", "development_rows",
                     "development_total_rows", "development_positives", "development_f2",
                     "development_flag_rate", "development_folds")
OP_COLUMNS = ("fold", "faculty", "model", "policy", "status", "threshold", "total_rows", "rows",
              "missing_scores", "positives", "negatives", "ap", "auc", "tp", "fp", "fn", "tn",
              "precision", "recall", "f1", "f2", "flagged", "flag_rate", "accuracy")


def _score_columns(frame):
    known = ["committee", *CORE_MODELS, *REFERENCE_ALIASES, *frame.attrs.get("reference_columns", [])]
    known += [str(column) for column in frame.columns if str(column).startswith("suite")]
    return list(dict.fromkeys(column for column in known if column in frame.columns))


def _mean(values):
    values = np.asarray(values, dtype=float)
    values = values[np.isfinite(values)]
    return float(values.mean()) if len(values) else math.nan


def _aggregate_cells(cells, model, **identifiers):
    result = dict(identifiers, model=model, cells=len(cells),
                  valid_ap_cells=int(cells.ap.notna().sum()),
                  valid_auc_cells=int(cells.auc.notna().sum()),
                  total_rows=int(cells.total_rows.sum()), rows=int(cells.rows.sum()),
                  missing_scores=int(cells.missing_scores.sum()),
                  positives=int(cells.positives.sum()), negatives=int(cells.negatives.sum()))
    result["coverage"] = result["rows"] / result["total_rows"] if result["total_rows"] else math.nan
    excluded = {"fold", "faculty", "model", *result}
    for column in cells.columns:
        if column not in excluded:
            if column.startswith(("tp_at_", "fp_at_", "tn_at_", "fn_at_", "flagged_at_")):
                result[column] = int(cells[column].sum())
            else:
                result[column] = _mean(cells[column])
    return result


def _comparison_pairs(models):
    pairs = []
    if "suite27_mcgb" in models:
        pairs.extend(("suite27_mcgb", other) for other in models if other != "suite27_mcgb")
    pairs.extend((a, b) for a, b, _ in COMPONENTS if a in models and b in models)
    # A reduced variant run without MCGB can still be compared to Committee.
    if "suite27_mcgb" not in models and "committee" in models:
        pairs.extend((a, "committee") for a in models if a != "committee")
    return list(dict.fromkeys(pairs))


def _paired_cells(frame, pairs):
    records = []
    for (fold, faculty), cell in _cell_groups(frame):
        for a, b in pairs:
            available_a = np.isfinite(cell[a].to_numpy(float))
            available_b = np.isfinite(cell[b].to_numpy(float))
            shared = available_a & available_b
            y = cell.actual.to_numpy()[shared]
            ma, mb = metrics(y, cell[a].to_numpy()[shared]), metrics(y, cell[b].to_numpy()[shared])
            record = {"fold": fold, "faculty": faculty, "model_a": a, "model_b": b,
                      "total_rows": len(cell), "rows": int(shared.sum()),
                      "students": int(cell.loc[shared, "student_id"].nunique()) if "student_id" in cell else 0,
                      "available_a": int(available_a.sum()), "available_b": int(available_b.sum()),
                      "coverage": float(shared.mean()) if len(cell) else math.nan}
            for metric in ("ap", "auc", "f2_at_20", "brier", "ece10"):
                record[f"{metric}_a"] = ma[metric]
                record[f"{metric}_b"] = mb[metric]
                record[f"delta_{metric}"] = ma[metric] - mb[metric]
            records.append(record)
    return pd.DataFrame(records, columns=PAIR_COLUMNS)


def _paired_summary(paired, splits, pairs):
    records = []
    for split, folds in splits.items():
        for a, b in pairs:
            cell = paired[(paired.model_a == a) & (paired.model_b == b) & paired.fold.isin(folds)]
            records.append({"split": split, "model_a": a, "model_b": b, "cells": len(cell),
                            "valid_ap_cells": int(cell.delta_ap.notna().sum()),
                            "rows": int(cell.rows.sum()), "total_rows": int(cell.total_rows.sum()),
                            "wins": int((cell.delta_ap > 0).sum()), "ties": int((cell.delta_ap == 0).sum()),
                            **{f"mean_delta_{key}": _mean(cell[f"delta_{key}"])
                               for key in ("ap", "auc", "f2_at_20", "brier", "ece10")}})
    return pd.DataFrame(records, columns=["split", "model_a", "model_b", "cells", "valid_ap_cells",
                                         "rows", "total_rows", "wins", "ties", "mean_delta_ap",
                                         "mean_delta_auc", "mean_delta_f2_at_20", "mean_delta_brier",
                                         "mean_delta_ece10"])


def _bootstrap_pair(frame, a, b, n_boot, seed):
    """Paired student-cluster resampling of a fixed set of eligible cells.

    A sampled student's multiplicity is reused for every occurrence in every
    faculty and fold. A replicate is undefined if any originally eligible cell
    loses a class; its omission is counted, never silently changes the cell set.
    """
    base = {"model_a": a, "model_b": b, "split": "holdout",
            "statistic": "macro_faculty_fold_ap_delta", "delta_ap": math.nan,
            "ci_low": math.nan, "ci_high": math.nan, "cells": 0, "excluded_cells": 0,
            "rows": 0, "students": 0, "requested_replicates": n_boot,
            "valid_replicates": 0, "invalid_replicates": 0, "seed": seed, "status": "no_eligible_cells"}
    if frame.empty:
        base["status"] = "no_holdout_rows"
        return base, []
    common = np.isfinite(frame[a].to_numpy(float)) & np.isfinite(frame[b].to_numpy(float))
    shared = frame.loc[common].copy()
    eligible = []
    point_deltas = []
    total_cells = frame.groupby(["fold", "faculty"]).ngroups
    for key, cell in _cell_groups(shared):
        y = cell.actual.to_numpy(np.int8)
        if len(np.unique(y)) < 2:
            continue
        sa, sb = cell[a].to_numpy(float), cell[b].to_numpy(float)
        point_deltas.append(float(average_precision_score(y, sa) - average_precision_score(y, sb)))
        eligible.append((key, cell, y, sa, sb))
    base.update(cells=len(eligible), excluded_cells=total_cells - len(eligible),
                rows=sum(len(cell) for _, cell, *_ in eligible), delta_ap=_mean(point_deltas))
    if not eligible:
        return base, []
    if "student_id" not in shared or shared.student_id.isna().any():
        base["status"] = "missing_student_id"
        return base, []
    normalized = [cell.student_id.map(lambda value: _canonical_id(value, "student_id"))
                  for _, cell, *_ in eligible]
    if any(ids.isna().any() for ids in normalized):
        base["status"] = "missing_student_id"
        return base, []
    students = sorted(set(student for ids in normalized for student in ids))
    student_map = {student: i for i, student in enumerate(students)}
    base["students"] = len(students)
    if not n_boot:
        base["status"] = "disabled"
        return base, []
    if len(students) < 2:
        base["status"] = "insufficient_students"
        return base, []
    cell_arrays = [(y, sa, sb, ids.map(student_map).to_numpy(int))
                   for (_, _, y, sa, sb), ids in zip(eligible, normalized)]
    rng = np.random.default_rng(seed)
    values, records = [], []
    for replicate in range(n_boot):
        counts = np.bincount(rng.integers(0, len(students), size=len(students)), minlength=len(students))
        deltas = []
        for y, sa, sb, student_indices in cell_arrays:
            weights = counts[student_indices]
            active = weights > 0
            if not active.any() or len(np.unique(y[active])) < 2:
                continue
            deltas.append(float(average_precision_score(y, sa, sample_weight=weights) -
                                average_precision_score(y, sb, sample_weight=weights)))
        value = _mean(deltas) if len(deltas) == len(cell_arrays) else math.nan
        if np.isfinite(value):
            values.append(value)
        records.append({"model_a": a, "model_b": b, "split": "holdout", "replicate": replicate,
                        "delta_ap": value, "valid_cells": len(deltas), "required_cells": len(cell_arrays),
                        "unique_sampled_students": int(np.count_nonzero(counts))})
    base.update(valid_replicates=len(values), invalid_replicates=n_boot - len(values),
                status="ok" if len(values) == n_boot else ("partial_replicates" if values else "no_valid_replicates"))
    if values:
        base.update(ci_low=float(np.quantile(values, .025)), ci_high=float(np.quantile(values, .975)))
    return base, records


def _development_threshold(y, score):
    """Exact F2 maximum over score ties, preferring fewer warnings on ties."""
    order = np.argsort(-score, kind="stable")
    sorted_y, sorted_score = y[order], score[order]
    end = np.r_[np.flatnonzero(sorted_score[:-1] != sorted_score[1:]), len(score) - 1]
    tp = np.cumsum(sorted_y)[end].astype(float)
    flagged = end + 1
    fp, fn = flagged - tp, float(y.sum()) - tp
    f2 = 5 * tp / np.maximum(5 * tp + fp + 4 * fn, 1)
    best = int(np.argmax(f2))
    threshold = float(sorted_score[end[best]])
    return threshold, float(f2[best]), float(flagged[best] / len(y))


def _check_threshold_chronology(frame, dev_folds, holdout_folds):
    """Reject thresholds calibrated on contemporaneous or future targets.

    Canonical semesters determine time whenever present. Legacy evaluation
    fixtures without semester use numeric fold order explicitly as a fallback.
    An absent development or holdout split has no threshold application to check.
    """
    dev = frame[frame.fold.isin(dev_folds)]
    holdout = frame[frame.fold.isin(holdout_folds)]
    if dev.empty or holdout.empty:
        return
    key = 'semester' if 'semester' in frame.columns else 'fold'
    arrays = [pd.to_numeric(rows[key], errors='raise').to_numpy(float) for rows in (dev, holdout)]
    if any(not np.isfinite(values).all() or (values != np.floor(values)).any() for values in arrays):
        raise ValueError(f'Operational threshold chronology requires finite integer {key} values')
    latest_dev, earliest_holdout = int(arrays[0].max()), int(arrays[1].min())
    if latest_dev >= earliest_holdout:
        raise ValueError(f'Operational threshold chronology violated: development {key} {latest_dev} must strictly precede holdout {key} {earliest_holdout}')


def _operational_tables(frame, models, dev_folds, holdout_folds):
    _check_threshold_chronology(frame, dev_folds, holdout_folds)
    thresholds, operational = [], []
    for faculty, faculty_frame in frame.groupby("faculty", sort=True):
        dev = faculty_frame[faculty_frame.fold.isin(dev_folds)]
        holdout = faculty_frame[faculty_frame.fold.isin(holdout_folds)]
        for model in models:
            available = np.isfinite(dev[model].to_numpy(float))
            y, scores = dev.actual.to_numpy(np.int8)[available], dev[model].to_numpy(float)[available]
            record = {"faculty": faculty, "model": model, "threshold": math.nan,
                      "development_rows": len(y), "development_total_rows": len(dev),
                      "development_positives": int(y.sum()), "development_f2": math.nan,
                      "development_flag_rate": math.nan,
                      "development_folds": ",".join(map(str, sorted(dev_folds)))}
            if not len(y):
                record["status"] = "no_development_scores"
            elif len(np.unique(y)) < 2:
                record["status"] = "one_class_development"
            else:
                threshold, f2, flag_rate = _development_threshold(y, scores)
                record.update(status="selected", threshold=threshold,
                              development_f2=f2, development_flag_rate=flag_rate)
            thresholds.append(record)
            groups = [(fold, rows) for fold, rows in holdout.groupby("fold", sort=True)]
            if len(holdout):
                groups.append(("POOLED", holdout))
            for fold, rows in groups:
                available = np.isfinite(rows[model].to_numpy(float))
                yy = rows.actual.to_numpy(np.int8)[available]
                ss = rows[model].to_numpy(float)[available]
                ap, auc = _rank_metrics(yy, ss)
                op = {"fold": fold, "faculty": faculty, "model": model, "policy": "fixed",
                      "status": record["status"], "threshold": record["threshold"],
                      "total_rows": len(rows), "rows": len(yy), "missing_scores": len(rows) - len(yy),
                      "positives": int(yy.sum()), "negatives": len(yy) - int(yy.sum()), "ap": ap, "auc": auc}
                if record["status"] == "selected" and len(yy):
                    op.update(_confusion(yy, ss >= record["threshold"]))
                    op["status"] = "applied"
                else:
                    op.update({key: math.nan for key in _confusion(np.array([], int), np.array([], bool))})
                    if record["status"] == "selected":
                        op["status"] = "no_holdout_scores"
                operational.append(op)
    return (pd.DataFrame(thresholds, columns=THRESHOLD_COLUMNS),
            pd.DataFrame(operational, columns=OP_COLUMNS))


def _is_it(faculty):
    name = str(faculty).lower().replace(" ", "_").replace("-", "_")
    return name in {"it", "cntt"} or "cong_nghe_thong_tin" in name or "information_technology" in name


def _json_safe(value):
    if isinstance(value, dict):
        return {str(key): _json_safe(item) for key, item in value.items()}
    if isinstance(value, (list, tuple, np.ndarray)):
        return [_json_safe(item) for item in value]
    if isinstance(value, np.generic):
        return _json_safe(value.item())
    if isinstance(value, float) and not math.isfinite(value):
        return None
    if value is pd.NA:
        return None
    if isinstance(value, Path):
        return str(value)
    return value


WORKLOAD_COLUMNS = ("fold", "faculty", "model", "policy", "status", "threshold", "rows", "students",
                    "flagged_courses", "flagged_students", "student_flag_rate", "mean_flags_per_student",
                    "max_flags_per_student", "flagged_positive_students", "flagged_negative_only_students")


def student_workload(frame, models, thresholds, holdout_folds):
    """Course-level fixed/budget flags aggregated by student within faculty/fold.

    Pooled rows count each student once across configured holdout folds. Fixed
    thresholds are learned only from development labels. Undefined thresholds
    remain unavailable; an empty holdout still writes the complete schema.
    """
    records = []
    holdout = frame[frame.fold.isin(holdout_folds)]
    if "student_id" not in frame:
        return pd.DataFrame(columns=WORKLOAD_COLUMNS)
    for faculty, faculty_rows in holdout.groupby("faculty", sort=True):
        groups = list(faculty_rows.groupby("fold", sort=True)) + [("POOLED", faculty_rows)]
        for fold, cell in groups:
            for model in models:
                keep = np.isfinite(cell[model].to_numpy(float))
                rows = cell.loc[keep]
                selected = thresholds[(thresholds.faculty == faculty) & (thresholds.model == model)]
                threshold = float(selected.iloc[0].threshold) if len(selected) else math.nan
                status = str(selected.iloc[0].status) if len(selected) else "no_development_scores"
                for policy in ("fixed", *(f"budget_{rate}" for rate in BUDGETS)):
                    record = dict(fold=fold, faculty=faculty, model=model, policy=policy,
                                  status=status if policy == "fixed" else "applied",
                                  threshold=threshold if policy == "fixed" else math.nan,
                                  rows=len(rows), students=int(rows.student_id.nunique()))
                    if not len(rows) or (policy == "fixed" and not np.isfinite(threshold)):
                        record.update({key: math.nan for key in WORKLOAD_COLUMNS[8:]})
                        if not len(rows):
                            record["status"] = "no_holdout_scores"
                    else:
                        scores = rows[model].to_numpy(float)
                        if policy == "fixed":
                            flags = scores >= threshold
                            record["status"] = "applied"
                        else:
                            flags = np.zeros(len(rows), bool)
                            count = max(1, round(int(policy.split("_")[1]) * len(rows) / 100))
                            flags[np.argsort(-scores, kind="stable")[:count]] = True
                        counts = pd.Series(flags.astype(int), index=rows.student_id).groupby(level=0).sum()
                        flagged_ids = set(rows.loc[flags, "student_id"])
                        positive_ids = set(rows.loc[flags & (rows.actual.to_numpy() == 1), "student_id"])
                        record.update(flagged_courses=int(flags.sum()), flagged_students=len(flagged_ids),
                                      student_flag_rate=len(flagged_ids) / max(record["students"], 1),
                                      mean_flags_per_student=float(counts.mean()), max_flags_per_student=int(counts.max()),
                                      flagged_positive_students=len(positive_ids),
                                      flagged_negative_only_students=len(flagged_ids - positive_ids))
                    records.append(record)
    return pd.DataFrame(records, columns=WORKLOAD_COLUMNS)


def write_report(frame, out_dir, dev_folds=(1, 2, 3, 4, 5), holdout_folds=(6, 7), bootstrap=200, seed=42):
    """Write complete CSV/JSON reports without selecting models on target labels.

    Returns a JSON-safe summary and artifact paths. Operational policy is always
    a fixed per-faculty, per-model development F2 threshold; unavailable thresholds
    remain unavailable. Bootstrap CIs describe paired holdout macro AP differences.
    """
    dev_folds = tuple(sorted(set(int(fold) for fold in dev_folds)))
    holdout_folds = tuple(sorted(set(int(fold) for fold in holdout_folds)))
    if set(dev_folds) & set(holdout_folds):
        raise ValueError("Development and holdout folds overlap; thresholds would leak holdout labels")
    if int(bootstrap) != bootstrap or bootstrap < 0:
        raise ValueError("bootstrap must be a nonnegative integer")
    bootstrap, seed = int(bootstrap), int(seed)
    if not {"fold", "faculty", "actual"}.issubset(frame.columns):
        raise ValueError("Reporting requires fold, faculty, and actual columns")
    work = frame.copy()
    numeric_fold = pd.to_numeric(work.fold, errors="raise")
    if not np.isfinite(numeric_fold).all() or not (numeric_fold == np.floor(numeric_fold)).all():
        raise ValueError("Fold IDs must be finite integers")
    work["fold"] = numeric_fold.astype(int)
    _check_threshold_chronology(work, dev_folds, holdout_folds)
    if work.faculty.isna().any():
        raise ValueError("Faculty identifiers cannot be missing")
    _vectors(work.actual, np.full(len(work), np.nan))
    models = _score_columns(work)
    for model in models:
        _, values = _vectors(work.actual, work[model])
        work[model] = values
    out_dir = Path(out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    all_folds = tuple(sorted(work.fold.unique()))
    splits = {"all": all_folds, "development": dev_folds, "holdout": holdout_folds}
    cell_records = [{"fold": fold, "faculty": faculty, "model": model,
                     **metrics(cell.actual, cell[model])}
                    for (fold, faculty), cell in _cell_groups(work) for model in models]
    metric_columns = ["fold", "faculty", "model", *metrics([], []).keys()]
    cells = pd.DataFrame(cell_records, columns=metric_columns)
    macro_records = [_aggregate_cells(cells[(cells.model == model) & cells.fold.isin(folds)], model, split=split)
                     for split, folds in splits.items() for model in models]
    macro_columns = ["split", "model", "cells", "valid_ap_cells", "valid_auc_cells", "total_rows", "rows",
                     "missing_scores", "positives", "negatives", "coverage", *[
                         key for key in metrics([], []) if key not in {"total_rows", "rows", "missing_scores",
                                                                      "positives", "negatives", "coverage"}]]
    macro = pd.DataFrame(macro_records, columns=macro_columns)
    fold_records = [_aggregate_cells(cells[(cells.model == model) & (cells.fold == fold)], model, fold=fold)
                    for fold in all_folds for model in models]
    by_fold = pd.DataFrame(fold_records, columns=["fold", *macro_columns[1:]])
    faculty_records = [_aggregate_cells(cells[(cells.model == model) & (cells.faculty == faculty) &
                                             cells.fold.isin(folds)], model, split=split, faculty=faculty)
                       for split, folds in splits.items() for faculty in sorted(work.faculty.unique())
                       for model in models]
    by_faculty = pd.DataFrame(faculty_records, columns=["split", "faculty", *macro_columns[1:]])
    pairs = _comparison_pairs(models)
    paired = _paired_cells(work, pairs)
    paired_summary = _paired_summary(paired, splits, pairs)
    components = []
    for a, b, name in COMPONENTS:
        selected = paired[(paired.model_a == a) & (paired.model_b == b)].copy()
        selected["component"] = name
        components.append(selected)
    components = pd.concat(components, ignore_index=True) if components else pd.DataFrame(columns=[*PAIR_COLUMNS, "component"])
    thresholds, operational = _operational_tables(work, models, dev_folds, holdout_folds)
    it = operational.loc[operational.faculty.map(_is_it).astype(bool)].copy()
    boot_records, draw_records = [], []
    holdout = work[work.fold.isin(holdout_folds)]
    for a, b in pairs:
        result, draws = _bootstrap_pair(holdout, a, b, bootstrap, seed)
        boot_records.append(result)
        draw_records.extend(draws)
    boots = pd.DataFrame(boot_records, columns=BOOT_COLUMNS)
    draws = pd.DataFrame(draw_records, columns=DRAW_COLUMNS)
    alignment = pd.DataFrame(work.attrs.get("reference_alignment_audit", []), columns=AUDIT_COLUMNS)
    workload = student_workload(work, models, thresholds, holdout_folds)
    aligned_cells = paired[["fold", "faculty", "model_a", "model_b", "total_rows", "rows", "students",
                            "available_a", "available_b", "coverage"]].copy()
    aligned_cells["eligible_ap"] = paired.delta_ap.notna()
    tables = {"predictions": work, "per_cell_metrics": cells, "macro_metrics": macro,
              "per_fold_metrics": by_fold, "per_faculty_metrics": by_faculty,
              "paired_metrics": paired, "paired_summary": paired_summary,
              "component_uplifts": components, "aligned_cells": aligned_cells,
              "thresholds": thresholds, "operational_metrics": operational, "it_confusion": it,
              "bootstrap_ap": boots, "bootstrap_replicates": draws, "reference_alignment_audit": alignment,
              "student_workload": workload}
    paths = {}
    from runtime import atomic_csv, atomic_json, print_table
    for name, table in tables.items():
        path = out_dir / f"{name}.csv"
        # Explicit formatting handles NaN as blank without NumPy cast warnings.
        atomic_csv(path, table)
        paths[name] = str(path.resolve())
    paths["report"] = str((out_dir / "report.json").resolve())
    warnings = []
    if not work.fold.isin(dev_folds).any():
        warnings.append("No development rows: fixed operational thresholds are unavailable")
    if not work.fold.isin(holdout_folds).any():
        warnings.append("No configured holdout rows: holdout operational metrics and bootstrap are unavailable")
    if len(cells) and (cells.missing_scores > 0).any():
        warnings.append("Score coverage differs; compare paired metrics on shared rows and inspect support counts")
    if len(boots) and (boots.invalid_replicates > 0).any():
        warnings.append("Some bootstrap draws lost a class in an eligible cell; invalid draws are counted and omitted from CIs")
    summary = _json_safe({"rows": len(work), "models": models, "folds": all_folds,
                          "dev_folds": dev_folds, "holdout_folds": holdout_folds,
                          "macro": macro.to_dict(orient="records"),
                          "paired_summary": paired_summary.to_dict(orient="records"),
                          "thresholds": thresholds.to_dict(orient="records"),
                          "bootstrap": boots.to_dict(orient="records"),
                          "reference_alignment": alignment.to_dict(orient="records"),
                          "warnings": warnings, "paths": paths,
                          "conventions": {
                              "primary": "suite27_mcgb (prespecified; no architecture selection by this report)",
                              "macro": "Unweighted mean over eligible faculty-fold cells; one-class AP/AUC undefined",
                              "paired": "Each model pair is evaluated on exactly its shared scored rows within each cell",
                              "budget": "Stable descending score order; round(rate*N), minimum one when N>0",
                              "threshold": "Per-faculty/model F2 maximization using development folds only; ties prefer fewer warnings",
                              "threshold_chronology": "Latest development canonical semester strictly precedes earliest holdout semester; numeric fold order is used only when semester is absent",
                              "bootstrap": "Paired student-cluster resampling, reusing each student's multiplicity across all folds/faculties; fixed eligible cells",
                              "bootstrap_interval": "2.5/97.5 percentile of valid replicates; class-degenerate replicates explicitly excluded",
                              "historical_holdouts": "F6/F7 are historical comparative benchmarks with acknowledged prior project-level reuse",
                              "missing_scores": "Never imputed; coverage, matched rows and eligible cells are saved"}})
    atomic_json(paths["report"], summary)
    print(f"Suite27 report: {len(work):,} rows, {len(models)} models, folds {list(all_folds)}", flush=True)
    for name, table in tables.items():
        if name != "predictions":
            print_table(name.replace("_", " ").title(), table)
    for warning in warnings:
        print(f"Report note: {warning}", flush=True)
    return summary
