"""Metrics and protocol-aware, seed-balanced research reporting for PACE.

Alert budgets select ceil(N * budget) rows, sorted by descending probability and
then lexicographic canonical row ID. Without row IDs, input position breaks ties.
Undefined metrics are NaN with a companion reason, never a training-failure flag.
"""
from __future__ import annotations

import json
import math
from pathlib import Path
import re
from typing import Callable

import numpy as np
import pandas as pd
from sklearn.metrics import average_precision_score, roc_auc_score


_KEYS = ["faculty", "fold", "seed", "arm", "arch", "task"]
_CELL = ["stage", "faculty", "fold", "arm", "arch", "task"]
_VARIANT = ["stage", "arm", "arch", "task"]
_GENERIC = {"", "none", "nan", "null", "na", "n/a", "baseline", "shared", "all"}
_METRICS = {"ap", "auc", "brier", "ece15", "f1", "f2", "precision", "recall", "tp", "fp",
            "tn", "fn", "n", "n_positive", "n_negative", "prevalence", "threshold_f1",
            "threshold_f2", "predicted_positive"}


def _arrays(y, p):
    y = np.asarray(y)
    p = np.asarray(p, dtype=float)
    if y.ndim != 1 or p.ndim != 1 or len(y) != len(p):
        raise ValueError("y and p must be aligned one-dimensional arrays")
    if not np.isin(y, [0, 1]).all():
        raise ValueError("y must contain binary 0/1 labels")
    if not np.isfinite(p).all() or ((p < 0) | (p > 1)).any():
        raise ValueError("p must contain finite probabilities in [0, 1]")
    return y.astype(np.int64), p


def _division(numerator, denominator, reason):
    return (float(numerator / denominator), "") if denominator else (float("nan"), reason)


def _threshold_metrics(y, p, threshold, beta):
    alert = p >= threshold
    positive = y == 1
    tp = int((alert & positive).sum())
    fp = int((alert & ~positive).sum())
    fn = int((~alert & positive).sum())
    tn = int((~alert & ~positive).sum())
    precision, precision_reason = _division(tp, tp + fp, "no_predicted_positives")
    recall, recall_reason = _division(tp, tp + fn, "no_positive_labels")
    if not len(y):
        score, score_reason = float("nan"), "empty_input"
        precision_reason = recall_reason = "empty_input"
    elif not positive.any():
        score, score_reason = float("nan"), "no_positive_labels"
    else:
        score = (1 + beta ** 2) * tp / ((1 + beta ** 2) * tp + beta ** 2 * fn + fp)
        score_reason = ""
    return dict(tp=tp, fp=fp, tn=tn, fn=fn, precision=precision, recall=recall,
                precision_reason=precision_reason, recall_reason=recall_reason,
                predicted_positive=int(alert.sum()), score=float(score), score_reason=score_reason)


def evaluate(y, p, thresholds=None, row_ids=None):
    """Return flat probability, threshold, and exact top-budget metrics.

    ``precision``, ``recall`` and unprefixed confusion counts use the F1 threshold;
    ``f2_*`` uses the independently selected F2 threshold. AP is undefined without
    positives; AUC requires both classes. F scores/recall without positives are
    marked undefined. No-alert precision is also undefined. Empty inputs remain
    valid reporting inputs and have explicit reasons.
    """
    y, p = _arrays(y, p)
    n = len(y)
    thresholds = {"f1": .5, "f2": .5, **({} if thresholds is None else dict(thresholds))}
    for key in ("f1", "f2"):
        value = float(thresholds[key])
        if not math.isfinite(value) or not 0 <= value <= 1:
            raise ValueError(f"{key} threshold must be finite and in [0, 1]")
        thresholds[key] = value
    if row_ids is None:
        tie = np.arange(n)
    else:
        ids = np.asarray(row_ids, dtype=object)
        if ids.ndim != 1 or len(ids) != n or pd.isna(ids).any():
            raise ValueError("row_ids must be nonmissing and aligned with y")
        tie = np.asarray([str(v) for v in ids])
        if len(set(tie)) != n:
            raise ValueError("canonical row_ids must be unique within an evaluation")
    positives = int(y.sum())
    negatives = n - positives
    out = dict(n=n, n_positive=positives, n_negative=negatives,
               prevalence=float(y.mean()) if n else float("nan"),
               threshold_f1=thresholds["f1"], threshold_f2=thresholds["f2"])
    out["ap_reason"] = "empty_input" if not n else "no_positive_labels" if not positives else ""
    out["auc_reason"] = ("empty_input" if not n else "no_positive_labels" if not positives
                         else "no_negative_labels" if not negatives else "")
    out["ap"] = float(average_precision_score(y, p)) if not out["ap_reason"] else float("nan")
    out["auc"] = float(roc_auc_score(y, p)) if not out["auc_reason"] else float("nan")
    out["brier"] = float(np.mean((p - y) ** 2)) if n else float("nan")
    out["brier_reason"] = "" if n else "empty_input"
    if n:
        bins = np.minimum((p * 15).astype(int), 14)
        pred_sum = np.bincount(bins, weights=p, minlength=15)
        obs_sum = np.bincount(bins, weights=y, minlength=15)
        out["ece15"] = float(np.abs(pred_sum - obs_sum).sum() / n)
    else:
        out["ece15"] = float("nan")
    out["ece15_reason"] = "" if n else "empty_input"
    for name, beta in (("f1", 1), ("f2", 2)):
        values = _threshold_metrics(y, p, thresholds[name], beta)
        out[name], out[name + "_reason"] = values.pop("score"), values.pop("score_reason")
        prefix = "" if name == "f1" else "f2_"
        out.update({prefix + key: value for key, value in values.items()})
    order = np.lexsort((tie, -p))
    for percent in (5, 10, 20):
        k = min(n, int(math.ceil(n * percent / 100)))
        tp = int(y[order[:k]].sum())
        precision, precision_reason = _division(tp, k, "empty_input")
        recall, recall_reason = _division(tp, positives, "no_positive_labels" if n else "empty_input")
        out.update({f"alerts_{percent}pct": k, f"alert_rate_{percent}pct": k / n if n else float("nan"),
                    f"tp_at_{percent}pct": tp, f"fp_at_{percent}pct": k - tp,
                    f"precision_at_{percent}pct": precision, f"recall_at_{percent}pct": recall,
                    f"precision_at_{percent}pct_reason": precision_reason,
                    f"recall_at_{percent}pct_reason": recall_reason})
    return out


def tune_thresholds(y, p):
    """Tune separate F1/F2 thresholds on supplied development predictions only.

    Equal optima choose the highest threshold. All-negative or empty inputs use
    the prespecified neutral .5 threshold because F-score tuning is undefined.
    All-positive and constant-score inputs are handled without special warnings.
    """
    y, p = _arrays(y, p)
    if not len(y) or not y.sum():
        return {"f1": .5, "f2": .5}
    order = np.argsort(-p, kind="stable")
    scores, labels = p[order], y[order]
    endpoints = np.flatnonzero(np.r_[scores[:-1] != scores[1:], True])
    tp = np.cumsum(labels)[endpoints]
    fp = endpoints + 1 - tp
    fn = y.sum() - tp
    out = {}
    for name, beta in (("f1", 1), ("f2", 2)):
        values = (1 + beta ** 2) * tp / ((1 + beta ** 2) * tp + beta ** 2 * fn + fp)
        out[name] = float(scores[endpoints[int(np.argmax(values))]])
    return out


def _fold_number(value):
    text = str(value).strip()
    match = re.fullmatch(r"[Ff]?(\d+)(?:\.0)?", text)
    if match is None or int(match.group(1)) < 1:
        raise ValueError(f"Fold must identify chronological F1, F2, ...; got {value!r}")
    return int(match.group(1))


def _normalize(frame, predictions=False):
    frame = pd.DataFrame(frame).copy()
    required = _KEYS + (["row_id", "y", "p"] if predictions else ["ap"])
    if frame.empty:
        for key in required + ["stage"]:
            if key not in frame:
                frame[key] = pd.Series(dtype="object")
        return frame
    for key in ("arch", "task"):
        if key not in frame:
            frame[key] = "none"
        frame[key] = frame[key].fillna("none").astype(str)
    missing = set(required) - set(frame.columns)
    if missing:
        raise ValueError("Reporting input lacks columns: " + ", ".join(sorted(missing)))
    for key in ("faculty", "arm", "seed"):
        if frame[key].isna().any():
            raise ValueError(f"Reporting identity {key} cannot be missing")
        frame[key] = frame[key].astype(str)
    frame["fold"] = frame["fold"].map(_fold_number)
    stages = np.where(frame["fold"] <= 5, "development", "retrospective")
    if "stage" in frame:
        aliases = {"dev": "development", "development": "development", "retro": "retrospective",
                   "retrospective": "retrospective", "holdout": "retrospective"}
        declared = frame["stage"].fillna("").astype(str).str.lower().map(aliases)
        if declared.isna().any() or (declared.to_numpy() != stages).any():
            raise ValueError("Declared stage contradicts development F1-F5 / retrospective F6+ folds")
    frame["stage"] = stages
    if predictions:
        if frame["row_id"].isna().any():
            raise ValueError("Prediction row_id cannot be missing")
        frame["row_id"] = frame["row_id"].astype(str)
        _arrays(frame["y"].to_numpy(), frame["p"].to_numpy())
    else:
        frame["ap"] = pd.to_numeric(frame["ap"], errors="raise")
        if ((frame.ap < 0) | (frame.ap > 1) | np.isinf(frame.ap)).any():
            raise ValueError("Reported AP must be in [0, 1] or undefined (NaN)")
        if frame.duplicated(_KEYS).any():
            raise ValueError("Duplicate faculty/fold/seed/arm/arch/task result identities")
    return frame.sort_values(_KEYS + (["row_id"] if predictions else []), kind="stable").reset_index(drop=True)


def _metric_columns(frame):
    return [column for column in frame if column in _METRICS or column.startswith(("f2_", "alerts_", "alert_rate_", "tp_at_", "fp_at_", "precision_at_", "recall_at_"))
            if not column.endswith("_reason") and pd.api.types.is_numeric_dtype(frame[column])]


def _interval(values, bootstrap):
    values = np.asarray(values, dtype=float)
    values = values[np.isfinite(values)]
    if not len(values) or bootstrap <= 0:
        return float("nan"), float("nan")
    rng = np.random.default_rng(35035)
    means = np.asarray([rng.choice(values, len(values), replace=True).mean() for _ in range(bootstrap)])
    return tuple(float(x) for x in np.quantile(means, [.025, .975]))


def _aggregate(results, bootstrap, extra_groups=()):
    """Average seeds first, then give each observed faculty-fold cell equal weight."""
    metrics = _metric_columns(results)
    cell_keys, variant_keys = _CELL + list(extra_groups), _VARIANT + list(extra_groups)
    cells, macro = [], []
    for key, group in results.groupby(cell_keys, dropna=False, sort=True):
        row = dict(zip(cell_keys, key))
        row["n_seeds"] = int(group.seed.nunique())
        for metric in metrics:
            values = pd.to_numeric(group[metric], errors="coerce")
            row[metric] = float(values.mean())
            row[metric + "_n_seeds"] = int(values.notna().sum())
        row["ap_seed_std"] = float(group.ap.std(ddof=1)) if len(group) > 1 else float("nan")
        cells.append(row)
    cells = pd.DataFrame(cells)
    if cells.empty:
        return pd.DataFrame(columns=cell_keys + metrics + ["n_seeds"]), pd.DataFrame(columns=variant_keys + metrics + ["n_cells_ap"])
    for key, group in cells.groupby(variant_keys, dropna=False, sort=True):
        row = dict(zip(variant_keys, key))
        row.update(n_cells_total=len(group), n_faculties=int(group.faculty.nunique()),
                   min_seeds=int(group.n_seeds.min()), max_seeds=int(group.n_seeds.max()),
                   ap_seed_std_mean=float(group.ap_seed_std.mean()))
        for metric in metrics:
            row[metric] = float(group[metric].mean())
            row["n_cells_" + metric] = int(group[metric].notna().sum())
        row["ap_ci_low"], row["ap_ci_high"] = _interval(group.ap, bootstrap)
        macro.append(row)
    return cells, pd.DataFrame(macro)


def _control_variant(rows, target_arch, target_task, control):
    """Prefer exact variants; generic controls may be shared across SSL tasks."""
    available = rows.loc[rows.arm == control]
    if available.empty:
        return None
    arch_values = available.arch.unique()
    if target_arch in arch_values:
        arch = target_arch
    else:
        generic = sorted(x for x in arch_values if str(x).lower() in _GENERIC)
        if not generic:
            return None
        arch = generic[0]
    available = available.loc[available.arch == arch]
    task_values = available.task.unique()
    if target_task in task_values:
        task = target_task
    else:
        generic = sorted(x for x in task_values if str(x).lower() in (_GENERIC | {"random", "supervised", "scratch"}))
        if not generic:
            return None
        task = generic[0]
    return arch, task


def _prediction_lookup(predictions):
    return {tuple(key): group for key, group in predictions.groupby(["stage"] + _KEYS, dropna=False, sort=False)}


def _reconcile_ap(results, predictions):
    """Make saved predictions authoritative for AP while retaining input values.

    Unverified result-only AP is retained for descriptive tables, but cannot enter
    a paired decision because support validation requires saved predictions.
    """
    frame = results.copy()
    frame["ap_reported"] = frame["ap"]
    frame["ap_from_predictions"] = np.nan
    frame["ap_source"] = "reported_unverified"
    frame["ap_consistency_status"] = "missing_predictions"
    if "ap_reason" not in frame:
        frame["ap_reason"] = ""
    frame["ap_reason"] = frame["ap_reason"].astype(object)
    # Required support is independent of whether a usable numerical AP survives
    # reconciliation. Unknown/missing/corrupt predictions cannot remove a failed
    # cell or seed from the promotion denominator. Only confirmed no-positive or
    # empty inputs are non-estimable; saved labels supersede reported reasons.
    nonestimable = frame.ap.isna() & frame.ap_reason.isin(["no_positive_labels", "empty_input"])
    frame["ap_support_required"] = ~nonestimable
    frame["ap_support_reason"] = np.where(nonestimable, "reported_nonestimable", "unverified_required")
    lookup = _prediction_lookup(predictions)
    for index, row in frame.iterrows():
        key = tuple(row[column] for column in ["stage"] + _KEYS)
        group = lookup.get(key)
        if group is None:
            continue
        has_positive = bool(group.y.sum())
        frame.at[index, "ap_support_required"] = has_positive
        frame.at[index, "ap_support_reason"] = "saved_positive_labels" if has_positive else "saved_no_positive_labels"
        if group.row_id.duplicated().any():
            frame.at[index, "ap"] = np.nan
            frame.at[index, "ap_reason"] = "duplicate_prediction_row_ids"
            frame.at[index, "ap_source"] = "invalid_saved_predictions"
            frame.at[index, "ap_consistency_status"] = "duplicate_prediction_row_ids"
            continue
        direct = float(average_precision_score(group.y, group.p)) if group.y.sum() else float("nan")
        reported = row["ap"]
        consistent = ((pd.isna(reported) and pd.isna(direct)) or
                      (pd.notna(reported) and pd.notna(direct) and np.isclose(reported, direct, rtol=0, atol=1e-12)))
        frame.at[index, "ap"] = direct
        frame.at[index, "ap_from_predictions"] = direct
        frame.at[index, "ap_reason"] = "" if pd.notna(direct) else "no_positive_labels"
        frame.at[index, "ap_source"] = "saved_predictions"
        frame.at[index, "ap_consistency_status"] = "consistent" if consistent else "reported_ap_mismatch_recomputed"
    return frame


def _support(lookup, stage, faculty, fold, seed, arm, arch, task, control, ca, ct, n_target, n_control):
    target = lookup.get((stage, faculty, fold, seed, arm, arch, task))
    baseline = lookup.get((stage, faculty, fold, seed, control, ca, ct))
    if target is None or baseline is None:
        return "missing_predictions"
    if target.row_id.duplicated().any() or baseline.row_id.duplicated().any():
        return "duplicate_row_ids"
    a = target.sort_values("row_id", kind="stable")
    b = baseline.sort_values("row_id", kind="stable")
    if a.row_id.tolist() != b.row_id.tolist():
        return "row_ids_mismatch"
    if not np.array_equal(a.y.to_numpy(), b.y.to_numpy()):
        return "labels_mismatch"
    if (pd.notna(n_target) and float(n_target) != len(a)) or (pd.notna(n_control) and float(n_control) != len(b)):
        return "result_prediction_count_mismatch"
    return "aligned"


# Ordered, predeclared contrasts. Each target is matched to its own architecture
# where relevant; control AP is never substituted from a different encoder.
_CONTRASTS = {"H2": ("H0", "H1", "HR"), "H1": ("H0",), "HR": ("H0",),
              "HS": ("H2", "H0"), "N1": ("N0",), "N2": ("N1", "N0"),
              "N3": ("N1", "N2", "N0")}


def _paired_tables(results, predictions, bootstrap):
    lookup, pairs = _prediction_lookup(predictions), []
    compare_metrics = [x for x in _metric_columns(results) if x in {"ap", "auc", "brier", "ece15", "f1", "f2", "precision", "recall"} or x.startswith(("precision_at_", "recall_at_"))]
    ids = ["stage", "faculty", "fold", "seed"]
    for (stage, arm, arch, task), target in results.groupby(_VARIANT, sort=True):
        stage_rows = results.loc[results.stage == stage]
        control_specs = []
        for control in _CONTRASTS.get(arm, ()):
            variant = _control_variant(stage_rows, arch, task, control)
            ca, ct = variant if variant else ("missing", "missing")
            control_specs.append((control, ca, ct, f"{arm}_vs_{control}"))
        # Predeclared task ablations hold architecture fixed. Architecture
        # ablations hold the SSL objective fixed; all share canonical row checks.
        if arm == "H2":
            available = set(zip(stage_rows.loc[stage_rows.arm == "H2", "arch"],
                                stage_rows.loc[stage_rows.arm == "H2", "task"]))
            for baseline_task in {"joint": ("grade", "course"), "grade": ("course",)}.get(task, ()):
                if (arch, baseline_task) in available:
                    control_specs.append(("H2", arch, baseline_task, f"H2_{task}_vs_{baseline_task}"))
            baseline_arch = {"gru": "mlp", "transformer": "gru"}.get(arch)
            if baseline_arch is not None and (baseline_arch, task) in available:
                control_specs.append(("H2", baseline_arch, task, f"H2_{arch}_vs_{baseline_arch}"))
        for control, ca, ct, contrast_name in control_specs:
            baseline = results.loc[(results.stage == stage) & (results.arm == control) & (results.arch == ca) & (results.task == ct)]
            merged = target.merge(baseline, on=ids, how="left", suffixes=("_target", "_control"), indicator=True)
            for _, source in merged.iterrows():
                row = {key: source[key] for key in ids}
                row.update(arm=arm, arch=arch, task=task, control=control, control_arch=ca,
                           control_task=ct, contrast=contrast_name)
                status = ("missing_control_result" if source["_merge"] != "both" else
                          _support(lookup, stage, row["faculty"], row["fold"], row["seed"], arm, arch, task,
                                   control, ca, ct, source.get("n_target", np.nan), source.get("n_control", np.nan)))
                row["support_status"] = status
                row["ap_consistency_target"] = source.get("ap_consistency_status_target", "unverified")
                row["ap_consistency_control"] = source.get("ap_consistency_status_control", "unverified")
                for metric in compare_metrics:
                    a, b = source.get(metric + "_target", np.nan), source.get(metric + "_control", np.nan)
                    valid = status == "aligned" and pd.notna(a) and pd.notna(b)
                    row[metric + "_target"] = float(a) if pd.notna(a) else float("nan")
                    row[metric + "_control"] = float(b) if pd.notna(b) else float("nan")
                    row[metric + "_delta"] = float(a - b) if valid else float("nan")
                row["included"] = pd.notna(row.get("ap_delta", np.nan))
                row["exclusion_reason"] = "" if row["included"] else (status if status != "aligned" else "undefined_ap")
                pairs.append(row)
    pair_keys = _VARIANT + ["control", "control_arch", "control_task", "contrast"]
    seed = pd.DataFrame(pairs)
    if seed.empty:
        columns = pair_keys + ["faculty", "fold", "ap_delta", "support_status", "included", "exclusion_reason"]
        return pd.DataFrame(columns=columns), pd.DataFrame(columns=columns), pd.DataFrame(columns=pair_keys + ["ap_delta"]), pd.DataFrame(columns=pair_keys + ["faculty", "harm_cells"])
    cell_rows = []
    for key, group in seed.groupby(pair_keys + ["faculty", "fold"], sort=True, dropna=False):
        row = dict(zip(pair_keys + ["faculty", "fold"], key))
        row.update(n_target_seeds=len(group), n_paired_seeds=int(group.included.sum()),
                   n_support_exclusions=int((group.support_status != "aligned").sum()),
                   n_undefined_ap=int((group.exclusion_reason == "undefined_ap").sum()))
        for metric in compare_metrics:
            valid = group[metric + "_delta"].notna()
            for suffix in ("target", "control", "delta"):
                row[metric + "_" + suffix] = float(group.loc[valid, metric + "_" + suffix].mean())
        cell_rows.append(row)
    cells = pd.DataFrame(cell_rows)
    summary_rows, faculty_rows = [], []
    for key, group in cells.groupby(pair_keys, sort=True, dropna=False):
        row = dict(zip(pair_keys, key))
        valid = group.ap_delta.notna()
        row.update(n_target_cells=len(group), n_paired_cells=int(valid.sum()),
                   n_paired_seeds=int(group.n_paired_seeds.sum()),
                   n_support_exclusions=int(group.n_support_exclusions.sum()),
                   positive_cells=int((group.ap_delta > 0).sum()),
                   harm_cells=int((group.ap_delta < 0).sum()), tie_cells=int((group.ap_delta == 0).sum()),
                   positive_cell_fraction=float((group.loc[valid, "ap_delta"] > 0).mean()) if valid.any() else float("nan"))
        for metric in compare_metrics:
            for suffix in ("target", "control", "delta"):
                row[metric + "_" + suffix] = float(group[metric + "_" + suffix].mean())
        row["ap_delta_ci_low"], row["ap_delta_ci_high"] = _interval(group.ap_delta, bootstrap)
        summary_rows.append(row)
        for faculty, local in group.groupby("faculty", sort=True):
            finite = local.ap_delta.notna()
            faculty_rows.append({**dict(zip(pair_keys, key)), "faculty": faculty,
                                 "paired_cells": int(finite.sum()), "mean_ap_delta": float(local.ap_delta.mean()),
                                 "win_cells": int((local.ap_delta > 0).sum()),
                                 "harm_cells": int((local.ap_delta < 0).sum()),
                                 "tie_cells": int((local.ap_delta == 0).sum()),
                                 "undefined_or_unpaired_cells": int((~finite).sum())})
    return seed, cells, pd.DataFrame(summary_rows), pd.DataFrame(faculty_rows)


def _candidate_evidence(results, pairs, stage, arch, task, expected_cells):
    """Use the identical seed support across all three controls before averaging."""
    controls = ("H0", "H1", "HR")
    subset = pairs.loc[(pairs.stage == stage) & (pairs.arm == "H2") & (pairs.arch == arch) & (pairs.task == task)]
    by_control = {}
    for control in controls:
        group = subset.loc[(subset.control == control) & subset.included]
        by_control[control] = {(r.faculty, r.fold, r.seed): r for r in group.itertuples()}
    common = set.intersection(*(set(by_control[c]) for c in controls))
    # A missing target seed must not disappear from the denominator merely
    # because only the successful target rows were supplied to this report.
    required = results.loc[(results.stage == stage) & results.arm.isin(["H0", "H1", "H2", "HR"]) & results.ap_support_required]
    expected_seed_keys = set(zip(required.faculty, required.fold, required.seed))
    observed_cells = {(key[0], key[1]) for key in common}
    complete = bool(expected_cells) and observed_cells == expected_cells and common == expected_seed_keys
    per_cell = []
    for faculty, fold in sorted(observed_cells):
        keys = [key for key in common if key[:2] == (faculty, fold)]
        values = {control: float(np.mean([by_control[control][key].ap_delta for key in keys])) for control in controls}
        per_cell.append(dict(faculty=faculty, fold=fold, **values))
    gains = {control: float(np.mean([row[control] for row in per_cell])) if per_cell else float("nan") for control in controls}
    score = min(gains.values()) if per_cell else float("nan")
    positive_fraction = float(np.mean([min(row[c] for c in controls) > 0 for row in per_cell])) if per_cell else float("nan")
    return dict(arch=arch, task=task, n_cells=len(per_cell), expected_cells=len(expected_cells),
                n_paired_seeds=len(common), n_expected_seeds=len(expected_seed_keys), complete_support=complete,
                gains=gains, worst_control_gain=score, positive_cell_fraction=positive_fraction,
                positive_cells=sum(min(row[c] for c in controls) > 0 for row in per_cell),
                harm_cells=sum(min(row[c] for c in controls) < 0 for row in per_cell))


def _decision(results, pairs):
    decision = dict(selection_stage="development", winner_arch=None, winner_task=None,
                    selected_arch=None, selected_task=None, promoted=False, promote=False,
                    complete_support=False, expected_cells=0, n_cells=0, n_expected_seeds=0, worst_control_gain=None,
                    positive_cell_fraction=None, gains={}, candidates=[], holdout_positive_evidence=False,
                    operational_recommendation="not_supported", reasons=[])
    dev = results.loc[results.stage == "development"]
    # Invalid saved predictions remain required support even when their AP was
    # invalidated. Only confirmed non-estimable no-positive/empty cells drop out.
    estimable = dev.loc[dev.arm.isin(["H0", "H1", "H2", "HR"]) & dev.ap_support_required]
    expected = set(zip(estimable.faculty, estimable.fold))
    decision["expected_cells"] = len(expected)
    decision["n_expected_seeds"] = len(set(zip(estimable.faculty, estimable.fold, estimable.seed)))
    variants = dev.loc[dev.arm == "H2", ["arch", "task"]].drop_duplicates()
    candidates = [_candidate_evidence(results, pairs, "development", row.arch, row.task, expected)
                  for row in variants.itertuples()]
    decision["candidates"] = candidates
    estimable_candidates = [candidate for candidate in candidates if np.isfinite(candidate["worst_control_gain"])]
    if not estimable_candidates:
        decision["reasons"] = ["No development H2 candidate has estimable, row-aligned evidence against all H0/H1/HR controls."]
        return decision
    # Complete candidates have comparable coverage. If none is complete, retain a
    # diagnostic best candidate for a freeze but explicitly block its promotion.
    complete = [candidate for candidate in estimable_candidates if candidate["complete_support"]]
    winner = sorted(complete or estimable_candidates,
                    key=lambda candidate: (-candidate["worst_control_gain"], candidate["arch"], candidate["task"]))[0]
    decision.update({key: winner[key] for key in ("complete_support", "expected_cells", "n_cells", "n_expected_seeds", "worst_control_gain", "positive_cell_fraction", "gains")})
    decision.update(winner_arch=winner["arch"], winner_task=winner["task"],
                    selected_arch=winner["arch"], selected_task=winner["task"])
    gain_ok = all(value >= .01 - 1e-12 for value in winner["gains"].values())
    wins_ok = winner["positive_cell_fraction"] >= .60
    promoted = bool(winner["complete_support"] and gain_ok and wins_ok)
    decision["promoted"] = decision["promote"] = promoted
    if not winner["complete_support"]:
        decision["reasons"].append("Incomplete development faculty-fold/seed or aligned control support blocks promotion.")
    if not gain_ok:
        decision["reasons"].append("Development mean AP gain is below .01 against at least one H0/H1/HR control.")
    if not wins_ok:
        decision["reasons"].append("H2 beats the strongest control in fewer than 60% of paired development faculty-fold cells.")
    if promoted:
        decision["reasons"].append("Development AP gains meet .01 against every control and wins meet 60%, with complete aligned support.")
    retro = results.loc[(results.stage == "retrospective") & results.arm.isin(["H0", "H1", "H2", "HR"]) & results.ap_support_required]
    retro_expected = set(zip(retro.faculty, retro.fold))
    holdout = _candidate_evidence(results, pairs, "retrospective", winner["arch"], winner["task"], retro_expected)
    decision["retrospective_evidence"] = holdout
    decision["holdout_positive_evidence"] = bool(holdout["complete_support"] and all(np.isfinite(v) and v > 0 for v in holdout["gains"].values()))
    if promoted and decision["holdout_positive_evidence"]:
        decision["operational_recommendation"] = "consider_monitored_pilot"
    return decision


def _diagnostics(predictions, results, bootstrap):
    outputs = {}
    definitions = {
        "history": ("n_history", lambda values: np.select([values == 0, values <= 5, values <= 20], ["0_no_history", "1_to_5", "6_to_20"], default="21_plus")),
        "oov": ("oov_rate", lambda values: np.select([values == 0, values < 1], ["0_known_only", "mixed_oov"], default="1_all_oov")),
        "no_history": ("n_history", lambda values: np.where(values == 0, "no_history", "has_history")),
    }
    threshold_lookup = {tuple(getattr(row, key) for key in ["stage"] + _KEYS):
                        {"f1": getattr(row, "threshold_f1", .5), "f2": getattr(row, "threshold_f2", .5)}
                        for row in results.itertuples()}
    for name, (source, bucket_fn) in definitions.items():
        rows = []
        if source in predictions:
            data = predictions.copy()
            values = pd.to_numeric(data[source], errors="coerce")
            valid = values.notna() & (values >= 0)
            if source == "oov_rate":
                valid &= values <= 1
            data["stratum"] = np.where(valid, bucket_fn(values), "unknown")
            for key, group in data.groupby(["stage"] + _KEYS + ["stratum"], dropna=False, sort=True):
                identity = dict(zip(["stage"] + _KEYS + ["stratum"], key))
                thresholds = threshold_lookup.get(tuple(key[:-1]), {"f1": .5, "f2": .5})
                thresholds = {metric: value if pd.notna(value) else .5 for metric, value in thresholds.items()}
                if group.row_id.duplicated().any():
                    # Keep support defects explicit, without interpreting duplicate
                    # observations as independent diagnostic enrollment rows.
                    rows.append({**identity, "ap": np.nan, "ap_reason": "duplicate_row_ids", "n": len(group)})
                    continue
                rows.append({**identity, **evaluate(group.y.to_numpy(), group.p.to_numpy(), thresholds, group.row_id.to_numpy())})
        detail = pd.DataFrame(rows)
        if detail.empty:
            detail = pd.DataFrame(columns=["stage"] + _KEYS + ["stratum", "ap", "ap_reason", "n"])
            macro = pd.DataFrame(columns=_VARIANT + ["stratum", "ap", "n_cells_ap"])
        else:
            _, macro = _aggregate(detail, bootstrap, ["stratum"])
        outputs[name + "_diagnostics"] = detail
        outputs[name + "_diagnostics_macro"] = macro
    return outputs


def _coverage_table(results, predictions):
    """Expose absent result rows, undefined AP, and prediction support defects."""
    lookup = {tuple(getattr(row, key) for key in ["stage"] + _KEYS): row._asdict()
              for row in results.itertuples(index=False)}
    pred_lookup = _prediction_lookup(predictions)
    identities = pd.concat([results[["stage"] + _KEYS], predictions[["stage"] + _KEYS]], ignore_index=True).drop_duplicates()
    rows = []
    for stage, data in identities.groupby("stage", sort=True):
        cells = data[["faculty", "fold", "seed"]].drop_duplicates().sort_values(["faculty", "fold", "seed"])
        variants = data[["arm", "arch", "task"]].drop_duplicates().sort_values(["arm", "arch", "task"])
        for variant in variants.itertuples(index=False):
            for cell in cells.itertuples(index=False):
                key = (stage, cell.faculty, cell.fold, cell.seed, variant.arm, variant.arch, variant.task)
                result = lookup.get(key)
                pred = pred_lookup.get(key)
                present = result is not None
                estimable = present and pd.notna(result["ap"])
                reason = "missing_result" if not present else result.get("ap_reason", "")
                if pd.isna(reason):
                    reason = ""
                if present and not estimable and not reason:
                    reason = "undefined_ap_unspecified"
                n_predictions = len(pred) if pred is not None else 0
                n_unique = int(pred.row_id.nunique()) if pred is not None else 0
                rows.append(dict(zip(["stage"] + _KEYS, key), result_present=present,
                                 ap_estimable=bool(estimable), ap_reason=reason,
                                 n_predictions=n_predictions, n_unique_row_ids=n_unique,
                                 duplicate_prediction_ids=n_predictions - n_unique))
    return pd.DataFrame(rows, columns=["stage"] + _KEYS + ["result_present", "ap_estimable", "ap_reason", "n_predictions", "n_unique_row_ids", "duplicate_prediction_ids"])


def _json_clean(value):
    if isinstance(value, dict):
        return {str(key): _json_clean(item) for key, item in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_clean(item) for item in value]
    if isinstance(value, (np.integer,)):
        return int(value)
    if isinstance(value, (float, np.floating)):
        return float(value) if np.isfinite(value) else None
    if isinstance(value, (np.bool_,)):
        return bool(value)
    return value


def _markdown(frame):
    if frame.empty:
        return "No estimable or available rows."
    def fmt(value):
        if pd.isna(value):
            return "undefined"
        if isinstance(value, (float, np.floating)):
            return f"{value:.6g}"
        return str(value).replace("|", "\\|").replace("\n", " ")
    lines = ["| " + " | ".join(map(str, frame.columns)) + " |", "| " + " | ".join("---" for _ in frame.columns) + " |"]
    lines.extend("| " + " | ".join(fmt(value) for value in row) + " |" for row in frame.itertuples(index=False, name=None))
    return "\n".join(lines)


def _interpretation(decision, results):
    text = ["All estimates describe these observed chronological folds; they do not establish a causal benefit from self-supervision.",
            "Primary estimates average seeds within faculty-fold first, then weight each faculty-fold equally. Missing and undefined cells are counted explicitly; undefined AP/AUC do not mean training failed.",
            "AP confidence intervals and paired-delta intervals use descriptive faculty-fold cluster resampling after seed averaging. Chronological folds are correlated, so these intervals are not independent-fold significance tests.",
            "AP is recomputed directly from saved prediction labels/probabilities wherever available; paired decisions require those predictions. Original reported AP and any discrepancy remain in the metric-integrity audit.",
            "Threshold metrics use separate supplied F1/F2 thresholds. Diagnostic strata inherit those thresholds; reporting never retunes them. Exact alert budgets use ceil(N × budget), with lexicographic row-ID ties."]
    if decision["winner_arch"] is None:
        text.append("No development winner can be selected from complete estimable paired evidence against all controls. Retrospective outcomes cannot resolve this development selection gap.")
    else:
        candidate = f"H2/{decision['winner_arch']}/{decision['winner_task']}"
        text.append(f"The development candidate is {candidate}; its worst paired mean AP gain against H0/H1/HR is {decision['worst_control_gain']:.6g}, and it beats all controls in {decision['positive_cell_fraction']:.1%} of paired faculty-fold cells.")
        text.append("Promotion criterion: " + ("met." if decision["promoted"] else "not met.") + " " + " ".join(decision["reasons"]))
        if (results.stage == "retrospective").any():
            if decision["holdout_positive_evidence"]:
                text.append("The development-selected candidate has positive retrospective mean AP deltas against every control on aligned complete support. This is evidence for this held-out setting; it does not guarantee gains elsewhere.")
            else:
                text.append("Retrospective evidence for the development-selected candidate is mixed, nonpositive, unavailable, or incompletely aligned. An operational recommendation is not supported by these results.")
            text.append("Retrospective evaluation did not alter architecture/task selection or the development promotion decision; other retrospective candidates are diagnostic only.")
        else:
            text.append("No retrospective evidence is available. Development promotion alone does not support an operational recommendation.")
    text.extend(["Positive AP deltas favor the target; lower Brier/ECE deltas favor the target. Seed dispersion measures training variability, while faculty wins/harms expose uneven transfer.",
                 "History/OOV/no-history strata describe ranking and calibration on each observed subgroup. Different subgroup prevalence and support can change AP; subgroup differences do not identify causes. New-course OOV rate is not evidence of curriculum change.",
                 "Task and encoder ablations compare observed transfer on shared downstream support under the declared information and training budgets; these budgets do not imply exact parameter matching.",
                 "Pretext reconstruction success alone is not evidence of downstream transfer. A synthetic demonstration supports software verification only and makes no research claim."])
    return text


def _observed_findings(decision, contrasts, faculties, diagnostics):
    """Describe observed contrasts conditionally, without causal extrapolation."""
    findings = []
    labels = {"N2_vs_N1": ("Frozen SSL", "supervised history"),
              "N3_vs_N2": ("Fine-tuning", "frozen SSL"),
              "H1_vs_H0": ("Deterministic history summaries", "the full engineered HGB baseline"),
              "H2_joint_vs_grade": ("Joint grade/course pretraining", "grade-only pretraining"),
              "H2_joint_vs_course": ("Joint grade/course pretraining", "course-only pretraining"),
              "H2_grade_vs_course": ("Grade-only pretraining", "course-only pretraining"),
              "H2_gru_vs_mlp": ("GRU frozen SSL", "MLP frozen SSL"),
              "H2_transformer_vs_gru": ("Transformer frozen SSL", "GRU frozen SSL")}
    if not contrasts.empty:
        for row in contrasts.loc[contrasts.contrast.isin(labels)].itertuples():
            if not np.isfinite(row.ap_delta):
                continue
            target, control = labels[row.contrast]
            direction = "higher" if row.ap_delta > 0 else "lower" if row.ap_delta < 0 else "equal"
            findings.append(f"{target} has {direction} mean AP than {control} in {row.stage} for {row.arch}/{row.task} "
                            f"(paired delta={row.ap_delta:.6g}; {row.n_paired_cells} faculty-fold cells). "
                            "This comparison describes the aligned evaluated support.")
    if not faculties.empty:
        harms = faculties.loc[faculties.harm_cells > 0]
        group_keys = ["stage", "arm", "arch", "task", "control", "control_arch", "control_task", "contrast"]
        for key, group in harms.groupby(group_keys, sort=True):
            stage, arm, arch, task, control, control_arch, control_task, _ = key
            control_name = f"{control}/{control_arch}/{control_task}"
            names = ", ".join(sorted(group.faculty.astype(str)))
            findings.append(f"Observed faculty harms for {stage} {arm}/{arch}/{task} versus {control_name}: "
                            f"{int(group.harm_cells.sum())} lower-AP faculty-fold cells across {names}. "
                            "Any overall gain should be read alongside this uneven performance.")
    if decision["winner_arch"] is not None:
        for name in ("no_history", "oov"):
            table = diagnostics[name + "_diagnostics_macro"]
            if table.empty:
                continue
            subset = table.loc[(table.arm == "H2") & (table.arch == decision["winner_arch"]) & (table.task == decision["winner_task"])]
            if name == "no_history":
                subset = subset.loc[subset.stratum == "no_history"]
            for row in subset.itertuples():
                if name == "no_history" and not np.isfinite(row.ap):
                    findings.append(f"No-history AP is undefined for the selected candidate in {row.stage}; "
                                    "the no-history rows remain included in overall evaluation and their reasons appear in the stratum table.")
                elif np.isfinite(row.ap):
                    findings.append(f"Selected-candidate {row.stage} {row.stratum} AP is {row.ap:.6g} "
                                    f"across {row.n_cells_ap} estimable faculty-fold cells. "
                                    "Subgroup prevalence and sample size limit comparisons with other strata.")
    return findings


def report_results(results_df, predictions_df, outdir, log: Callable[[str], None], bootstrap=200):
    """Write and print complete research tables, returning a dev-only decision.

    AP is recomputed from saved predictions and reported AP discrepancies are audited.
    Contrasts require identical canonical enrollment IDs and labels within each
    faculty/fold/seed. Seeds are paired before averaging. Candidate selection uses
    the common support of all three H0/H1/HR controls, then the worst mean AP gain;
    ties break by architecture/task name. No retrospective row enters selection.
    ``bootstrap`` is a deterministic descriptive cell-resampling count, not an
    independent-fold inference procedure. Files are overwritten once per report.
    """
    if not isinstance(bootstrap, (int, np.integer)) or bootstrap < 0:
        raise ValueError("bootstrap must be a nonnegative integer")
    results, predictions = _normalize(results_df), _normalize(predictions_df, predictions=True)
    results = _reconcile_ap(results, predictions)
    outdir = Path(outdir)
    outdir.mkdir(parents=True, exist_ok=True)
    cells, macro = _aggregate(results, int(bootstrap))
    pairs, paired_cells, contrasts, faculties = _paired_tables(results, predictions, int(bootstrap))
    decision = _json_clean(_decision(results, pairs))
    tables = {"results_by_seed": results, "faculty_fold_metrics": cells, "macro_metrics": macro,
              "paired_seed_contrasts": pairs, "paired_faculty_fold_contrasts": paired_cells,
              "paired_contrasts": contrasts, "faculty_wins_harms": faculties,
              "coverage_audit": _coverage_table(results, predictions),
              "metric_integrity_audit": results[["stage"] + _KEYS + ["ap", "ap_reported", "ap_from_predictions", "ap_source", "ap_consistency_status", "ap_reason", "ap_support_required", "ap_support_reason"]]}
    support_columns = [x for x in ["stage"] + _KEYS + ["control", "control_arch", "control_task", "contrast", "support_status", "included", "exclusion_reason", "ap_consistency_target", "ap_consistency_control"] if x in pairs]
    tables["support_audit"] = pairs[support_columns]
    diagnostics = _diagnostics(predictions, results, int(bootstrap))
    tables.update(diagnostics)
    for name, table in tables.items():
        table.to_csv(outdir / (name + ".csv"), index=False)
    predictions.to_csv(outdir / "predictions_all.csv", index=False)
    (outdir / "decision.json").write_text(json.dumps(decision, indent=2, allow_nan=False) + "\n")
    narrative = _interpretation(decision, results) + _observed_findings(decision, contrasts, faculties, diagnostics)
    markdown = ["# Suite35 PACE research report", *narrative]
    plain = ["Suite35 PACE research report", *narrative]
    # Every row/column is printed once per report; do not call this per epoch.
    # CSVs retain machine precision, while printed/readable tables use six digits.
    for name, table in tables.items():
        title = name.replace("_", " ").title()
        shown = table.to_string(index=False, max_rows=None, max_cols=None, float_format=lambda value: f"{value:.6g}") if not table.empty else "No available rows."
        log(title + "\n" + shown)
        markdown += ["## " + title, _markdown(table)]
        plain += [title, shown]
    for line in narrative:
        log(line)
    (outdir / "report.md").write_text("\n\n".join(markdown) + "\n")
    (outdir / "report.txt").write_text("\n\n".join(plain) + "\n")
    return decision
