#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Post-hoc comparison of test2.py and final_train_v2.py warning scores.

This script performs two analyses without retraining final_train_v2:

1. Reconstruct the existing operating policies:
   - test2: pooled-development F1 threshold + its drift-aware warning ratio.
   - final_train_v2: fixed/drift warnings already stored in its fold CSV files.

2. Compare both score rankings at identical per-semester warning budgets.
   Each fold independently flags exactly the top 10%, 15%, 20%, or user-defined
   fraction. The fold decisions are then concatenated for pooled metrics.

The modified test2_equal_budget.py must be run once first so that per-row
classification probabilities are available on disk.
"""
from __future__ import annotations

import argparse
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Sequence, Tuple

import numpy as np
import pandas as pd
from sklearn.metrics import average_precision_score, roc_auc_score


@dataclass
class ScoreRecord:
    model: str
    fold: int
    y_failure: np.ndarray
    score: np.ndarray
    target_grade10: Optional[np.ndarray] = None
    frame: Optional[pd.DataFrame] = None

    def __post_init__(self) -> None:
        self.y_failure = np.asarray(self.y_failure, dtype=np.int64).reshape(-1)
        self.score = np.asarray(self.score, dtype=np.float64).reshape(-1)
        if self.y_failure.shape != self.score.shape:
            raise ValueError(
                f"{self.model} fold {self.fold}: label and score lengths differ"
            )
        if not np.isfinite(self.score).all():
            raise ValueError(f"{self.model} fold {self.fold}: non-finite scores")
        if self.target_grade10 is not None:
            self.target_grade10 = np.asarray(
                self.target_grade10, dtype=np.float64
            ).reshape(-1)
            if self.target_grade10.shape != self.y_failure.shape:
                raise ValueError(
                    f"{self.model} fold {self.fold}: target length differs"
                )


def parse_int_list(spec: str) -> List[int]:
    values = [int(value.strip()) for value in spec.split(",") if value.strip()]
    if not values:
        raise ValueError("At least one fold is required")
    return values


def parse_rates(spec: str) -> Tuple[float, ...]:
    rates = tuple(float(value.strip()) for value in spec.split(",") if value.strip())
    if not rates:
        raise ValueError("At least one warning budget is required")
    if any(not 0.0 < rate < 1.0 for rate in rates):
        raise ValueError("Every warning budget must be between 0 and 1")
    return rates


def safe_ap(y: np.ndarray, score: np.ndarray) -> float:
    y = np.asarray(y, dtype=np.int64)
    return (
        float(average_precision_score(y, score))
        if np.unique(y).size == 2
        else float("nan")
    )


def safe_auc(y: np.ndarray, score: np.ndarray) -> float:
    y = np.asarray(y, dtype=np.int64)
    return (
        float(roc_auc_score(y, score))
        if np.unique(y).size == 2
        else float("nan")
    )


def binary_metrics(y: np.ndarray, warning: np.ndarray) -> Dict[str, float]:
    y = np.asarray(y, dtype=np.int64).reshape(-1)
    warning = np.asarray(warning, dtype=np.int64).reshape(-1)
    if y.shape != warning.shape:
        raise ValueError("Labels and warnings must have identical shapes")

    target = y.astype(bool)
    predicted = warning.astype(bool)
    tp = int(np.sum(target & predicted))
    fp = int(np.sum(~target & predicted))
    fn = int(np.sum(target & ~predicted))
    tn = int(np.sum(~target & ~predicted))
    n = tp + fp + fn + tn

    precision = tp / (tp + fp) if (tp + fp) else 0.0
    recall = tp / (tp + fn) if (tp + fn) else 0.0
    specificity = tn / (tn + fp) if (tn + fp) else 0.0
    f1 = (
        2.0 * precision * recall / (precision + recall)
        if (precision + recall)
        else 0.0
    )
    return {
        "n": n,
        "tn": tn,
        "fp": fp,
        "fn": fn,
        "tp": tp,
        "flagged": tp + fp,
        "flag_rate": (tp + fp) / n if n else 0.0,
        "accuracy": (tn + tp) / n if n else 0.0,
        "precision": precision,
        "recall": recall,
        "specificity": specificity,
        "f1": f1,
    }


def warn_by_exact_rate(score: np.ndarray, rate: float) -> np.ndarray:
    """Flag exactly round(rate*n) highest scores, with deterministic tie order."""
    score = np.asarray(score, dtype=np.float64).reshape(-1)
    if len(score) == 0 or rate <= 0.0:
        return np.zeros(len(score), dtype=np.int64)
    if not np.isfinite(score).all():
        raise ValueError("Scores contain NaN or infinity")
    k = min(len(score), max(1, int(round(float(rate) * len(score)))))
    order = np.argsort(-score, kind="mergesort")
    warning = np.zeros(len(score), dtype=np.int64)
    warning[order[:k]] = 1
    return warning


def warn_by_test2_rate(score: np.ndarray, rate: float) -> np.ndarray:
    """Reproduce test2.py's historical top-K threshold/tie behavior."""
    score = np.asarray(score, dtype=np.float64).reshape(-1)
    if len(score) == 0 or rate <= 0.0:
        return np.zeros(len(score), dtype=np.int64)
    k = min(len(score), max(1, int(round(float(rate) * len(score)))))
    threshold = np.partition(score, -k)[-k]
    return (score >= threshold).astype(np.int64)


def best_f1_threshold(
    y: np.ndarray, score: np.ndarray, min_flags: int = 5
) -> Tuple[float, float, Dict[str, float]]:
    """Replicate test2.py's pooled development F1 threshold search."""
    y = np.asarray(y, dtype=np.int64).reshape(-1)
    score = np.asarray(score, dtype=np.float64).reshape(-1)
    candidates = np.unique(np.round(score, 5))
    if len(candidates) > 400:
        candidates = np.quantile(score, np.linspace(0.01, 0.99, 400))

    best_threshold = float(np.median(score))
    best_metrics = binary_metrics(y, score >= best_threshold)
    best_f1 = -1.0
    for threshold in candidates:
        metrics = binary_metrics(y, score >= threshold)
        if metrics["flagged"] >= min_flags and metrics["f1"] > best_f1:
            best_f1 = metrics["f1"]
            best_threshold = float(threshold)
            best_metrics = metrics
    return best_threshold, best_metrics["flag_rate"], best_metrics


def load_test2_record(directory: Path, faculty: str, fold: int) -> ScoreRecord:
    path = directory / f"{faculty}_fold_{fold}_test2_predictions.csv"
    if not path.exists():
        raise FileNotFoundError(
            f"Missing test2 predictions: {path}\n"
            "Run test2_equal_budget.py once with --predictions_dir first."
        )
    frame = pd.read_csv(path, low_memory=False)
    required = {"actual_failure_lt5", "classification_probability"}
    missing = required - set(frame.columns)
    if missing:
        raise ValueError(f"{path} is missing columns: {sorted(missing)}")
    grade = (
        frame["target_grade10"].to_numpy(dtype=float)
        if "target_grade10" in frame.columns
        else None
    )
    return ScoreRecord(
        model="test2",
        fold=fold,
        y_failure=frame["actual_failure_lt5"].to_numpy(dtype=np.int64),
        score=frame["classification_probability"].to_numpy(dtype=float),
        target_grade10=grade,
        frame=frame,
    )


def load_v2_record(directory: Path, faculty: str, fold: int) -> ScoreRecord:
    path = directory / f"{faculty}_fold_{fold}_test_results.csv"
    if not path.exists():
        raise FileNotFoundError(f"Missing final_train_v2 results: {path}")
    frame = pd.read_csv(path, low_memory=False)
    score_column = next(
        (
            column
            for column in (
                "classifier_operational_score",
                "failure_probability_lt5",
            )
            if column in frame.columns
        ),
        None,
    )
    if score_column is None or "actual_failure_lt5" not in frame.columns:
        raise ValueError(
            f"{path} must contain actual_failure_lt5 and a classifier score column"
        )
    grade = (
        frame["target_grade10"].to_numpy(dtype=float)
        if "target_grade10" in frame.columns
        else None
    )
    return ScoreRecord(
        model="final_train_v2",
        fold=fold,
        y_failure=frame["actual_failure_lt5"].to_numpy(dtype=np.int64),
        score=frame[score_column].to_numpy(dtype=float),
        target_grade10=grade,
        frame=frame,
    )


def assert_aligned(test2: ScoreRecord, v2: ScoreRecord) -> None:
    if test2.fold != v2.fold:
        raise ValueError("Cannot align records from different folds")
    if not np.array_equal(test2.y_failure, v2.y_failure):
        raise ValueError(
            f"Fold {test2.fold}: failure labels differ between model outputs"
        )
    if test2.target_grade10 is not None and v2.target_grade10 is not None:
        if not np.allclose(
            test2.target_grade10,
            v2.target_grade10,
            rtol=0.0,
            atol=1e-10,
            equal_nan=True,
        ):
            raise ValueError(
                f"Fold {test2.fold}: target row order differs between outputs"
            )


def concatenate(records: Sequence[ScoreRecord]) -> Tuple[np.ndarray, np.ndarray]:
    return (
        np.concatenate([record.y_failure for record in records]),
        np.concatenate([record.score for record in records]),
    )


def evaluate_equal_budget(
    records: Sequence[ScoreRecord], rate: float
) -> Dict[str, float]:
    all_y: List[np.ndarray] = []
    all_warning: List[np.ndarray] = []
    for record in records:
        all_y.append(record.y_failure)
        all_warning.append(warn_by_exact_rate(record.score, rate))
    y = np.concatenate(all_y)
    warning = np.concatenate(all_warning)
    metrics = binary_metrics(y, warning)
    metrics["budget"] = float(rate)
    return metrics


def test2_policy(
    development: Sequence[ScoreRecord],
) -> Tuple[float, float, float, Dict[int, float]]:
    y_dev, score_dev = concatenate(development)
    threshold, flag_rate, _ = best_f1_threshold(y_dev, score_dev)
    prevalence = float(np.mean(y_dev))
    warning_ratio = flag_rate / max(prevalence, 1e-12)
    prevalence_map = {
        record.fold: float(np.mean(record.y_failure)) for record in development
    }
    return threshold, warning_ratio, prevalence, prevalence_map


def evaluate_test2_current_policy(
    holdout: Sequence[ScoreRecord],
    threshold: float,
    warning_ratio: float,
    default_prevalence: float,
    prevalence_map: Dict[int, float],
) -> List[Dict[str, float]]:
    rows: List[Dict[str, float]] = []
    pooled_y, pooled_score, pooled_fixed, pooled_drift = [], [], [], []
    for record in holdout:
        previous = prevalence_map.get(record.fold - 1, default_prevalence)
        drift_rate = float(np.clip(warning_ratio * previous, 1e-4, 0.99))
        fixed = (record.score >= threshold).astype(np.int64)
        drift = warn_by_test2_rate(record.score, drift_rate)
        rows.append(
            current_policy_row(record, fixed, drift, drift_rate, "F1-dev")
        )
        prevalence_map[record.fold] = float(np.mean(record.y_failure))
        pooled_y.append(record.y_failure)
        pooled_score.append(record.score)
        pooled_fixed.append(fixed)
        pooled_drift.append(drift)
    rows.append(
        pooled_policy_row(
            "test2",
            pooled_y,
            pooled_score,
            pooled_fixed,
            pooled_drift,
            "F1-dev",
        )
    )
    return rows


def evaluate_v2_current_policy(
    holdout: Sequence[ScoreRecord],
) -> List[Dict[str, float]]:
    rows: List[Dict[str, float]] = []
    pooled_y, pooled_score, pooled_fixed, pooled_drift = [], [], [], []
    for record in holdout:
        if record.frame is None:
            raise ValueError("V2 record frame is required")
        frame = record.frame
        if "classifier_warning_fixed" not in frame.columns:
            raise ValueError(
                "V2 CSV is missing classifier_warning_fixed; use the completed V2 output"
            )
        fixed = frame["classifier_warning_fixed"].to_numpy(dtype=np.int64)
        if "classifier_warning_drift" in frame.columns:
            drift = frame["classifier_warning_drift"].to_numpy(dtype=np.int64)
        else:
            if "classifier_drift_rate" not in frame.columns:
                raise ValueError("V2 CSV has no drift warning or drift rate")
            drift_rate = float(frame["classifier_drift_rate"].iloc[0])
            drift = warn_by_exact_rate(record.score, drift_rate)
        drift_rate = float(np.mean(drift))
        rows.append(current_policy_row(record, fixed, drift, drift_rate, "F2-dev"))
        pooled_y.append(record.y_failure)
        pooled_score.append(record.score)
        pooled_fixed.append(fixed)
        pooled_drift.append(drift)
    rows.append(
        pooled_policy_row(
            "final_train_v2",
            pooled_y,
            pooled_score,
            pooled_fixed,
            pooled_drift,
            "F2-dev",
        )
    )
    return rows


def current_policy_row(
    record: ScoreRecord,
    fixed: np.ndarray,
    drift: np.ndarray,
    drift_rate: float,
    objective: str,
) -> Dict[str, float]:
    fixed_metrics = binary_metrics(record.y_failure, fixed)
    drift_metrics = binary_metrics(record.y_failure, drift)
    return {
        "model": record.model,
        "fold": str(record.fold),
        "n": len(record.y_failure),
        "prevalence": float(np.mean(record.y_failure)),
        "ap": safe_ap(record.y_failure, record.score),
        "auc": safe_auc(record.y_failure, record.score),
        "objective": objective,
        **{f"fixed_{key}": value for key, value in fixed_metrics.items()},
        **{f"drift_{key}": value for key, value in drift_metrics.items()},
        "drift_rate_requested": drift_rate,
    }


def pooled_policy_row(
    model: str,
    ys: Sequence[np.ndarray],
    scores: Sequence[np.ndarray],
    fixed: Sequence[np.ndarray],
    drift: Sequence[np.ndarray],
    objective: str,
) -> Dict[str, float]:
    y = np.concatenate(ys)
    score = np.concatenate(scores)
    fixed_warning = np.concatenate(fixed)
    drift_warning = np.concatenate(drift)
    fixed_metrics = binary_metrics(y, fixed_warning)
    drift_metrics = binary_metrics(y, drift_warning)
    return {
        "model": model,
        "fold": "POOLED",
        "n": len(y),
        "prevalence": float(np.mean(y)),
        "ap": safe_ap(y, score),
        "auc": safe_auc(y, score),
        "objective": objective,
        **{f"fixed_{key}": value for key, value in fixed_metrics.items()},
        **{f"drift_{key}": value for key, value in drift_metrics.items()},
        "drift_rate_requested": float(np.mean(drift_warning)),
    }


def print_current_policy_table(rows: Sequence[Dict[str, float]]) -> None:
    print("\n" + "=" * 154)
    print("CURRENT OPERATING-POLICY COMPARISON")
    print("test2 uses its pooled-development F1 threshold; V2 uses its pooled-development F2 threshold.")
    print("=" * 154)
    print(
        f"{'model':<16} {'fold':>7} {'test':>7} {'base%':>7} | {'AP':>7} {'AUC':>7} | "
        f"{'ACC@t':>7} {'P@t':>7} {'R@t':>7} {'F1@t':>7} {'rate@t':>8} | "
        f"{'ACC@r':>7} {'P@r':>7} {'R@r':>7} {'F1@r':>7} {'rate@r':>8}"
    )
    for row in rows:
        print(
            f"{row['model']:<16} {row['fold']:>7} {int(row['n']):7d} "
            f"{100*row['prevalence']:6.2f}% | {row['ap']:7.4f} {row['auc']:7.4f} | "
            f"{row['fixed_accuracy']:7.4f} {row['fixed_precision']:7.4f} "
            f"{row['fixed_recall']:7.4f} {row['fixed_f1']:7.4f} "
            f"{100*row['fixed_flag_rate']:7.2f}% | "
            f"{row['drift_accuracy']:7.4f} {row['drift_precision']:7.4f} "
            f"{row['drift_recall']:7.4f} {row['drift_f1']:7.4f} "
            f"{100*row['drift_flag_rate']:7.2f}%"
        )


def equal_budget_rows(
    records_by_model: Dict[str, Sequence[ScoreRecord]],
    rates: Iterable[float],
) -> List[Dict[str, float]]:
    rows: List[Dict[str, float]] = []
    for rate in rates:
        for model, records in records_by_model.items():
            for record in records:
                warning = warn_by_exact_rate(record.score, rate)
                metrics = binary_metrics(record.y_failure, warning)
                rows.append(
                    {
                        "model": model,
                        "fold": str(record.fold),
                        "budget": rate,
                        "ap": safe_ap(record.y_failure, record.score),
                        "auc": safe_auc(record.y_failure, record.score),
                        "prevalence": float(np.mean(record.y_failure)),
                        **metrics,
                    }
                )
            pooled = evaluate_equal_budget(records, rate)
            y, score = concatenate(records)
            rows.append(
                {
                    "model": model,
                    "fold": "POOLED",
                    "budget": rate,
                    "ap": safe_ap(y, score),
                    "auc": safe_auc(y, score),
                    "prevalence": float(np.mean(y)),
                    **pooled,
                }
            )
    return rows


def print_equal_budget_table(rows: Sequence[Dict[str, float]]) -> None:
    print("\n" + "=" * 145)
    print("EQUAL PER-SEMESTER WARNING-BUDGET COMPARISON")
    print("Each fold independently flags exactly the requested top-risk percentage.")
    print("=" * 145)
    print(
        f"{'budget':>8} {'model':<16} {'fold':>7} {'test':>7} {'base%':>7} | "
        f"{'AP':>7} {'AUC':>7} | {'flagged':>8} {'ACC':>7} {'P':>7} {'R':>7} "
        f"{'F1':>7} {'SPEC':>7} {'TP':>7} {'FP':>7} {'FN':>7}"
    )
    for row in rows:
        print(
            f"{100*row['budget']:7.1f}% {row['model']:<16} {row['fold']:>7} "
            f"{int(row['n']):7d} {100*row['prevalence']:6.2f}% | "
            f"{row['ap']:7.4f} {row['auc']:7.4f} | {int(row['flagged']):8d} "
            f"{row['accuracy']:7.4f} {row['precision']:7.4f} "
            f"{row['recall']:7.4f} {row['f1']:7.4f} "
            f"{row['specificity']:7.4f} {int(row['tp']):7d} "
            f"{int(row['fp']):7d} {int(row['fn']):7d}"
        )


def print_pooled_deltas(rows: Sequence[Dict[str, float]]) -> None:
    pooled = [row for row in rows if row["fold"] == "POOLED"]
    print("\n### POOLED V2 - TEST2 DELTAS AT EQUAL BUDGET")
    print(
        f"{'budget':>8} {'ΔTP':>7} {'ΔFP':>7} {'ΔFN':>7} "
        f"{'ΔP':>9} {'ΔR':>9} {'ΔF1':>9}"
    )
    budgets = sorted({row["budget"] for row in pooled})
    for budget in budgets:
        group = {row["model"]: row for row in pooled if row["budget"] == budget}
        if "test2" not in group or "final_train_v2" not in group:
            continue
        a, b = group["test2"], group["final_train_v2"]
        print(
            f"{100*budget:7.1f}% {int(b['tp']-a['tp']):7d} "
            f"{int(b['fp']-a['fp']):7d} {int(b['fn']-a['fn']):7d} "
            f"{b['precision']-a['precision']:+9.4f} "
            f"{b['recall']-a['recall']:+9.4f} "
            f"{b['f1']-a['f1']:+9.4f}"
        )


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Compare test2 and final_train_v2 at current policies and equal budgets"
    )
    parser.add_argument("--faculty", default="khoa_cong_nghe_thong_tin")
    parser.add_argument("--test2_dir", required=True, type=Path)
    parser.add_argument("--v2_dir", required=True, type=Path)
    parser.add_argument("--dev_folds", default="1,2,3,4,5")
    parser.add_argument("--holdout_folds", default="6,7")
    parser.add_argument("--budgets", default="0.10,0.15,0.20")
    parser.add_argument(
        "--output_csv",
        type=Path,
        default=Path("warning_budget_comparison.csv"),
    )
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    dev_folds = parse_int_list(args.dev_folds)
    holdout_folds = parse_int_list(args.holdout_folds)
    budgets = parse_rates(args.budgets)

    test2_dev = [
        load_test2_record(args.test2_dir, args.faculty, fold) for fold in dev_folds
    ]
    test2_holdout = [
        load_test2_record(args.test2_dir, args.faculty, fold)
        for fold in holdout_folds
    ]
    v2_holdout = [
        load_v2_record(args.v2_dir, args.faculty, fold)
        for fold in holdout_folds
    ]
    for test2_record, v2_record in zip(test2_holdout, v2_holdout):
        assert_aligned(test2_record, v2_record)

    threshold, ratio, dev_prevalence, prevalence_map = test2_policy(test2_dev)
    print(
        f"test2 reconstructed development policy: threshold={threshold:.6f}, "
        f"warning_ratio={ratio:.4f}, prevalence={dev_prevalence:.4%}"
    )

    current_rows = evaluate_test2_current_policy(
        test2_holdout,
        threshold,
        ratio,
        dev_prevalence,
        prevalence_map,
    ) + evaluate_v2_current_policy(v2_holdout)
    current_rows.sort(
        key=lambda row: (
            999 if row["fold"] == "POOLED" else int(row["fold"]),
            row["model"],
        )
    )
    print_current_policy_table(current_rows)

    budget_rows = equal_budget_rows(
        {"test2": test2_holdout, "final_train_v2": v2_holdout},
        budgets,
    )
    budget_rows.sort(
        key=lambda row: (
            row["budget"],
            99 if row["fold"] == "POOLED" else int(row["fold"]),
            row["model"],
        )
    )
    print_equal_budget_table(budget_rows)
    print_pooled_deltas(budget_rows)

    output = args.output_csv.expanduser()
    output.parent.mkdir(parents=True, exist_ok=True)
    pd.DataFrame(
        [
            {"analysis": "current_policy", **row} for row in current_rows
        ]
        + [{"analysis": "equal_budget", **row} for row in budget_rows]
    ).to_csv(output, index=False)
    print(f"\nSaved comparison CSV: {output.resolve()}")


if __name__ == "__main__":
    main()
