#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Audit failure prevalence across all faculty expanding-fold test semesters.

Why test files only?
--------------------
The expanding training sets overlap heavily across folds. Counting every train
file would count the same historical registrations repeatedly. Each fold's test
file represents its target semester, so pooling test files gives a clean
semester-by-semester prevalence audit.

Failure definition: target grade < boundary (default: target_grade10 < 5).
"""
from __future__ import annotations

import argparse
from pathlib import Path
from typing import Iterable

import numpy as np
import pandas as pd


def parse_folds(text: str) -> list[int]:
    return [int(x.strip()) for x in text.split(",") if x.strip()]


def discover_faculties(splits_dir: Path, folds: Iterable[int]) -> list[str]:
    faculties: set[str] = set()
    for fold in folds:
        fold_dir = splits_dir / f"fold_{fold}"
        if not fold_dir.exists():
            continue
        for p in fold_dir.glob("*_test.csv"):
            faculties.add(p.name[:-len("_test.csv")])
    return sorted(faculties)


def read_fold_prevalence(
    path: Path,
    target: str,
    failure_boundary: float,
) -> dict[str, float | int]:
    # Read only the target column to keep the audit fast even for wide files.
    frame = pd.read_csv(path, usecols=[target], low_memory=False)
    grade = pd.to_numeric(frame[target], errors="coerce").dropna().to_numpy(float)
    n = int(len(grade))
    failures = int(np.sum(grade < failure_boundary))
    normals = n - failures
    prevalence = failures / n if n else np.nan
    normal_per_failure = normals / failures if failures else np.inf
    return {
        "rows": n,
        "failures": failures,
        "normals": normals,
        "prevalence": prevalence,
        "normal_per_failure": normal_per_failure,
    }


def _aggregate(rows: pd.DataFrame) -> dict[str, float | int]:
    n = int(rows["rows"].sum()) if len(rows) else 0
    failures = int(rows["failures"].sum()) if len(rows) else 0
    normals = n - failures
    prevalence = failures / n if n else np.nan
    return {
        "rows": n,
        "failures": failures,
        "normals": normals,
        "prevalence": prevalence,
        "normal_per_failure": normals / failures if failures else np.inf,
    }


def audit_faculties(
    splits_dir: str | Path,
    dev_folds: list[int],
    holdout_folds: list[int],
    target: str = "target_grade10",
    failure_boundary: float = 5.0,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    root = Path(splits_dir)
    all_folds = list(dict.fromkeys([*dev_folds, *holdout_folds]))
    faculties = discover_faculties(root, all_folds)
    if not faculties:
        raise FileNotFoundError(
            f"No '*_test.csv' files found under {root}/fold_<n>. "
            "Check --splits_dir and fold numbers."
        )

    fold_rows: list[dict] = []
    for faculty in faculties:
        for fold in all_folds:
            path = root / f"fold_{fold}" / f"{faculty}_test.csv"
            if not path.exists():
                continue
            try:
                stats = read_fold_prevalence(path, target, failure_boundary)
            except ValueError as exc:
                raise KeyError(f"Missing target column '{target}' in {path}") from exc
            fold_rows.append({
                "faculty": faculty,
                "fold": fold,
                "split": "development" if fold in dev_folds else "holdout",
                **stats,
            })

    by_fold = pd.DataFrame(fold_rows).sort_values(["faculty", "fold"]).reset_index(drop=True)
    summary_rows: list[dict] = []
    for faculty, g in by_fold.groupby("faculty", sort=True):
        dev = _aggregate(g[g["fold"].isin(dev_folds)])
        hold = _aggregate(g[g["fold"].isin(holdout_folds)])
        all_stats = _aggregate(g)
        finite_prev = g["prevalence"].replace([np.inf, -np.inf], np.nan).dropna()
        dev_prev = float(dev["prevalence"])
        hold_prev = float(hold["prevalence"])
        drift_ratio = (
            hold_prev / dev_prev
            if np.isfinite(dev_prev) and dev_prev > 0 and np.isfinite(hold_prev)
            else np.nan
        )
        summary_rows.append({
            "faculty": faculty,
            "dev_rows": dev["rows"],
            "dev_failures": dev["failures"],
            "dev_prevalence": dev_prev,
            "dev_normal_per_failure": dev["normal_per_failure"],
            "holdout_rows": hold["rows"],
            "holdout_failures": hold["failures"],
            "holdout_prevalence": hold_prev,
            "holdout_normal_per_failure": hold["normal_per_failure"],
            "all_rows": all_stats["rows"],
            "all_failures": all_stats["failures"],
            "all_prevalence": all_stats["prevalence"],
            "all_normal_per_failure": all_stats["normal_per_failure"],
            "min_fold_prevalence": float(finite_prev.min()) if len(finite_prev) else np.nan,
            "max_fold_prevalence": float(finite_prev.max()) if len(finite_prev) else np.nan,
            "holdout_to_dev_prevalence_ratio": drift_ratio,
            "available_folds": int(g["fold"].nunique()),
        })

    # Rank primarily by development prevalence because that is the positive
    # supervision available before holdout deployment. Higher = less imbalance.
    summary = pd.DataFrame(summary_rows).sort_values(
        ["dev_prevalence", "dev_failures", "holdout_prevalence"],
        ascending=[False, False, False],
    ).reset_index(drop=True)
    summary.insert(0, "rank_by_dev_prevalence", np.arange(1, len(summary) + 1))
    return summary, by_fold


def _pct(x: float) -> str:
    return "   n/a" if not np.isfinite(x) else f"{100*x:6.2f}%"


def _ratio(x: float) -> str:
    if np.isnan(x):
        return " n/a "
    if np.isinf(x):
        return " inf "
    return f"{x:5.1f}"


def print_summary(summary: pd.DataFrame, failure_boundary: float) -> None:
    print("=" * 142)
    print(f"FACULTY FAILURE-PREVALENCE AUDIT | failure = target grade < {failure_boundary:g}")
    print("Ranked by DEVELOPMENT prevalence (higher prevalence = less severe class imbalance)")
    print("=" * 142)
    print(
        f"{'#':>2}  {'faculty':46s} | {'dev fail':>9s} {'dev prev':>8s} {'N/F':>5s} | "
        f"{'hold fail':>9s} {'hold prev':>9s} {'N/F':>5s} | {'all prev':>8s} | "
        f"{'min fold':>8s} {'max fold':>8s} {'H/D':>6s}"
    )
    print("-" * 142)
    for _, r in summary.iterrows():
        print(
            f"{int(r['rank_by_dev_prevalence']):2d}  {r['faculty'][:46]:46s} | "
            f"{int(r['dev_failures']):4d}/{int(r['dev_rows']):4d} {_pct(r['dev_prevalence']):>8s} {_ratio(r['dev_normal_per_failure']):>5s} | "
            f"{int(r['holdout_failures']):4d}/{int(r['holdout_rows']):4d} {_pct(r['holdout_prevalence']):>9s} {_ratio(r['holdout_normal_per_failure']):>5s} | "
            f"{_pct(r['all_prevalence']):>8s} | {_pct(r['min_fold_prevalence']):>8s} {_pct(r['max_fold_prevalence']):>8s} "
            f"{_ratio(r['holdout_to_dev_prevalence_ratio']):>6s}"
        )
    print("=" * 142)
    print("N/F = number of normal registrations per failure. Lower is better balanced.")
    print("H/D = holdout prevalence / development prevalence. Values far from 1 indicate prevalence drift.")
    print("Use development prevalence/failure count for model-training feasibility; inspect H/D before deployment.")


def print_by_fold(by_fold: pd.DataFrame) -> None:
    print("\nPER-FOLD DETAIL")
    print("=" * 105)
    for faculty, g in by_fold.groupby("faculty", sort=True):
        parts = []
        for _, r in g.sort_values("fold").iterrows():
            parts.append(
                f"F{int(r['fold'])}:{100*r['prevalence']:.2f}% "
                f"({int(r['failures'])}/{int(r['rows'])})"
            )
        print(f"{faculty:46s} | " + "  ".join(parts))


def parse_args() -> argparse.Namespace:
    p = argparse.ArgumentParser(description="Audit class imbalance across all faculties.")
    p.add_argument("--splits_dir", required=True, help="Expanding-fold root containing fold_1, ..., fold_7")
    p.add_argument("--dev_folds", default="1,2,3,4,5")
    p.add_argument("--holdout_folds", default="6,7")
    p.add_argument("--target", default="target_grade10")
    p.add_argument("--failure_boundary", type=float, default=5.0)
    p.add_argument("--output_dir", default=None, help="Optional directory for summary/detail CSV files")
    p.add_argument("--show_folds", action="store_true", help="Print per-fold prevalence after the ranking table")
    return p.parse_args()


def main() -> None:
    args = parse_args()
    dev_folds = parse_folds(args.dev_folds)
    holdout_folds = parse_folds(args.holdout_folds)
    summary, by_fold = audit_faculties(
        args.splits_dir,
        dev_folds,
        holdout_folds,
        args.target,
        args.failure_boundary,
    )
    print_summary(summary, args.failure_boundary)
    if args.show_folds:
        print_by_fold(by_fold)

    if args.output_dir:
        out = Path(args.output_dir)
        out.mkdir(parents=True, exist_ok=True)
        summary_path = out / "faculty_failure_prevalence_summary.csv"
        detail_path = out / "faculty_failure_prevalence_by_fold.csv"
        summary.to_csv(summary_path, index=False)
        by_fold.to_csv(detail_path, index=False)
        print(f"\nSaved: {summary_path}")
        print(f"Saved: {detail_path}")


if __name__ == "__main__":
    main()
