#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Comprehensive text-first EDA for the academic early-warning dataset.

The script is designed for servers without a graphical desktop. It prints all
important results as plain-text tables and also writes detailed CSV files.

Expected directory layout:
    data/splits/<strategy>/fold_<n>/<faculty>_train.csv
    data/splits/<strategy>/fold_<n>/<faculty>_test.csv

Example:
    python academic_early_warning_eda.py \
        --faculty khoa_cong_nghe_thong_tin \
        --strategy expanding \
        --dev_folds 1,2,3,4,5 \
        --test_folds 6,7 \
        --output_dir eda_outputs_it \
        > eda_it.log 2>&1

Dependencies: numpy, pandas only.
"""
from __future__ import annotations

import argparse
import json
import math
import sys
import time
import warnings
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Sequence

import numpy as np
import pandas as pd

warnings.filterwarnings("ignore", category=RuntimeWarning)

TARGET = "target_grade10"
DEFAULT_EVENT = 5.0

# Keep this aligned with the current SoTA training script.
EXCLUDE_COLS = {
    "dataset_split", "student_faculty_key", "student_faculty_name", "IDSV",
    "target_NHHK", "target_semester_ordinal", "IDKQDK", "IDToHoc", "IDMH",
    "target_IDCongDiem", "target_grade10", "target_grade4", "target_status",
    "base_row_id", "course_cluster_label",
}

ID_COLUMNS = ["base_row_id", "IDKQDK", "IDSV", "IDMH", "IDToHoc"]
SEMESTER_COLUMNS = ["target_semester_ordinal", "target_NHHK"]


# ---------------------------------------------------------------------------
# Reporting and output helpers
# ---------------------------------------------------------------------------
class Reporter:
    def __init__(self, report_path: Path):
        self.report_path = report_path
        self.report_path.parent.mkdir(parents=True, exist_ok=True)
        self._fh = self.report_path.open("w", encoding="utf-8")

    def close(self) -> None:
        self._fh.close()

    def line(self, text: str = "") -> None:
        print(text)
        self._fh.write(text + "\n")
        self._fh.flush()

    def section(self, title: str, char: str = "=") -> None:
        width = max(90, min(150, len(title) + 12))
        self.line("\n" + char * width)
        self.line(title)
        self.line(char * width)

    def table(self, df: pd.DataFrame, title: str | None = None,
              max_rows: int | None = None, float_format: str = "{:.4f}") -> None:
        if title:
            self.line(f"\n{title}")
        if df is None or df.empty:
            self.line("  [No rows]")
            return
        shown = df if max_rows is None else df.head(max_rows)
        with pd.option_context(
            "display.max_columns", None,
            "display.width", 240,
            "display.max_colwidth", 72,
        ):
            self.line(shown.to_string(index=False, float_format=lambda x: float_format.format(x)))
        if max_rows is not None and len(df) > max_rows:
            self.line(f"  ... {len(df) - max_rows:,} additional rows saved to CSV.")


@dataclass
class OutputStore:
    root: Path

    def __post_init__(self) -> None:
        self.root.mkdir(parents=True, exist_ok=True)

    def save(self, name: str, df: pd.DataFrame) -> Path:
        path = self.root / f"{name}.csv"
        df.to_csv(path, index=False, encoding="utf-8-sig")
        return path

    def save_json(self, name: str, value: dict) -> Path:
        path = self.root / f"{name}.json"
        path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
        return path


# ---------------------------------------------------------------------------
# Generic numerical helpers
# ---------------------------------------------------------------------------
def _average_ranks(values: np.ndarray) -> np.ndarray:
    values = np.asarray(values, dtype=float)
    order = np.argsort(values, kind="mergesort")
    sorted_values = values[order]
    ranks = np.arange(1, len(values) + 1, dtype=float)
    i = 0
    while i < len(values):
        j = i
        while j + 1 < len(values) and sorted_values[j + 1] == sorted_values[i]:
            j += 1
        if j > i:
            ranks[i:j + 1] = (i + 1 + j + 1) / 2.0
        i = j + 1
    out = np.empty(len(values), dtype=float)
    out[order] = ranks
    return out


def binary_auc(y_binary: Sequence[int | bool], score: Sequence[float]) -> float:
    y = np.asarray(y_binary).astype(bool)
    s = np.asarray(score, dtype=float)
    valid = np.isfinite(s)
    y, s = y[valid], s[valid]
    n_pos = int(y.sum())
    n_neg = int((~y).sum())
    if n_pos == 0 or n_neg == 0:
        return float("nan")
    ranks = _average_ranks(s)
    return float((ranks[y].sum() - n_pos * (n_pos + 1) / 2.0) / (n_pos * n_neg))


def _safe_std(values: np.ndarray) -> float:
    if len(values) < 2:
        return float("nan")
    return float(np.nanstd(values, ddof=1))


def standardized_mean_difference(x0: np.ndarray, x1: np.ndarray) -> float:
    x0 = np.asarray(x0, dtype=float)
    x1 = np.asarray(x1, dtype=float)
    x0 = x0[np.isfinite(x0)]
    x1 = x1[np.isfinite(x1)]
    if len(x0) < 2 or len(x1) < 2:
        return float("nan")
    v0 = np.var(x0, ddof=1)
    v1 = np.var(x1, ddof=1)
    denom = len(x0) + len(x1) - 2
    if denom <= 0:
        return float("nan")
    pooled = math.sqrt(max(((len(x0) - 1) * v0 + (len(x1) - 1) * v1) / denom, 0.0))
    if pooled == 0:
        return 0.0 if np.mean(x0) == np.mean(x1) else float("inf")
    return float((np.mean(x1) - np.mean(x0)) / pooled)


def _quantile_edges(values: np.ndarray, bins: int = 10) -> np.ndarray:
    values = np.asarray(values, dtype=float)
    values = values[np.isfinite(values)]
    if len(values) == 0:
        return np.array([-np.inf, np.inf])
    qs = np.linspace(0.0, 1.0, bins + 1)
    edges = np.unique(np.quantile(values, qs))
    if len(edges) < 2:
        value = float(edges[0])
        return np.array([-np.inf, value, np.inf])
    edges = edges.astype(float)
    edges[0] = -np.inf
    edges[-1] = np.inf
    return edges


def numeric_psi(reference: Sequence[float], current: Sequence[float], bins: int = 10,
                epsilon: float = 1e-6) -> float:
    """Population Stability Index using reference quantile bins.

    Missingness is analyzed separately, so NaNs are omitted here.
    """
    ref = np.asarray(reference, dtype=float)
    cur = np.asarray(current, dtype=float)
    ref = ref[np.isfinite(ref)]
    cur = cur[np.isfinite(cur)]
    if len(ref) < 2 or len(cur) < 2:
        return float("nan")
    edges = _quantile_edges(ref, bins=bins)
    ref_counts, _ = np.histogram(ref, bins=edges)
    cur_counts, _ = np.histogram(cur, bins=edges)
    ref_p = np.clip(ref_counts / max(ref_counts.sum(), 1), epsilon, None)
    cur_p = np.clip(cur_counts / max(cur_counts.sum(), 1), epsilon, None)
    return float(np.sum((cur_p - ref_p) * np.log(cur_p / ref_p)))


def binned_overlap_and_iv(x: np.ndarray, y: np.ndarray, bins: int = 10,
                          epsilon: float = 0.5) -> tuple[float, float]:
    """Return distribution overlap and Information Value using quantile bins."""
    x = np.asarray(x, dtype=float)
    y = np.asarray(y).astype(bool)
    valid = np.isfinite(x)
    xv, yv = x[valid], y[valid]
    if yv.sum() == 0 or (~yv).sum() == 0 or len(xv) < 4:
        return float("nan"), float("nan")
    edges = _quantile_edges(xv, bins=bins)
    pos_counts, _ = np.histogram(xv[yv], bins=edges)
    neg_counts, _ = np.histogram(xv[~yv], bins=edges)
    pos_p = (pos_counts + epsilon) / (pos_counts.sum() + epsilon * len(pos_counts))
    neg_p = (neg_counts + epsilon) / (neg_counts.sum() + epsilon * len(neg_counts))
    overlap = float(np.minimum(pos_p, neg_p).sum())
    iv = float(np.sum((pos_p - neg_p) * np.log(pos_p / neg_p)))
    return overlap, iv


def effective_sample_size(weights: Sequence[float]) -> float:
    w = np.asarray(weights, dtype=float)
    w = w[np.isfinite(w) & (w >= 0)]
    if len(w) == 0 or np.sum(w * w) == 0:
        return float("nan")
    return float(np.sum(w) ** 2 / np.sum(w * w))


# ---------------------------------------------------------------------------
# File discovery and loading
# ---------------------------------------------------------------------------
def parse_fold_spec(spec: str, available: Sequence[int]) -> list[int]:
    if spec.strip().lower() == "all":
        return sorted(set(int(x) for x in available))
    requested = [int(x.strip()) for x in spec.split(",") if x.strip()]
    return [x for x in requested if x in set(available)]


def discover_folds(splits_dir: Path, faculty: str) -> list[int]:
    folds: list[int] = []
    for path in sorted(splits_dir.glob("fold_*")):
        try:
            fold = int(path.name.split("_")[1])
        except (IndexError, ValueError):
            continue
        if (path / f"{faculty}_train.csv").exists() and (path / f"{faculty}_test.csv").exists():
            folds.append(fold)
    return sorted(folds)


def read_fold(fold: int, faculty: str, splits_dir: Path) -> tuple[pd.DataFrame, pd.DataFrame]:
    base = splits_dir / f"fold_{fold}"
    train_path = base / f"{faculty}_train.csv"
    test_path = base / f"{faculty}_test.csv"
    if not train_path.exists() or not test_path.exists():
        raise FileNotFoundError(f"Missing fold {fold}: {train_path} or {test_path}")
    train = pd.read_csv(train_path, low_memory=False)
    test = pd.read_csv(test_path, low_memory=False)
    return train, test


def valid_target_frame(df: pd.DataFrame, target: str = TARGET) -> pd.DataFrame:
    out = df.copy()
    if target not in out.columns:
        raise KeyError(f"Required target column missing: {target}")
    out[target] = pd.to_numeric(out[target], errors="coerce")
    return out.loc[out[target].notna()].copy()


def choose_dedup_key(df: pd.DataFrame) -> list[str]:
    for candidate in (["base_row_id"], ["IDKQDK"]):
        if all(c in df.columns for c in candidate) and df[candidate].notna().all(axis=1).any():
            return list(candidate)
    combo = [c for c in ["IDSV", "IDMH", "IDToHoc", "target_NHHK"] if c in df.columns]
    return combo


def build_canonical_snapshot(train: pd.DataFrame, test: pd.DataFrame) -> tuple[pd.DataFrame, list[str]]:
    combined = pd.concat([train, test], ignore_index=True, sort=False)
    key = choose_dedup_key(combined)
    if key:
        combined = combined.drop_duplicates(subset=key, keep="last")
    else:
        combined = combined.drop_duplicates(keep="last")
    return combined.reset_index(drop=True), key


def model_feature_columns(df: pd.DataFrame) -> list[str]:
    return [c for c in df.columns if c not in EXCLUDE_COLS]


def _id_overlap(train: pd.DataFrame, test: pd.DataFrame, col: str) -> tuple[int, float]:
    if col not in train.columns or col not in test.columns:
        return 0, float("nan")
    tr = set(train[col].dropna().astype(str))
    te = set(test[col].dropna().astype(str))
    overlap = len(tr & te)
    return overlap, overlap / max(len(te), 1)


def analyze_fold(fold: int, faculty: str, splits_dir: Path,
                 event: float = DEFAULT_EVENT) -> tuple[dict, pd.DataFrame]:
    train_raw, test_raw = read_fold(fold, faculty, splits_dir)
    train = valid_target_frame(train_raw)
    test = valid_target_frame(test_raw)

    ytr = train[TARGET].to_numpy(float)
    yte = test[TARGET].to_numpy(float)
    train_features = model_feature_columns(train)
    test_features = model_feature_columns(test)
    common_features = sorted(set(train_features) & set(test_features))

    train_students = set(train["IDSV"].dropna().astype(str)) if "IDSV" in train else set()
    test_students = test["IDSV"].dropna().astype(str) if "IDSV" in test else pd.Series(dtype=str)
    train_courses = set(train["IDMH"].dropna().astype(str)) if "IDMH" in train else set()
    test_courses = test["IDMH"].dropna().astype(str) if "IDMH" in test else pd.Series(dtype=str)

    new_student_rate = float((~test_students.isin(train_students)).mean()) if len(test_students) else float("nan")
    new_course_rate = float((~test_courses.isin(train_courses)).mean()) if len(test_courses) else float("nan")

    key = choose_dedup_key(pd.concat([train, test], ignore_index=True, sort=False))
    train_duplicates = int(train.duplicated(subset=key).sum()) if key else int(train.duplicated().sum())
    test_duplicates = int(test.duplicated(subset=key).sum()) if key else int(test.duplicated().sum())

    registration_overlap, registration_overlap_rate = _id_overlap(train, test, "IDKQDK")
    student_overlap, student_overlap_rate = _id_overlap(train, test, "IDSV")
    course_overlap, course_overlap_rate = _id_overlap(train, test, "IDMH")

    target_semesters = ",".join(map(str, sorted(test.get("target_NHHK", pd.Series(dtype=str)).dropna().unique())))
    summary = {
        "fold": fold,
        "train_rows_raw": len(train_raw),
        "test_rows_raw": len(test_raw),
        "train_rows_valid_target": len(train),
        "test_rows_valid_target": len(test),
        "test_semesters": target_semesters,
        "train_failure_count": int((ytr < event).sum()),
        "train_failure_rate": float((ytr < event).mean()),
        "test_failure_count": int((yte < event).sum()),
        "test_failure_rate": float((yte < event).mean()),
        "train_mean_grade": float(np.mean(ytr)),
        "test_mean_grade": float(np.mean(yte)),
        "train_unique_students": int(train["IDSV"].nunique()) if "IDSV" in train else np.nan,
        "test_unique_students": int(test["IDSV"].nunique()) if "IDSV" in test else np.nan,
        "train_unique_courses": int(train["IDMH"].nunique()) if "IDMH" in train else np.nan,
        "test_unique_courses": int(test["IDMH"].nunique()) if "IDMH" in test else np.nan,
        "new_student_rate": new_student_rate,
        "new_course_rate": new_course_rate,
        "registration_id_overlap": registration_overlap,
        "registration_id_overlap_rate": registration_overlap_rate,
        "student_overlap_count": student_overlap,
        "student_overlap_rate": student_overlap_rate,
        "course_overlap_count": course_overlap,
        "course_overlap_rate": course_overlap_rate,
        "train_duplicate_keys": train_duplicates,
        "test_duplicate_keys": test_duplicates,
        "train_feature_count": len(train_features),
        "test_feature_count": len(test_features),
        "common_feature_count": len(common_features),
        "train_only_feature_count": len(set(train_features) - set(test_features)),
        "test_only_feature_count": len(set(test_features) - set(train_features)),
    }

    drift_rows: list[dict] = []
    for feature in common_features:
        tr_num = pd.to_numeric(train[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
        te_num = pd.to_numeric(test[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
        tr_missing = float(tr_num.isna().mean())
        te_missing = float(te_num.isna().mean())
        tr_values = tr_num.dropna().to_numpy(float)
        te_values = te_num.dropna().to_numpy(float)
        if len(tr_values) >= 2 and len(te_values) >= 2:
            pooled_sd = np.std(tr_values)
            standardized_shift = (
                abs(float(np.mean(te_values) - np.mean(tr_values))) / pooled_sd
                if pooled_sd > 0 else (0.0 if np.mean(te_values) == np.mean(tr_values) else float("inf"))
            )
            psi = numeric_psi(tr_values, te_values)
        else:
            standardized_shift = float("nan")
            psi = float("nan")
        drift_rows.append({
            "fold": fold,
            "feature": feature,
            "train_missing_rate": tr_missing,
            "test_missing_rate": te_missing,
            "missing_rate_change": te_missing - tr_missing,
            "abs_missing_rate_change": abs(te_missing - tr_missing),
            "standardized_mean_shift": standardized_shift,
            "psi_train_to_test": psi,
            "train_all_missing": bool(tr_num.isna().all()),
            "test_all_missing": bool(te_num.isna().all()),
        })
    return summary, pd.DataFrame(drift_rows)


# ---------------------------------------------------------------------------
# Target and semester analyses
# ---------------------------------------------------------------------------
def semester_summary(df: pd.DataFrame, event: float) -> pd.DataFrame:
    work = valid_target_frame(df)
    semester_key = "target_semester_ordinal" if "target_semester_ordinal" in work else "target_NHHK"
    group_cols = [semester_key]
    if "target_NHHK" in work and semester_key != "target_NHHK":
        group_cols.append("target_NHHK")

    rows: list[dict] = []
    for keys, group in work.groupby(group_cols, dropna=False, sort=True):
        if not isinstance(keys, tuple):
            keys = (keys,)
        y = group[TARGET].to_numpy(float)
        row = {col: value for col, value in zip(group_cols, keys)}
        row.update({
            "rows": len(group),
            "unique_students": int(group["IDSV"].nunique()) if "IDSV" in group else np.nan,
            "unique_courses": int(group["IDMH"].nunique()) if "IDMH" in group else np.nan,
            "failures": int((y < event).sum()),
            "failure_rate": float((y < event).mean()),
            "grade_zero_rate": float((y == 0).mean()),
            "grade_below_3_rate": float((y < 3).mean()),
            "near_fail_4_to_5_rate": float(((y >= 4) & (y < 5)).mean()),
            "mean_grade": float(np.mean(y)),
            "median_grade": float(np.median(y)),
            "grade_std": float(np.std(y)),
            "grade_q05": float(np.quantile(y, 0.05)),
            "grade_q25": float(np.quantile(y, 0.25)),
            "grade_q75": float(np.quantile(y, 0.75)),
            "grade_q95": float(np.quantile(y, 0.95)),
        })
        rows.append(row)
    return pd.DataFrame(rows).sort_values(group_cols).reset_index(drop=True)


def grade_band_summary(df: pd.DataFrame) -> pd.DataFrame:
    work = valid_target_frame(df)
    y = work[TARGET]
    labels = pd.Series(index=work.index, dtype="string")
    labels.loc[y == 0] = "grade = 0"
    labels.loc[(y > 0) & (y < 3)] = "0 < grade < 3"
    labels.loc[(y >= 3) & (y < 4)] = "3 <= grade < 4"
    labels.loc[(y >= 4) & (y < 5)] = "4 <= grade < 5"
    labels.loc[(y >= 5) & (y < 6)] = "5 <= grade < 6"
    labels.loc[(y >= 6) & (y < 7)] = "6 <= grade < 7"
    labels.loc[(y >= 7) & (y < 8)] = "7 <= grade < 8"
    labels.loc[(y >= 8) & (y < 9)] = "8 <= grade < 9"
    labels.loc[(y >= 9) & (y <= 10)] = "9 <= grade <= 10"
    labels.loc[(y < 0) | (y > 10)] = "outside [0,10]"
    order = [
        "grade = 0", "0 < grade < 3", "3 <= grade < 4", "4 <= grade < 5",
        "5 <= grade < 6", "6 <= grade < 7", "7 <= grade < 8",
        "8 <= grade < 9", "9 <= grade <= 10", "outside [0,10]",
    ]
    out = labels.value_counts(dropna=False).reindex(order, fill_value=0).rename_axis("grade_band").reset_index(name="rows")
    out["rate"] = out["rows"] / max(len(work), 1)
    return out


def target_status_cross_tab(df: pd.DataFrame, event: float) -> pd.DataFrame:
    if "target_status" not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    work["class"] = np.where(work[TARGET] < event, "failure_<5", "normal_>=5")
    tab = pd.crosstab(work["target_status"].fillna("<MISSING>"), work["class"], margins=True)
    return tab.reset_index()


# ---------------------------------------------------------------------------
# Feature profiling, class separability, and feature-family summaries
# ---------------------------------------------------------------------------
def build_feature_profile(df: pd.DataFrame, feature_cols: Sequence[str]) -> pd.DataFrame:
    rows: list[dict] = []
    n = len(df)
    for feature in feature_cols:
        raw = df[feature]
        raw_nonmissing = raw.notna()
        numeric_before_inf = pd.to_numeric(raw, errors="coerce")
        inf_count = int(np.isinf(numeric_before_inf.to_numpy(dtype=float, na_value=np.nan)).sum())
        numeric = numeric_before_inf.replace([np.inf, -np.inf], np.nan)
        conversion_loss = raw_nonmissing & numeric_before_inf.isna()
        values = numeric.dropna()
        unique_values = np.sort(values.unique()) if len(values) else np.array([])
        is_binary = len(unique_values) <= 2 and set(unique_values.tolist()).issubset({0.0, 1.0})
        value_counts = values.value_counts(dropna=False)
        dominant_rate = float(value_counts.iloc[0] / len(values)) if len(values) else float("nan")
        q25 = float(values.quantile(0.25)) if len(values) else float("nan")
        q75 = float(values.quantile(0.75)) if len(values) else float("nan")
        iqr = q75 - q25 if np.isfinite(q25) and np.isfinite(q75) else float("nan")
        if len(values) and np.isfinite(iqr) and iqr > 0:
            outlier_rate = float(((values < q25 - 1.5 * iqr) | (values > q75 + 1.5 * iqr)).mean())
        else:
            outlier_rate = 0.0 if len(values) else float("nan")
        rows.append({
            "feature": feature,
            "rows": n,
            "raw_missing_count": int(raw.isna().sum()),
            "numeric_missing_count": int(numeric.isna().sum()),
            "numeric_missing_rate": float(numeric.isna().mean()),
            "numeric_conversion_loss_count": int(conversion_loss.sum()),
            "numeric_conversion_loss_rate_of_raw_nonmissing": float(conversion_loss.sum() / max(raw_nonmissing.sum(), 1)),
            "infinite_value_count": inf_count,
            "numeric_unique": int(values.nunique()),
            "is_constant": bool(values.nunique() <= 1),
            "is_binary": bool(is_binary),
            "is_integer_like": bool(len(values) == 0 or np.allclose(values, np.round(values))),
            "zero_rate_nonmissing": float((values == 0).mean()) if len(values) else float("nan"),
            "dominant_value_rate_nonmissing": dominant_rate,
            "near_zero_variance": bool(np.isfinite(dominant_rate) and dominant_rate >= 0.99),
            "mean": float(values.mean()) if len(values) else float("nan"),
            "std": float(values.std()) if len(values) > 1 else float("nan"),
            "min": float(values.min()) if len(values) else float("nan"),
            "q01": float(values.quantile(0.01)) if len(values) else float("nan"),
            "q25": q25,
            "median": float(values.median()) if len(values) else float("nan"),
            "q75": q75,
            "q99": float(values.quantile(0.99)) if len(values) else float("nan"),
            "max": float(values.max()) if len(values) else float("nan"),
            "skew": float(values.skew()) if len(values) > 2 else float("nan"),
            "iqr_outlier_rate": outlier_rate,
        })
    return pd.DataFrame(rows)


def compute_univariate_separability(df: pd.DataFrame, y_binary: Sequence[int | bool],
                                    feature_cols: Sequence[str], min_class_count: int = 20) -> pd.DataFrame:
    y = np.asarray(y_binary).astype(bool)
    rows: list[dict] = []
    for feature in feature_cols:
        x_series = pd.to_numeric(df[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
        x = x_series.to_numpy(float)
        valid = np.isfinite(x)
        xv, yv = x[valid], y[valid]
        pos = xv[yv]
        neg = xv[~yv]
        missing_pos = float(np.mean(~valid[y])) if y.sum() else float("nan")
        missing_neg = float(np.mean(~valid[~y])) if (~y).sum() else float("nan")
        if len(pos) < min_class_count or len(neg) < min_class_count:
            auc = float("nan")
            d = float("nan")
            corr = float("nan")
            overlap = float("nan")
            iv = float("nan")
        else:
            auc = binary_auc(yv, xv)
            d = standardized_mean_difference(neg, pos)
            corr = float(np.corrcoef(xv, yv.astype(float))[0, 1]) if np.std(xv) > 0 else 0.0
            overlap, iv = binned_overlap_and_iv(xv, yv)
        rows.append({
            "feature": feature,
            "valid_rows": int(valid.sum()),
            "valid_failure_rows": int(len(pos)),
            "valid_normal_rows": int(len(neg)),
            "mean_failure": float(np.mean(pos)) if len(pos) else float("nan"),
            "mean_normal": float(np.mean(neg)) if len(neg) else float("nan"),
            "median_failure": float(np.median(pos)) if len(pos) else float("nan"),
            "median_normal": float(np.median(neg)) if len(neg) else float("nan"),
            "auc_raw": auc,
            "auc_directionless": max(auc, 1.0 - auc) if np.isfinite(auc) else float("nan"),
            "predictive_direction": "higher=failure" if np.isfinite(auc) and auc >= 0.5 else "lower=failure",
            "cohens_d_failure_minus_normal": d,
            "point_biserial_corr": corr,
            "distribution_overlap": overlap,
            "information_value": iv,
            "missing_rate_failure": missing_pos,
            "missing_rate_normal": missing_neg,
            "missing_rate_gap_failure_minus_normal": missing_pos - missing_neg,
            "abs_missing_rate_gap": abs(missing_pos - missing_neg) if np.isfinite(missing_pos) and np.isfinite(missing_neg) else float("nan"),
        })
    return pd.DataFrame(rows)


def feature_family(feature: str) -> str:
    f = feature.lower()
    if f.startswith("course_status_"):
        return "course_status_one_hot"
    if "same_course" in f:
        return "same_course_history"
    if "component" in f:
        return "assessment_component_history"
    if "warning" in f:
        return "academic_warning_history"
    if "conduct" in f:
        return "conduct_history"
    if f.startswith("reg_") or "days_from_open" in f or "deadline" in f:
        return "registration_timing"
    if "target_semester" in f:
        return "target_semester_workload"
    if f.startswith("prev_"):
        return "previous_semester"
    if f.startswith("recent3_"):
        return "recent_three_semesters"
    if f.startswith("hist_course") or f.startswith("student_prior"):
        return "student_course_history"
    if f.startswith("hist_hp"):
        return "historical_course_components"
    if f.startswith("hist_"):
        return "other_historical"
    if "course_" in f or f.startswith("course"):
        return "course_context"
    return "other"


# ---------------------------------------------------------------------------
# Temporal feature statistics and drift
# ---------------------------------------------------------------------------
def meaningful_semesters(df: pd.DataFrame, min_rows: int = 100,
                         min_fraction: float = 0.005) -> list[object]:
    semester_col = "target_semester_ordinal" if "target_semester_ordinal" in df else "target_NHHK"
    counts = df.groupby(semester_col, dropna=False).size().sort_index()
    threshold = max(min_rows, int(math.ceil(min_fraction * len(df))))
    return counts[counts >= threshold].index.tolist()


def compute_feature_semester_statistics(df: pd.DataFrame, feature_cols: Sequence[str],
                                        event: float, min_class_count: int = 10) -> pd.DataFrame:
    work = valid_target_frame(df)
    semester_col = "target_semester_ordinal" if "target_semester_ordinal" in work else "target_NHHK"
    rows: list[dict] = []
    for semester, group in work.groupby(semester_col, dropna=False, sort=True):
        y = (group[TARGET].to_numpy(float) < event)
        semester_code = group["target_NHHK"].iloc[0] if "target_NHHK" in group else semester
        for feature in feature_cols:
            x_s = pd.to_numeric(group[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
            x = x_s.to_numpy(float)
            valid = np.isfinite(x)
            xv, yv = x[valid], y[valid]
            if yv.sum() >= min_class_count and (~yv).sum() >= min_class_count and np.std(xv) > 0:
                auc = binary_auc(yv, xv)
                d = standardized_mean_difference(xv[~yv], xv[yv])
            else:
                auc = float("nan")
                d = float("nan")
            values = x_s.dropna()
            rows.append({
                "semester": semester_code,
                "semester_ordinal": semester,
                "feature": feature,
                "rows": len(group),
                "failure_count": int(y.sum()),
                "failure_rate": float(y.mean()),
                "missing_rate": float(x_s.isna().mean()),
                "mean": float(values.mean()) if len(values) else float("nan"),
                "std": float(values.std()) if len(values) > 1 else float("nan"),
                "median": float(values.median()) if len(values) else float("nan"),
                "q25": float(values.quantile(0.25)) if len(values) else float("nan"),
                "q75": float(values.quantile(0.75)) if len(values) else float("nan"),
                "auc_raw": auc,
                "auc_directionless": max(auc, 1.0 - auc) if np.isfinite(auc) else float("nan"),
                "cohens_d": d,
            })
    return pd.DataFrame(rows)


def _sample_finite(values: pd.Series, max_rows: int, rng: np.random.Generator) -> np.ndarray:
    arr = pd.to_numeric(values, errors="coerce").replace([np.inf, -np.inf], np.nan).dropna().to_numpy(float)
    if len(arr) > max_rows:
        arr = rng.choice(arr, size=max_rows, replace=False)
    return arr


def compute_adjacent_semester_drift(df: pd.DataFrame, feature_cols: Sequence[str],
                                    min_semester_rows: int = 100,
                                    min_semester_fraction: float = 0.005,
                                    max_rows_per_semester: int = 10000,
                                    seed: int = 42) -> pd.DataFrame:
    semester_col = "target_semester_ordinal" if "target_semester_ordinal" in df else "target_NHHK"
    counts = df.groupby(semester_col, dropna=False).size().sort_index()
    threshold = max(min_semester_rows, int(math.ceil(min_semester_fraction * len(df))))
    sems = counts[counts >= threshold].index.tolist()
    rng = np.random.default_rng(seed)
    rows: list[dict] = []
    for prev_sem, cur_sem in zip(sems[:-1], sems[1:]):
        prev = df.loc[df[semester_col] == prev_sem]
        cur = df.loc[df[semester_col] == cur_sem]
        prev_code = prev["target_NHHK"].iloc[0] if "target_NHHK" in prev else prev_sem
        cur_code = cur["target_NHHK"].iloc[0] if "target_NHHK" in cur else cur_sem
        for feature in feature_cols:
            prev_numeric = pd.to_numeric(prev[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
            cur_numeric = pd.to_numeric(cur[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
            prev_values = _sample_finite(prev_numeric, max_rows_per_semester, rng)
            cur_values = _sample_finite(cur_numeric, max_rows_per_semester, rng)
            missing_change = float(cur_numeric.isna().mean() - prev_numeric.isna().mean())
            if len(prev_values) >= 2 and len(cur_values) >= 2:
                prev_sd = float(np.std(prev_values))
                mean_shift = (
                    abs(float(np.mean(cur_values) - np.mean(prev_values))) / prev_sd
                    if prev_sd > 0 else (0.0 if np.mean(cur_values) == np.mean(prev_values) else float("inf"))
                )
                psi = numeric_psi(prev_values, cur_values)
            else:
                mean_shift = float("nan")
                psi = float("nan")
            rows.append({
                "previous_semester": prev_code,
                "current_semester": cur_code,
                "previous_ordinal": prev_sem,
                "current_ordinal": cur_sem,
                "feature": feature,
                "previous_rows": len(prev),
                "current_rows": len(cur),
                "psi": psi,
                "standardized_mean_shift": mean_shift,
                "missing_rate_change": missing_change,
                "abs_missing_rate_change": abs(missing_change),
            })
    return pd.DataFrame(rows)


def summarize_temporal_drift(drift: pd.DataFrame) -> pd.DataFrame:
    if drift.empty:
        return pd.DataFrame()
    rows = []
    for feature, group in drift.groupby("feature"):
        psi_valid = group["psi"].replace([np.inf, -np.inf], np.nan)
        mean_valid = group["standardized_mean_shift"].replace([np.inf, -np.inf], np.nan)
        miss_valid = group["abs_missing_rate_change"]
        psi_idx = psi_valid.idxmax() if psi_valid.notna().any() else None
        mean_idx = mean_valid.idxmax() if mean_valid.notna().any() else None
        miss_idx = miss_valid.idxmax() if miss_valid.notna().any() else None
        rows.append({
            "feature": feature,
            "adjacent_pairs": len(group),
            "psi_median": float(psi_valid.median()),
            "psi_max": float(psi_valid.max()),
            "psi_max_pair": (
                f"{group.loc[psi_idx, 'previous_semester']}->{group.loc[psi_idx, 'current_semester']}"
                if psi_idx is not None else ""
            ),
            "standardized_mean_shift_median": float(mean_valid.median()),
            "standardized_mean_shift_max": float(mean_valid.max()),
            "mean_shift_max_pair": (
                f"{group.loc[mean_idx, 'previous_semester']}->{group.loc[mean_idx, 'current_semester']}"
                if mean_idx is not None else ""
            ),
            "abs_missing_change_median": float(miss_valid.median()),
            "abs_missing_change_max": float(miss_valid.max()),
            "missing_change_max_pair": (
                f"{group.loc[miss_idx, 'previous_semester']}->{group.loc[miss_idx, 'current_semester']}"
                if miss_idx is not None else ""
            ),
        })
    return pd.DataFrame(rows)


def summarize_target_relationship_stability(stats: pd.DataFrame) -> pd.DataFrame:
    if stats.empty:
        return pd.DataFrame()
    rows = []
    for feature, group in stats.groupby("feature"):
        valid = group.loc[group["auc_raw"].notna()].copy()
        if valid.empty:
            rows.append({"feature": feature, "valid_semesters": 0})
            continue
        direction = np.where(valid["auc_raw"] >= 0.5, 1, -1)
        sign_changes = int(np.sum(direction[1:] != direction[:-1])) if len(direction) > 1 else 0
        rows.append({
            "feature": feature,
            "valid_semesters": len(valid),
            "auc_directionless_mean": float(valid["auc_directionless"].mean()),
            "auc_directionless_std": float(valid["auc_directionless"].std(ddof=0)),
            "auc_directionless_min": float(valid["auc_directionless"].min()),
            "auc_directionless_max": float(valid["auc_directionless"].max()),
            "auc_raw_min": float(valid["auc_raw"].min()),
            "auc_raw_max": float(valid["auc_raw"].max()),
            "predictive_direction_changes": sign_changes,
            "cohens_d_abs_mean": float(valid["cohens_d"].abs().mean()),
            "cohens_d_abs_std": float(valid["cohens_d"].abs().std(ddof=0)),
        })
    return pd.DataFrame(rows)


# ---------------------------------------------------------------------------
# Correlation and redundancy
# ---------------------------------------------------------------------------
def correlation_pairs(df: pd.DataFrame, feature_cols: Sequence[str], threshold: float = 0.95,
                      max_rows: int = 25000, seed: int = 42) -> pd.DataFrame:
    numeric = df[list(feature_cols)].apply(pd.to_numeric, errors="coerce").replace([np.inf, -np.inf], np.nan)
    keep = [c for c in numeric.columns if numeric[c].notna().sum() >= 2 and numeric[c].nunique(dropna=True) > 1]
    numeric = numeric[keep]
    if len(numeric) > max_rows:
        numeric = numeric.sample(max_rows, random_state=seed)
    numeric = numeric.fillna(numeric.median()).fillna(0.0)
    if numeric.shape[1] < 2:
        return pd.DataFrame()
    corr = numeric.corr(method="pearson")
    arr = corr.to_numpy()
    rows = []
    for i in range(len(keep)):
        for j in range(i + 1, len(keep)):
            value = arr[i, j]
            if np.isfinite(value) and abs(value) >= threshold:
                rows.append({
                    "feature_a": keep[i],
                    "feature_b": keep[j],
                    "pearson_corr": float(value),
                    "abs_corr": abs(float(value)),
                })
    return pd.DataFrame(rows).sort_values("abs_corr", ascending=False).reset_index(drop=True) if rows else pd.DataFrame()


# ---------------------------------------------------------------------------
# Course, student, cold-start, and binned analyses
# ---------------------------------------------------------------------------
def course_summary(df: pd.DataFrame, event: float, prior_strength: float = 50.0) -> pd.DataFrame:
    if "IDMH" not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    work["failure"] = work[TARGET] < event
    global_rate = float(work["failure"].mean())
    out = (
        work.groupby("IDMH", dropna=False)
        .agg(
            rows=(TARGET, "size"),
            unique_students=("IDSV", "nunique") if "IDSV" in work else (TARGET, "size"),
            semesters=("target_NHHK", "nunique") if "target_NHHK" in work else (TARGET, "size"),
            failures=("failure", "sum"),
            failure_rate=("failure", "mean"),
            mean_grade=(TARGET, "mean"),
            median_grade=(TARGET, "median"),
        )
        .reset_index()
    )
    out["smoothed_failure_rate"] = (out["failures"] + prior_strength * global_rate) / (out["rows"] + prior_strength)
    return out.sort_values(["smoothed_failure_rate", "rows"], ascending=[False, False]).reset_index(drop=True)


def course_temporal_volatility(df: pd.DataFrame, event: float, min_rows_per_semester: int = 20) -> pd.DataFrame:
    if "IDMH" not in df.columns or "target_NHHK" not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    work["failure"] = work[TARGET] < event
    grouped = (
        work.groupby(["IDMH", "target_NHHK"], dropna=False)
        .agg(rows=(TARGET, "size"), failures=("failure", "sum"), failure_rate=("failure", "mean"))
        .reset_index()
    )
    grouped = grouped.loc[grouped["rows"] >= min_rows_per_semester]
    rows = []
    for course, group in grouped.groupby("IDMH"):
        if len(group) < 2:
            continue
        rows.append({
            "IDMH": course,
            "valid_semesters": len(group),
            "total_rows_in_valid_semesters": int(group["rows"].sum()),
            "failure_rate_mean": float(group["failure_rate"].mean()),
            "failure_rate_std": float(group["failure_rate"].std(ddof=0)),
            "failure_rate_min": float(group["failure_rate"].min()),
            "failure_rate_max": float(group["failure_rate"].max()),
            "failure_rate_range": float(group["failure_rate"].max() - group["failure_rate"].min()),
        })
    return pd.DataFrame(rows).sort_values("failure_rate_range", ascending=False).reset_index(drop=True) if rows else pd.DataFrame()


def student_summary(df: pd.DataFrame, event: float) -> pd.DataFrame:
    if "IDSV" not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    work["failure"] = work[TARGET] < event
    return (
        work.groupby("IDSV", dropna=False)
        .agg(
            registrations=(TARGET, "size"),
            unique_courses=("IDMH", "nunique") if "IDMH" in work else (TARGET, "size"),
            semesters=("target_NHHK", "nunique") if "target_NHHK" in work else (TARGET, "size"),
            failures=("failure", "sum"),
            failure_rate=("failure", "mean"),
            mean_grade=(TARGET, "mean"),
        )
        .reset_index()
    )


def group_target_summary(df: pd.DataFrame, group_col: str, event: float) -> pd.DataFrame:
    if group_col not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    work["failure"] = work[TARGET] < event
    out = (
        work.groupby(group_col, dropna=False)
        .agg(
            rows=(TARGET, "size"),
            failures=("failure", "sum"),
            failure_rate=("failure", "mean"),
            mean_grade=(TARGET, "mean"),
            median_grade=(TARGET, "median"),
        )
        .reset_index()
    )
    return out.sort_values("rows", ascending=False).reset_index(drop=True)


def quantile_binned_target_summary(df: pd.DataFrame, feature: str, event: float,
                                   q: int = 10) -> pd.DataFrame:
    if feature not in df.columns:
        return pd.DataFrame()
    work = valid_target_frame(df)
    x = pd.to_numeric(work[feature], errors="coerce").replace([np.inf, -np.inf], np.nan)
    valid = x.notna()
    if valid.sum() < 20 or x[valid].nunique() < 2:
        return pd.DataFrame()
    try:
        bins = pd.qcut(x[valid], q=min(q, int(x[valid].nunique())), duplicates="drop")
    except ValueError:
        return pd.DataFrame()
    temp = work.loc[valid, [TARGET]].copy()
    temp["bin"] = bins.astype(str)
    temp["failure"] = temp[TARGET] < event
    out = (
        temp.groupby("bin", observed=False)
        .agg(rows=(TARGET, "size"), failures=("failure", "sum"), failure_rate=("failure", "mean"), mean_grade=(TARGET, "mean"))
        .reset_index()
    )
    out.insert(0, "feature", feature)
    return out


# ---------------------------------------------------------------------------
# Automated interpretation tables
# ---------------------------------------------------------------------------
def build_family_summary(profile: pd.DataFrame, separability: pd.DataFrame,
                         drift_summary: pd.DataFrame) -> pd.DataFrame:
    merged = profile.merge(separability, on="feature", how="left")
    if not drift_summary.empty:
        merged = merged.merge(drift_summary, on="feature", how="left")
    merged["family"] = merged["feature"].map(feature_family)
    agg = {
        "feature": "count",
        "numeric_missing_rate": "mean",
        "dominant_value_rate_nonmissing": "median",
        "auc_directionless": "median",
        "information_value": "median",
        "abs_missing_rate_gap": "median",
    }
    if "psi_max" in merged:
        agg["psi_max"] = "median"
    out = merged.groupby("family", dropna=False).agg(agg).rename(columns={"feature": "n_features"}).reset_index()
    max_auc = merged.groupby("family")["auc_directionless"].max().rename("best_auc_directionless").reset_index()
    out = out.merge(max_auc, on="family", how="left")
    return out.sort_values("best_auc_directionless", ascending=False).reset_index(drop=True)


def class_separability_distribution(separability: pd.DataFrame) -> pd.DataFrame:
    auc = separability["auc_directionless"].dropna()
    bands = [0.5, 0.52, 0.55, 0.60, 0.65, 0.70, 1.000001]
    labels = ["0.50-0.52", "0.52-0.55", "0.55-0.60", "0.60-0.65", "0.65-0.70", ">=0.70"]
    cats = pd.cut(auc, bins=bands, labels=labels, right=False, include_lowest=True)
    out = cats.value_counts(sort=False).rename_axis("directionless_auc_band").reset_index(name="feature_count")
    out["feature_rate"] = out["feature_count"] / max(len(auc), 1)
    return out


def automated_candidate_tables(profile: pd.DataFrame, separability: pd.DataFrame,
                               stability: pd.DataFrame, drift_summary: pd.DataFrame) -> dict[str, pd.DataFrame]:
    merged = profile.merge(separability, on="feature", how="left")
    if not stability.empty:
        merged = merged.merge(stability, on="feature", how="left")
    if not drift_summary.empty:
        merged = merged.merge(drift_summary, on="feature", how="left")

    high_missing = merged.loc[merged["numeric_missing_rate"] >= 0.90].sort_values("numeric_missing_rate", ascending=False)
    near_constant = merged.loc[merged["dominant_value_rate_nonmissing"] >= 0.995].sort_values("dominant_value_rate_nonmissing", ascending=False)
    weak = merged.loc[
        (merged["auc_directionless"].fillna(0.5) < 0.53)
        & (merged["cohens_d_failure_minus_normal"].abs().fillna(0.0) < 0.10)
        & (merged["abs_missing_rate_gap"].fillna(0.0) < 0.02)
    ].sort_values(["dominant_value_rate_nonmissing", "numeric_missing_rate"], ascending=False)
    strong_stable = merged.loc[
        (merged.get("auc_directionless_mean", pd.Series(index=merged.index, dtype=float)).fillna(0.5) >= 0.60)
        & (merged.get("auc_directionless_std", pd.Series(index=merged.index, dtype=float)).fillna(1.0) <= 0.08)
        & (merged.get("predictive_direction_changes", pd.Series(index=merged.index, dtype=float)).fillna(99) == 0)
    ].sort_values("auc_directionless_mean", ascending=False)
    unstable = merged.loc[
        (merged.get("auc_directionless_mean", pd.Series(index=merged.index, dtype=float)).fillna(0.5) >= 0.56)
        & (
            (merged.get("predictive_direction_changes", pd.Series(index=merged.index, dtype=float)).fillna(0) > 0)
            | (merged.get("auc_directionless_std", pd.Series(index=merged.index, dtype=float)).fillna(0) >= 0.10)
        )
    ].sort_values(["predictive_direction_changes", "auc_directionless_std"], ascending=False)
    suspicious = merged.loc[
        (merged["auc_directionless"].fillna(0.5) >= 0.95)
        | (merged["point_biserial_corr"].abs().fillna(0) >= 0.90)
        | merged["feature"].str.lower().str.contains(r"target_grade|final_grade|ket_qua|result", regex=True)
    ].sort_values("auc_directionless", ascending=False)
    highly_skewed = merged.loc[
        (merged["min"].fillna(-1) >= 0)
        & (merged["skew"].abs().fillna(0) >= 5)
        & (~merged["is_binary"])
    ].sort_values("skew", ascending=False)
    return {
        "candidates_high_missing": high_missing,
        "candidates_near_constant": near_constant,
        "candidates_univariate_weak": weak,
        "candidates_strong_stable": strong_stable,
        "candidates_temporally_unstable": unstable,
        "candidates_possible_leakage_review": suspicious,
        "candidates_log_transform_review": highly_skewed,
    }


# ---------------------------------------------------------------------------
# Main pipeline
# ---------------------------------------------------------------------------
def run_eda(args: argparse.Namespace) -> None:
    start = time.time()
    splits_dir = Path(args.splits_dir) if args.splits_dir else Path(__file__).resolve().parent / "data" / "splits" / args.strategy
    available = discover_folds(splits_dir, args.faculty)
    if not available:
        raise SystemExit(f"No folds found for faculty '{args.faculty}' in {splits_dir}")
    dev_folds = parse_fold_spec(args.dev_folds, available)
    test_folds = parse_fold_spec(args.test_folds, available)
    selected_folds = sorted(set(dev_folds + test_folds))
    if not dev_folds:
        raise SystemExit("At least one development fold is required.")

    output_root = Path(args.output_dir) / args.faculty
    store = OutputStore(output_root)
    reporter = Reporter(output_root / "eda_report.txt")

    try:
        reporter.section("ACADEMIC EARLY-WARNING DATA EDA — TEXT REPORT")
        reporter.line(f"Faculty: {args.faculty}")
        reporter.line(f"Splits directory: {splits_dir}")
        reporter.line(f"Strategy: {args.strategy}")
        reporter.line(f"Failure definition: {TARGET} < {args.event}")
        reporter.line(f"Development folds: {dev_folds}")
        reporter.line(f"Holdout/diagnostic folds: {test_folds}")
        reporter.line("IMPORTANT: feature-engineering decisions should primarily use DEVELOPMENT results. Holdout sections are diagnostics, not a feature-selection oracle.")

        # 1. Fold inventory and train-test drift.
        reporter.section("1. FOLD INVENTORY, TARGET DRIFT, AND INTEGRITY CHECKS")
        fold_summaries = []
        fold_drift_parts = []
        for fold in selected_folds:
            reporter.line(f"Analyzing fold {fold} ...")
            summary, drift = analyze_fold(fold, args.faculty, splits_dir, event=args.event)
            fold_summaries.append(summary)
            fold_drift_parts.append(drift)
        fold_summary_df = pd.DataFrame(fold_summaries).sort_values("fold")
        fold_drift_df = pd.concat(fold_drift_parts, ignore_index=True) if fold_drift_parts else pd.DataFrame()
        store.save("01_fold_summary", fold_summary_df)
        store.save("02_fold_train_test_feature_drift", fold_drift_df)
        reporter.table(fold_summary_df, "Fold-level summary")
        if not fold_drift_df.empty:
            fold_drift_top = fold_drift_df.sort_values(["fold", "psi_train_to_test"], ascending=[True, False]).groupby("fold").head(args.top_n)
            reporter.table(fold_drift_top, f"Top {args.top_n} train→test feature shifts per fold", max_rows=args.top_n * len(selected_folds))
            all_missing_test = fold_drift_df.loc[fold_drift_df["test_all_missing"], ["fold", "feature", "train_missing_rate", "test_missing_rate"]]
            reporter.table(all_missing_test, "Features entirely missing in a fold test set", max_rows=100)

        # 2. Canonical snapshots.
        latest_dev = max(dev_folds)
        latest_all = max(selected_folds)
        dev_train, dev_test = read_fold(latest_dev, args.faculty, splits_dir)
        full_train, full_test = read_fold(latest_all, args.faculty, splits_dir)
        dev_snapshot, dev_key = build_canonical_snapshot(dev_train, dev_test)
        full_snapshot, full_key = build_canonical_snapshot(full_train, full_test)
        dev_snapshot = valid_target_frame(dev_snapshot)
        full_snapshot = valid_target_frame(full_snapshot)
        reporter.section("2. CANONICAL NON-DUPLICATED SNAPSHOTS")
        reporter.line(f"Development snapshot: fold {latest_dev} train + test, deduplicated by {dev_key or '[all columns]'}: {len(dev_snapshot):,} rows")
        reporter.line(f"Full diagnostic snapshot: fold {latest_all} train + test, deduplicated by {full_key or '[all columns]'}: {len(full_snapshot):,} rows")
        reporter.line("The latest expanding fold is used instead of concatenating all fold training files, because expanding training files repeat historical rows.")

        # 3. Target and semester analysis.
        reporter.section("3. TARGET DISTRIBUTION AND SEMESTER EVOLUTION")
        sem_full = semester_summary(full_snapshot, args.event)
        sem_dev = semester_summary(dev_snapshot, args.event)
        grade_bands_dev = grade_band_summary(dev_snapshot)
        status_tab_dev = target_status_cross_tab(dev_snapshot, args.event)
        store.save("03_semester_summary_full_diagnostic", sem_full)
        store.save("04_semester_summary_development", sem_dev)
        store.save("05_grade_band_summary_development", grade_bands_dev)
        if not status_tab_dev.empty:
            store.save("06_target_status_by_class_development", status_tab_dev)
        reporter.table(sem_full, "All available semesters — sample size, failure prevalence, and grades")
        reporter.table(grade_bands_dev, "Development grade distribution by bands")
        reporter.table(status_tab_dev, "Development target_status × binary class", max_rows=100)
        meaningful = meaningful_semesters(full_snapshot, args.min_semester_rows, args.min_semester_fraction)
        reporter.line(f"Meaningful semesters for adjacent drift analysis: {meaningful}")
        tiny = sem_full.loc[sem_full["rows"] < max(args.min_semester_rows, math.ceil(args.min_semester_fraction * len(full_snapshot)))]
        reporter.table(tiny, "Small/special semesters requiring data-quality interpretation")

        # 4. Feature profile on development snapshot.
        reporter.section("4. FEATURE QUALITY PROFILE — DEVELOPMENT DATA")
        feature_cols = model_feature_columns(dev_snapshot)
        profile = build_feature_profile(dev_snapshot, feature_cols)
        store.save("07_feature_profile_development", profile)
        reporter.line(f"Model feature count in development snapshot: {len(feature_cols):,}")
        summary_quality = pd.DataFrame({
            "measure": [
                "constant", "binary", "near-zero variance", "missing >=90%", "conversion loss >0",
                "nonnegative high-skew candidates", "features with infinities converted to missing",
            ],
            "feature_count": [
                int(profile["is_constant"].sum()),
                int(profile["is_binary"].sum()),
                int(profile["near_zero_variance"].sum()),
                int((profile["numeric_missing_rate"] >= 0.90).sum()),
                int((profile["numeric_conversion_loss_count"] > 0).sum()),
                int(((profile["min"] >= 0) & (profile["skew"].abs() >= 5) & (~profile["is_binary"])).sum()),
                int((profile["infinite_value_count"] > 0).sum()),
            ],
        })
        reporter.table(summary_quality, "Feature-quality counts")
        profile_print_cols = [
            "feature", "numeric_missing_rate", "numeric_unique", "is_constant", "is_binary",
            "dominant_value_rate_nonmissing", "numeric_conversion_loss_count",
            "infinite_value_count", "skew", "iqr_outlier_rate",
        ]
        reporter.table(profile.sort_values("numeric_missing_rate", ascending=False)[profile_print_cols], f"Top {args.top_n} features by missingness", max_rows=args.top_n)
        reporter.table(profile.sort_values("dominant_value_rate_nonmissing", ascending=False)[profile_print_cols], f"Top {args.top_n} sparse/near-constant features", max_rows=args.top_n)
        reporter.table(profile.sort_values("skew", ascending=False)[profile_print_cols], f"Top {args.top_n} positively skewed features", max_rows=args.top_n)

        # 5. Class separability.
        reporter.section("5. FAILURE-vs-NORMAL CLASS SEPARABILITY — DEVELOPMENT DATA")
        y_dev = (dev_snapshot[TARGET].to_numpy(float) < args.event)
        separability = compute_univariate_separability(dev_snapshot, y_dev, feature_cols, args.min_class_count)
        store.save("08_feature_class_separability_development", separability)
        sep_dist = class_separability_distribution(separability)
        store.save("09_class_separability_auc_distribution", sep_dist)
        reporter.table(sep_dist, "How many individual features separate the two classes?")
        sep_print_cols = [
            "feature", "valid_failure_rows", "valid_normal_rows", "mean_failure", "mean_normal",
            "auc_raw", "auc_directionless", "predictive_direction",
            "cohens_d_failure_minus_normal", "point_biserial_corr",
            "distribution_overlap", "information_value", "abs_missing_rate_gap",
        ]
        reporter.table(separability.sort_values("auc_directionless", ascending=False)[sep_print_cols], f"Top {args.top_n} features by directionless univariate AUC", max_rows=args.top_n)
        reporter.table(separability.reindex(separability["cohens_d_failure_minus_normal"].abs().sort_values(ascending=False).index)[sep_print_cols], f"Top {args.top_n} features by absolute standardized mean difference", max_rows=args.top_n)
        reporter.table(separability.sort_values("information_value", ascending=False)[sep_print_cols], f"Top {args.top_n} features by binned Information Value", max_rows=args.top_n)
        reporter.table(separability.sort_values("abs_missing_rate_gap", ascending=False)[sep_print_cols], f"Top {args.top_n} features where missingness differs by class", max_rows=args.top_n)
        valid_auc = separability["auc_directionless"].dropna()
        reporter.line(f"Best single-feature directionless AUC: {valid_auc.max():.4f}" if len(valid_auc) else "No valid univariate AUC values.")
        reporter.line(f"Features with directionless AUC >= 0.60: {(valid_auc >= 0.60).sum():,} / {len(valid_auc):,}")
        reporter.line("Low univariate separation does not prove a feature is useless; interactions may still matter. These tables are evidence for review, not automatic deletion rules.")

        # 6. Semester-by-semester feature/target relationships and drift.
        reporter.section("6. TEMPORAL FEATURE DRIFT AND FEATURE-TARGET STABILITY")
        sem_stats_full = compute_feature_semester_statistics(full_snapshot, model_feature_columns(full_snapshot), args.event, args.min_class_count_per_semester)
        drift_full = compute_adjacent_semester_drift(
            full_snapshot,
            model_feature_columns(full_snapshot),
            min_semester_rows=args.min_semester_rows,
            min_semester_fraction=args.min_semester_fraction,
            max_rows_per_semester=args.max_drift_rows,
            seed=args.seed,
        )
        drift_summary = summarize_temporal_drift(drift_full)
        stability = summarize_target_relationship_stability(sem_stats_full)
        store.save("10_feature_semester_statistics_full_diagnostic", sem_stats_full)
        store.save("11_adjacent_semester_drift_full_diagnostic", drift_full)
        store.save("12_feature_temporal_drift_summary", drift_summary)
        store.save("13_feature_target_stability_by_semester", stability)
        drift_print_cols = [
            "feature", "adjacent_pairs", "psi_median", "psi_max", "psi_max_pair",
            "standardized_mean_shift_max", "mean_shift_max_pair",
            "abs_missing_change_max", "missing_change_max_pair",
        ]
        reporter.table(drift_summary.sort_values("psi_max", ascending=False)[drift_print_cols], f"Top {args.top_n} features by maximum adjacent-semester PSI", max_rows=args.top_n)
        reporter.table(drift_summary.sort_values("standardized_mean_shift_max", ascending=False)[drift_print_cols], f"Top {args.top_n} features by standardized mean shift", max_rows=args.top_n)
        reporter.table(drift_summary.sort_values("abs_missing_change_max", ascending=False)[drift_print_cols], f"Top {args.top_n} features by semester missingness change", max_rows=args.top_n)
        stability_print_cols = [
            "feature", "valid_semesters", "auc_directionless_mean", "auc_directionless_std",
            "auc_directionless_min", "auc_directionless_max", "predictive_direction_changes",
            "cohens_d_abs_mean", "cohens_d_abs_std",
        ]
        stable_show = stability.loc[stability["valid_semesters"] >= 3].sort_values(["auc_directionless_mean", "auc_directionless_std"], ascending=[False, True])
        reporter.table(stable_show[stability_print_cols], f"Top {args.top_n} temporally strong feature-target relationships", max_rows=args.top_n)
        unstable_show = stability.loc[stability["valid_semesters"] >= 3].sort_values(["predictive_direction_changes", "auc_directionless_std"], ascending=False)
        reporter.table(unstable_show[stability_print_cols], f"Top {args.top_n} temporally unstable or direction-changing relationships", max_rows=args.top_n)

        # 7. Correlation and redundancy.
        reporter.section("7. FEATURE CORRELATION, REDUNDANCY, AND POSSIBLE LEAKAGE REVIEW")
        corr_pairs = correlation_pairs(dev_snapshot, feature_cols, args.corr_threshold, args.max_corr_rows, args.seed)
        store.save("14_high_correlation_feature_pairs_development", corr_pairs)
        reporter.table(corr_pairs, f"Feature pairs with |Pearson correlation| >= {args.corr_threshold}", max_rows=max(args.top_n, 100))
        reporter.line(f"High-correlation pair count: {len(corr_pairs):,}")
        reporter.line("Highly correlated pairs are redundancy candidates, but retain domain meaning and temporal stability when choosing which member to keep.")

        # 8. Course/student/cohort analyses.
        reporter.section("8. COURSE, STUDENT, COLD-START, AND COHORT ANALYSES")
        courses = course_summary(dev_snapshot, args.event, args.course_prior_strength)
        course_vol = course_temporal_volatility(full_snapshot, args.event, args.min_course_semester_rows)
        students = student_summary(dev_snapshot, args.event)
        store.save("15_course_summary_development", courses)
        store.save("16_course_temporal_volatility_full_diagnostic", course_vol)
        store.save("17_student_summary_development", students)
        supported_courses = courses.loc[courses["rows"] >= args.min_course_rows]
        reporter.table(supported_courses.sort_values("smoothed_failure_rate", ascending=False), f"Highest-risk courses with at least {args.min_course_rows} rows (smoothed rate)", max_rows=args.top_n)
        reporter.table(supported_courses.sort_values("smoothed_failure_rate", ascending=True), f"Lowest-risk courses with at least {args.min_course_rows} rows (smoothed rate)", max_rows=args.top_n)
        reporter.table(course_vol, f"Courses with the largest semester-to-semester failure-rate variation", max_rows=args.top_n)
        if not students.empty:
            student_registration_dist = students["registrations"].describe(percentiles=[0.25, 0.5, 0.75, 0.9, 0.95, 0.99]).rename_axis("stat").reset_index(name="registrations")
            reporter.table(student_registration_dist, "Registrations per student — development snapshot")

        group_features = [
            "student_gpa_history_missing_flag", "course_history_missing_flag", "is_summer_target",
            "hist_warning_in_last_semester", "hist_warning_in_last2_consecutive", "isDKCung",
        ]
        group_parts = []
        for feature in group_features:
            result = group_target_summary(dev_snapshot, feature, args.event)
            if not result.empty:
                result.insert(0, "analysis_feature", feature)
                group_parts.append(result)
                reporter.table(result, f"Failure rate grouped by {feature}")
        grouped_flags = pd.concat(group_parts, ignore_index=True) if group_parts else pd.DataFrame()
        store.save("18_binary_group_target_summaries_development", grouped_flags)

        binned_features = [
            "prev_semester_gpa10", "prev_cumulative_gpa10", "hist_semester_count",
            "hist_course_grade10_count", "hist_course_grade10_mean", "hist_same_course_attempts",
            "hist_failed_credits_sum", "prev_failed_credits", "course_fail_rate_clipped",
            "target_semester_registered_credits", "target_semester_average_course_difficulty",
            "target_semester_unseen_course_rate", "hist_warning_rate", "prev_conduct_score",
        ]
        binned_parts = []
        for feature in binned_features:
            result = quantile_binned_target_summary(dev_snapshot, feature, args.event, q=args.quantile_bins)
            if not result.empty:
                binned_parts.append(result)
                reporter.table(result, f"Binned failure relationship: {feature}")
        binned_all = pd.concat(binned_parts, ignore_index=True) if binned_parts else pd.DataFrame()
        store.save("19_binned_feature_target_relationships_development", binned_all)

        # 9. Feature-family and automated candidate tables.
        reporter.section("9. FEATURE-FAMILY SUMMARY AND FEATURE-ENGINEERING CANDIDATES")
        family = build_family_summary(profile, separability, drift_summary)
        store.save("20_feature_family_summary", family)
        reporter.table(family, "Feature-family summary")

        candidates = automated_candidate_tables(profile, separability, stability, drift_summary)
        candidate_print_cols = [
            "feature", "numeric_missing_rate", "numeric_unique", "dominant_value_rate_nonmissing",
            "auc_directionless", "cohens_d_failure_minus_normal", "information_value",
            "abs_missing_rate_gap", "valid_semesters", "auc_directionless_mean",
            "auc_directionless_std", "predictive_direction_changes", "psi_max",
            "abs_missing_change_max", "skew",
        ]
        for name, table in candidates.items():
            store.save(f"21_{name}", table)
            visible = [c for c in candidate_print_cols if c in table.columns]
            reporter.table(table[visible] if visible else table, name.replace("_", " ").title(), max_rows=args.top_n)

        # 10. Final automated findings.
        reporter.section("10. AUTOMATED BIG-PICTURE FINDINGS")
        holdout_rows = fold_summary_df.loc[fold_summary_df["fold"].isin(test_folds)]
        if len(holdout_rows) >= 2:
            first = holdout_rows.iloc[0]
            last = holdout_rows.iloc[-1]
            reporter.line(
                f"Failure prevalence changes from {first['test_failure_rate']:.2%} in fold {int(first['fold'])} "
                f"to {last['test_failure_rate']:.2%} in fold {int(last['fold'])}."
            )
        reporter.line(f"Development features with >=90% missingness: {(profile['numeric_missing_rate'] >= 0.90).sum():,}")
        reporter.line(f"Development near-zero-variance features: {profile['near_zero_variance'].sum():,}")
        reporter.line(f"Development high-correlation pairs (|r|>={args.corr_threshold}): {len(corr_pairs):,}")
        reporter.line(f"Features with univariate directionless AUC >=0.60: {(separability['auc_directionless'] >= 0.60).sum():,}")
        reporter.line(f"Features with at least one predictive-direction change across semesters: {(stability.get('predictive_direction_changes', pd.Series(dtype=float)) > 0).sum():,}")
        reporter.line(f"Features with maximum adjacent-semester PSI >=0.25: {(drift_summary.get('psi_max', pd.Series(dtype=float)) >= 0.25).sum():,}")
        reporter.line("Interpretation order for the next research step:")
        reporter.line("  1. Validate special/tiny semesters and extraction completeness.")
        reporter.line("  2. Identify whether failures are concentrated in courses, cold-start students, or missing-history cohorts.")
        reporter.line("  3. Prefer features with stable class separation across development semesters.")
        reporter.line("  4. Review highly missing, near-constant, redundant, and direction-changing features; do not remove them automatically.")
        reporter.line("  5. Form one feature-engineering hypothesis at a time and re-evaluate with the unchanged temporal backtest.")

        metadata = {
            "faculty": args.faculty,
            "splits_dir": str(splits_dir),
            "strategy": args.strategy,
            "dev_folds": dev_folds,
            "test_folds": test_folds,
            "event": args.event,
            "development_snapshot_rows": len(dev_snapshot),
            "full_snapshot_rows": len(full_snapshot),
            "development_feature_count": len(feature_cols),
            "runtime_seconds": time.time() - start,
            "output_directory": str(output_root.resolve()),
        }
        store.save_json("00_run_metadata", metadata)
        reporter.section("EDA COMPLETE")
        reporter.line(f"Runtime: {metadata['runtime_seconds']:.1f} seconds")
        reporter.line(f"Text report: {(output_root / 'eda_report.txt').resolve()}")
        reporter.line(f"Detailed CSV/JSON outputs: {output_root.resolve()}")
    finally:
        reporter.close()


def build_arg_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Comprehensive text-first EDA for student-course failure prediction."
    )
    parser.add_argument("--faculty", default="khoa_cong_nghe_thong_tin")
    parser.add_argument("--strategy", default="expanding", choices=["expanding", "sliding"])
    parser.add_argument("--splits_dir", default=None,
                        help="Default: <script_dir>/data/splits/<strategy>")
    parser.add_argument("--dev_folds", default="1,2,3,4,5")
    parser.add_argument("--test_folds", default="6,7")
    parser.add_argument("--event", type=float, default=DEFAULT_EVENT)
    parser.add_argument("--output_dir", default="eda_outputs")
    parser.add_argument("--top_n", type=int, default=30)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--min_class_count", type=int, default=20,
                        help="Minimum valid rows per class for global univariate metrics.")
    parser.add_argument("--min_class_count_per_semester", type=int, default=10)
    parser.add_argument("--min_semester_rows", type=int, default=100)
    parser.add_argument("--min_semester_fraction", type=float, default=0.005)
    parser.add_argument("--max_drift_rows", type=int, default=10000)
    parser.add_argument("--corr_threshold", type=float, default=0.95)
    parser.add_argument("--max_corr_rows", type=int, default=25000)
    parser.add_argument("--min_course_rows", type=int, default=50)
    parser.add_argument("--min_course_semester_rows", type=int, default=20)
    parser.add_argument("--course_prior_strength", type=float, default=50.0)
    parser.add_argument("--quantile_bins", type=int, default=10)
    return parser


def main() -> None:
    args = build_arg_parser().parse_args()
    run_eda(args)


if __name__ == "__main__":
    main()