# -*- coding: utf-8 -*-
"""Shared utilities for AQ breakthrough experiments.

This module intentionally imports the canonical data preparation, base-expert
configuration, temporal cross-fitting, and metrics from ``train_v3.py`` so the
new experiments do not silently change the benchmark committee.
"""
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Sequence
import math
import random

import numpy as np
import pandas as pd
import torch
from sklearn.metrics import average_precision_score, roc_auc_score

import train_v3 as core

BASE_NAMES = list(core.BASE_NAMES)
TARGET = core.TARGET
FAILURE_BOUNDARY = core.FAILURE_BOUNDARY


class OOVEncoder:
    """Deterministic categorical encoder with index 0 reserved for OOV/missing."""
    def __init__(self):
        self.mapping_: dict[str, int] = {}

    @staticmethod
    def _norm(v) -> str:
        if v is None:
            return "<MISSING>"
        try:
            if pd.isna(v):
                return "<MISSING>"
        except Exception:
            pass
        return str(v)

    def fit(self, values: Iterable) -> "OOVEncoder":
        vals = sorted({self._norm(v) for v in values if self._norm(v) != "<MISSING>"})
        self.mapping_ = {v: i + 1 for i, v in enumerate(vals)}
        return self

    def transform(self, values: Iterable) -> np.ndarray:
        if self.mapping_ is None:
            raise RuntimeError("OOVEncoder not fitted")
        return np.asarray([self.mapping_.get(self._norm(v), 0) for v in values], dtype=np.int64)

    def fit_transform(self, values: Iterable) -> np.ndarray:
        return self.fit(values).transform(values)

    @property
    def size(self) -> int:
        return len(self.mapping_) + 1


def seed_all(seed: int) -> None:
    core.seed_all(int(seed))


def safe_ap(y, score) -> float:
    y = np.asarray(y, int).reshape(-1)
    score = np.asarray(score, float).reshape(-1)
    return float("nan") if len(y) == 0 or np.unique(y).size < 2 else float(average_precision_score(y, score))


def safe_auc(y, score) -> float:
    y = np.asarray(y, int).reshape(-1)
    score = np.asarray(score, float).reshape(-1)
    return float("nan") if len(y) == 0 or np.unique(y).size < 2 else float(roc_auc_score(y, score))


def make_committee_models(seed: int, n_estimators: int = 500, hist_max_iter: int = 250):
    return core.make_base_models(seed, n_estimators, hist_max_iter)


def committee_meta_features(expert_probabilities: np.ndarray) -> tuple[np.ndarray, list[str]]:
    p = np.asarray(expert_probabilities, float)
    stats = core.committee_statistics(p)
    z = np.column_stack([
        p,
        stats["committee_mean"], stats["committee_min"], stats["committee_max"],
        stats["committee_std"], stats["committee_spread"], stats["committee_entropy"],
        stats["committee_vote_fraction"],
    ])
    names = [f"expert_probability_{n}" for n in BASE_NAMES] + list(core.UNCERTAINTY_NAMES)
    return np.nan_to_num(z, nan=0.0, posinf=0.0, neginf=0.0), names


def load_binary_fold(splits_dir, faculty: str, fold: int):
    data = core.load_fold(splits_dir, faculty, int(fold))
    if data is None:
        return None
    Xtr, Xte, gtr, gte, train, test, names = data
    ytr = (np.asarray(gtr, float) < FAILURE_BOUNDARY).astype(np.int8)
    yte = (np.asarray(gte, float) < FAILURE_BOUNDARY).astype(np.int8)
    return core.clean_X(Xtr), core.clean_X(Xte), ytr, yte, train, test, list(names)


def fit_production_committee(Xtr, ytr, Xte, *, seed=42, n_estimators=500, hist_max_iter=250):
    models = make_committee_models(seed, n_estimators, hist_max_iter)
    for m in models.values():
        m.fit(core.clean_X(Xtr), np.asarray(ytr, int))
    p = core.predict_experts(models, core.clean_X(Xte))
    q = core.committee_statistics(p)["committee_mean"]
    return models, p, q


def crossfit_historical_committee(
    X, y, frame: pd.DataFrame, *, seed=42, n_estimators=500, hist_max_iter=250,
    crossfit_folds=5, crossfit_min_train_rows=100, min_oof_coverage=0.40,
    verbose=False,
):
    groups, source = core.extract_temporal_groups(frame)
    return core.generate_crossfit_predictions(
        X, y, groups, source,
        seed=seed, n_estimators=n_estimators, hist_max_iter=hist_max_iter,
        crossfit_folds=crossfit_folds, min_train_rows=crossfit_min_train_rows,
        min_oof_coverage=min_oof_coverage, verbose=verbose,
    )


def top_raw_selector(X, y, feature_names, k=48):
    return core.RawFeatureSelector(k).fit(X, y, feature_names)


def top_raw_transform(selector, X):
    return selector.transform(X)


def binary_fbeta_at_best_threshold(y, score, beta=2.0) -> dict:
    y = np.asarray(y, int)
    score = np.asarray(score, float)
    if len(y) == 0:
        return {"threshold": np.nan, "precision": np.nan, "recall": np.nan, "f1": np.nan, "fbeta": np.nan, "flag_rate": np.nan}
    unique = np.unique(score)
    thresholds = unique if len(unique) <= 800 else np.unique(np.quantile(unique, np.linspace(0.001, 0.999, 800)))
    best = None
    for t in thresholds:
        pred = (score >= t).astype(int)
        m = core.report(y, score, pred, beta)
        key = (m["fbeta"], m["recall"], m["precision"], -m["flag_rate"])
        if best is None or key > best[0]:
            best = (key, float(t), m)
    _, t, m = best
    return {"threshold": t, **{k: m[k] for k in ["precision", "recall", "f1", "fbeta", "flag_rate"]}}


def score_row(y, score, *, prefix="", beta=2.0) -> dict:
    b = binary_fbeta_at_best_threshold(y, score, beta)
    return {
        f"{prefix}n": int(len(y)),
        f"{prefix}failures": int(np.asarray(y, int).sum()),
        f"{prefix}prevalence": float(np.mean(y)) if len(y) else np.nan,
        f"{prefix}ap": safe_ap(y, score),
        f"{prefix}auc": safe_auc(y, score),
        f"{prefix}best_f2": b["fbeta"],
        f"{prefix}best_precision": b["precision"],
        f"{prefix}best_recall": b["recall"],
        f"{prefix}best_flag_rate": b["flag_rate"],
    }


def novelty_groups(train_frame: pd.DataFrame, target_frame: pd.DataFrame) -> np.ndarray:
    """Return four stable subgroup labels using only identifiers available before target labels."""
    tr_students = set(train_frame["IDSV"].astype(str)) if "IDSV" in train_frame else set()
    tr_courses = set(train_frame["IDMH"].astype(str)) if "IDMH" in train_frame else set()
    students = target_frame["IDSV"].astype(str).to_numpy() if "IDSV" in target_frame else np.array(["<NA>"] * len(target_frame))
    courses = target_frame["IDMH"].astype(str).to_numpy() if "IDMH" in target_frame else np.array(["<NA>"] * len(target_frame))
    new_s = np.asarray([s not in tr_students for s in students])
    new_c = np.asarray([c not in tr_courses for c in courses])
    out = np.empty(len(target_frame), dtype=object)
    out[(~new_s) & (~new_c)] = "returning_seen"
    out[(~new_s) & new_c] = "returning_new_course"
    out[new_s & (~new_c)] = "new_student_seen_course"
    out[new_s & new_c] = "new_student_new_course"
    return out


def novelty_flags(train_frame: pd.DataFrame, target_frame: pd.DataFrame) -> np.ndarray:
    groups = novelty_groups(train_frame, target_frame)
    return np.column_stack([
        np.isin(groups, ["new_student_seen_course", "new_student_new_course"]).astype(float),
        np.isin(groups, ["returning_new_course", "new_student_new_course"]).astype(float),
    ])


def subgroup_metric_rows(groups, y, committee, candidate, extra: dict | None = None) -> list[dict]:
    groups = np.asarray(groups, object)
    y = np.asarray(y, int)
    committee = np.asarray(committee, float)
    candidate = np.asarray(candidate, float)
    rows = []
    for g in ["returning_seen", "returning_new_course", "new_student_seen_course", "new_student_new_course"]:
        m = groups == g
        if not m.any():
            continue
        row = {
            "group": g,
            "rows": int(m.sum()),
            "failures": int(y[m].sum()),
            "prevalence": float(y[m].mean()),
            "committee_ap": safe_ap(y[m], committee[m]),
            "candidate_ap": safe_ap(y[m], candidate[m]),
            "delta_ap": safe_ap(y[m], candidate[m]) - safe_ap(y[m], committee[m]) if np.unique(y[m]).size == 2 else np.nan,
            "committee_auc": safe_auc(y[m], committee[m]),
            "candidate_auc": safe_auc(y[m], candidate[m]),
        }
        if extra:
            row.update(extra)
        rows.append(row)
    return rows


def save_csv(rows, path):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    pd.DataFrame(rows).to_csv(path, index=False)


def parse_folds(s: str) -> list[int]:
    return core.parse_folds(s)


def parse_float_grid(s: str) -> list[float]:
    return sorted({float(x.strip()) for x in str(s).split(",") if x.strip()})
