# -*- coding: utf-8 -*-
"""Shared utilities for suite2 direct-improvement experiments.

All experiments deliberately reuse the canonical fold loading, target definition,
base experts, and temporal cross-fitting from ``train_v3.py``.  This module adds
only experiment-level meta representations and diagnostics.
"""
from __future__ import annotations
import sys
from pathlib import Path as _BootstrapPath
_PARENT = str(_BootstrapPath(__file__).resolve().parents[1])
if _PARENT not in sys.path:
    sys.path.insert(0, _PARENT)

from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Sequence
import math

import numpy as np
import pandas as pd
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.inspection import permutation_importance
from sklearn.metrics import average_precision_score, roc_auc_score

import train_v3 as core
import breakthrough_common as bc

BASE_NAMES = list(core.BASE_NAMES)
FAILURE_BOUNDARY = core.FAILURE_BOUNDARY
TARGET = core.TARGET


def safe_ap(y, p) -> float:
    return bc.safe_ap(y, p)


def safe_auc(y, p) -> float:
    return bc.safe_auc(y, p)


def clip_probability(p):
    return np.clip(np.asarray(p, float), 1e-7, 1.0 - 1e-7)


def v2_meta_features(probability_matrix: np.ndarray) -> tuple[np.ndarray, list[str]]:
    """Original V2-style probability/logit/statistics representation.

    For ``m`` experts this returns ``2*m + 7`` columns.  With the original
    three experts that is 13 columns, exactly matching final_train_v2.
    """
    p = clip_probability(np.asarray(probability_matrix, float))
    if p.ndim != 2:
        raise ValueError("probability_matrix must be 2-D")
    logits = np.log(p / (1.0 - p))
    mean = p.mean(axis=1, keepdims=True)
    minimum = p.min(axis=1, keepdims=True)
    maximum = p.max(axis=1, keepdims=True)
    std = p.std(axis=1, keepdims=True)
    spread = maximum - minimum
    vote_fraction = (p >= 0.5).mean(axis=1, keepdims=True)
    entropy = -(mean * np.log(mean + 1e-12) + (1.0 - mean) * np.log(1.0 - mean + 1e-12))
    names = (
        [f"probability__expert_{i}" for i in range(p.shape[1])]
        + [f"logit__expert_{i}" for i in range(p.shape[1])]
        + [
            "probability_mean", "probability_min", "probability_max",
            "probability_std", "probability_spread", "vote_fraction",
            "ensemble_entropy",
        ]
    )
    z = np.column_stack([p, logits, mean, minimum, maximum, std, spread, vote_fraction, entropy])
    return np.nan_to_num(z, nan=0.0, posinf=0.0, neginf=0.0), names


def _temporal_codes_for_frames(history: pd.DataFrame, target: pd.DataFrame | None = None):
    if target is None:
        groups, source = core.extract_temporal_groups(history)
        return groups, None, source
    n = len(history)
    combined = pd.concat([history, target], ignore_index=True, sort=False)
    groups, source = core.extract_temporal_groups(combined)
    return groups[:n], groups[n:], source


def chronological_support_features(history: pd.DataFrame, target: pd.DataFrame | None = None) -> pd.DataFrame:
    """Chronology-safe student/course support using *strictly prior* semesters.

    When ``target`` is supplied, only history rows with a temporal code smaller
    than each target row can contribute.  Rows from the same target semester do
    not count as prior evidence for one another.
    """
    hist = history.reset_index(drop=True)
    tgt = None if target is None else target.reset_index(drop=True)
    gh, gt, _ = _temporal_codes_for_frames(hist, tgt)

    def values(frame, col, default):
        if col in frame:
            return frame[col].astype("string").fillna(default).astype(str).to_numpy()
        return np.asarray([default] * len(frame), object)

    hs = values(hist, "IDSV", "<NO_STUDENT>")
    hc = values(hist, "IDMH", "<NO_COURSE>")

    if tgt is None:
        out_rows = len(hist)
        ts, tc, tg = hs, hc, gh
        update_with_target = True
    else:
        out_rows = len(tgt)
        ts = values(tgt, "IDSV", "<NO_STUDENT>")
        tc = values(tgt, "IDMH", "<NO_COURSE>")
        tg = gt
        update_with_target = False

    # If temporal information is unavailable, fall back to static history counts
    # for external target rows; for historical rows every row has zero prior
    # support rather than leaking arbitrary row order.
    if not np.isfinite(gh).any() or (tgt is not None and not np.isfinite(tg).any()):
        if tgt is None:
            return pd.DataFrame({
                "history_semesters": np.zeros(out_rows, int),
                "history_rows": np.zeros(out_rows, int),
                "new_student": np.ones(out_rows, float),
                "course_prior_rows": np.zeros(out_rows, int),
                "new_course": np.ones(out_rows, float),
            })
        student_rows = pd.Series(hs).value_counts().to_dict()
        course_rows = pd.Series(hc).value_counts().to_dict()
        return pd.DataFrame({
            "history_semesters": [0 if s not in student_rows else 1 for s in ts],
            "history_rows": [student_rows.get(s, 0) for s in ts],
            "new_student": [float(s not in student_rows) for s in ts],
            "course_prior_rows": [course_rows.get(c, 0) for c in tc],
            "new_course": [float(c not in course_rows) for c in tc],
        })

    # For a single historical frame process block-by-block: query before update.
    if tgt is None:
        result = np.zeros((len(hist), 5), float)
        student_rows: dict[str, int] = {}
        student_semesters: dict[str, set] = {}
        course_rows: dict[str, int] = {}
        finite_blocks = sorted(v for v in pd.unique(gh[np.isfinite(gh)]))
        for b in finite_blocks:
            idx = np.where(gh == b)[0]
            for i in idx:
                s, c = hs[i], hc[i]
                sems = student_semesters.get(s, set())
                result[i] = [
                    len(sems), student_rows.get(s, 0), float(student_rows.get(s, 0) == 0),
                    course_rows.get(c, 0), float(course_rows.get(c, 0) == 0),
                ]
            # Same-semester rows become available only after all queries in b.
            for i in idx:
                s, c = hs[i], hc[i]
                student_rows[s] = student_rows.get(s, 0) + 1
                student_semesters.setdefault(s, set()).add(float(b))
                course_rows[c] = course_rows.get(c, 0) + 1
        # NaN temporal rows remain conservative/no-history.
        return pd.DataFrame(result, columns=[
            "history_semesters", "history_rows", "new_student", "course_prior_rows", "new_course"
        ]).astype({"history_semesters": int, "history_rows": int, "course_prior_rows": int})

    # External targets may be at multiple future blocks.  Build cumulative
    # dictionaries from historical rows only and snapshot before each target b.
    result = np.zeros((len(tgt), 5), float)
    student_rows: dict[str, int] = {}
    student_semesters: dict[str, set] = {}
    course_rows: dict[str, int] = {}
    hist_blocks = sorted(v for v in pd.unique(gh[np.isfinite(gh)]))
    target_blocks = sorted(v for v in pd.unique(tg[np.isfinite(tg)]))
    hpos = 0
    for b in target_blocks:
        while hpos < len(hist_blocks) and hist_blocks[hpos] < b:
            hb = hist_blocks[hpos]
            for i in np.where(gh == hb)[0]:
                s, c = hs[i], hc[i]
                student_rows[s] = student_rows.get(s, 0) + 1
                student_semesters.setdefault(s, set()).add(float(hb))
                course_rows[c] = course_rows.get(c, 0) + 1
            hpos += 1
        for i in np.where(tg == b)[0]:
            s, c = ts[i], tc[i]
            result[i] = [
                len(student_semesters.get(s, set())), student_rows.get(s, 0), float(student_rows.get(s, 0) == 0),
                course_rows.get(c, 0), float(course_rows.get(c, 0) == 0),
            ]
    # Target rows lacking a temporal code still get support from all history.
    nan_idx = np.where(~np.isfinite(tg))[0]
    if len(nan_idx):
        # add remaining historical blocks
        while hpos < len(hist_blocks):
            hb = hist_blocks[hpos]
            for i in np.where(gh == hb)[0]:
                s, c = hs[i], hc[i]
                student_rows[s] = student_rows.get(s, 0) + 1
                student_semesters.setdefault(s, set()).add(float(hb))
                course_rows[c] = course_rows.get(c, 0) + 1
            hpos += 1
        for i in nan_idx:
            s, c = ts[i], tc[i]
            result[i] = [len(student_semesters.get(s, set())), student_rows.get(s,0), float(student_rows.get(s,0)==0), course_rows.get(c,0), float(course_rows.get(c,0)==0)]
    return pd.DataFrame(result, columns=[
        "history_semesters", "history_rows", "new_student", "course_prior_rows", "new_course"
    ]).astype({"history_semesters": int, "history_rows": int, "course_prior_rows": int})


def history_regime_labels(history_semesters: Iterable[int]) -> np.ndarray:
    d = np.asarray(list(history_semesters), int)
    out = np.empty(len(d), dtype=object)
    out[d <= 0] = "history0"
    out[(d >= 1) & (d <= 2)] = "history1_2"
    out[d >= 3] = "history3plus"
    return out


def support_matrix(support: pd.DataFrame) -> tuple[np.ndarray, list[str]]:
    cols = ["history_semesters", "history_rows", "new_student", "course_prior_rows", "new_course"]
    z = support.reindex(columns=cols, fill_value=0).to_numpy(float)
    # Log counts improve numerical behavior for linear stackers while preserving flags.
    z[:, 0] = np.log1p(np.maximum(z[:, 0], 0))
    z[:, 1] = np.log1p(np.maximum(z[:, 1], 0))
    z[:, 3] = np.log1p(np.maximum(z[:, 3], 0))
    return np.nan_to_num(z), ["log1p_history_semesters", "log1p_history_rows", "new_student", "log1p_course_prior_rows", "new_course"]


def relational_competence_features(p4: np.ndarray, support: pd.DataFrame) -> tuple[np.ndarray, list[str]]:
    p4 = clip_probability(np.asarray(p4, float))
    if p4.ndim != 2 or p4.shape[1] != 4:
        raise ValueError("p4 must contain exactly 4 expert columns (3 tree + relational)")
    meta, names = v2_meta_features(p4)
    sup, sup_names = support_matrix(support)
    q3 = p4[:, :3].mean(axis=1)
    rel = p4[:, 3]
    extra = np.column_stack([rel - q3, np.abs(rel - q3)])
    extra_names = ["relational_minus_tree_committee", "abs_relational_tree_gap"]
    return np.column_stack([meta, sup, extra]), names + sup_names + extra_names


def full_meta_features(X_raw: np.ndarray, expert_p: np.ndarray, support: pd.DataFrame) -> tuple[np.ndarray, list[str]]:
    meta, meta_names = v2_meta_features(expert_p)
    sup, sup_names = support_matrix(support)
    X = core.clean_X(X_raw)
    names = [f"raw_{i}" for i in range(X.shape[1])] + meta_names + sup_names
    return np.column_stack([X, meta, sup]), names


def blend_probability(committee, specialist, authority: float):
    q = clip_probability(committee)
    s = clip_probability(specialist)
    a = float(authority)
    return clip_probability((1.0 - a) * q + a * s)


def window_indices_before_block(blocks, target_block, recent_blocks: int | None):
    b = np.asarray(blocks, float)
    eligible = sorted(v for v in pd.unique(b[np.isfinite(b) & (b < float(target_block))]))
    if recent_blocks is not None:
        eligible = eligible[-int(recent_blocks):]
    return np.where(np.isin(b, eligible))[0]


def fit_hgb_classifier(X, y, *, seed=42, max_iter=250, max_leaf_nodes=31,
                       min_samples_leaf=20, learning_rate=0.05,
                       l2_regularization=1.0, sample_weight=None):
    X = core.clean_X(X)
    y = np.asarray(y, int)
    model = HistGradientBoostingClassifier(
        learning_rate=float(learning_rate), max_iter=int(max_iter),
        max_leaf_nodes=int(max_leaf_nodes), min_samples_leaf=int(min_samples_leaf),
        l2_regularization=float(l2_regularization), early_stopping=True,
        random_state=int(seed),
    )
    model.fit(X, y, sample_weight=sample_weight)
    return model


def predict_positive(model, X):
    p = np.asarray(model.predict_proba(core.clean_X(X)), float)
    cls = np.asarray(model.classes_)
    idx = np.where(cls == 1)[0]
    if len(idx) != 1:
        raise RuntimeError("positive class 1 not exposed")
    return clip_probability(p[:, int(idx[0])])


def effective_weights(y, beta=0.9999):
    w, stats = core.effective_sample_weights(np.asarray(y, int), beta)
    w = np.asarray(w, float)
    return w / max(w.mean(), 1e-12), stats


def select_authority(y, q, specialist, grid=(0, .25, .5, .75, 1), beta=2.0):
    rows = []
    for a in grid:
        s = blend_probability(q, specialist, a)
        best = bc.binary_fbeta_at_best_threshold(y, s, beta)
        rows.append({
            "authority": float(a), "ap": safe_ap(y, s), "auc": safe_auc(y, s),
            "best_f2": best["fbeta"], "best_precision": best["precision"],
            "best_recall": best["recall"], "best_flag_rate": best["flag_rate"],
        })
    # AP primary; AUC and F2 tie-break; lower authority wins exact ties.
    best = max(rows, key=lambda r: (np.nan_to_num(r["ap"], nan=-1), np.nan_to_num(r["auc"], nan=-1), np.nan_to_num(r["best_f2"], nan=-1), -r["authority"]))
    return best, rows


def hgb_parameter_grid(max_iter=250):
    # Small, deliberately interpretable grid: enough to test tree complexity
    # without turning every fold into a broad hyperparameter sweep.
    return [
        {"learning_rate": .05, "max_iter": max_iter, "max_leaf_nodes": 15, "min_samples_leaf": 20, "l2_regularization": 1.0},
        {"learning_rate": .05, "max_iter": max_iter, "max_leaf_nodes": 31, "min_samples_leaf": 20, "l2_regularization": 1.0},
        {"learning_rate": .03, "max_iter": max_iter, "max_leaf_nodes": 31, "min_samples_leaf": 40, "l2_regularization": 2.0},
    ]


def select_hgb_with_authority(Xtr, ytr, Xva, yva, qva, *, seed=42,
                              max_iter=250, authority_grid=(0,.25,.5,.75,1),
                              beta=0.9999, param_grid=None):
    param_grid = hgb_parameter_grid(max_iter) if param_grid is None else list(param_grid)
    w, _ = effective_weights(ytr, beta)
    candidates = []
    best_bundle = None
    for j, params in enumerate(param_grid):
        model = fit_hgb_classifier(Xtr, ytr, seed=seed + j * 17, sample_weight=w, **params)
        pv = predict_positive(model, Xva)
        best_a, arows = select_authority(yva, qva, pv, authority_grid)
        row = {"candidate": j, **params, **{f"selected_{k}": v for k,v in best_a.items()}}
        candidates.append(row)
        key = (best_a["ap"], best_a["auc"], best_a["best_f2"], -best_a["authority"])
        if best_bundle is None or key > best_bundle[0]:
            best_bundle = (key, model, params, best_a, arows, pv)
    _, model, params, best_a, arows, pv = best_bundle
    return {
        "model": model, "params": params, "authority": best_a["authority"],
        "validation": best_a, "authority_rows": arows, "candidate_rows": candidates,
        "validation_specialist_probability": pv,
    }


def permutation_rows(model, X, y, feature_names, *, seed=42, max_rows=4000, top_k=25):
    y = np.asarray(y, int)
    X = core.clean_X(X)
    if len(y) < 50 or np.unique(y).size < 2:
        return []
    if len(y) > max_rows:
        rng = np.random.default_rng(seed)
        # stratified-ish sample keeping all positives when feasible
        pos = np.where(y == 1)[0]
        neg = np.where(y == 0)[0]
        npos = min(len(pos), max_rows // 2)
        nneg = min(len(neg), max_rows - npos)
        idx = np.r_[rng.choice(pos, npos, replace=False), rng.choice(neg, nneg, replace=False)]
        X, y = X[idx], y[idx]
    try:
        r = permutation_importance(model, X, y, scoring="average_precision", n_repeats=1, random_state=seed, n_jobs=1)
        order = np.argsort(-r.importances_mean)[:top_k]
        return [{"feature": feature_names[i] if i < len(feature_names) else f"f{i}", "importance_ap_drop": float(r.importances_mean[i]), "importance_std": float(r.importances_std[i]), "rank": k+1} for k,i in enumerate(order)]
    except Exception:
        return []


def score_metrics(y, score, beta=2.0):
    return bc.score_row(y, score, beta=beta)


def equal_budget_metrics(y, score, rate=0.20, beta=2.0):
    y = np.asarray(y, int)
    score = np.asarray(score, float)
    pred = core.warn_by_rate(score, float(rate))
    m = core.report(y, score, pred, beta)
    return {"budget_rate": float(rate), **m}


def print_metric_line(label, y, score, q=None):
    ap, auc = safe_ap(y, score), safe_auc(y, score)
    s = f"{label:28s} AP/AUC={ap:.4f}/{auc:.4f}"
    if q is not None:
        s += f" | ΔAP={ap-safe_ap(y,q):+.4f} ΔAUC={auc-safe_auc(y,q):+.4f}"
    print(s)


def save_csv(rows, path):
    bc.save_csv(rows, path)


def parse_folds(s):
    return bc.parse_folds(s)


def load_binary_fold(*args, **kwargs):
    return bc.load_binary_fold(*args, **kwargs)


def crossfit_committee(*args, **kwargs):
    return bc.crossfit_historical_committee(*args, **kwargs)


def fit_production_committee(*args, **kwargs):
    return bc.fit_production_committee(*args, **kwargs)
