"""Canonical-cache loading and strict, global chronological student history.

The canonical caches and Suite12 structured caches are trusted local joblib
artifacts. This module never writes them. Raw features retain their cache values;
all imputation, scaling, and course-vocabulary fitting happen in Preprocessor on
explicitly selected training rows. Semester integers in Table are ordinals, not
local cross-fitting block identifiers.
"""
from __future__ import annotations

from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
import re
from typing import Sequence
from urllib.parse import quote

import joblib
import numpy as np
import pandas as pd
import torch
from torch.utils.data import Dataset


COURSE_STATIC = (
    "SoTinChi", "SoTietMH", "SoTietTuan", "SiSoMax", "SiSoMin", "KhgTinhDTB",
    "IsMHTN", "IsMonKT", "IsMHKN", "IsMHDacThu", "IsGDTiengAnh", "IsMonNgoaiNgu",
)
COURSE_HIST = ("hist_count", "hist_mean", "hist_std", "hist_fail_rate", "hist_pass_rate")
COURSE_FEATURES = COURSE_STATIC + COURSE_HIST
GRADE_COL = "target_grade10"
GRADE_AVAILABILITY_ASSUMPTION = (
    "Observed previous-semester grades are assumed available by the next "
    "semester; the canonical caches do not supply grade-release timestamps."
)


def _numeric(value, name: str, ndim: int) -> np.ndarray:
    try:
        result = np.asarray(value, dtype=np.float64)
    except (TypeError, ValueError) as exc:
        raise ValueError(f"{name} must contain numeric values") from exc
    if result.ndim != ndim:
        raise ValueError(f"{name} must be {ndim}-dimensional, got {result.shape}")
    return result


def _integers(value, name: str) -> np.ndarray:
    result = _numeric(value, name, 1)
    if not np.isfinite(result).all() or (result != np.floor(result)).any():
        raise ValueError(f"{name} must contain finite integer values")
    if ((result < np.iinfo(np.int64).min) | (result >= 2.0**63)).any():
        raise ValueError(f"{name} contains integers outside the supported range")
    return result.astype(np.int64)


def _labels(value, name="labels") -> np.ndarray:
    result = _numeric(value, name, 1)
    if not np.isfinite(result).all() or not np.isin(result, [0, 1]).all():
        raise ValueError(f"{name} must contain finite binary labels (failure=1)")
    return result.astype(np.int8)


def _canonical_id(value, name: str) -> str:
    if isinstance(value, (bool, np.bool_)) or value is None or value is pd.NA:
        raise ValueError(f"{name} requires a meaningful nonmissing ID, got {value!r}")
    if isinstance(value, (int, np.integer)):
        return str(int(value))
    if isinstance(value, (float, np.floating)):
        if not np.isfinite(value):
            raise ValueError(f"{name} requires a finite nonmissing ID, got {value!r}")
        return str(int(value)) if float(value).is_integer() else str(float(value))
    text = str(value).strip()
    if text.casefold() in {"", "nan", "none", "null", "<na>", "nat", "inf", "-inf", "+inf"}:
        raise ValueError(f"{name} requires a meaningful nonmissing ID, got {value!r}")
    # CSV/frame coercion commonly produces "17.0" for numeric ID 17. Preserve
    # padded identifiers such as "001", whose leading zeros can be meaningful.
    if re.fullmatch(r"[+-]?(?:0|[1-9][0-9]*)\.0+", text):
        return str(int(text.split(".", 1)[0]))
    return text


def _ids(value, name: str) -> np.ndarray:
    raw = np.asarray(value, dtype=object)
    if raw.ndim != 1:
        raise ValueError(f"{name} must be a one-dimensional ID array")
    return np.asarray([_canonical_id(v, name) for v in raw], dtype=object)


def _indices(value, n: int, name="indices") -> np.ndarray:
    if value is None:
        return np.arange(n, dtype=np.int64)
    raw = np.asarray(value)
    if raw.ndim != 1:
        raise ValueError(f"{name} must be a one-dimensional index array or boolean mask")
    if raw.dtype.kind == "b":
        if len(raw) != n:
            raise ValueError(f"{name} boolean mask has length {len(raw)}, expected {n}")
        return np.flatnonzero(raw)
    result = _integers(raw, name)
    if ((result < 0) | (result >= n)).any():
        raise ValueError(f"{name} must be in [0, {n}), got {result.tolist()[:10]}")
    return result


def _validate_grades(grade: np.ndarray, y: np.ndarray, name: str):
    if np.isinf(grade).any() or ((grade < 0) | (grade > 10)).any():
        raise ValueError(f"{name}: grade must be in 0..10 or missing")
    known = np.isfinite(grade)
    inconsistent = known & ((grade < 5).astype(np.int8) != y)
    if inconsistent.any():
        rows = np.flatnonzero(inconsistent)[:8].tolist()
        raise ValueError(f"{name}: grade/failure consistency violated at rows {rows}; failure means grade < 5")


@dataclass
class Table:
    """Unstandardized row table; x includes raw features and 17 course features.

    Rows need not be ordered. IDs are canonical nonmissing strings; semester is a
    globally comparable integer ordinal. NaN/inf numeric inputs are treated as
    missing by Preprocessor. Known grades must agree with binary failure labels.
    Target tables produced by load_fold deliberately contain no observed grades.
    """

    x: np.ndarray
    course_meta: np.ndarray
    y: np.ndarray
    grade: np.ndarray
    semester: np.ndarray
    student: np.ndarray
    course: np.ndarray
    faculty: np.ndarray
    sample_id: np.ndarray
    committee: np.ndarray
    faculty_names: tuple[str, ...]
    feature_names: tuple[str, ...]

    def __post_init__(self):
        self.x = _numeric(self.x, "x", 2)
        self.course_meta = _numeric(self.course_meta, "course_meta", 2)
        self.y = _labels(self.y)
        self.grade = _numeric(self.grade, "grade", 1)
        self.semester = _integers(self.semester, "semester")
        self.student = _ids(self.student, "student")
        self.course = _ids(self.course, "course")
        self.faculty = _integers(self.faculty, "faculty")
        self.sample_id = _ids(self.sample_id, "sample_id")
        self.committee = _numeric(self.committee, "committee", 1)
        self.faculty_names = tuple(_canonical_id(n, "faculty name") for n in self.faculty_names)
        self.feature_names = tuple(str(n) for n in self.feature_names)
        n = len(self.y)
        for name in ("x", "course_meta", "grade", "semester", "student", "course",
                     "faculty", "sample_id", "committee"):
            if len(getattr(self, name)) != n:
                raise ValueError(f"{name} row count does not match labels ({n})")
        if self.course_meta.shape[1] != len(COURSE_FEATURES):
            raise ValueError("course_meta must contain the canonical 17 course features")
        if self.x.shape[1] != len(self.feature_names) or len(set(self.feature_names)) != len(self.feature_names):
            raise ValueError("feature_names must be unique and match x columns")
        if not self.faculty_names or len(set(self.faculty_names)) != len(self.faculty_names):
            raise ValueError("faculty_names must be nonempty and unique")
        if ((self.faculty < 0) | (self.faculty >= len(self.faculty_names))).any():
            raise ValueError("faculty index is outside faculty_names")
        if len(set(self.sample_id)) != n:
            raise ValueError("sample_id must be unique for each canonical row")
        if np.isinf(self.committee).any() or ((self.committee < 0) | (self.committee > 1)).any():
            raise ValueError("committee must contain probabilities in [0,1] or NaN")
        _validate_grades(self.grade, self.y, "Table")

    def __len__(self) -> int:
        return len(self.y)

    def subset(self, indices) -> "Table":
        """Select rows by integer indices (preserving order) or boolean mask."""
        selected = _indices(indices, len(self))
        arrays = {name: getattr(self, name)[selected] for name in
                  ("x", "course_meta", "y", "grade", "semester", "student", "course",
                   "faculty", "sample_id", "committee")}
        return Table(**arrays, faculty_names=self.faculty_names, feature_names=self.feature_names)


def discover_faculties(cache_dir, fold=1) -> list[str]:
    suffix = f"_fold_{int(fold)}.joblib"
    root = Path(cache_dir)
    names = [p.name[:-len(suffix)] for p in sorted(root.glob(f"*{suffix}")) if p.is_file()]
    if not names:
        raise FileNotFoundError(f"No canonical cache files matching *{suffix} under {root}")
    return names


def _feature_names(cache: dict, source: str) -> list[str]:
    names = list(cache["feature_names"])
    if not names or any(not isinstance(n, str) or not n.strip() for n in names) or len(set(names)) != len(names):
        raise ValueError(f"{source}: feature_names must be nonempty, unique strings")
    return names


def _common_features(caches: dict, faculties: Sequence[str]) -> list[str]:
    if not faculties:
        raise ValueError("At least one faculty must be selected")
    first = _feature_names(caches[faculties[0]], faculties[0])
    shared = set(first)
    for faculty in faculties[1:]:
        shared.intersection_update(_feature_names(caches[faculty], faculty))
    result = [name for name in first if name in shared]
    if not result:
        raise ValueError("No common raw features across selected faculties")
    return result


def common_feature_names(cache_dir, faculties, fold=1) -> list[str]:
    """Return the raw-feature intersection in the first faculty's column order."""
    names = list(faculties)
    caches = {f: joblib.load(Path(cache_dir) / f"{f}_fold_{int(fold)}.joblib") for f in names}
    return _common_features(caches, names)


def _semesters(frame: pd.DataFrame, source: str) -> np.ndarray:
    if "target_NHHK" not in frame:
        raise ValueError(f"{source}: missing semester column target_NHHK")
    code = _integers(frame["target_NHHK"], f"{source}.target_NHHK semester")
    year, term = code // 10, code % 10
    bad = (year <= 0) | ~np.isin(term, [1, 2, 3])
    if bad.any():
        raise ValueError(f"{source}: unparseable semester target_NHHK {code[bad][:8].tolist()}; expected year*10 + term (1,2,3)")
    return year * 3 + term - 1


def _frame_grades(frame: pd.DataFrame, y: np.ndarray, source: str) -> np.ndarray:
    if GRADE_COL not in frame:
        return np.full(len(frame), np.nan)
    raw = frame[GRADE_COL]
    grade = pd.to_numeric(raw, errors="coerce").to_numpy(dtype=np.float64, na_value=np.nan)
    missing = raw.isna() | raw.astype(str).str.strip().str.lower().isin(["", "nan", "none", "null", "<na>"])
    if (np.isnan(grade) & ~missing.to_numpy()).any():
        raise ValueError(f"{source}: grade contains nonnumeric nonmissing values")
    _validate_grades(grade, y, source)
    return grade


def _sample_ids(frame: pd.DataFrame, faculty: str, student, course, semester):
    """Use an explicit row ID when available; otherwise use event identity + occurrence."""
    prefix = quote(faculty, safe="")
    if "sample_id" in frame:
        supplied = _ids(frame["sample_id"].to_numpy(), "sample_id")
        ids = np.asarray([f"{prefix}|id|{quote(s, safe='')}" for s in supplied], dtype=object)
        if len(set(ids)) != len(ids):
            raise ValueError(f"{faculty}: explicit sample_id is not unique")
        return ids, 0
    seen = defaultdict(int)
    ids = []
    duplicate_events = 0
    for sid, cid, sem in zip(student, course, semester):
        key = (sid, cid, int(sem))
        occurrence = seen[key]
        seen[key] += 1
        duplicate_events += int(occurrence > 0)
        ids.append(f"{prefix}|event|{quote(sid, safe='')}|{quote(cid, safe='')}|{sem}|{occurrence}")
    return np.asarray(ids, dtype=object), duplicate_events


def _committee(cache: dict, n: int, train: bool) -> np.ndarray:
    keys = ("cf_probabilities",) if train else ("qte", "pte")
    for key in keys:
        if key not in cache:
            continue
        values = np.asarray(cache[key], dtype=np.float64)
        if values.ndim not in (1, 2) or len(values) != n:
            raise ValueError(f"{key} committee row count/shape mismatch")
        if values.ndim == 2:
            if values.shape[1] == 0:
                raise ValueError(f"{key} has no committee member columns")
            valid = np.isfinite(values).all(axis=1)
            out = np.full(n, np.nan)
            out[valid] = values[valid].mean(axis=1)
        else:
            out = np.where(np.isfinite(values), values, np.nan)
        if train and "cf_covered_mask" in cache:
            mask = np.asarray(cache["cf_covered_mask"])
            if mask.shape != (n,) or not np.isin(mask, [0, 1]).all():
                raise ValueError("cf_covered_mask must be a boolean mask matching canonical train rows")
            out[~mask.astype(bool)] = np.nan
        if ((out < 0) | (out > 1)).any():
            raise ValueError(f"{key} contains committee probabilities outside [0,1]")
        return out
    return np.full(n, np.nan)


def _raw_features(cache, key: str, row_count: int, common: Sequence[str], source: str):
    names = _feature_names(cache, source)
    matrix = _numeric(cache[key], f"{source}.{key}", 2)
    if matrix.shape != (row_count, len(names)):
        raise ValueError(f"{source}.{key}: shape {matrix.shape} does not match frame/features")
    missing = set(common) - set(names)
    if missing:
        raise ValueError(f"{source}: missing common raw features {sorted(missing)}")
    return matrix[:, [names.index(n) for n in common]]


def _course_array(value, n: int, name: str) -> np.ndarray:
    array = _numeric(value, name, 2)
    if array.shape != (n, len(COURSE_FEATURES)):
        raise ValueError(f"{name} must have shape ({n}, 17), got {array.shape}")
    return array.copy()


def _reconstruct_history(parts: list[dict]) -> None:
    """As-of running course statistics, updated only after a whole semester.

    hist_count counts labeled events, grade mean/std use observed grades, and
    std is sample standard deviation (ddof=1). Grade-free label rates are still
    observed. A course with no prior observations has count 0 and missing rates.
    Course IDs are globally shared across the selected faculties.
    """
    semester = np.concatenate([p["semester"] for p in parts])
    course = np.concatenate([p["course"] for p in parts])
    grades = np.concatenate([p["grade"] for p in parts])
    labels = np.concatenate([p["y"] for p in parts])
    missing = np.concatenate([~p["covered"] for p in parts])
    meta = np.concatenate([p["course_meta"] for p in parts], axis=0)
    by_semester = np.argsort(semester, kind="stable")
    stats = {}  # course -> [labeled count, failures, grade count, mean, M2]
    start = 0
    while start < len(by_semester):
        stop = start + 1
        sem = semester[by_semester[start]]
        while stop < len(by_semester) and semester[by_semester[stop]] == sem:
            stop += 1
        rows = by_semester[start:stop]
        for i in rows[missing[rows]]:
            state = stats.get(course[i])
            if state is None:
                values = [0., np.nan, np.nan, np.nan, np.nan]
            else:
                n, failures, ng, mean, m2 = state
                values = [n, mean if ng else np.nan,
                          np.sqrt(max(m2, 0.) / (ng - 1)) if ng > 1 else np.nan,
                          failures / n, 1. - failures / n]
            meta[i, len(COURSE_STATIC):] = values
        for i in rows:
            state = stats.setdefault(course[i], [0, 0, 0, 0., 0.])
            state[0] += 1
            state[1] += int(labels[i])
            if np.isfinite(grades[i]):
                state[2] += 1
                delta = grades[i] - state[3]
                state[3] += delta / state[2]
                state[4] += delta * (grades[i] - state[3])
        start = stop
    start = 0
    for part in parts:
        stop = start + len(part["y"])
        part["course_meta"] = meta[start:stop]
        start = stop


def load_fold(cache_dir, structured_dir, faculties, fold, common_features=None):
    """Load all canonical history and target rows plus a JSON-compatible audit.

    Structured train features are mapped by train_row_index, even if reordered.
    Those covered values and all structured target values are preserved exactly.
    Uncovered historical rows get static metadata only from their own row and
    course statistics from strictly prior semesters. No labels are dropped based
    on CF coverage. Target grades are validated, then discarded from model data.
    """
    faculties = tuple(_canonical_id(f, "faculty name") for f in faculties)
    if not faculties or len(set(faculties)) != len(faculties):
        raise ValueError("Select at least one faculty, without duplicate names")
    caches = {f: joblib.load(Path(cache_dir) / f"{f}_fold_{int(fold)}.joblib") for f in faculties}
    common = _common_features(caches, faculties) if common_features is None else list(common_features)
    if not common or len(set(common)) != len(common):
        raise ValueError("common_features must be a nonempty list of unique feature names")
    hist, targets, audits = [], [], []
    for fi, faculty in enumerate(faculties):
        cache = caches[faculty]
        source = f"{faculty} fold {int(fold)}"
        structured_path = Path(structured_dir) / f"{faculty}_fold_{int(fold)}.joblib"
        if not structured_path.is_file():
            raise FileNotFoundError(f"Missing Suite12 structured cache: {structured_path}")
        structured = joblib.load(structured_path)
        frames = [cache["train_frame"].reset_index(drop=True), cache["test_frame"].reset_index(drop=True)]
        y_train, y_target = _labels(cache["ytr"], f"{source}.ytr"), _labels(cache["yte"], f"{source}.yte")
        if len(frames[0]) != len(y_train) or len(frames[1]) != len(y_target):
            raise ValueError(f"{source}: canonical frame/label row count mismatch")
        n_train, n_target = len(y_train), len(y_target)
        covered_idx = _indices(structured["train_row_index"], n_train, f"{source}.train_row_index")
        if len(set(covered_idx)) != len(covered_idx):
            raise ValueError(f"{source}: train_row_index must not contain duplicate canonical rows")
        covered = np.zeros(n_train, dtype=bool)
        covered[covered_idx] = True
        committee_train = _committee(cache, n_train, True)
        if "cf_covered_mask" in cache and "cf_probabilities" in cache:
            expected = np.isfinite(committee_train)
            if not np.array_equal(covered, expected):
                raise ValueError(f"{source}: train_row_index does not align with finite CF-covered canonical rows")
        train_meta = np.full((n_train, len(COURSE_FEATURES)), np.nan)
        for j, name in enumerate(COURSE_STATIC):
            if name in frames[0]:
                train_meta[:, j] = pd.to_numeric(frames[0][name], errors="coerce").to_numpy(dtype=float, na_value=np.nan)
        train_meta[covered_idx] = _course_array(structured["train"]["course_target"], len(covered_idx), f"{source}.train.course_target")
        target_meta = _course_array(structured["target"]["course_target"], n_target, f"{source}.target.course_target")
        target_alignment = "canonical row order (structured target has no explicit row index)"
        if "target_row_index" in structured:
            target_idx = _indices(structured["target_row_index"], n_target, f"{source}.target_row_index")
            if len(target_idx) != n_target or len(set(target_idx)) != n_target:
                raise ValueError(f"{source}: target_row_index must be a one-to-one permutation")
            reordered = np.empty_like(target_meta)
            reordered[target_idx] = target_meta
            target_meta = reordered
            target_alignment = "explicit target_row_index"
        duplicate_keys = []
        for is_train, frame, y, meta in ((True, frames[0], y_train, train_meta),
                                       (False, frames[1], y_target, target_meta)):
            mode = "train" if is_train else "target"
            row_source = f"{source}.{mode}"
            for key in ("IDSV", "IDMH"):
                if key not in frame:
                    raise ValueError(f"{row_source}: missing required ID column {key}")
            students = _ids(frame["IDSV"].to_numpy(), f"{row_source}.student IDSV")
            courses = _ids(frame["IDMH"].to_numpy(), f"{row_source}.course IDMH")
            semester = _semesters(frame, row_source)
            grade = _frame_grades(frame, y, row_source)
            sample_id, duplicate_count = _sample_ids(frame, faculty, students, courses, semester)
            duplicate_keys.append(duplicate_count)
            part = dict(
                raw=_raw_features(cache, "Xtr" if is_train else "Xte", len(y), common, row_source),
                course_meta=meta, y=y, grade=grade if is_train else np.full(len(y), np.nan),
                semester=semester, student=students, course=courses,
                faculty=np.full(len(y), fi, dtype=np.int64), sample_id=sample_id,
                committee=committee_train if is_train else _committee(cache, len(y), False),
            )
            if is_train:
                part["covered"] = covered
                hist.append(part)
            else:
                targets.append(part)
        audits.append(dict(faculty=faculty, history_rows=n_train, target_rows=n_target,
                           structured_covered_rows=int(covered.sum()),
                           reconstructed_history_rows=int((~covered).sum()),
                           duplicate_event_occurrences=duplicate_keys,
                           target_alignment=target_alignment))
    _reconstruct_history(hist)
    feature_names = tuple(common) + tuple(f"course::{name}" for name in COURSE_FEATURES)

    def concatenate(parts):
        values = {key: np.concatenate([p[key] for p in parts], axis=0)
                  for key in ("course_meta", "y", "grade", "semester", "student", "course",
                              "faculty", "sample_id", "committee")}
        values["x"] = np.concatenate([np.concatenate([p["raw"], p["course_meta"]], axis=1) for p in parts], axis=0)
        return Table(**values, faculty_names=faculties, feature_names=feature_names)

    history, target = concatenate(hist), concatenate(targets)
    assert_future(history, target)
    reconstructed = np.concatenate([~p["covered"] for p in hist])
    audit = dict(
        fold=int(fold), history_rows=len(history), target_rows=len(target),
        structured_covered_rows=int((~reconstructed).sum()),
        reconstructed_history_rows=int(reconstructed.sum()),
        history_grade_known_rows=int(np.isfinite(history.grade).sum()),
        history_grade_missing_rows=int((~np.isfinite(history.grade)).sum()),
        history_course_missing_by_feature={name: int((~np.isfinite(history.course_meta[:, j])).sum())
                                           for j, name in enumerate(COURSE_FEATURES)},
        reconstructed_static_missing_values=int((~np.isfinite(history.course_meta[reconstructed, :len(COURSE_STATIC)])).sum()),
        target_course_missing_by_feature={name: int((~np.isfinite(target.course_meta[:, j])).sum())
                                          for j, name in enumerate(COURSE_FEATURES)},
        history_input_nonfinite_values=int((~np.isfinite(history.x)).sum()),
        target_input_nonfinite_values=int((~np.isfinite(target.x)).sum()),
        history_semester_range=[int(history.semester.min()), int(history.semester.max())],
        target_semester_range=[int(target.semester.min()), int(target.semester.max())],
        common_raw_features=list(common), course_features=list(COURSE_FEATURES),
        course_statistics="prior labeled-event count and label rates; observed-grade mean and sample std (ddof=1)",
        id_policy="globally canonical student/course strings; sample_id uses explicit ID or faculty/student/course/semester/occurrence",
        course_scope="global across selected faculties", grade_failure_cutoff=5.,
        target_grades_discarded=True, availability_assumption=GRADE_AVAILABILITY_ASSUMPTION,
        faculties=audits,
    )
    print(f"[Data assumption] {GRADE_AVAILABILITY_ASSUMPTION}")
    return history, target, audit


def chronological_split(table: Table, val_semesters=3):
    """Split by one global cutoff, retaining at least one older training semester."""
    count = int(val_semesters)
    if count < 1 or count != val_semesters:
        raise ValueError("val_semesters must be a positive integer")
    unique = np.unique(table.semester)
    if len(unique) < 2:
        raise ValueError("Chronological selection requires at least two distinct historical semesters; use earlier history")
    validation_semesters = unique[-min(count, len(unique) - 1):]
    boundary = validation_semesters[0]
    train = np.flatnonzero(table.semester < boundary)
    validation = np.flatnonzero(table.semester >= boundary)
    if table.semester[train].max() >= table.semester[validation].min():
        raise ValueError("Global chronology violation between training and validation semesters")
    audit = dict(train_rows=len(train), validation_rows=len(validation),
                 train_semesters=np.unique(table.semester[train]).astype(int).tolist(),
                 validation_semesters=validation_semesters.astype(int).tolist(),
                 cutoff_semester=int(boundary),
                 faculty_rows={name: {"train": int((table.faculty[train] == fi).sum()),
                                      "validation": int((table.faculty[validation] == fi).sum())}
                               for fi, name in enumerate(table.faculty_names)},
                 validation_history_policy="fixed origin: training partition only")
    return train, validation, audit


def assert_future(history: Table, target: Table):
    """Reject overlapping global calendar windows, including cross-faculty overlap."""
    if not len(history) or not len(target):
        raise ValueError("Global chronology requires nonempty history and target tables")
    maximum, minimum = int(history.semester.max()), int(target.semester.min())
    if maximum >= minimum:
        hist_ranges = {f: [int(history.semester[history.faculty == i].min()),
                           int(history.semester[history.faculty == i].max())]
                       for i, f in enumerate(history.faculty_names) if (history.faculty == i).any()}
        target_ranges = {f: [int(target.semester[target.faculty == i].min()),
                             int(target.semester[target.faculty == i].max())]
                         for i, f in enumerate(target.faculty_names) if (target.faculty == i).any()}
        raise ValueError(f"Global chronology violation: max history semester {maximum} >= min target semester {minimum}; "
                         f"history faculty ranges={hist_ranges}; target faculty ranges={target_ranges}. "
                         "Regenerate caches with a shared earlier training cutoff.")


def _fit_stats(values: np.ndarray):
    values = np.asarray(values, dtype=np.float64)
    finite = np.isfinite(values)
    count = finite.sum(axis=0)
    mean = np.divide(np.where(finite, values, 0.).sum(axis=0), count,
                     out=np.zeros(values.shape[1]), where=count > 0)
    centered = np.where(finite, values - mean, 0.)
    variance = np.divide(np.square(centered).sum(axis=0), count,
                         out=np.zeros(values.shape[1]), where=count > 0)
    scale = np.sqrt(variance)
    scale = np.where(scale < 1e-6, 1., scale)
    if not np.isfinite(mean).all() or not np.isfinite(scale).all():
        raise ValueError("Training numeric values are too large to fit finite preprocessing statistics")
    return mean, scale


def _standardize(values, mean, scale):
    values = np.asarray(values, dtype=np.float64)
    result = (np.where(np.isfinite(values), values, mean) - mean) / scale
    # Float32 is the model input dtype. Explicitly fail if a finite source value
    # cannot be represented after scaling rather than silently overflow to inf.
    if not np.isfinite(result).all() or (np.abs(result) > np.finfo(np.float32).max).any():
        raise ValueError("Standardized numeric inputs exceed finite float32 range")
    return result.astype(np.float32)


@dataclass
class Prepared:
    """Model-ready finite arrays and the original, unmodified Table."""
    x: np.ndarray
    course_meta: np.ndarray
    course_id: np.ndarray
    table: Table

    def __len__(self):
        return len(self.table)


@dataclass
class Preprocessor:
    """Training-only scaling, mean imputation, and course-ID vocabulary.

    fit(table, indices) accepts integer indices or a boolean training mask.
    Every x column has a presence flag after the standardized values, followed
    by faculty indicators. Missing metadata also has standardized value zero
    and a separate zero presence flag. The unknown course code is 0;
    n_courses includes that reserved code.
    to_dict/from_dict produce a JSON-compatible serialization of all state.
    """
    x_mean: np.ndarray
    x_scale: np.ndarray
    meta_mean: np.ndarray
    meta_scale: np.ndarray
    course_vocabulary: dict[str, int]
    faculty_names: tuple[str, ...]
    feature_names: tuple[str, ...]

    @classmethod
    def fit(cls, table: Table, indices) -> "Preprocessor":
        chosen = _indices(indices, len(table), "training indices")
        if len(chosen) == 0:
            raise ValueError("Preprocessor.fit requires nonempty training indices")
        x_mean, x_scale = _fit_stats(table.x[chosen])
        meta_mean, meta_scale = _fit_stats(table.course_meta[chosen])
        vocabulary = {course: i + 1 for i, course in enumerate(sorted(set(table.course[chosen])))}
        return cls(x_mean, x_scale, meta_mean, meta_scale, vocabulary,
                   table.faculty_names, table.feature_names)

    @property
    def n_courses(self) -> int:
        return len(self.course_vocabulary) + 1

    def transform(self, table: Table) -> Prepared:
        if table.feature_names != self.feature_names or table.faculty_names != self.faculty_names:
            raise ValueError("Preprocessor transform feature/faculty schema differs from fitted schema")
        x = _standardize(table.x, self.x_mean, self.x_scale)
        x_presence = np.isfinite(table.x).astype(np.float32)
        faculty = np.eye(len(self.faculty_names), dtype=np.float32)[table.faculty]
        meta = _standardize(table.course_meta, self.meta_mean, self.meta_scale)
        presence = np.isfinite(table.course_meta).astype(np.float32)
        ids = np.asarray([self.course_vocabulary.get(c, 0) for c in table.course], dtype=np.int64)
        return Prepared(np.concatenate([x, x_presence, faculty], axis=1),
                        np.concatenate([meta, presence], axis=1), ids, table)

    def to_dict(self) -> dict:
        return {"version": 2, "x_mean": self.x_mean.tolist(), "x_scale": self.x_scale.tolist(),
                "meta_mean": self.meta_mean.tolist(), "meta_scale": self.meta_scale.tolist(),
                "course_vocabulary": dict(self.course_vocabulary),
                "faculty_names": list(self.faculty_names), "feature_names": list(self.feature_names)}

    @classmethod
    def from_dict(cls, state: dict) -> "Preprocessor":
        if state.get("version") != 2:
            raise ValueError("Unsupported Preprocessor state version")
        arrays = {key: _numeric(state[key], key, 1) for key in ("x_mean", "x_scale", "meta_mean", "meta_scale")}
        if any(not np.isfinite(v).all() for v in arrays.values()):
            raise ValueError("Serialized preprocessing statistics must be finite")
        if (arrays["x_scale"] <= 0).any() or (arrays["meta_scale"] <= 0).any():
            raise ValueError("Serialized preprocessing scales must be positive")
        features, faculties = tuple(state["feature_names"]), tuple(state["faculty_names"])
        if (len(arrays["x_mean"]) != len(features) or len(arrays["x_scale"]) != len(features)
                or len(arrays["meta_mean"]) != 17 or len(arrays["meta_scale"]) != 17):
            raise ValueError("Serialized preprocessing dimensions do not match schema")
        vocabulary = {str(key): int(value) for key, value in state["course_vocabulary"].items()}
        if sorted(vocabulary.values()) != list(range(1, len(vocabulary) + 1)):
            raise ValueError("Serialized course vocabulary must use contiguous codes starting at 1")
        return cls(**arrays, course_vocabulary=vocabulary, faculty_names=faculties, feature_names=features)


@dataclass
class HistoryIndex:
    """Padded bank row indices and label-independent history support counts."""
    indices: np.ndarray
    recency: np.ndarray
    same_course: np.ndarray
    full_count: np.ndarray
    retained_count: np.ndarray

    @property
    def max_history(self) -> int:
        return self.indices.shape[1]


def build_history(bank: Table, query: Table, max_history=32, allowed=None, embargo=0) -> HistoryIndex:
    """Retain the most recent H strictly earlier same-student events.

    Selection and tie-breaking depend only on semester and stable sample_id,
    never grades or failure labels. Events must precede query.semester-embargo;
    embargo is a nonnegative number of full semester ordinals. allowed is a
    boolean bank-row mask; selection validation must pass its training-partition
    mask for fixed-origin forecasts.
    """
    width = int(max_history)
    if width < 1 or width != max_history:
        raise ValueError("max_history must be a positive integer")
    try:
        embargo_value = int(embargo)
    except (TypeError, ValueError, OverflowError) as exc:
        raise ValueError("embargo must be a nonnegative integer") from exc
    if embargo_value < 0 or embargo_value != embargo:
        raise ValueError("embargo must be a nonnegative integer")
    if allowed is None:
        allowed_mask = np.ones(len(bank), dtype=bool)
    else:
        allowed_mask = np.asarray(allowed)
        if allowed_mask.shape != (len(bank),) or allowed_mask.dtype.kind != "b":
            raise ValueError("allowed must be a boolean mask matching the history bank")
    groups = defaultdict(list)
    for i in np.flatnonzero(allowed_mask):
        groups[bank.student[i]].append(int(i))
    ordered = {}
    for student, rows in groups.items():
        indices = np.asarray(sorted(rows, key=lambda i: (int(bank.semester[i]), str(bank.sample_id[i]))), dtype=np.int64)
        ordered[student] = (indices, bank.semester[indices])
    ids = np.full((len(query), width), -1, dtype=np.int64)
    recency = np.zeros((len(query), width), dtype=np.float32)
    same = np.zeros((len(query), width), dtype=np.float32)
    full_count, retained = np.zeros(len(query), dtype=np.int64), np.zeros(len(query), dtype=np.int64)
    for i, student in enumerate(query.student):
        group = ordered.get(student)
        if group is None:
            continue
        rows, semesters = group
        count = int(np.searchsorted(semesters, int(query.semester[i]) - embargo_value, side="left"))
        selected = rows[max(0, count - width):count]
        full_count[i], retained[i] = count, len(selected)
        ids[i, :len(selected)] = selected
        recency[i, :len(selected)] = query.semester[i] - bank.semester[selected]
        same[i, :len(selected)] = bank.course[selected] == query.course[i]
    return HistoryIndex(ids, recency, same, full_count, retained)


def _validate_history(bank: Table, query: Table, history: HistoryIndex):
    ids = np.asarray(history.indices)
    if ids.ndim != 2 or ids.shape[0] != len(query) or ids.shape[1] < 1 or ids.dtype.kind not in "iu":
        raise ValueError("History indices must be an integer [query rows, max_history] matrix")
    if ((ids < -1) | (ids >= len(bank))).any():
        raise ValueError("History indices contain invalid bank row positions")
    valid = ids >= 0
    if history.recency.shape != ids.shape or history.same_course.shape != ids.shape:
        raise ValueError("History metadata shapes do not match history indices")
    for count in (history.full_count, history.retained_count):
        if np.asarray(count).shape != (len(query),):
            raise ValueError("History count shape does not match query rows")
    if not np.array_equal(valid.sum(axis=1), history.retained_count) or (history.full_count < history.retained_count).any():
        raise ValueError("History full/retained counts are inconsistent")
    if valid.any():
        query_rows = np.nonzero(valid)[0]
        bank_rows = ids[valid]
        distances = query.semester[query_rows] - bank.semester[bank_rows]
        if (distances <= 0).any():
            raise ValueError("History contains same-semester or future events")
        if not np.array_equal(history.recency[valid], distances):
            raise ValueError("History recency does not match query/bank semester distances")
    if not np.isfinite(history.recency).all() or not np.isfinite(history.same_course).all():
        raise ValueError("History recency and same_course must be finite")


def shuffle_history(bank: Table, query: Table, history: HistoryIndex, seed):
    """Exchange histories between different students in the same faculty/semester.

    Balanced groups use a permutation. With imbalanced repeated students, donors
    may be reused to avoid same-student donors whenever an alternative exists;
    the returned audit reports reuse and groups that cannot be shuffled. The
    candidate bank remains exactly the original fixed-origin history bank.
    """
    _validate_history(bank, query, history)
    rng = np.random.default_rng(seed)
    groups = defaultdict(list)
    for i in range(len(query)):
        groups[(int(query.faculty[i]), int(query.semester[i]))].append(i)
    donor = np.arange(len(query), dtype=np.int64)
    different, unshuffleable = 0, 0
    for rows in groups.values():
        by_student = defaultdict(list)
        for i in rows:
            by_student[query.student[i]].append(i)
        if len(by_student) < 2:
            unshuffleable += len(rows)
            continue
        students = list(by_student)
        rng.shuffle(students)
        ordered = []
        for student in students:
            members = np.asarray(by_student[student])
            rng.shuffle(members)
            ordered.extend(members.tolist())
        shift = max(len(v) for v in by_student.values())
        rotated = np.roll(np.asarray(ordered), shift)
        for recipient, candidate in zip(ordered, rotated):
            if query.student[recipient] == query.student[candidate]:
                alternatives = [i for i in rows if query.student[i] != query.student[recipient]]
                candidate = int(rng.choice(alternatives))
            donor[recipient] = candidate
            different += 1
    ids = history.indices[donor].copy()
    valid = ids >= 0
    recency = np.zeros_like(history.recency)
    same = np.zeros_like(history.same_course)
    if valid.any():
        qi = np.nonzero(valid)[0]
        recency[valid] = query.semester[qi] - bank.semester[ids[valid]]
        same[valid] = query.course[qi] == bank.course[ids[valid]]
    result = HistoryIndex(ids, recency, same, history.full_count[donor].copy(), history.retained_count[donor].copy())
    _validate_history(bank, query, result)
    audit = dict(seed=int(seed), rows=len(query), different_student_rows=different,
                 unshuffleable_rows=unshuffleable, donor_reuse_rows=len(donor) - len(np.unique(donor)),
                 changed_history_rows=int(np.any(ids != history.indices, axis=1).sum()),
                 donor_rows=donor.tolist(), grouping="same query faculty and semester",
                 policy="different-student permutation with donor reuse when needed")
    return result, audit


class MemoryDataset(Dataset):
    """Finite tensor batches from Prepared query/bank arrays and HistoryIndex.

    indices optionally selects query rows without rebuilding their history.
    Padded numeric values are zero, padded course ID is unknown 0, and h_mask is
    false. Missing historical grades use value 0 with h_grade_present=0 so the
    model can use the observed binary outcome as its readiness fallback.
    """
    def __init__(self, query: Prepared, bank: Prepared, history: HistoryIndex, indices=None):
        _validate_history(bank.table, query.table, history)
        if query.x.shape[1] != bank.x.shape[1] or query.course_meta.shape[1] != bank.course_meta.shape[1]:
            raise ValueError("Prepared query and bank feature dimensions must match")
        for prepared in (query, bank):
            if not np.isfinite(prepared.x).all() or not np.isfinite(prepared.course_meta).all():
                raise ValueError("Prepared model inputs must be finite")
        self.query, self.bank, self.history = query, bank, history
        self.indices = _indices(indices, len(query), "query indices")

    def __len__(self):
        return len(self.indices)

    def __getitem__(self, index):
        i = int(self.indices[index])
        row_ids = self.history.indices[i]
        mask = row_ids >= 0
        selected = row_ids[mask]
        width = len(row_ids)
        history_x = np.zeros((width, self.bank.x.shape[1]), dtype=np.float32)
        history_meta = np.zeros((width, self.bank.course_meta.shape[1]), dtype=np.float32)
        history_course = np.zeros(width, dtype=np.int64)
        history_y = np.zeros(width, dtype=np.float32)
        history_grade = np.zeros(width, dtype=np.float32)
        grade_present = np.zeros(width, dtype=np.float32)
        if len(selected):
            history_x[mask] = self.bank.x[selected]
            history_meta[mask] = self.bank.course_meta[selected]
            history_course[mask] = self.bank.course_id[selected]
            history_y[mask] = self.bank.table.y[selected]
            grade = self.bank.table.grade[selected]
            present = np.isfinite(grade)
            history_grade[mask] = np.where(present, grade / 10., 0.)
            grade_present[mask] = present
        return {
            "x": torch.as_tensor(self.query.x[i], dtype=torch.float32),
            "course_meta": torch.as_tensor(self.query.course_meta[i], dtype=torch.float32),
            "course_id": torch.tensor(int(self.query.course_id[i]), dtype=torch.long),
            "faculty": torch.tensor(int(self.query.table.faculty[i]), dtype=torch.long),
            "y": torch.tensor(float(self.query.table.y[i]), dtype=torch.float32),
            "sample_index": torch.tensor(i, dtype=torch.long),
            "h_x": torch.from_numpy(history_x), "h_meta": torch.from_numpy(history_meta),
            "h_course_id": torch.from_numpy(history_course), "h_y": torch.from_numpy(history_y),
            "h_grade": torch.from_numpy(history_grade), "h_grade_present": torch.from_numpy(grade_present),
            "h_mask": torch.from_numpy(mask),
            "h_recency": torch.as_tensor(np.where(mask, self.history.recency[i], 0.), dtype=torch.float32),
            "h_same": torch.as_tensor(np.where(mask, self.history.same_course[i], 0.), dtype=torch.float32),
        }
