"""Read the native AQ canonical caches without requiring intermediate exports.

The input joblib artifacts are trusted local files. They are read, never written.
Preparation scans one cache at a time, retains identities/grades rather than raw
matrices, and produces a re-iterable collection of metadata-only cohort handles.
A handle loads its own train/test matrices on the first corresponding access.

AQ caches expose semester codes, not certified grade-release dates. Timestamps
below are INTERNAL ORDINAL CARRIERS, not actual dates: a semester's origin is
1900-01-01 UTC plus its global ordinal in days, and its release is twelve hours
later. This represents the inherited assumption that earlier-semester observed
grades are available by the next semester; it cannot establish publication time.
"""
from __future__ import annotations

from collections import defaultdict
from collections.abc import Sequence
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import re
from urllib.parse import quote

import joblib
import numpy as np
import pandas as pd

COURSE_STATIC = (
    'SoTinChi', 'SoTietMH', 'SoTietTuan', 'SiSoMax', 'SiSoMin', 'KhgTinhDTB',
    'IsMHTN', 'IsMonKT', 'IsMHKN', 'IsMHDacThu', 'IsGDTiengAnh', 'IsMonNgoaiNgu',
)
COURSE_HIST = ('hist_count', 'hist_mean', 'hist_std', 'hist_fail_rate', 'hist_pass_rate')
COURSE_FEATURES = COURSE_STATIC + COURSE_HIST
# The established train_v3 exclusion set, with explicit sample identity added.
CSV_EXCLUDE = {
    'dataset_split', 'student_faculty_key', 'student_faculty_name', 'IDSV',
    'target_NHHK', 'target_semester_ordinal', 'is_summer_target', 'IDKQDK',
    'IDToHoc', 'IDMH', 'target_IDCongDiem', 'target_grade10', 'target_grade4',
    'target_status', 'base_row_id', 'course_cluster_label', 'sample_id',
}
AVAILABILITY_ASSUMPTION = (
    'Observed previous-semester numeric grades are assumed available by the next '
    'semester; AQ canonical caches provide no grade-release timestamps.'
)
TIMESTAMP_SEMANTICS = (
    'INTERNAL ORDINAL CARRIERS, not actual dates: 1900-01-01 UTC + semester '
    'ordinal days; labels/events available at origin + 12 hours.'
)
_EPOCH = pd.Timestamp('1900-01-01', tz='UTC')
_MISSING_TEXT = {'', 'nan', 'none', 'null', '<na>', 'nat', 'inf', '-inf', '+inf'}


def _canonical_id(value, name):
    if value is None or value is pd.NA or isinstance(value, (bool, np.bool_)):
        raise ValueError(f'{name}: meaningful nonmissing ID required, got {value!r}')
    if isinstance(value, (int, np.integer)):
        return str(int(value))
    if isinstance(value, (float, np.floating)):
        if not np.isfinite(value):
            raise ValueError(f'{name}: finite nonmissing ID required, got {value!r}')
        return str(int(value)) if float(value).is_integer() else str(float(value))
    text = str(value).strip()
    if text.casefold() in _MISSING_TEXT:
        raise ValueError(f'{name}: meaningful nonmissing ID required, got {value!r}')
    # Preserve zero padding: "001" and "001.0" remain distinct identifiers.
    if re.fullmatch(r'[+-]?(?:0|[1-9][0-9]*)\.0+', text):
        return str(int(text.split('.')[0]))
    return text


def _missing(value):
    if value is None or value is pd.NA:
        return True
    if isinstance(value, (float, np.floating)):
        return bool(np.isnan(value))
    return str(value).strip().casefold() in {'', 'nan', 'none', 'null', '<na>', 'nat'}


def _integers(value, name):
    try:
        result = np.asarray(value, dtype=float)
    except (TypeError, ValueError) as exc:
        raise ValueError(f'{name}: numeric integer values required') from exc
    if result.ndim != 1 or not np.isfinite(result).all() or (result != np.floor(result)).any():
        raise ValueError(f'{name}: finite one-dimensional integer values required')
    if ((result < np.iinfo(np.int64).min) | (result >= 2.**63)).any():
        raise ValueError(f'{name}: integer outside supported range')
    return result.astype(np.int64)


def semester_ordinal(code):
    """AQ year*10+term code -> globally comparable ordinal, term in 1,2,3."""
    number = int(_integers([code], 'target_NHHK semester')[0])
    year, term = divmod(number, 10)
    if year <= 0 or term not in (1, 2, 3):
        raise ValueError(f'Unparseable target_NHHK semester {code!r}; expected year*10+term (term 1,2,3)')
    return year * 3 + term - 1


def _semesters(frame, source):
    if 'target_NHHK' not in frame:
        raise ValueError(f'{source}: missing required target_NHHK column')
    codes = _integers(frame['target_NHHK'], f'{source}.target_NHHK')
    unique = {int(code): semester_ordinal(code) for code in np.unique(codes)}
    return np.asarray([str(int(c)) for c in codes], dtype=object), np.asarray([unique[int(c)] for c in codes], dtype=np.int64)


def _origin(order):
    return _EPOCH + pd.to_timedelta(order, unit='D')


def _labels(value, n, source):
    try:
        values = np.asarray(value, dtype=float)
    except (TypeError, ValueError) as exc:
        raise ValueError(f'{source}: binary ytr/yte labels required') from exc
    if values.shape != (n,) or not np.isfinite(values).all() or not np.isin(values, (0., 1.)).all():
        raise ValueError(f'{source}: binary labels must align to all canonical frame rows; failure=1')
    return values.astype(np.int8)


def _grades(frame, labels, source):
    if 'target_grade10' not in frame:
        return np.full(len(frame), np.nan)
    raw = frame['target_grade10']
    result = pd.to_numeric(raw, errors='coerce').to_numpy(dtype=float, na_value=np.nan)
    missing = np.asarray([_missing(v) for v in raw], dtype=bool)
    if (np.isnan(result) & ~missing).any():
        raise ValueError(f'{source}: target_grade10 has nonnumeric nonmissing values; provide numeric 0..10 grades')
    if np.isinf(result).any() or ((result < 0) | (result > 10)).any():
        raise ValueError(f'{source}: target_grade10 grade outside 0..10; missing grades must remain missing')
    known = np.isfinite(result)
    if labels is not None and (known & ((result < 5).astype(np.int8) != labels)).any():
        rows = np.flatnonzero(known & ((result < 5).astype(np.int8) != labels))[:8].tolist()
        raise ValueError(f'{source}: grade/failure consistency violated at rows {rows}; failure means grade < 5')
    return result


def _feature_names(cache, source):
    if 'feature_names' not in cache:
        raise ValueError(f'{source}: canonical cache missing feature_names')
    names = list(cache['feature_names'])
    if not names or any(not isinstance(n, str) or not n.strip() for n in names) or len(set(names)) != len(names):
        raise ValueError(f'{source}: feature_names must be nonempty unique strings')
    forbidden = set(names) & CSV_EXCLUDE
    if forbidden:
        raise ValueError(f'{source}: identity/target metadata found in feature_names: {sorted(forbidden)}')
    if any(n.startswith('course::') for n in names):
        raise ValueError(f'{source}: raw feature names may not use reserved course:: namespace')
    return names


def _matrix(cache, key, n, names, source, common=None):
    if key not in cache:
        raise ValueError(f'{source}: missing canonical {key}')
    matrix = np.asarray(cache[key])
    if matrix.ndim != 2 or matrix.shape != (n, len(names)):
        raise ValueError(f'{source}.{key}: full-frame-aligned shape ({n}, {len(names)}) required; got {matrix.shape}')
    if common is None:
        # Check shape without retaining or copying all raw matrix values in preparation.
        return None
    lookup = {name: i for i, name in enumerate(names)}
    try:
        result = np.asarray(matrix[:, [lookup[name] for name in common]], dtype=float)
    except (TypeError, ValueError, KeyError) as exc:
        raise ValueError(f'{source}.{key}: common raw features must be numeric and present') from exc
    if np.isinf(result).any():
        raise ValueError(f'{source}.{key}: infinite raw features; use finite numeric values or NaN')
    return result


def _numeric_column(frame, name, source):
    raw = frame[name]
    result = pd.to_numeric(raw, errors='coerce').to_numpy(dtype=float, na_value=np.nan)
    malformed = np.isnan(result) & ~np.asarray([_missing(v) for v in raw])
    if malformed.any() or np.isinf(result).any():
        raise ValueError(f'{source}.{name}: numeric values or missing values required')
    return result


def _static_features(cache, frame, key, names, source):
    result = np.full((len(frame), len(COURSE_STATIC)), np.nan)
    # Own canonical frame takes priority; own raw cache column is the only fallback.
    for j, name in enumerate(COURSE_STATIC):
        if name in frame:
            result[:, j] = _numeric_column(frame, name, source)
        elif name in names:
            try:
                result[:, j] = np.asarray(cache[key])[:, names.index(name)].astype(float)
            except (TypeError, ValueError) as exc:
                raise ValueError(f'{source}.{name}: raw static feature must be numeric') from exc
            if np.isinf(result[:, j]).any():
                raise ValueError(f'{source}.{name}: infinite static course feature')
    return result


def _identities(frame, source):
    for column in ('IDSV', 'IDMH'):
        if column not in frame:
            raise ValueError(f'{source}: missing required {column}; course_target is 17 numeric metadata, not course IDs')
    students = np.asarray([_canonical_id(v, f'{source}.IDSV') for v in frame['IDSV']], dtype=object)
    courses = np.asarray([_canonical_id(v, f'{source}.IDMH') for v in frame['IDMH']], dtype=object)
    semesters, orders = _semesters(frame, source)
    sections = np.asarray([
        _canonical_id(v, f'{source}.IDToHoc') if not _missing(v) else ''
        for v in frame['IDToHoc']
    ], dtype=object) if 'IDToHoc' in frame else np.full(len(frame), '', dtype=object)
    occurrences = defaultdict(int)
    ids, strategies = [], defaultdict(int)
    for i, (student, course, semester, section) in enumerate(zip(students, courses, semesters, sections)):
        key = (student, course, semester, section)
        occurrence = occurrences[key]
        occurrences[key] += 1
        for column in ('IDKQDK', 'sample_id'):
            if column in frame and not _missing(frame[column].iloc[i]):
                value = _canonical_id(frame[column].iloc[i], f'{source}.{column}')
                ids.append(column + '|' + quote(value, safe=''))
                strategies[column] += 1
                break
        else:
            ids.append('event|' + '|'.join(quote(v, safe='') for v in key) + f'|{occurrence}')
            strategies['student_course_semester_section_occurrence'] += 1
    if len(set(ids)) != len(ids):
        raise ValueError(f'{source}: duplicate stable row identity within canonical snapshot')
    grade_ids = np.asarray([
        None if _missing(v) else _canonical_id(v, f'{source}.target_IDCongDiem')
        for v in frame['target_IDCongDiem']
    ], dtype=object) if 'target_IDCongDiem' in frame else np.full(len(frame), None, dtype=object)
    return dict(row_id=np.asarray(ids, dtype=object), student_id=students, course_id=courses,
                semester=semesters, order=orders, section=sections, grade_record_id=grade_ids), dict(strategies)


def _coverage(cache, n, source, required=False):
    present = ('cf_covered_mask' in cache, 'cf_probabilities' in cache)
    if not any(present):
        if required:
            raise ValueError(f'{source}: sidecar/covered history requires cf_covered_mask and cf_probabilities')
        return np.zeros(n, dtype=bool)
    if not all(present):
        raise ValueError(f'{source}: provide both cf_covered_mask and cf_probabilities to validate finite CF coverage')
    mask = np.asarray(cache['cf_covered_mask'])
    if mask.shape != (n,) or not np.isin(mask, (False, True)).all():
        raise ValueError(f'{source}: cf_covered_mask must match all canonical train rows')
    try:
        probabilities = np.asarray(cache['cf_probabilities'], dtype=float)
    except (TypeError, ValueError) as exc:
        raise ValueError(f'{source}: cf_probabilities must be numeric') from exc
    if probabilities.ndim == 1:
        probabilities = probabilities[:, None]
    if probabilities.ndim != 2 or probabilities.shape[0] != n or probabilities.shape[1] < 1:
        raise ValueError(f'{source}: cf_probabilities must have canonical rows and at least one member column')
    finite = np.isfinite(probabilities)
    if ((probabilities[finite] < 0) | (probabilities[finite] > 1)).any():
        raise ValueError(f'{source}: finite CF probabilities outside [0,1]')
    return mask.astype(bool) & finite.all(axis=1)


def _indices(value, n, name):
    raw = np.asarray(value)
    if raw.dtype.kind == 'b':
        if raw.shape != (n,):
            raise ValueError(f'{name}: boolean index must match canonical rows')
        indices = np.flatnonzero(raw)
    else:
        indices = _integers(raw, name)
    if ((indices < 0) | (indices >= n)).any():
        raise ValueError(f'{name}: index outside [0,{n})')
    if len(set(indices.tolist())) != len(indices):
        raise ValueError(f'{name}: duplicate canonical row indices')
    return indices


def _course_array(value, n, name):
    try:
        result = np.asarray(value, dtype=float)
    except (TypeError, ValueError) as exc:
        raise ValueError(f'{name}: 17 numeric course metadata features required') from exc
    if result.shape != (n, len(COURSE_FEATURES)):
        raise ValueError(f'{name}: shape ({n},17) required, got {result.shape}; course_target is not a course ID')
    if np.isinf(result).any():
        raise ValueError(f'{name}: infinite course metadata values')
    return result


def _sidecar(path, covered, n_target, source):
    if path is None:
        return None
    structured = joblib.load(path)
    try:
        train_idx = _indices(structured['train_row_index'], len(covered), source + '.train_row_index')
        expected = np.flatnonzero(covered)
        if not np.array_equal(np.sort(train_idx), expected):
            raise ValueError(f'{source}: train_row_index must equal finite CF-covered canonical rows (cf_covered_mask AND all finite member probabilities)')
        train = _course_array(structured['train']['course_target'], len(train_idx), source + '.train.course_target')
        target = _course_array(structured['target']['course_target'], n_target, source + '.target.course_target')
        target_idx = _indices(structured.get('target_row_index', np.arange(n_target)), n_target, source + '.target_row_index')
        if len(target_idx) != n_target:
            raise ValueError(f'{source}.target_row_index: a full one-to-one permutation is required')
    except KeyError as exc:
        raise ValueError(f'{source}: missing proven Suite12 sidecar field {exc.args[0]}') from exc
    return train_idx, train, target_idx, target


@dataclass(frozen=True)
class _Source:
    faculty: str
    fold: int
    cache: Path | None
    train_csv: Path | None = None
    test_csv: Path | None = None
    structured: Path | None = None

    @property
    def name(self):
        return f'{self.faculty} fold {self.fold}'

    @property
    def files(self):
        return tuple(p for p in (self.cache, self.train_csv, self.test_csv, self.structured) if p is not None)


def _read_source(source):
    if source.cache is not None:
        cache = joblib.load(source.cache)
        if not isinstance(cache, dict):
            raise ValueError(f'{source.cache}: canonical cache must be a mapping')
        return cache, 0
    frames, dropped = [], 0
    for path in (source.train_csv, source.test_csv):
        frame = pd.read_csv(path, dtype=str, low_memory=False)
        if 'target_grade10' not in frame:
            raise ValueError(f'{path}: split CSV requires target_grade10; cannot infer grades from binary labels')
        grade = _grades(frame, None, str(path))
        valid = np.isfinite(grade)
        dropped += int((~valid).sum())
        # Match the established expanding-split loader: only absent targets drop rows.
        frame = frame.loc[valid].reset_index(drop=True)
        frames.append(frame)
    train, target = frames
    names = [c for c in train.columns if c not in CSV_EXCLUDE and c in target.columns]
    if not names:
        raise ValueError(f'{source.name}: no numeric predictors remain after established train_v3 exclusions')
    # Read IDs as strings, then recover numeric predictor types from training
    # values. Categorical vocabularies fit this historical train snapshot only;
    # target-only categories become all-zero, matching the predecessor encoder.
    numeric, categorical = [], []
    for name in names:
        parsed = pd.to_numeric(train[name],errors='coerce')
        malformed = parsed.isna() & ~train[name].map(_missing)
        (categorical if malformed.any() else numeric).append(name)
    matrices=[]
    for frame in frames:
        block=frame[numeric].apply(pd.to_numeric,errors='coerce').to_numpy(dtype=float)
        if np.isinf(block).any():raise ValueError(f'{source.name}: infinite numeric split predictor')
        matrices.append(block)
    encoded_names=[]
    if categorical:
        from sklearn.preprocessing import OneHotEncoder
        encoder=OneHotEncoder(handle_unknown='ignore',sparse_output=False,dtype=float)
        cat_train=train[categorical].astype('string').fillna('__MISSING__')
        cat_target=target[categorical].astype('string').fillna('__MISSING__')
        matrices[0]=np.column_stack([matrices[0],encoder.fit_transform(cat_train)])
        matrices[1]=np.column_stack([matrices[1],encoder.transform(cat_target)])
        encoded_names=encoder.get_feature_names_out(categorical).tolist()
    return dict(feature_names=numeric+encoded_names, Xtr=matrices[0], Xte=matrices[1],
                train_frame=train, test_frame=target,
                ytr=(_grades(train, None, source.name) < 5).astype(np.int8),
                yte=(_grades(target, None, source.name) < 5).astype(np.int8)), dropped


def _parts(cache, source):
    names = _feature_names(cache, source.name)
    output = []
    for mode, framekey, ykey, xkey in (('train', 'train_frame', 'ytr', 'Xtr'), ('target', 'test_frame', 'yte', 'Xte')):
        if framekey not in cache or not isinstance(cache[framekey], pd.DataFrame):
            raise ValueError(f'{source.name}: missing canonical DataFrame {framekey}')
        frame = cache[framekey].reset_index(drop=True)
        if not len(frame):
            raise ValueError(f'{source.name}.{mode}: nonempty canonical enrollment rows required')
        if ykey not in cache:
            raise ValueError(f'{source.name}: missing canonical {ykey}')
        y = _labels(cache[ykey], len(frame), source.name + '.' + ykey)
        grade = _grades(frame, y, source.name + '.' + mode)
        ids, strategies = _identities(frame, source.name + '.' + mode)
        _matrix(cache, xkey, len(frame), names, source.name)
        static = _static_features(cache, frame, xkey, names, source.name + '.' + mode)
        output.append(dict(frame=frame, y=y, grade=grade, ids=ids, strategies=strategies, static=static, xkey=xkey))
    train, target = output
    target_terms = np.unique(target['ids']['semester'])
    if len(target_terms) != 1:
        raise ValueError(f'{source.name}: target/test frame must contain exactly one target_NHHK semester')
    if train['ids']['order'].max() >= target['ids']['order'].min():
        raise ValueError(f'{source.name}: train contains current/future semester rows; require strictly earlier semesters')
    if set(train['ids']['row_id']) & set(target['ids']['row_id']):
        raise ValueError(f'{source.name}: train/target row identity overlap')
    return names, train, target


def _sha256(path):
    digest = hashlib.sha256()
    with path.open('rb') as handle:
        for block in iter(lambda: handle.read(1 << 20), b''):
            digest.update(block)
    return digest.hexdigest()


def _frame_hash(frame):
    # JSON fixes column order, preserves string IDs, and uses explicit nulls.
    return hashlib.sha256(frame.to_json(orient='split', date_format='iso', double_precision=15).encode()).hexdigest()


def _resolve(value, base):
    path = Path(value).expanduser()
    return (path if path.is_absolute() else base / path).resolve()


def _discover(cfg, base):
    root = _resolve(cfg.get('aq_root', '/root/disk_3tb/earlywarning/30072026'), base)
    cache_dir = _resolve(cfg['aq_cache_dir'], base) if cfg.get('aq_cache_dir') else root / 'results/cross_faculty_residual_moe/cache'
    structured_dir = _resolve(cfg['aq_structured_dir'], base) if cfg.get('aq_structured_dir') else root / 'results/suite12_structured_cache'
    folds_raw = cfg.get('aq_folds', list(range(1, 8)))
    if isinstance(folds_raw, str):
        folds_raw = [v.strip() for v in folds_raw.split(',') if v.strip()]
    folds = _integers(folds_raw, 'aq_folds').tolist()
    if not folds or any(f < 1 for f in folds) or len(set(folds)) != len(folds):
        raise ValueError('aq_folds must be nonempty unique positive integers')
    folds = sorted(folds)
    canonical = {}
    for path in sorted(cache_dir.glob('*_fold_*.joblib')):
        match = re.fullmatch(r'(.+)_fold_([1-9][0-9]*)\.joblib', path.name)
        if match and path.is_file():
            canonical[(match[1], int(match[2]))] = path
    if cfg.get('aq_splits_dir'):
        split_dir = _resolve(cfg['aq_splits_dir'], base)
    else:
        candidates = [root / 'data/splits/expanding', root.parent / 'data/splits/expanding',
                      root.parent.parent / 'data/splits/expanding']
        split_dir = next((p for p in candidates if p.is_dir()), candidates[0])
    csvs = {}
    for fold_path in sorted(split_dir.glob('fold_*')):
        match = re.fullmatch(r'fold_([1-9][0-9]*)', fold_path.name)
        if not match or not fold_path.is_dir():
            continue
        fold = int(match[1])
        for path in sorted(fold_path.glob('*_train.csv')):
            faculty = path.name[:-len('_train.csv')]
            target = fold_path / f'{faculty}_test.csv'
            if target.is_file():
                csvs[(faculty, fold)] = (path, target)
    available = sorted({f for f, k in set(canonical) | set(csvs) if k in folds})
    wanted = cfg.get('aq_faculties', 'all')
    if wanted is None or isinstance(wanted, str) and wanted.strip().lower() in ('', 'all', 'auto'):
        faculties = available
    elif isinstance(wanted, str):
        faculties = [v.strip() for v in wanted.split(',') if v.strip()]
    else:
        faculties = [_canonical_id(v, 'aq_faculties') for v in wanted]
    if not faculties or len(set(faculties)) != len(faculties):
        raise ValueError(f'No unique AQ faculties selected; searched {cache_dir} and {split_dir}')
    faculties = sorted(faculties)
    sources = []
    # Scan every available fold for selected faculties so full historical grades
    # do not depend on which supervised folds this invocation evaluates.
    for faculty, fold in sorted(set(canonical) | set(csvs), key=lambda x: (x[1], x[0])):
        if faculty not in faculties:
            continue
        path = canonical.get((faculty, fold))
        sidecar = structured_dir / f'{faculty}_fold_{fold}.joblib'
        if path is not None:
            sources.append(_Source(faculty, fold, path, structured=sidecar if sidecar.is_file() else None))
        else:
            tr, te = csvs[(faculty, fold)]
            sources.append(_Source(faculty, fold, None, tr, te))
    found = {(s.faculty, s.fold) for s in sources}
    missing = [(f, k) for k in folds for f in faculties if (f, k) not in found]
    if missing:
        raise FileNotFoundError(f'Missing native AQ faculty/folds {missing}; expected canonical {cache_dir}/FACULTY_fold_N.joblib or split {split_dir}/fold_N/FACULTY_train.csv and FACULTY_test.csv')
    requested = [s for s in sources if s.fold in folds]
    return sources, requested, faculties, folds, cache_dir, structured_dir, split_dir


def _merge_record(registry, event_id, identity, grade, label, credit, faculty, source, grade_record_id=None):
    record = registry.get(event_id)
    if record is None:
        registry[event_id] = dict(identity=identity, grade=float(grade), label=int(label),
                                  credit=float(credit), faculties={faculty}, grade_record_id=grade_record_id)
        return False
    old = record['identity']
    # Missing sections may coexist with the same stable registration identity.
    if old[:3] != identity[:3] or (old[3] and identity[3] and old[3] != identity[3]):
        raise ValueError(f'{source}: identity conflict across folds/faculties for {event_id}: {old} versus {identity}')
    if record['label'] != int(label):
        raise ValueError(f'{source}: label conflict across folds/faculties for {event_id}')
    if record['grade_record_id'] is not None and grade_record_id is not None and record['grade_record_id'] != grade_record_id:
        raise ValueError(f'{source}: grade record identity conflict for {event_id}')
    if record['grade_record_id'] is None:
        record['grade_record_id'] = grade_record_id
    if np.isfinite(record['grade']) and np.isfinite(grade) and not np.isclose(record['grade'], grade, rtol=0, atol=1e-9):
        raise ValueError(f'{source}: grade conflict across folds/faculties for {event_id}: {record["grade"]} versus {grade}')
    if np.isfinite(record['credit']) and np.isfinite(credit) and not np.isclose(record['credit'], credit, rtol=0, atol=1e-9):
        raise ValueError(f'{source}: credit conflict across folds/faculties for {event_id}')
    if not np.isfinite(record['grade']) and np.isfinite(grade):
        record['grade'] = float(grade)
    if not np.isfinite(record['credit']) and np.isfinite(credit):
        record['credit'] = float(credit)
    record['faculties'].add(faculty)
    return True


class _CourseHistory:
    """Compact course/semester cumulative sufficient statistics; strict as-of reads."""
    def __init__(self, records):
        grouped = defaultdict(lambda: defaultdict(lambda: np.zeros(5, dtype=float)))
        for record in records.values():
            _, course, semester, _ = record['identity']
            state = grouped[course][semester_ordinal(semester)]
            grade = record['grade']
            state[0] += 1
            state[1] += record['label']
            if np.isfinite(grade):
                state[2] += 1; state[3] += grade; state[4] += grade * grade
        self.courses = {}
        for course, semesters in grouped.items():
            orders = np.asarray(sorted(semesters), dtype=np.int64)
            self.courses[course] = (orders, np.cumsum(np.stack([semesters[int(o)] for o in orders]), axis=0))

    def for_rows(self, courses, orders):
        result = np.full((len(courses), len(COURSE_HIST)), np.nan)
        result[:, 0] = 0.
        # Cache repeated course/semester queries within the cohort only.
        memo = {}
        for i, (course, order) in enumerate(zip(courses, orders)):
            key = (course, int(order))
            if key not in memo:
                values = np.array([0., np.nan, np.nan, np.nan, np.nan])
                item = self.courses.get(course)
                if item is not None:
                    semesters, cumulative = item
                    j = np.searchsorted(semesters, order, side='left') - 1
                    if j >= 0:
                        n, failures, ng, total, squared = cumulative[j]
                        values = np.array([n, total / ng if ng else np.nan,
                                           np.sqrt(max(0., squared - total * total / ng) / ng) if ng else np.nan,
                                           failures / n, 1. - failures / n])
                memo[key] = values
            result[i] = memo[key]
        return result


def _normalize(part, source, raw, meta):
    ids = part['ids']
    origin = _origin(ids['order'])
    frame = pd.DataFrame({'row_id': ids['row_id'], 'student_id': ids['student_id'],
                          'course_id': ids['course_id'], 'semester': ids['semester'],
                          'faculty': source.faculty, 'label': part['y'],
                          'origin_at': origin, 'label_available_at': origin + pd.Timedelta(hours=12),
                          '_order': ids['order']})
    # Construct the feature block once to avoid dataframe fragmentation.
    for name, values in raw.items():
        frame[name] = values
    course = pd.DataFrame(meta, columns=['course::' + n for n in COURSE_FEATURES])
    return pd.concat([frame, course], axis=1)


def _load_cohort(source, common, history, support):
    cache, _ = _read_source(source)
    names, train, target = _parts(cache, source)
    covered = _coverage(cache, len(train['y']), source.name,
                        required=source.structured is not None or support == 'covered')
    sidecar = _sidecar(source.structured, covered, len(target['y']), source.name)
    frames = []
    for part in (train, target):
        ids = part['ids']
        meta = np.column_stack([part['static'], history.for_rows(ids['course_id'], ids['order'])])
        if sidecar is not None:
            indices, values = sidecar[:2] if part is train else sidecar[2:]
            meta[indices] = values
        values = _matrix(cache, part['xkey'], len(part['y']), names, source.name, common)
        raw = dict(zip(common, values.T))
        frames.append(_normalize(part, source, raw, meta))
    if support == 'covered':
        frames[0] = frames[0].loc[covered].reset_index(drop=True)
        if not len(frames[0]):
            raise ValueError(f'{source.name}: aq_history_support=covered selects no finite CF-covered rows; use full history support')
    return {'train': frames[0], 'test': frames[1]}


class LazyCohort(dict):
    """Dictionary metadata is cheap; matrices belong to this handle after access."""
    def __init__(self, metadata, loader):
        super().__init__(metadata)
        self._loader = loader

    def __getitem__(self, key):
        if key in ('train', 'test') and not dict.__contains__(self, key):
            self.update(self._loader())
            self._loader = None
        return dict.__getitem__(self, key)

    def get(self, key, default=None):
        if key in ('train', 'test'):
            return self[key]
        return dict.get(self, key, default)


class NativeCohorts(Sequence):
    """Each iteration makes fresh lazy handles and retains no loaded cohorts."""
    def __init__(self, sources, metadata, common, history, support):
        self._sources = tuple(sources)
        self._metadata = tuple(metadata)
        self._common = tuple(common)
        self._history = history
        self._support = support

    def __len__(self):
        return len(self._sources)

    def __getitem__(self, index):
        if isinstance(index, slice):
            return [self[i] for i in range(*index.indices(len(self)))]
        source = self._sources[index]
        return LazyCohort(dict(self._metadata[index]),
                          lambda: _load_cohort(source, self._common, self._history, self._support))

    def __iter__(self):
        for i in range(len(self)):
            yield self[i]


def load_native_inputs(cfg, base):
    """Return events, automatic calendar, lazy cohorts, fingerprints, source audit.

    Canonical files use the established feature_names/Xtr/Xte/ytr/yte and
    train_frame/test_frame schema. Missing canonical cohorts fall back to numeric
    expanding fold CSVs. Missing Suite12 sidecars reconstruct the canonical 17
    course features from own-row static inputs and strictly prior-semester global
    deduplicated training events. No upstream source is created or modified.
    """
    base = Path(base).resolve()
    support = cfg.get('aq_history_support', 'full')
    if support not in ('full', 'covered'):
        raise ValueError('aq_history_support must be full or covered')
    sources, requested, faculties, folds, cache_dir, structured_dir, split_dir = _discover(cfg, base)
    verbose=bool(cfg.get('aq_verbose',False))
    if verbose:print(f'AQ DISCOVERY faculties={len(faculties)} requested_cohorts={len(requested)} source_cohorts={len(sources)} cache={cache_dir} structured={structured_dir} splits={split_dir}',flush=True)
    selected = {(s.faculty, s.fold) for s in requested}
    common, meta_by_key = None, {}
    all_registry, train_registry = {}, {}
    semesters, target_by_fold = {}, {}
    fingerprints, cohort_audits = {}, []
    counts = defaultdict(int)
    strategy_counts = defaultdict(int)
    for position,source in enumerate(sources,1):
        for path in source.files:
            fingerprints['source:' + str(path.resolve())] = _sha256(path)
        cache, dropped = _read_source(source)
        counts['csv_rows_missing_target_dropped'] += dropped
        names, train, target = _parts(cache, source)
        is_requested = (source.faculty, source.fold) in selected
        if is_requested:
            common = names.copy() if common is None else [n for n in common if n in set(names)]
        covered = _coverage(cache, len(train['y']), source.name,
                            required=source.structured is not None or (is_requested and support == 'covered'))
        sidecar = _sidecar(source.structured, covered, len(target['y']), source.name)
        target_term = str(target['ids']['semester'][0])
        previous = target_by_fold.setdefault(source.fold, target_term)
        if previous != target_term:
            raise ValueError(f'All faculties must share the same global target semester in fold {source.fold}: {previous} versus {target_term}')
        for part in (train, target):
            for sem, order in zip(part['ids']['semester'], part['ids']['order']):
                semesters[str(sem)] = int(order)
            for strategy, count in part['strategies'].items():
                strategy_counts[strategy] += count
            ids = part['ids']
            for i, event_id in enumerate(ids['row_id']):
                identity = (ids['student_id'][i], ids['course_id'][i], ids['semester'][i], ids['section'][i])
                grade, label, credit = part['grade'][i], part['y'][i], part['static'][i, 0]
                if np.isfinite(credit) and credit < 0:
                    raise ValueError(f'{source.name}: SoTinChi credits must be nonnegative')
                grade_id = ids['grade_record_id'][i]
                counts['all_snapshot_duplicate_rows'] += _merge_record(all_registry, event_id, identity, grade, label, credit, source.faculty, source.name, grade_id)
                if part is train:
                    counts['history_snapshot_rows'] += 1
                    counts['history_duplicate_rows_removed'] += _merge_record(train_registry, event_id, identity, grade, label, credit, source.faculty, source.name, grade_id)
        target_order = int(target['ids']['order'][0])
        if is_requested:
            if support == 'covered' and not covered.any():
                raise ValueError(f'{source.name}: aq_history_support=covered selects no finite CF-covered rows; use full history support')
            meta_by_key[(source.faculty, source.fold)] = {
                'faculty': source.faculty, 'fold': source.fold,
                'target_semester': target_term, 'target_order': target_order,
                'origin_at': _origin(target_order),
                'train_rows': len(train['y']) if support == 'full' else int(covered.sum()),
                'test_rows': len(target['y']),
            }
        cohort_audits.append(dict(faculty=source.faculty, fold=source.fold, requested=is_requested,
                                 source_mode='canonical' if source.cache is not None else 'expanding_csv',
                                 train_rows=len(train['y']), target_rows=len(target['y']),
                                 history_grade_known_rows=int(np.isfinite(train['grade']).sum()),
                                 history_grade_missing_rows=int(np.isnan(train['grade']).sum()),
                                 finite_cf_covered_rows=int(covered.sum()),
                                 sidecar_present=sidecar is not None,
                                 preserved_sidecar_train_rows=int(covered.sum()) if sidecar is not None else 0,
                                 reconstructed_train_rows=int((~covered).sum()) if sidecar is not None else len(train['y']),
                                 reconstructed_target_rows=0 if sidecar is not None else len(target['y']),
                                 target_semester=target_term,
                                 target_alignment='validated sidecar order: canonical unless explicit target_row_index present' if sidecar is not None else 'canonical row order'))
        if verbose:print(f'AQ SCAN {position}/{len(sources)} {source.name} train={len(train["y"])} target={len(target["y"])} finite_CF={int(covered.sum())} sidecar={sidecar is not None} target_semester={target_term}',flush=True)
        # No raw matrices or frames are retained in the collection or audit.
        del cache, train, target, sidecar
    if not common:
        raise ValueError('No common raw features across all requested AQ faculty/folds')
    fold_orders = [semesters[target_by_fold[k]] for k in sorted(target_by_fold)]
    if any(a >= b for a, b in zip(fold_orders, fold_orders[1:])):
        raise ValueError('Fold numbers must strictly increase in global target semester order')
    # Test grades validate identity/label consistency above, but only train grades
    # populate this independent registry and the event corpus/course statistics.
    # A grade record can be referenced by several registration rows. Keep every
    # supervised row while counting its observed outcome only once in histories.
    grade_registry = {}
    for event_id, record in train_registry.items():
        key = ('grade|' + quote(record['grade_record_id'], safe='')) if record['grade_record_id'] is not None else event_id
        for faculty in sorted(record['faculties']):
            _merge_record(grade_registry, key, record['identity'], record['grade'], record['label'],
                          record['credit'], faculty, 'historical grade registry', record['grade_record_id'])
    counts['history_grade_identity_duplicates_removed'] = len(train_registry) - len(grade_registry)
    history = _CourseHistory(grade_registry)
    event_records = []
    for event_id, record in grade_registry.items():
        student, course, semester, _ = record['identity']
        memberships = sorted(record['faculties'])
        credit = record['credit']
        credits_missing = not np.isfinite(credit)
        if not np.isfinite(credit):
            credit = 0.
            counts['missing_credit_events_default_zero'] += 1
        origin = _origin(semesters[semester])
        event_records.append(dict(event_id=event_id, grade_record_id=record['grade_record_id'], failure_label=record['label'], student_id=student, course_id=course,
                                  semester=semester, grade=record['grade'], credits=credit, credits_missing=credits_missing,
                                  attempt=1, faculty=memberships[0],
                                  faculty_memberships=json.dumps(memberships, ensure_ascii=False),
                                  available_at=origin + pd.Timedelta(hours=12)))
    events = pd.DataFrame(event_records, columns=['event_id', 'grade_record_id', 'failure_label', 'student_id', 'course_id', 'semester', 'grade',
                                                 'credits', 'credits_missing', 'attempt', 'faculty', 'faculty_memberships', 'available_at'])
    if not len(events):
        raise ValueError('No native AQ training events found')
    # Stable attempt order uses semester, then registration identity for ties.
    attempt_order = events.assign(_ordinal=events.semester.map(semesters)).sort_values(
        ['student_id', 'course_id', '_ordinal', 'event_id'], kind='stable').index
    events.loc[attempt_order, 'attempt'] = events.loc[attempt_order].groupby(['student_id', 'course_id'], sort=False).cumcount().to_numpy() + 1
    events['attempt'] = events['attempt'].astype(np.int64)
    student_memberships = defaultdict(set)
    for record in all_registry.values():
        student_memberships[record['identity'][0]].update(record['faculties'])
    # Target identities are known registration metadata. They can join earlier
    # transcript records without making any target outcome an SSL event.
    events.attrs['student_faculty_memberships'] = {s:sorted(fs) for s,fs in sorted(student_memberships.items())}
    calendar = pd.DataFrame([dict(semester=sem, order=order, origin_at=_origin(order),
                                  labels_available_at=_origin(order) + pd.Timedelta(hours=12))
                             for sem, order in sorted(semesters.items(), key=lambda x: x[1])])
    features = list(common) + ['course::' + n for n in COURSE_FEATURES]
    metadata = []
    for source in requested:
        meta = meta_by_key[(source.faculty, source.fold)]
        meta['features'] = features.copy()
        metadata.append(meta)
    entries = NativeCohorts(requested, metadata, common, history, support)
    fingerprints['events'] = _frame_hash(events)
    fingerprints['eligible_student_memberships'] = hashlib.sha256(json.dumps(events.attrs['student_faculty_memberships'],sort_keys=True).encode()).hexdigest()
    fingerprints['calendar'] = _frame_hash(calendar)
    fingerprints['native_contract'] = hashlib.sha256(json.dumps(
        dict(faculties=faculties, folds=folds, history_support=support, features=features,
             assumption=AVAILABILITY_ASSUMPTION, timestamp_semantics=TIMESTAMP_SEMANTICS),
        sort_keys=True).encode()).hexdigest()
    audit = dict(input_mode='aq_native', history_support=support,
                 faculties=faculties, folds=folds, requested_cohorts=len(entries),
                 scanned_cohorts=len(sources), common_raw_features=common,
                 common_raw_feature_count=len(common), course_features=list(COURSE_FEATURES),
                 engineered_feature_count=len(features), history_events=len(events),
                 history_grade_known_events=int(events.grade.notna().sum()),
                 history_grade_missing_events=int(events.grade.isna().sum()),
                 missing_grades_excluded_from_ssl=int(events.grade.isna().sum()),
                 cross_faculty_events=sum(len(r['faculties']) > 1 for r in train_registry.values()),
                 target_grades_excluded_from_event_corpus=True,
                 event_source='full train_frame from every available fold for selected faculties; never test_frame grades',
                 id_policy='global normalized IDKQDK; sample_id fallback; deterministic student/course/semester/section/occurrence fallback',
                 id_strategy_snapshot_rows=dict(strategy_counts),
                 attempt_policy='1-based chronological event number per student/course; registration identity breaks within-semester ties',
                 course_statistics='strictly earlier semesters; deduplicated labeled-event count/failure rates; observed-grade mean and population std (ddof=0; singleton zero), matching preserved Suite17 sidecars',
                 availability_assumption=AVAILABILITY_ASSUMPTION, timestamp_semantics=TIMESTAMP_SEMANTICS,
                 availability_mode='semester_order', source_file_count=sum(k.startswith('source:') for k in fingerprints),
                 transcript_join_students=len(student_memberships),
                 cache_dir=str(cache_dir), structured_dir=str(structured_dir), splits_dir=str(split_dir),
                 cohorts=cohort_audits, **dict(counts))
    # Stable zero-count keys make machine-readable audits easy to consume.
    for key in ('csv_rows_missing_target_dropped', 'all_snapshot_duplicate_rows',
                'history_snapshot_rows', 'history_duplicate_rows_removed', 'missing_credit_events_default_zero'):
        audit.setdefault(key, 0)
    return events, calendar, entries, fingerprints, audit
