"""Explicit CSV and trusted AQ-cache adapters for Suite35 PACE.

No adapter infers numeric grades from labels, target courses from feature values,
row positions from lengths, or publication timestamps from semester names.
Joblib deserialization executes Python: inspect/export only trusted files.
"""
from __future__ import annotations

import hashlib
import json
import re
import tempfile
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any

import joblib
import numpy as np
import pandas as pd


class AdapterError(ValueError):
    """A source cannot be normalized without an explicit mapping or correction."""


_TRUST_WARNING = 'Joblib can execute Python code; load only files you trust.'
_EVENT_REQUIRED = ('event_id', 'student_id', 'course_id', 'semester', 'grade', 'credits', 'available_at')
_DEFAULT_KEYS = {name: name for name in (
    'Xtr', 'Xte', 'ytr', 'yte', 'train_frame', 'test_frame', 'feature_names', 'cf_covered_mask')}
_DEFAULT_COLUMNS = {'student_id': 'IDSV', 'course_id': 'IDMH', 'semester': 'target_NHHK'}
_RESERVED = {'row_id', 'student_id', 'course_id', 'semester', 'faculty', 'fold', 'split',
             'origin_at', 'label_available_at', 'label'}


def _config(path: str | Path) -> tuple[dict, Path]:
    path = Path(path).expanduser().resolve()
    try:
        value = json.loads(path.read_text(encoding='utf-8'))
    except (OSError, ValueError) as exc:
        raise AdapterError(f'Cannot read JSON configuration {path}: {exc}') from exc
    if not isinstance(value, dict):
        raise AdapterError(f'{path}: configuration must be a JSON object')
    return value, path.parent


def _path(value: str | Path, base: Path) -> Path:
    p = Path(value).expanduser()
    return p.resolve() if p.is_absolute() else (base / p).resolve()


def _hash(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open('rb') as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b''):
            digest.update(chunk)
    return digest.hexdigest()


def _write_json(path: Path, data: dict) -> None:
    path.write_text(json.dumps(data, indent=2, sort_keys=True, allow_nan=False) + '\n', encoding='utf-8')


def _read_csv(spec: dict, base: Path) -> tuple[pd.DataFrame, Path]:
    if not isinstance(spec, dict) or not spec.get('path'):
        raise AdapterError('CSV source requires an explicit path')
    path = _path(spec['path'], base)
    options = dict(spec.get('read_csv', {}))
    # Strings preserve leading zeroes and avoid treating identifiers as numbers.
    # Numeric grade/credit/calendar fields are converted only after mapping.
    dtype = options.get('dtype', 'string')
    if dtype not in ('str', 'string', 'object', str, object) and not isinstance(dtype, dict):
        raise AdapterError('read_csv.dtype must preserve string identifiers (use "string")')
    if isinstance(dtype, dict) and any(v not in ('str', 'string', 'object', str, object) for v in dtype.values()):
        raise AdapterError('read_csv.dtype must preserve string identifiers; numeric conversion happens after mapping')
    if options.get('converters'):
        raise AdapterError('read_csv.converters are not supported: preserve source identifiers as strings')
    # A partial dtype dict allows unspecified IDs to be inferred numerically.
    # Read all source fields as strings; numeric conversion is canonical and audited.
    options['dtype'] = 'string'
    options.setdefault('keep_default_na', False)
    if options.get('parse_dates'):
        raise AdapterError('Map timestamp columns explicitly; do not use read_csv.parse_dates')
    try:
        frame = pd.read_csv(path, **options)
    except Exception as exc:
        raise AdapterError(f'Cannot read CSV {path}: {exc}') from exc
    if frame.columns.duplicated().any():
        raise AdapterError(f'{path}: duplicate column names are not allowed')
    return frame, path


def _required_columns(frame: pd.DataFrame, columns: Sequence[str], context: str) -> None:
    missing = [c for c in columns if c not in frame]
    if missing:
        raise AdapterError(f'{context}: missing columns {missing}; available columns: {list(frame.columns)}')


def _strings(values: Any, name: str, *, allow_missing: bool = False) -> pd.Series:
    result = pd.Series(values).reset_index(drop=True).astype('string')
    result = result.mask(result.str.strip().eq(''), pd.NA)
    if not allow_missing and result.isna().any():
        raise AdapterError(f'{name}: {int(result.isna().sum())} missing identifiers/values')
    return result


def _numbers(values: Any, name: str, low: float, high: float | None = None,
             *, allow_missing: bool = False) -> pd.Series:
    raw = pd.Series(values).reset_index(drop=True)
    empty = raw.isna() | raw.astype('string').str.strip().eq('').fillna(False)
    result = pd.to_numeric(raw.mask(empty), errors='coerce').astype(float)
    invalid = (~empty & result.isna()) | (~np.isfinite(result) & ~empty)
    if not allow_missing:
        invalid |= empty
    invalid |= result.lt(low)
    if high is not None:
        invalid |= result.gt(high)
    if invalid.any():
        interval = f'[{low}, {high}]' if high is not None else f'>= {low}'
        raise AdapterError(f'{name}: {int(invalid.sum())} invalid values; expected numeric {interval}; never infer grades from binary labels')
    return result


def _timestamps(values: Any, name: str, fmt: str | None = None,
                timezone: str | None = None) -> pd.Series:
    raw = pd.Series(values).reset_index(drop=True)
    normalized = []
    for position, value in enumerate(raw):
        if fmt is None and isinstance(value, str) and not re.match(r'^\d{4}-\d{2}-\d{2}(?:$|[T ])', value):
            raise AdapterError(f'{name}: non-ISO timestamp at row position {position}; explicitly declare timestamp_format so date ordering is not guessed')
        try:
            parsed = pd.to_datetime(value, errors='raise', format=fmt or 'mixed')
            if pd.isna(parsed):
                raise ValueError('missing timestamp')
        except (ValueError, TypeError, OverflowError) as exc:
            raise AdapterError(f'{name}: missing/invalid timestamp at row position {position}; map actual availability explicitly') from exc
        if parsed.tzinfo is None:
            if not timezone:
                raise AdapterError(f'{name}: naive timestamp at row position {position}; explicitly declare timestamp_timezone or supply timezone-aware source timestamps')
            try:
                parsed = parsed.tz_localize(timezone, ambiguous='raise', nonexistent='raise')
            except Exception as exc:
                raise AdapterError(f'{name}: cannot apply declared timestamp_timezone {timezone!r} at row position {position}') from exc
        normalized.append(parsed.tz_convert('UTC').isoformat())
    return pd.Series(normalized, dtype='string')


def extract_events(mapping_path: str | Path, output_path: str | Path) -> dict:
    """Normalize raw numeric-grade events using explicit columns and stable joins.

    Paths in the JSON mapping are relative to that JSON. Joins must declare
    left_on/right_on and validate="many_to_one"; left joins retain every event.
    Missing numeric grades are retained by default, or explicitly excluded.
    Strict availability is the default. Explicit calendar_embargo mode omits
    available_at entirely; the runner applies the declared calendar proxy.
    Writes the CSV and a sibling ``<name>.audit.json`` and returns that audit.
    """
    cfg, base = _config(mapping_path)
    availability_mode = cfg.get('availability_mode', 'strict')
    if availability_mode not in ('strict', 'calendar_embargo'):
        raise AdapterError('availability_mode must be "strict" or explicitly "calendar_embargo"; availability timestamps are never guessed')
    columns = dict(cfg.get('columns', {}))
    ignored_available_mapping = availability_mode == 'calendar_embargo' and 'available_at' in columns
    if availability_mode == 'calendar_embargo':
        # The runner constructs the conservative, labeled proxy. Omitting the
        # field prevents entry time from masquerading as certified publication.
        columns.pop('available_at', None)
    required = _EVENT_REQUIRED if availability_mode == 'strict' else tuple(
        name for name in _EVENT_REQUIRED if name != 'available_at')
    missing = [name for name in required if not columns.get(name)]
    if missing:
        raise AdapterError(f'columns must explicitly map canonical fields: {missing}')
    if any(name not in _EVENT_REQUIRED + ('faculty', 'attempt') for name in columns):
        raise AdapterError('Supported event fields are event_id, student_id, course_id, semester, grade, credits, available_at, faculty, attempt')
    frame, source = _read_csv(cfg.get('source'), base)
    input_rows = len(frame)
    joins = []
    for index, spec in enumerate(cfg.get('joins', [])):
        if spec.get('validate') != 'many_to_one':
            raise AdapterError(f'join {index}: explicitly set validate="many_to_one"')
        left_on, right_on = spec.get('left_on'), spec.get('right_on')
        if isinstance(left_on, str):
            left_on = [left_on]
        if isinstance(right_on, str):
            right_on = [right_on]
        if not left_on or not right_on or len(left_on) != len(right_on):
            raise AdapterError(f'join {index}: declare equally sized left_on and right_on keys')
        if spec.get('how', 'left') != 'left':
            raise AdapterError(f'join {index}: only left joins preserve the event cohort; filter explicitly upstream')
        right, right_path = _read_csv(spec, base)
        _required_columns(frame, left_on, f'join {index} left')
        _required_columns(right, right_on, f'join {index} right')
        if right[right_on].isna().any().any() or right[right_on].eq('').any().any():
            raise AdapterError(f'join {index}: right keys may not be missing')
        indicator = f'__pace_join_{index}'
        try:
            frame = frame.merge(right, how='left', left_on=left_on, right_on=right_on,
                                validate='many_to_one', sort=False, indicator=indicator,
                                suffixes=tuple(spec.get('suffixes', ['', '_lookup'])))
        except (ValueError, pd.errors.MergeError) as exc:
            raise AdapterError(f'join {index}: many_to_one validation failed: {exc}') from exc
        unmatched = int(frame[indicator].eq('left_only').sum())
        frame = frame.drop(columns=indicator)
        if len(frame) != input_rows:
            raise AdapterError(f'join {index}: row count changed unexpectedly')
        joins.append({'path': str(right_path), 'rows': len(right), 'unmatched_event_rows': unmatched,
                      'sha256': _hash(right_path)})
    _required_columns(frame, list(columns.values()), 'event columns')
    output = pd.DataFrame({name: frame[source_name].reset_index(drop=True)
                           for name, source_name in columns.items()})
    for name in ('event_id', 'student_id', 'course_id', 'semester'):
        output[name] = _strings(output[name], name)
    if output.event_id.duplicated().any():
        raise AdapterError('event_id must uniquely identify a registration/attempt; duplicates require an explicit source correction')
    if 'faculty' in output:
        output['faculty'] = _strings(output['faculty'], 'faculty')
    output['grade'] = _numbers(output['grade'], 'grade', 0, 10, allow_missing=True)
    output['credits'] = _numbers(output['credits'], 'credits', 0)
    if 'attempt' in output:
        output['attempt'] = _numbers(output['attempt'], 'attempt', 1)
        if (output.attempt.dropna() % 1 != 0).any():
            raise AdapterError('A supplied attempt column must contain positive integers with no missing values')
    if availability_mode == 'strict':
        output['available_at'] = _timestamps(output['available_at'], 'available_at', cfg.get('timestamp_format'),
                                              cfg.get('timestamp_timezone'))
    missing_grade = int(output.grade.isna().sum())
    policy = cfg.get('missing_grade', 'retain')
    if policy not in ('retain', 'exclude'):
        raise AdapterError('missing_grade must be "retain" or "exclude"')
    if policy == 'exclude':
        output = output.loc[output.grade.notna()].reset_index(drop=True)
    destination = Path(output_path).expanduser().resolve()
    destination.parent.mkdir(parents=True, exist_ok=True)
    output.to_csv(destination, index=False)
    audit = {'adapter': 'raw_events', 'source': str(source), 'source_sha256': _hash(source),
             'input_rows': input_rows, 'output_rows': len(output), 'missing_grade_rows': missing_grade,
             'missing_grade_policy': policy, 'excluded_missing_grade_rows': missing_grade if policy == 'exclude' else 0,
             'joins': joins, 'output': str(destination), 'grade_semantics': 'explicit numeric grade; no binary-label inference',
             'availability_mode': availability_mode,
             'ignored_available_at_mapping': ignored_available_mapping,
             'availability_semantics': (
                 'explicit source available_at; no semester proxy inference' if availability_mode == 'strict'
                 else 'explicit calendar_embargo proxy; available_at omitted; runner must apply declared calendar embargo; no grade-entry timestamp is repurposed')}
    _write_json(destination.with_suffix('.audit.json'), audit)
    return audit


def resolve_key_path(source: Any, key_path: str) -> Any:
    """Resolve dotted mapping paths, preferring an exact literal dotted key.

    Errors expose available key names and types, never student data values.
    """
    if not isinstance(key_path, str) or not key_path:
        raise AdapterError('A nonempty explicit key path is required')
    current, remainder, traversed = source, key_path, []
    while True:
        if not isinstance(current, Mapping):
            raise AdapterError(f'Key path {key_path!r}: {".".join(traversed) or "<root>"} is {type(current).__name__}, expected mapping; available keys: []')
        if remainder in current:
            return current[remainder]
        head, separator, tail = remainder.partition('.')
        if head not in current:
            raise AdapterError(f'Key path {key_path!r}: missing {head!r}; available keys: {sorted(map(str, current.keys()))}')
        current = current[head]
        traversed.append(head)
        if not separator:
            return current
        remainder = tail


def _schema(value: Any, depth: int = 0) -> dict:
    result: dict[str, Any] = {'type': type(value).__name__}
    if isinstance(value, pd.DataFrame):
        result.update(shape=list(value.shape), columns=[str(c) for c in value.columns],
                      dtypes={str(c): str(dtype) for c, dtype in value.dtypes.items()})
    elif isinstance(value, pd.Series):
        result.update(shape=list(value.shape), dtype=str(value.dtype), name=str(value.name))
    elif isinstance(value, np.ndarray):
        result.update(shape=list(value.shape), dtype=str(value.dtype))
    elif isinstance(value, Mapping):
        result['keys'] = [str(key) for key in value]
        if depth < 12:
            result['children'] = {str(key): _schema(child, depth + 1) for key, child in value.items()}
        else:
            result['children_omitted'] = 'maximum schema depth 12'
    elif isinstance(value, (list, tuple)):
        result['shape'] = [len(value)]
        # No sequence items are printed: they may be student IDs or labels.
        result['item_types'] = sorted({type(item).__name__ for item in value})
    elif hasattr(value, 'shape'):
        result['shape'] = list(value.shape)
        result['dtype'] = str(getattr(value, 'dtype', 'unknown'))
    return result


def inspect_sources(paths: Sequence[str | Path] | str | Path) -> dict:
    """Return only keys, shapes and schemas; never print individual row values.

    Calling this on joblibs is an explicit request to deserialize trusted Python
    artifacts. CSV inspection reads headers only. The caller renders the report.
    """
    if isinstance(paths, (str, Path)):
        paths = [paths]
    report = {'warning': _TRUST_WARNING, 'sources': []}
    for value in paths:
        path = Path(value).expanduser().resolve()
        try:
            if path.suffix.lower() in ('.joblib', '.pkl', '.pickle'):
                schema = _schema(joblib.load(path))
            elif path.suffix.lower() == '.csv':
                schema = _schema(pd.read_csv(path, nrows=0))
                schema['shape'][0] = None  # Header-only inspection does not claim zero rows.
            elif path.suffix.lower() == '.json':
                schema = _schema(json.loads(path.read_text(encoding='utf-8')))
            else:
                raise AdapterError(f'Unsupported source type {path.suffix!r}; inspect .csv, .json or trusted .joblib')
        except AdapterError:
            raise
        except Exception as exc:
            raise AdapterError(f'Cannot inspect {path}: {type(exc).__name__}; verify the file and trusted loader') from exc
        report['sources'].append({'path': str(path), 'schema': schema})
    return report


def _load_joblib(path: Path) -> Any:
    try:
        return joblib.load(path)
    except Exception as exc:
        raise AdapterError(f'Cannot load trusted cache {path}: {type(exc).__name__}: {exc}') from exc


def _frame(value: Any, name: str) -> pd.DataFrame:
    if not isinstance(value, pd.DataFrame):
        raise AdapterError(f'{name}: expected pandas DataFrame, got {type(value).__name__}; configure its exact key path')
    if value.columns.duplicated().any():
        raise AdapterError(f'{name}: duplicate column names')
    return value.reset_index(drop=True).copy()


def _vector(value: Any, name: str) -> np.ndarray:
    result = np.asarray(value)
    if result.ndim == 2 and result.shape[1] == 1:
        result = result[:, 0]
    if result.ndim != 1:
        raise AdapterError(f'{name}: expected a vector, found shape {result.shape}; configure the exact column key path')
    return result


def _features(value: Any, names: list[str], name: str) -> np.ndarray:
    if isinstance(value, pd.DataFrame) and list(value.columns) != names:
        raise AdapterError(f'{name}: feature column order differs from feature_names; expected {names}, found {list(value.columns)}')
    array = np.asarray(value)
    if array.ndim != 2 or array.shape[1] != len(names):
        raise AdapterError(f'{name}: expected 2D features with {len(names)} columns, found shape {array.shape}')
    if array.dtype.kind not in 'biuf':
        raise AdapterError(f'{name}: numeric features required; found dtype {array.dtype}; encode categorical features upstream and declare the exact ordered feature_names')
    if np.isinf(array).any():
        raise AdapterError(f'{name}: infinite feature values are not supported; correct preprocessing upstream')
    return array


def _feature_names(value: Any, context: str) -> list[str]:
    names = list(_vector(value, context))
    if not names or any(not isinstance(name, str) or not name for name in names):
        raise AdapterError(f'{context}: expected a nonempty ordered vector of feature-name strings')
    if len(names) != len(set(names)) or set(names) & _RESERVED:
        raise AdapterError(f'{context}: names must be unique and may not collide with canonical enrollment columns')
    return names


def _labels(value: Any, expected: int, name: str) -> np.ndarray:
    array = _vector(value, name)
    if len(array) != expected:
        raise AdapterError(f'{name}: expected {expected} labels under declared coverage, found {len(array)}')
    if array.dtype.kind not in 'biuf' or not np.isin(array, [0, 1]).all():
        raise AdapterError(f'{name}: expected observed binary failure labels 0/1; grades are never inferred from labels')
    return array.astype(np.int64)


def _calendar(spec: dict, base: Path) -> tuple[pd.DataFrame, Path]:
    raw, source = _read_csv(spec, base)
    names = {name: name for name in ('semester', 'order', 'origin_at', 'labels_available_at')}
    names.update(spec.get('columns', {}))
    _required_columns(raw, list(names.values()), 'calendar')
    result = pd.DataFrame({name: raw[col] for name, col in names.items()})
    result['semester'] = _strings(result.semester, 'calendar semester')
    result['order'] = _numbers(result.order, 'calendar order', 0)
    if (result.order % 1 != 0).any() or result.order.duplicated().any() or result.semester.duplicated().any():
        raise AdapterError('calendar semester/order must be unique; order must contain integers and no ties')
    result['order'] = result.order.astype(np.int64)
    for name in ('origin_at', 'labels_available_at'):
        result[name] = _timestamps(result[name], f'calendar {name}', spec.get('timestamp_format'),
                                   spec.get('timestamp_timezone'))
    result = result.sort_values('order').reset_index(drop=True)
    origins = pd.to_datetime(result.origin_at, utc=True)
    if not origins.is_monotonic_increasing or origins.duplicated().any():
        raise AdapterError('calendar origin_at must increase strictly with order')
    if (pd.to_datetime(result.labels_available_at, utc=True) <= origins).any():
        raise AdapterError('calendar labels_available_at must strictly follow origin_at')
    return result, source


def _course_targets(frame: pd.DataFrame, positions: np.ndarray, sidecar_targets: Any | None,
                    sidecar_positions: np.ndarray | None, column: str, name: str) -> pd.Series:
    if column in frame:
        values = _strings(frame[column], f'{name} course_id', allow_missing=True)
    else:
        values = pd.Series(pd.NA, index=np.arange(len(frame)), dtype='string')
    if sidecar_targets is not None:
        targets = _strings(_vector(sidecar_targets, f'{name} course_target'), f'{name} sidecar course_target')
        assert sidecar_positions is not None
        if len(targets) != len(sidecar_positions):
            raise AdapterError(f'{name} course_target: {len(targets)} rows but {len(sidecar_positions)} explicitly indexed rows')
        previous = values.iloc[sidecar_positions].reset_index(drop=True)
        mismatch = previous.notna() & previous.ne(targets)
        if mismatch.any():
            raise AdapterError(f'{name}: sidecar course_target disagrees with frame {column} at {int(mismatch.sum())} explicit row positions')
        values.iloc[sidecar_positions] = targets.to_numpy()
    chosen = values.iloc[positions].reset_index(drop=True)
    if chosen.isna().any():
        raise AdapterError(f'{name}: {int(chosen.isna().sum())} missing target courses. Map columns.course_id or configure an aligned sidecar; full Xtr requires target courses for uncovered rows too')
    return chosen


def _normalize_rows(frame: pd.DataFrame, positions: np.ndarray, x: np.ndarray, y: np.ndarray,
                    targets: pd.Series, names: list[str], columns: dict, cfg: dict,
                    faculty: str, fold: int, split: str, calendar: pd.DataFrame) -> pd.DataFrame:
    selected = frame.iloc[positions].reset_index(drop=True)
    _required_columns(selected, [columns['student_id'], columns['semester']], f'{split}_frame')
    result = pd.DataFrame({
        'student_id': _strings(selected[columns['student_id']], f'{split} student_id'),
        'course_id': targets,
        'semester': _strings(selected[columns['semester']], f'{split} semester'),
        'faculty': faculty, 'fold': fold, 'split': split, 'label': y,
    })
    row_column = columns.get('row_id')
    if row_column:
        _required_columns(selected, [row_column], f'{split} row_id')
    elif 'row_id' in selected:
        row_column = 'row_id'
    if row_column:
        result['row_id'] = _strings(selected[row_column], f'{split} row_id')
    else:
        identity = cfg.get('identity_fields')
        if not isinstance(identity, list) or not identity or len(identity) != len(set(identity)):
            raise AdapterError('Without a provided row_id column, declare explicit nonempty unique identity_fields; no row identity is guessed')
        parts = []
        for field in identity:
            if field in result:
                parts.append(_strings(result[field], f'identity {field}'))
            elif field in selected:
                parts.append(_strings(selected[field], f'identity {field}'))
            else:
                raise AdapterError(f'identity_fields: missing {field!r}; available canonical fields {list(result.columns)}; frame columns {list(selected.columns)}')
        result['row_id'] = ['aq_' + hashlib.sha256(json.dumps(
            {'fields': identity, 'values': list(values)}, ensure_ascii=False, separators=(',', ':')).encode('utf-8')).hexdigest()
            for values in zip(*parts)]
    if result.row_id.duplicated().any():
        raise AdapterError(f'{faculty} fold {fold} {split}: duplicate row_id; include a stable registration/attempt identity')
    index = calendar.set_index('semester')
    missing = sorted(set(result.semester) - set(index.index))
    if missing:
        raise AdapterError(f'{split}: semesters absent from required calendar: {missing}')
    result['origin_at'] = result.semester.map(index.origin_at)
    result['label_available_at'] = result.semester.map(index.labels_available_at)
    result = result[['row_id', 'student_id', 'course_id', 'semester', 'faculty', 'fold', 'split',
                     'origin_at', 'label_available_at', 'label']]
    for i, name in enumerate(names):
        result[name] = x[:, i]
    return result


def export_aq(config_path: str | Path, output_dir: str | Path) -> dict:
    """Export each trusted AQ faculty/fold snapshot with exact numeric features.

    Required JSON fields: trusted_joblib=true, faculties, folds,
    xtr_coverage="full"|"covered", calendar.path. ``key_paths`` overrides the
    default cache keys. An optional sidecar_pattern requires all three explicit
    sidecar_key_paths: train_course_target, test_course_target, train_row_index.
    Sidecar train indices must equal flatnonzero(cf_covered_mask), in order.
    ``columns`` maps canonical frame identifiers; without row_id, declare
    identity_fields. Full and covered input conventions are never inferred.
    An optional ``structured_features`` object must explicitly enable appending
    sidecar matrices and declare train_key_path, test_key_path and
    feature_names_key_path. Uncovered full-frame rows receive missing values.
    """
    cfg, config_base = _config(config_path)
    if cfg.get('trusted_joblib') is not True:
        raise AdapterError(f'Set trusted_joblib=true only after verifying cache provenance. {_TRUST_WARNING}')
    base = _path(cfg.get('root', '.'), config_base)
    coverage = cfg.get('xtr_coverage')
    if coverage not in ('full', 'covered'):
        raise AdapterError('Explicit xtr_coverage="full" or "covered" is required; shapes alone do not establish row alignment')
    ycoverage = cfg.get('ytr_coverage', coverage)
    if ycoverage not in ('full', 'covered'):
        raise AdapterError('ytr_coverage must be "full" or "covered"')
    if coverage == 'full' and ycoverage == 'covered':
        raise AdapterError('Full Xtr cannot use covered-only ytr: provide full observed labels or export covered Xtr explicitly')
    faculties, folds = cfg.get('faculties'), cfg.get('folds')
    if not isinstance(faculties, list) or not faculties or len(set(faculties)) != len(faculties):
        raise AdapterError('faculties must explicitly list unique faculty filename values')
    if any(not isinstance(f, str) or not re.fullmatch(r'[A-Za-z0-9_-]+', f) for f in faculties):
        raise AdapterError('faculty filename values must contain only letters, digits, underscore or hyphen')
    if not isinstance(folds, list) or not folds or any(type(f) is not int or f < 1 for f in folds) or len(set(folds)) != len(folds):
        raise AdapterError('folds must explicitly list unique positive integers')
    if not isinstance(cfg.get('calendar'), dict):
        raise AdapterError('A required calendar CSV must explicitly supply semester, order, origin_at, labels_available_at')
    calendar, calendar_source = _calendar(cfg['calendar'], base)
    key_paths = dict(_DEFAULT_KEYS)
    key_paths.update(cfg.get('key_paths', {}))
    unknown = set(key_paths) - set(_DEFAULT_KEYS)
    if unknown:
        raise AdapterError(f'Unknown key_paths entries {sorted(unknown)}; expected {sorted(_DEFAULT_KEYS)}')
    columns = dict(_DEFAULT_COLUMNS)
    columns.update(cfg.get('columns', {}))
    sidecar_pattern = cfg.get('sidecar_pattern')
    skeys = cfg.get('sidecar_key_paths', {})
    if sidecar_pattern and any(not skeys.get(key) for key in ('train_course_target', 'test_course_target', 'train_row_index')):
        raise AdapterError('sidecar_pattern requires explicit sidecar_key_paths for train_course_target, test_course_target, train_row_index')
    structured = cfg.get('structured_features', {'enabled': False})
    if not isinstance(structured, dict) or type(structured.get('enabled')) is not bool:
        raise AdapterError('structured_features must explicitly declare enabled=true or enabled=false')
    if structured['enabled']:
        if not sidecar_pattern:
            raise AdapterError('structured_features requires an explicit trusted sidecar_pattern with validated train_row_index')
        for key in ('train_key_path', 'test_key_path', 'feature_names_key_path'):
            if not structured.get(key):
                raise AdapterError(f'structured_features requires explicit {key}; inspect the sidecar schema first')
        expected_structured = structured.get('expected_count')
        if expected_structured is not None and (type(expected_structured) is not int or expected_structured < 1):
            raise AdapterError('structured_features.expected_count must be a positive integer when declared')
    expected_total = cfg.get('expected_total_features')
    if expected_total is not None and (type(expected_total) is not int or expected_total < 1):
        raise AdapterError('expected_total_features must be a positive integer when declared')
    pattern = cfg.get('cache_pattern', 'results/cross_faculty_residual_moe/cache/{faculty}_fold_{fold}.joblib')
    destination = Path(output_dir).expanduser().resolve()
    destination.parent.mkdir(parents=True, exist_ok=True)
    with tempfile.TemporaryDirectory(prefix='pace-aq-', dir=destination.parent) as staging_name:
        stage_dir = Path(staging_name)
        staged, audits = [], []
        for faculty in faculties:
            for fold in folds:
                try:
                    cache_path = _path(pattern.format(faculty=faculty, fold=fold), base)
                except (ValueError, KeyError) as exc:
                    raise AdapterError(f'Invalid cache_pattern: {exc}') from exc
                cache = _load_joblib(cache_path)
                values = {key: resolve_key_path(cache, path) for key, path in key_paths.items()}
                train = _frame(values['train_frame'], 'train_frame')
                test = _frame(values['test_frame'], 'test_frame')
                names = _feature_names(values['feature_names'], 'feature_names')
                base_feature_count = len(names)
                xtr = _features(values['Xtr'], names, 'Xtr')
                xte = _features(values['Xte'], names, 'Xte')
                mask = _vector(values['cf_covered_mask'], 'cf_covered_mask')
                if len(mask) != len(train) or mask.dtype.kind not in 'biu' or not np.isin(mask, [False, True]).all():
                    raise AdapterError('cf_covered_mask must be a boolean/0-1 vector aligned to every train_frame row')
                covered = np.flatnonzero(mask.astype(bool))
                train_positions = np.arange(len(train)) if coverage == 'full' else covered
                if len(xtr) != len(train_positions):
                    raise AdapterError(f'Xtr: xtr_coverage={coverage!r} requires {len(train_positions)} rows; found {len(xtr)}')
                if len(xte) != len(test):
                    raise AdapterError(f'Xte must align with every test_frame row: {len(xte)} versus {len(test)}')
                ytr = _labels(values['ytr'], len(train) if ycoverage == 'full' else len(covered), 'ytr')
                if coverage == 'covered' and ycoverage == 'full':
                    ytr = ytr[covered]
                yte = _labels(values['yte'], len(test), 'yte')
                sidecar_path, target_train, target_test, sidecar = None, None, None, None
                if sidecar_pattern:
                    try:
                        sidecar_path = _path(sidecar_pattern.format(faculty=faculty, fold=fold), base)
                    except (ValueError, KeyError) as exc:
                        raise AdapterError(f'Invalid sidecar_pattern: {exc}') from exc
                    sidecar = _load_joblib(sidecar_path)
                    indices = _vector(resolve_key_path(sidecar, skeys['train_row_index']), 'train_row_index')
                    if indices.dtype.kind not in 'iu' or not np.array_equal(indices, covered):
                        raise AdapterError('sidecar train_row_index must equal flatnonzero(cf_covered_mask) exactly, including order; no row guessing')
                    target_train = resolve_key_path(sidecar, skeys['train_course_target'])
                    target_test = resolve_key_path(sidecar, skeys['test_course_target'])
                structured_count, structured_missing_rows = 0, 0
                if structured['enabled']:
                    structured_names = _feature_names(resolve_key_path(sidecar, structured['feature_names_key_path']),
                                                       'structured feature_names')
                    if set(names) & set(structured_names):
                        raise AdapterError('structured feature names overlap canonical feature_names; do not append features already present in Xtr/Xte')
                    structured_count = len(structured_names)
                    if structured.get('expected_count') is not None and structured_count != structured['expected_count']:
                        raise AdapterError(f'structured_features expected_count={structured["expected_count"]}, found {structured_count}')
                    extra_train = _features(resolve_key_path(sidecar, structured['train_key_path']),
                                            structured_names, 'structured train')
                    extra_test = _features(resolve_key_path(sidecar, structured['test_key_path']),
                                           structured_names, 'structured test')
                    if len(extra_train) != len(covered):
                        raise AdapterError(f'structured train: expected {len(covered)} rows aligned to validated train_row_index, found {len(extra_train)}')
                    if len(extra_test) != len(test):
                        raise AdapterError(f'structured test: expected {len(test)} rows aligned to test_frame, found {len(extra_test)}')
                    if coverage == 'full':
                        aligned_train = np.full((len(train), structured_count), np.nan, dtype=np.float64)
                        aligned_train[covered] = extra_train
                        structured_missing_rows = len(train) - len(covered)
                    else:
                        aligned_train = extra_train
                    xtr = np.concatenate([xtr, aligned_train], axis=1)
                    xte = np.concatenate([xte, extra_test], axis=1)
                    names = names + structured_names
                if expected_total is not None and len(names) != expected_total:
                    raise AdapterError(f'expected_total_features={expected_total}, found {len(names)} after explicitly configured feature assembly; existing complete caches must disable structured append')
                train_targets = _course_targets(train, train_positions, target_train, covered if sidecar_pattern else None,
                                                 columns['course_id'], 'train')
                test_positions = np.arange(len(test))
                test_targets = _course_targets(test, test_positions, target_test, test_positions if sidecar_pattern else None,
                                                columns['course_id'], 'test')
                normalized_train = _normalize_rows(train, train_positions, xtr, ytr, train_targets, names, columns,
                                                    cfg, faculty, fold, 'train', calendar)
                normalized_test = _normalize_rows(test, test_positions, xte, yte, test_targets, names, columns,
                                                   cfg, faculty, fold, 'test', calendar)
                if set(normalized_train.row_id) & set(normalized_test.row_id):
                    raise AdapterError(f'{faculty} fold {fold}: train/test row_id overlap; correct registration identities')
                stem = f'{faculty}_fold_{fold}'
                entry = {'faculty': faculty, 'fold': fold, 'train_path': stem + '_train.csv',
                         'test_path': stem + '_test.csv', 'features': names}
                staged.append(entry)
                # Seventeen digits also preserve the represented float32 value
                # when a downstream CSV reader promotes it to float64.
                normalized_train.to_csv(stage_dir / entry['train_path'], index=False, float_format='%.17g')
                normalized_test.to_csv(stage_dir / entry['test_path'], index=False, float_format='%.17g')
                audits.append({'faculty': faculty, 'fold': fold, 'cache': str(cache_path),
                               'cache_sha256': _hash(cache_path), 'sidecar': str(sidecar_path) if sidecar_path else None,
                               'sidecar_sha256': _hash(sidecar_path) if sidecar_path else None,
                               'xtr_coverage': coverage, 'ytr_coverage': ycoverage, 'train_frame_rows': len(train),
                               'covered_train_rows': len(covered), 'uncovered_train_rows': int(len(train) - len(covered)),
                               'excluded_uncovered_rows': int(len(train) - len(train_positions)),
                               'train_rows': len(normalized_train), 'test_rows': len(normalized_test),
                               'base_feature_count': base_feature_count,
                               'structured_feature_count': structured_count,
                               'structured_missing_train_rows': structured_missing_rows,
                               'feature_count': len(names)})
        # Stage one fold at a time, keeping memory bounded and validating every
        # source before making any normalized output visible.
        calendar.to_csv(stage_dir / 'calendar.csv', index=False)
        manifest = {'format': 'suite35_pace_cohort_manifest_v1', 'calendar': 'calendar.csv',
                    'entries': staged}
        _write_json(stage_dir / 'manifest.json', manifest)
        _write_json(stage_dir / 'schema.json', {'format': manifest['format'],
                                                'canonical_columns': sorted(_RESERVED),
                                                'label_semantics': 'provided binary failure label; 1=failure, 0=non-failure',
                                                'feature_order': 'exact per-entry features list in manifest.json',
                                                'grade_source': 'not present; extract numeric historical grades separately'})
        audit = {'adapter': 'aq', 'warning': _TRUST_WARNING, 'integration_status': 'validated against configured files only; no claim of raw AQ source compatibility',
                 'calendar_source': str(calendar_source), 'calendar_sha256': _hash(calendar_source),
                 'manifest': str(destination / 'manifest.json'), 'entries': audits}
        _write_json(stage_dir / 'audit.json', audit)
        destination.mkdir(parents=True, exist_ok=True)
        for staged_file in sorted(stage_dir.iterdir()):
            if staged_file.name != 'manifest.json':
                staged_file.replace(destination / staged_file.name)
        (stage_dir / 'manifest.json').replace(destination / 'manifest.json')
        return audit
