"""Train six Suite27 comparisons with supported earlier origins and hard-tree scores."""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
import resource
import shutil
import time

import numpy as np
import pandas as pd
import joblib
import torch

from data import Preprocessor, _semesters, build_history, common_feature_names, discover_faculties, load_fold
from evaluation import _check_threshold_chronology, add_references, metrics, write_report
from experiment import Config, VARIANTS, interventions, refit_variant, regimes, select_recipes
from protocol import UnsupportedOriginsError, choose_origins
from runtime import (atomic_csv, atomic_json, atomic_torch, code_hashes, faculty_support, file_hash, json_hash,
                     print_table, safe_json, score_frame, seed_all, versions)


REFERENCE_DIRECTORIES = {
    'suite14': 'suite14_authority_safe_collaborative_moe',
    'suite16': 'suite16_conditional_tree_boosting',
    'suite17': 'suite17_robust_forward_specialist_boosting',
    'suite18': 'suite18_fmm_forward_specialist_boosting',
    'suite20': 'suite20_semester_target_transformer',
    'suite21': 'suite21_guarded_temporal_residual',
    'suite22': 'suite22_sftr_tabm', 'suite22_primary': 'suite22_sftr_tabm',
    'suite22_relational': 'suite22_sftr_tabm',
    'suite23': 'suite23_cs_hagt', 'suite23_graph': 'suite23_cs_hagt',
    'suite24': 'suite24_target_conditioned_structural_residual',
    'suite24_posthoc': 'suite24_target_conditioned_structural_residual',
    'suite25': 'suite25_structure_aware_chronological_retrieval',
    'suite25_blend': 'suite25_structure_aware_chronological_retrieval',
    'suite26_mint': 'suite26_joint_tree_memory', 'suite26_memory': 'suite26_joint_tree_memory',
    'suite26_tree': 'suite26_joint_tree_memory',
    'suite26_original': 'suite26_tree_native_student_memory',
    'suite26_original_memory': 'suite26_tree_native_student_memory',
    'suite26_original_tree': 'suite26_tree_native_student_memory',
    'suite26_original_joint_head': 'suite26_tree_native_student_memory',
    'council': 'suite4/2_competence_council', 'v2': 'final_train_v2',
}


def _list(values, cast=str):
    return [cast(item.strip()) for value in values for item in str(value).split(',') if item.strip()]


def parser():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument('--root', default='/root/disk_3tb/earlywarning/30072026')
    p.add_argument('--cache-dir', '--cache_dir')
    p.add_argument('--structured-dir', '--structured_cache_dir')
    p.add_argument('--out-dir', '--out_dir')
    p.add_argument('--faculties', nargs='+', default=['all'])
    p.add_argument('--folds', nargs='+', default=['1', '2', '3', '4', '5', '6', '7'])
    p.add_argument('--dev-folds', nargs='+', default=['1', '2', '3', '4', '5'])
    p.add_argument('--holdout-folds', nargs='+', default=['6', '7'])
    p.add_argument('--seeds', nargs='+', default=['20261002'])
    p.add_argument('--seed', type=int, help='Single-seed alias, overrides --seeds')
    p.add_argument('--variants', nargs='+', default=list(VARIANTS))
    p.add_argument('--depth-grid', nargs='+', default=['3', '4'])
    p.add_argument('--tree-grid', '--trees-grid', nargs='+', default=['32', '64', '96', '128'])
    p.add_argument('--path-grid', nargs='+', default=['0', '1.5'])
    p.add_argument('--max-trees', type=int, default=128)
    p.add_argument('--bins', type=int, default=63)
    p.add_argument('--min-leaf', type=int, default=40)
    p.add_argument('--min-hessian', type=float, default=1.)
    p.add_argument('--l2', type=float, default=5.)
    p.add_argument('--learning-rate', type=float, default=.05)
    p.add_argument('--leaf-clip', type=float, default=2.)
    p.add_argument('--max-history', type=int, default=32)
    p.add_argument('--history-embargo', type=int, default=0)
    p.add_argument('--hidden', type=int, default=24)
    p.add_argument('--reader-epochs', type=int, default=3)
    p.add_argument('--reader-max-rows', type=int, default=120000)
    p.add_argument('--reader-batch', type=int, default=512)
    p.add_argument('--reader-lr', type=float, default=.001)
    p.add_argument('--adapt-epochs', type=int, default=1)
    p.add_argument('--adapt-max-rows', type=int, default=10000)
    p.add_argument('--adapt-batch', type=int, default=128)
    p.add_argument('--adapt-lr', type=float, default=.0002)
    p.add_argument('--adapt-trees', type=int, default=8)
    p.add_argument('--batch-size', type=int, default=2048)
    p.add_argument('--origins', '--n-origins', type=int, default=2)
    p.add_argument('--origin-min-rows', type=int, default=2000, help='Minimum TOTAL validation rows in an origin semester')
    p.add_argument('--origin-min-faculty-rows', type=int, default=30, help='Minimum rows per eligible validation faculty and its earlier training bank')
    p.add_argument('--origin-min-positive', '--origin-min-positives', type=int, default=5)
    p.add_argument('--origin-min-negative', '--origin-min-negatives', type=int, default=20)
    p.add_argument('--origin-min-train-rows', type=int, default=5000)
    p.add_argument('--origin-coverage', type=float, default=1.)
    p.add_argument('--reference-dirs', nargs='*', default=None, help='name=directory or known prior suite directory')
    p.add_argument('--bootstrap', type=int, default=200)
    p.add_argument('--threads', type=int, default=min(os.cpu_count() or 1, 8))
    p.add_argument('--device', default='cpu', help='cpu, cuda, cuda:0 or explicit auto selection')
    p.add_argument('--preflight-only', action='store_true', help='Validate all schemas, inputs and origin support; write no results')
    p.add_argument('--resume', action='store_true', help='Reuse only verified complete folds with identical manifest')
    p.add_argument('--no-diagnostics', '--skip-diagnostics', action='store_true')
    return p


def resolve_args(argv=None):
    p = parser()
    args = p.parse_args(argv)
    args.root = str(Path(args.root).expanduser().resolve())
    for name, default in [('cache_dir', 'results/cross_faculty_residual_moe/cache'),
                          ('structured_dir', 'results/suite12_structured_cache'),
                          ('out_dir', 'results/suite27_memory_conditioned_boosting')]:
        setattr(args, name, str(Path(getattr(args, name) or Path(args.root) / default).expanduser().resolve()))
    for name in ('folds', 'dev_folds', 'holdout_folds', 'seeds', 'depth_grid', 'tree_grid'):
        setattr(args, name, _list(getattr(args, name), int))
    args.path_grid = _list(args.path_grid, float)
    args.variants, args.faculties = _list(args.variants), _list(args.faculties)
    if args.seed is not None:
        args.seeds = [args.seed]
    args.seed = args.seeds[0] if args.seeds else 20261002
    for name in ('folds', 'seeds', 'variants', 'depth_grid', 'tree_grid', 'path_grid'):
        values = getattr(args, name)
        if not values or len(set(values)) != len(values):
            p.error(f'{name.replace("_", "-")} must be nonempty and unique')
    if any(fold < 1 for fold in args.folds) or min(args.seeds) < 0 or max(args.seeds) >= 2**32:
        p.error('folds must be positive; seeds must be in [0, 2**32)')
    if set(args.dev_folds) & set(args.holdout_folds):
        p.error('Development and holdout folds must be disjoint')
    if set(args.variants) - set(VARIANTS):
        p.error(f'variants must be chosen from {VARIANTS}')
    if args.faculties == ['all']:
        args.faculties = discover_faculties(args.cache_dir, args.folds[0])
    if not args.faculties or len(set(args.faculties)) != len(args.faculties):
        p.error('faculties must be nonempty and unique')
    for name in ('max_trees', 'bins', 'min_leaf', 'hidden', 'max_history', 'reader_max_rows', 'reader_batch',
                 'adapt_max_rows', 'adapt_batch', 'adapt_trees', 'batch_size', 'origins', 'threads',
                 'origin_min_rows', 'origin_min_faculty_rows', 'origin_min_train_rows'):
        if getattr(args, name) <= 0:
            p.error(f'{name.replace("_", "-")} must be positive')
    if min(args.depth_grid) < 1 or max(args.depth_grid) > 16 or min(args.tree_grid) < 1 or max(args.tree_grid) > args.max_trees:
        p.error('depth-grid must be in 1..16; tree-grid must be in 1..max-trees')
    if not 2 <= args.bins <= 255 or min(args.path_grid) < 0 or not all(np.isfinite(args.path_grid)):
        p.error('bins must be in 2..255 and path-grid finite/nonnegative')
    for name in ('history_embargo', 'reader_epochs', 'adapt_epochs', 'bootstrap', 'origin_min_positive', 'origin_min_negative', 'l2', 'min_hessian'):
        if getattr(args, name) < 0 or not np.isfinite(getattr(args, name)):
            p.error(f'{name.replace("_", "-")} must be finite and nonnegative')
    for name in ('reader_lr', 'adapt_lr', 'learning_rate', 'leaf_clip'):
        if not np.isfinite(getattr(args, name)) or getattr(args, name) <= 0:
            p.error(f'{name.replace("_", "-")} must be finite and positive')
    if not 0 < args.origin_coverage <= 1:
        p.error('origin-coverage must be in (0,1]')
    if args.device == 'auto':
        args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
    try:
        device = torch.device(args.device)
    except (RuntimeError, ValueError) as exc:
        p.error(str(exc))
    if device.type not in ('cpu', 'cuda') or (device.type == 'cuda' and not torch.cuda.is_available()):
        p.error('Requested device is unsupported or unavailable; explicitly use an available CPU/CUDA device')
    if args.reference_dirs is None:
        args.reference_dirs = {name: str(Path(args.root) / 'results' / directory)
                               for name, directory in REFERENCE_DIRECTORIES.items()
                               if (Path(args.root) / 'results' / directory).is_dir()}
    else:
        references = {}
        for item in args.reference_dirs:
            if '=' in item:
                name, directory = item.split('=', 1)
                if not name:
                    p.error('Reference names cannot be empty')
                references[name] = str(Path(directory).expanduser().resolve())
            else:
                directory = Path(item).expanduser().resolve()
                names = [name for name, known in REFERENCE_DIRECTORIES.items() if Path(known).name == directory.name]
                for name in names or [directory.name]:
                    references[name] = str(directory)
        args.reference_dirs = references
    return args


def configuration(args):
    return Config(**{name: tuple(getattr(args, name)) if name in ('depth_grid', 'tree_grid', 'path_grid', 'seeds', 'variants')
                    else getattr(args, name) for name in Config.__dataclass_fields__})


def common_features_for_run(args):
    ordered = common_feature_names(args.cache_dir, args.faculties, args.folds[0])
    shared = set(ordered)
    for fold in args.folds[1:]:
        shared.intersection_update(common_feature_names(args.cache_dir, args.faculties, fold))
    names = [name for name in ordered if name in shared]
    if not names:
        raise ValueError('No common numeric features across all requested faculties and folds')
    return names


def validate_threshold_chronology(args):
    """Read target time keys before any model selection/refit or result writing."""
    records = []
    for fold in args.folds:
        fold_semesters = []
        for faculty in args.faculties:
            path = Path(args.cache_dir) / f'{faculty}_fold_{fold}.joblib'
            cache = joblib.load(path)
            values = np.unique(_semesters(cache['test_frame'], str(path) + '.test_frame'))
            target_rows = len(cache['test_frame'])
            del cache
            if len(values) > 1:
                raise ValueError(f'Canonical target fold must contain exactly one semester: {path}')
            if len(values):
                fold_semesters.append(int(values[0]))
            records.append(dict(fold=fold, faculty=faculty, semester=int(values[0]) if len(values) else None,
                                target_rows=target_rows))
        if not fold_semesters:
            raise ValueError(f'Canonical target fold has no target rows: fold {fold}')
        if len(set(fold_semesters)) != 1:
            raise ValueError(f'Canonical target semester differs across faculties in fold {fold}')
    frame = pd.DataFrame(records, columns=['fold', 'faculty', 'semester', 'target_rows'])
    _check_threshold_chronology(frame[frame.target_rows > 0], args.dev_folds, args.holdout_folds)
    print_table('Operational target chronology, every requested fold/faculty', frame)
    return frame


def make_manifest(args, common):
    config = {key: value for key, value in vars(args).items() if key not in ('resume', 'preflight_only', 'out_dir')}
    inputs = {}
    for fold in args.folds:
        for faculty in args.faculties:
            for parent in (args.cache_dir, args.structured_dir):
                path = Path(parent) / f'{faculty}_fold_{fold}.joblib'
                if not path.is_file():
                    raise FileNotFoundError(f'Missing required input: {path}')
                inputs[str(path)] = dict(bytes=path.stat().st_size, sha256=file_hash(path))
    references = {}
    for name, root in args.reference_dirs.items():
        root = Path(root)
        paths = [root] if root.is_file() else sorted(root.rglob('*.csv'))
        references[name] = {str(path): dict(bytes=path.stat().st_size, sha256=file_hash(path)) for path in paths}
    code, runtime = code_hashes(), versions()
    return safe_json(dict(format_version=1, configuration=config, common_features=common, inputs=inputs,
                          reference_inputs=references, code=code, runtime=runtime,
                          configuration_sha256=json_hash(config), schema_sha256=json_hash(common),
                          code_sha256=json_hash(code), runtime_sha256=json_hash(runtime),
                          input_sha256=json_hash(dict(canonical=inputs, references=references))))


def origins_for_fold(args, history, target):
    semesters = np.unique(target.semester)
    if len(semesters) != 1:
        raise ValueError('Each canonical target fold must be exactly one semester')
    return choose_origins(history, int(semesters[0]), n_origins=args.origins,
                          min_rows=args.origin_min_rows, min_faculty_rows=args.origin_min_faculty_rows,
                          min_positive=args.origin_min_positive, min_negative=args.origin_min_negative,
                          min_train_rows=args.origin_min_train_rows, coverage=args.origin_coverage)


def validate_complete(folder, manifest):
    marker = folder / 'complete.json'
    completed = json.loads(marker.read_text())
    if completed.get('status') != 'complete' or completed.get('manifest_sha256') != json_hash(manifest) or not completed.get('files'):
        raise ValueError(f'Invalid or mismatched completion marker: {marker}')
    expected = completed['files']
    actual = {str(path.relative_to(folder)) for path in folder.rglob('*') if path.is_file() and path != marker}
    if set(expected) != actual:
        raise ValueError(f'Completed fold artifact inventory differs: {folder}')
    for name, digest in expected.items():
        path = folder / name
        if not path.is_file() or file_hash(path) != digest:
            raise ValueError(f'Completed fold has missing or modified artifact: {path}')


def _resource_row(fold, variant, seed, phase, started, device, checkpoint_bytes=0):
    return dict(fold=fold, variant=variant, seed=seed, phase=phase, seconds=time.perf_counter() - started,
                peak_host_rss_mb=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
                peak_host_rss_semantics='lifetime_process_peak', checkpoint_bytes=int(checkpoint_bytes),
                peak_cuda_mb=torch.cuda.max_memory_allocated(device) / 2**20 if str(device).startswith('cuda') else 0.)


def run_fold(args, fold, common, manifest):
    started = time.perf_counter()
    folder = Path(args.out_dir) / f'fold_{fold}'
    history, target, data_audit = load_fold(args.cache_dir, args.structured_dir, args.faculties, fold, common)
    support = faculty_support(target, args.faculties)
    print_table(f'F{fold} target support, every requested faculty', support)
    data_audit['target_faculty_support'] = support.to_dict('records')
    try:
        origins, audit = origins_for_fold(args, history, target)
    except UnsupportedOriginsError as exc:
        print_table(f'F{fold} failed origin support', exc.audit)
        folder.mkdir(parents=True, exist_ok=True)
        atomic_csv(folder / 'origin_support_failure.csv', exc.audit)
        raise
    print_table(f'F{fold} full origin support', audit)
    if folder.exists():
        shutil.rmtree(folder)
    folder.mkdir(parents=True)
    atomic_csv(folder / 'origin_support.csv', audit)
    atomic_csv(folder / 'target_support.csv', support)
    atomic_json(folder / 'data_audit.json', data_audit)
    cfg = configuration(args)
    selection_started = time.perf_counter()
    recipes = select_recipes(history, origins, audit, cfg, folder)
    resource_rows = [_resource_row(fold, 'all', '', 'selection', selection_started, args.device)]
    frame = score_frame(target, fold)
    reader_log = pd.read_csv(folder / 'reader_training.csv').to_dict('records')
    adapt_log = pd.read_csv(folder / 'adaptation.csv').to_dict('records')
    forest_log = pd.read_csv(folder / 'forest_training.csv').to_dict('records')
    seed_frames, tree_summaries, tree_nodes, diagnostic_rows = [], [], [], []
    regime_rows, intervention_rows, intervention_scores = [], [], []
    scores = {variant: [] for variant in args.variants}
    for seed in args.seeds:
        seed_frame = score_frame(target, fold)
        for variant in args.variants:
            refit_started = time.perf_counter()
            if str(args.device).startswith('cuda'):
                torch.cuda.reset_peak_memory_stats(args.device)
            seed_all(seed)
            bundle, score, artifacts = refit_variant(history, target, variant, recipes[variant], cfg, seed)
            scores[variant].append(np.asarray(score, float))
            score_column = f'suite27_{variant}'
            seed_frame[score_column] = score
            provenance = dict(fold=fold, faculties=args.faculties, common_features=common,
                              schema_sha256=manifest['schema_sha256'], code_hashes=manifest['code'],
                              runtime=manifest['runtime'], configuration=manifest['configuration'],
                              input_fingerprints=manifest['inputs'], manifest_sha256=json_hash(manifest))
            bundle['provenance'] = provenance
            checkpoint_path = folder / f'model_{variant}_seed_{seed}.pt'
            atomic_torch(checkpoint_path, bundle)
            keyed = score_frame(target, fold, {score_column: score})
            keyed['seed'] = seed
            atomic_csv(folder / f'predictions_{variant}_seed_{seed}.csv', keyed)
            context = dict(fold=fold, variant=variant, seed=seed, phase='refit')
            reader_log.extend(dict(row, **context) for row in artifacts.get('reader_log', []))
            adapt_log.extend(dict(row, **context) for row in artifacts.get('adapt_log', []))
            forest_log.extend(dict(row, **context) for row in artifacts.get('forest_log', []))
            tree_summaries.append(dict(artifacts.get('tree_summary', {}), **context))
            nodes = artifacts.get('tree_nodes', pd.DataFrame()).copy()
            for key, value in context.items():
                nodes[key] = value
            tree_nodes.append(nodes)
            if not args.no_diagnostics:
                model, provider = artifacts['model'], artifacts['provider']
                diagnostics = artifacts.get('memory_diagnostics', pd.DataFrame()).copy()
                if len(diagnostics):
                    diagnostics['sample_id'] = target.sample_id
                    for key, value in context.items():
                        diagnostics[key] = value
                    diagnostic_rows.append(diagnostics)
                regime_rows.append(regimes(target, score, provider, variant, seed).assign(fold=fold))
                intervention, changed = interventions(model, provider, target.y, variant, seed)
                intervention_rows.append(intervention.assign(fold=fold))
                if len(changed):
                    for key in ('sample_id', 'student_id', 'course_id', 'semester', 'actual', 'faculty'):
                        if key not in changed:
                            changed[key] = keyed[key].to_numpy()
                    intervention_scores.append(changed.assign(fold=fold))
            resource_rows.append(_resource_row(fold, variant, seed, 'refit_and_diagnostics', refit_started, args.device,
                                               checkpoint_bytes=checkpoint_path.stat().st_size))
            print_table(f'F{fold} {variant} seed {seed} target faculty metrics',
                        pd.DataFrame([dict(fold=fold, faculty=faculty, variant=variant, seed=seed,
                                           **metrics(keyed.loc[keyed.faculty == faculty, 'actual'],
                                                     keyed.loc[keyed.faculty == faculty, score_column]))
                                      for faculty in args.faculties]))
            # Artifact dataframes/logs are retained above; release prepared bank,
            # provider and model references before the next variant is fitted.
            del artifacts, bundle
            if not args.no_diagnostics:
                del model, provider
        seed_frame['seed'] = seed
        atomic_csv(folder / f'predictions_seed_{seed}.csv', seed_frame)
        seed_frames.append(seed_frame)
    for variant, values in scores.items():
        frame[f'suite27_{variant}'] = np.mean(values, axis=0)
        frame[f'seed_sd_{variant}'] = np.std(values, axis=0)
    seed_predictions = pd.concat(seed_frames, ignore_index=True)
    atomic_csv(folder / 'seed_predictions.csv', seed_predictions)
    atomic_csv(folder / 'predictions.csv', frame)
    for faculty, part in frame.groupby('faculty', sort=True):
        atomic_csv(folder / f'{faculty}_scores.csv', part)
        atomic_csv(folder / f'{faculty}_fold_{fold}_scores.csv', part)
    log_tables = dict(reader_training=pd.DataFrame(reader_log), adaptation=pd.DataFrame(adapt_log),
                      forest_training=pd.DataFrame(forest_log),
                      tree_summary=pd.DataFrame(tree_summaries), tree_nodes=_concat(tree_nodes),
                      memory_diagnostics=_concat(diagnostic_rows), regime_metrics=_concat(regime_rows),
                      intervention_metrics=_concat(intervention_rows), intervention_predictions=_concat(intervention_scores),
                      resource_timing=pd.DataFrame(resource_rows))
    for name, table in log_tables.items():
        if len(table.columns) == 0:
            table = pd.DataFrame(columns=['fold', 'variant', 'seed', 'status'])
        atomic_csv(folder / f'{name}.csv', table)
        if name in ('resource_timing', 'reader_training', 'adaptation', 'regime_metrics', 'intervention_metrics', 'tree_summary'):
            print_table(f'F{fold} {name}', table)
    atomic_json(folder / 'fold_summary.json', dict(fold=fold, origins=origins, recipes=recipes,
                                                  seeds=args.seeds, variants=args.variants,
                                                  seconds=time.perf_counter() - started,
                                                  history_rows=len(history), target_rows=len(target)))
    files = {str(path.relative_to(folder)): file_hash(path) for path in sorted(folder.rglob('*')) if path.is_file()}
    atomic_json(folder / 'complete.json', dict(status='complete', manifest_sha256=json_hash(manifest), files=files))
    return frame


def _concat(frames):
    return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()


def main(argv=None):
    args = resolve_args(argv)
    configuration(args)
    torch.set_num_threads(args.threads)
    seed_all(args.seed)
    common = common_features_for_run(args)
    validate_threshold_chronology(args)
    print('Suite27: primary scores are intercept plus hard-tree leaf increments.', flush=True)
    print(f'Device={args.device}; faculties={args.faculties}; folds={args.folds}; variants={args.variants}; seeds={args.seeds}', flush=True)
    if args.preflight_only:
        make_manifest(args, common)
        for fold in args.folds:
            history, target, _ = load_fold(args.cache_dir, args.structured_dir, args.faculties, fold, common)
            print_table(f'PREFLIGHT F{fold} target support, every requested faculty', faculty_support(target, args.faculties))
            try:
                origins, audit = origins_for_fold(args, history, target)
            except UnsupportedOriginsError as exc:
                print_table(f'PREFLIGHT F{fold} failed origin support', exc.audit)
                raise
            print_table(f'PREFLIGHT F{fold} full origin support', audit)
            pre = Preprocessor.fit(history, np.arange(len(history)))
            pre.transform(history)
            pre.transform(target)
            build_history(history, target, max_history=args.max_history, embargo=args.history_embargo)
            print(f'[PREFLIGHT F{fold}] OK: {len(history):,} history, {len(target):,} target, origins={origins}', flush=True)
        print('Preflight complete; no result files created.', flush=True)
        return
    output = Path(args.out_dir)
    if output.exists() and any(output.iterdir()) and not args.resume:
        raise FileExistsError(f'Result directory is nonempty: {output}. Choose a new --out-dir or use --resume')
    manifest = make_manifest(args, common)
    manifest_path = output / 'run_manifest.json'
    if args.resume and output.exists() and any(output.iterdir()):
        if not manifest_path.is_file() or json.loads(manifest_path.read_text()) != manifest:
            raise ValueError('Resume configuration, source code, runtime, schema or input fingerprints do not match; use a new output directory')
    output.mkdir(parents=True, exist_ok=True)
    atomic_json(manifest_path, manifest)
    frames = []
    for fold in args.folds:
        folder = output / f'fold_{fold}'
        if args.resume and (folder / 'complete.json').is_file():
            validate_complete(folder, manifest)
            print(f'[RESUME F{fold}] Verified complete fold; reusing keyed predictions.', flush=True)
            frames.append(pd.read_csv(folder / 'predictions.csv', dtype={'sample_id': str, 'student_id': str, 'course_id': str, 'faculty': str}))
        else:
            frames.append(run_fold(args, fold, common, manifest))
    frame = pd.concat(frames, ignore_index=True)
    frame, reference_audit = add_references(frame, args.reference_dirs)
    atomic_csv(output / 'reference_alignment.csv', reference_audit)
    seed_frames = [pd.read_csv(output / f'fold_{fold}' / 'seed_predictions.csv', dtype={'sample_id': str, 'student_id': str, 'course_id': str, 'faculty': str}) for fold in args.folds]
    seed_predictions = pd.concat(seed_frames, ignore_index=True)
    atomic_csv(output / 'seed_predictions.csv', seed_predictions)
    variation = pd.DataFrame([dict(seed=seed, fold=fold, faculty=faculty, model=f'suite27_{variant}',
                                   **metrics(part.actual, part[f'suite27_{variant}']))
                              for (seed, fold, faculty), part in seed_predictions.groupby(['seed', 'fold', 'faculty'], sort=True)
                              for variant in args.variants])
    atomic_csv(output / 'seed_variation.csv', variation)
    print_table('Seed variation, every faculty/fold/seed', variation)
    report = write_report(frame, output, dev_folds=args.dev_folds, holdout_folds=args.holdout_folds,
                          bootstrap=args.bootstrap, seed=args.seed)
    print(f'Complete. Reports and reloadable checkpoints: {output}', flush=True)
    return report


if __name__ == '__main__':
    main()
