#!/usr/bin/env python3
"""Suite32: Chronology-validated directed faculty borrowing and native-data recovery audit.

All learned policies are fit using origins older than the OUTER target. No Council, Committee,
Suite22 or Suite25 probabilities are consumed by the learned Suite32 models.
"""
from __future__ import annotations
import argparse,json,os,platform,sys,time,traceback,hashlib
from pathlib import Path
import numpy as np
import pandas as pd
from s32_data import names_in,load_fold,status_audit,feature_ledger,student_overlap_ledger,note,stable_seed
from s32_model import fit_weighted,predict
from s32_policy import donor_grid,replay_policy,select_final,language_donors,random_sources
from s32_metrics import score,macro,paired,operational,summary_lines

ROOT='/root/disk_3tb/earlywarning/30072026'

def parse():
    p=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description='Suite32 SELECT — selective historical borrowing + faculty-native information controls')
    p.add_argument('--stage',choices=['preflight','development','retrospective','all','report'],default='preflight')
    p.add_argument('--root',default=ROOT);p.add_argument('--cache-dir');p.add_argument('--structured-dir');p.add_argument('--output-dir')
    p.add_argument('--faculties',default='all');p.add_argument('--folds',default=None)
    p.add_argument('--seed',type=int,default=42)
    p.add_argument('--max-iter',type=int,default=100);p.add_argument('--min-samples-leaf',type=int,default=20)
    p.add_argument('--rho',type=float,default=.25,help='Total normalized mass allocated to donated history')
    p.add_argument('--donor-budget',type=int,default=4096,help='Cap on total nonlocal rows in each fit; per-source divided among donors')
    p.add_argument('--selection-origins',type=int,default=4)
    p.add_argument('--min-history',type=int,default=100);p.add_argument('--min-history-failures',type=int,default=5)
    p.add_argument('--min-validation',type=int,default=30);p.add_argument('--min-validation-failures',type=int,default=2)
    p.add_argument('--min-origins',type=int,default=3);p.add_argument('--min-replays',type=int,default=1)
    p.add_argument('--min-delta',type=float,default=.001);p.add_argument('--min-win-rate',type=float,default=.5)
    p.add_argument('--min-replay-delta',type=float,default=0.);p.add_argument('--max-replay-harm',type=float,default=.01)
    p.add_argument('--max-selected',type=int,default=3)
    p.add_argument('--release-gap',type=int,default=0,help='Calendar-semester proxy release gap (NOT verified publication timestamps)')
    p.add_argument('--mask-audit',action='store_true',help='Run cache-NaN/zero/placebo indicator ablations; do not claim true provenance')
    p.add_argument('--max-targets',type=int,default=0,help='Smoke/pilot only: limit target faculties; not valid all-faculty benchmark')
    p.add_argument('--max-donor-candidates',type=int,default=0,help='Reserved; full directed grid is mandatory by default')
    p.add_argument('--force',action='store_true',help='Replace saved fold CSV files')
    return p.parse_args()

def paths(a):
    root=Path(a.root)
    a.cache_dir=str(Path(a.cache_dir or root/'results/cross_faculty_residual_moe/cache'))
    a.structured_dir=str(Path(a.structured_dir or root/'results/suite12_structured_cache'))
    a.output_dir=str(Path(a.output_dir or root/'results/suite32_selective_borrowing'))
    return a

def selected_folds(a):
    if a.folds:return [int(i) for i in a.folds.split(',') if i.strip()]
    return {'preflight':[1,2,3,4,5,6,7],'development':[1,2,3,4,5],'retrospective':[6,7],
            'all':[1,2,3,4,5,6,7],'report':[]}[a.stage]

def atomic_csv(df,path):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
    temp=path.with_suffix(path.suffix+'.tmp');df.to_csv(temp,index=False);temp.replace(path)

def save_json(d,path):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
    tmp=path.with_suffix(path.suffix+'.tmp')
    tmp.write_text(json.dumps(d,indent=2,default=lambda x:x.item() if isinstance(x,np.generic) else str(x)))
    tmp.replace(path)

def preds_to_metrics(preds):
    rows=[]
    if len(preds)==0:return pd.DataFrame()
    for (fold,fac,model),v in preds.groupby(['fold','faculty','model'],sort=True):
        rows.append({'fold':int(fold),'faculty':fac,'model':model,**score(v.y.to_numpy(),v.p.to_numpy())})
    return pd.DataFrame(rows)

def make_fit(data,target,name,config,origin,donors=(),representation='common',mask_mode=None,rho=0.,equal=False):
    cap=max(1,config.donor_budget//max(1,len(donors))) if donors else 0
    m,mat,diag=fit_weighted(data,target,origin,donors,rho,representation=representation,
       mask_mode=mask_mode,max_donor_rows=cap,seed=config.seed,
       max_iter=config.max_iter,min_samples_leaf=config.min_samples_leaf,
       release_gap=config.release_gap,donor_weight_mode='size' if name=='hgb_pool_all' else ('equal' if equal else 'provided'))
    prob=predict(m,mat,data.target[target])
    diag.update({'fold':data.fold,'model':name,'faculty':target,'prediction_seconds':0.})
    return prob,diag

def one_faculty(data,target,a):
    start=time.monotonic(); origin=int(min(data.target[target].semester))
    edges,origins=donor_grid(data,target,a)
    replay=replay_policy(data,target,edges,origins,a)
    selected,reason=select_final(data,target,edges,origins,replay,a)
    sel=[x for x,w in selected]
    note('SELECTION_FINAL',fold=data.fold,faculty=target,selected=sel,reason=reason.replace(' ','_'),
         replay_mean=float(replay.delta_ap.mean()) if not replay.empty else float('nan'))
    all_donors=[(f,1.) for f in data.faculties() if f!=target]
    langs=[(f,1.) for f in language_donors(target,data.faculties())]
    rand=random_sources(data,target,len(selected) if selected else 1,a.seed)
    experiments=[
      ('hgb_local_common',[],0.,'common',None,False),
      ('hgb_local_native',[],0.,'native',None,False),
      ('hgb_pool_all',all_donors,a.rho,'common',None,False),
      ('hgb_pool_equal',all_donors,a.rho,'common',None,True),
      ('hgb_language_prior',langs,a.rho if langs else 0.,'common',None,True),
      ('hgb_random_donor',rand,a.rho,'common',None,True),
      ('suite32_selective_common',selected,a.rho if selected else 0.,'common',None,False),
      ('suite32_selective_native',selected,a.rho if selected else 0.,'native',None,False),
    ]
    if a.mask_audit:
        experiments += [('native_cache_nan_indicator',[],0.,'native','observed',False),
                        ('native_naive_zero_indicator',[],0.,'native','naive_zero',False),
                        ('native_placebo_indicator',[],0.,'native','placebo',False),
                        ('suite32_selective_native_cache_nan',selected,a.rho if selected else 0.,'native','observed',False)]
    target_part=data.target[target]
    preds=[];training=[]
    for name,donors,rho,repr_,mask,equal in experiments:
        try:
            p,diag=make_fit(data,target,name,a,origin,donors,repr_,mask,rho,equal)
            training.append(diag)
            frame=pd.DataFrame({'fold':data.fold,'faculty':target,'model':name,
              'row_index':target_part.row_index,'student':target_part.student,'course':target_part.course_id,
              'origin':target_part.semester,'y':target_part.y,'p':p})
            preds.append(frame)
            met=score(target_part.y,p)
            note('FACULTY_RESULTS',fold=data.fold,faculty=target,model=name,
               rows=len(target_part),failures=int(target_part.y.sum()),prevalence=float(target_part.y.mean()),
               ap=met['ap'],auc=met['auc'],f2_at20=met['f2_at_20'],brier=met['brier'],ece10=met['ece10'],
               rho=diag['rho'],donor_rows=diag['n_donor'],effective_n=diag['weight_ess'],seconds=diag['seconds'])
            note('TRAINING_INSIGHT',fold=data.fold,faculty=target,model=name,
                 in_sample_bce=diag['training_weighted_bce'],local_bce=diag['training_local_bce'],
                 donor_bce=diag['training_donor_bce'],in_sample_local_ap=diag['training_local_ap_IN_SAMPLE'],
                 fitted_iterations=diag['fitted_iterations'],note='not_validation_metrics')
            note('SOURCE_MASS',fold=data.fold,faculty=target,model=name,mass=diag['source_mass'],
                 source_failures=diag['source_failures'],positive_ess=diag['positive_weight_ess'])
        except Exception:
            note('ERROR',fold=data.fold,faculty=target,model=name)
            raise
    if np.isfinite(target_part.committee).all() and np.all((target_part.committee>=0)&(target_part.committee<=1)):
        preds.append(pd.DataFrame({'fold':data.fold,'faculty':target,'model':'committee_reference',
          'row_index':target_part.row_index,'student':target_part.student,'course':target_part.course_id,
          'origin':target_part.semester,'y':target_part.y,'p':target_part.committee}))
        note('FACULTY_RESULTS',fold=data.fold,faculty=target,model='committee_reference',
             ap=score(target_part.y,target_part.committee)['ap'])
    else:note('REFERENCE_MISSING',fold=data.fold,faculty=target,reference='committee')
    decisions={'fold':data.fold,'faculty':target,'origin':origin,'donors':sel,
        'weights':dict(selected),'reason':reason,'historical_origins':origins,
        'historical_replay_delta':float(replay.delta_ap.mean()) if not replay.empty else None,
        'elapsed_seconds':time.monotonic()-start,'availability_policy':'semester proxy, release gap '+str(a.release_gap)}
    p=pd.concat(preds,ignore_index=True)
    return p,edges,replay,pd.DataFrame(training),decisions

def historical_reference_compare(df):
    """Metric-only descriptive reference, conditional on same row/failure counts.
    Same counts never prove row-key matching. No paired-student significance test.
    """
    file=Path(__file__).parent/'reference/historical_model_ap_cells.csv'
    if not file.exists():return pd.DataFrame()
    old=pd.read_csv(file)
    old=old[old.ap.notna()].rename(columns={'model':'reference','ap':'reference_ap',
             'rows':'reference_rows','positives':'reference_failures'})
    new=df[df.model.isin(['hgb_local_common','hgb_local_native','suite32_selective_common','suite32_selective_native'])]
    z=new.merge(old[['fold','faculty','reference','reference_ap','reference_rows','reference_failures']],on=['fold','faculty'])
    if z.empty:return z
    z['same_count_check']=(z.rows==z.reference_rows)&(z.failures==z.reference_failures)
    z['delta_ap']=z.ap-z.reference_ap
    z['alignment']='METRIC_ONLY; same_count_check_is_not_row_alignment'
    return z

def historical_council_compare(df,root):
    file=Path(__file__).parent/'reference/council_13_faculties.csv'
    if not file.exists():return pd.DataFrame()
    c=pd.read_csv(file)
    c=c[c.fold.isin([6,7])].rename(columns={'ap':'council_ap'})
    m=df[(df.fold.isin([6,7]))][['fold','faculty','model','ap']]
    z=m.merge(c[['faculty','fold','council_ap']],on=['faculty','fold'])
    if z.empty:return pd.DataFrame()
    z['delta_ap']=z.ap-z.council_ap
    # These are archived rounded AP values; not row-aligned predictions.
    z['comparison_scope']='ARCHIVED_METRIC_ONLY_NO_ROW_PAIRED_CI'
    return z

def report(out):
    out=Path(out);pred_files=sorted((out/'predictions').glob('fold_*__*.csv'))
    if not pred_files:raise FileNotFoundError(f'No prediction artifacts under {out}')
    pred=[pd.read_csv(f) for f in pred_files]
    all_pred=pd.concat(pred,ignore_index=True)
    metrics=preds_to_metrics(all_pred)
    atomic_csv(metrics,out/'reports/faculty_fold_metrics.csv')
    all_models=sorted(metrics.model.unique())
    print('\n'+'='*116+'\nSUITE32 DEVELOPMENT F1-F5 — 14-FACULTY MACRO\n'+'='*116)
    print(summary_lines(metrics,[1,2,3,4,5]))
    print('\n'+'='*116+'\nSUITE32 RETROSPECTIVE F6-F7 — 14-FACULTY MACRO\n'+'='*116)
    print(summary_lines(metrics,[6,7]))
    atomic_csv(macro(metrics,[1,2,3,4,5]),out/'reports/macro_dev.csv')
    atomic_csv(macro(metrics,[6,7]),out/'reports/macro_holdout.csv')
    for fold in [1,2,3,4,5,6,7]:
        d=metrics[metrics.fold==fold]
        if len(d):note('FOLD_MACRO',fold=fold,rows=int(d[d.model==d.model.iloc[0]].rows.sum()),summary={k:round(float(v),6) for k,v in d.groupby('model').ap.mean().items()})
    comparisons=[]
    for fset,stage in [([1,2,3,4,5],'development'),([6,7],'retrospective')]:
        for name in all_models:
            for ref in ['hgb_local_common','hgb_local_native','hgb_pool_all','hgb_pool_equal','committee_reference']:
                if name==ref:continue
                v=paired(metrics,name,ref,fset)
                if v:v['stage']=stage;comparisons.append(v)
    diff=pd.DataFrame(comparisons);atomic_csv(diff,out/'reports/paired_cell_comparisons.csv')
    if len(diff):
        z=diff[(diff.stage=='retrospective') & (diff.candidate.isin(['suite32_selective_common','suite32_selective_native']))]
        if not z.empty:
            print('\n[PAIRED HOLDOUT] Same faculty-fold cell AP changes (not independent paired-student CIs)')
            print(z.to_string(index=False,float_format=lambda x:f'{x:.6f}'))
    fac=metrics[metrics.fold.isin([6,7])].groupby(['faculty','model'],as_index=False).agg(ap=('ap','mean'),
            f2_at_20=('f2_at_20','mean'),mean_brier=('brier','mean'))
    atomic_csv(fac,out/'reports/faculty_holdout.csv')
    subset=fac[fac.model.isin(['hgb_local_common','hgb_local_native','suite32_selective_native','committee_reference'])]
    if not subset.empty:
        print('\n[HOLDOUT FACULTY MATRIX] mean over F6,F7 (not pooled AP)')
        print(subset.pivot(index='faculty',columns='model',values='ap').round(6).to_string())
    refs=historical_reference_compare(metrics)
    if len(refs):
        atomic_csv(refs,out/'reports/historical_model_metric_comparison.csv')
        print('\n[HISTORICAL MODEL REFERENCE] Archive AP cell comparisons, NOT student-paired')
        late=refs[refs.fold.isin([6,7]) & refs.same_count_check]
        for candidate in ('suite32_selective_native','suite32_selective_common'):
            sub=late[late.model==candidate]
            for ref in ('suite25_blend','suite22','suite25','suite17','suite27','committee','v2'):
                v=sub[sub.reference==ref]
                if not v.empty:
                    note('HISTORICAL_HEAD_TO_HEAD',candidate=candidate,reference=ref,
                        comparable_count_cells=len(v),mean_candidate_ap=float(v.ap.mean()),
                        mean_reference_ap=float(v.reference_ap.mean()),mean_delta_ap=float(v.delta_ap.mean()),
                        wins=int((v.delta_ap>0).sum()),note='METRICS_ONLY_NOT_ROW_PAIR')
    c=historical_council_compare(metrics,out)
    if len(c):
        atomic_csv(c,out/'reports/council_archived_metric_comparison.csv')
        print('\n[COUNCIL ARCHIVED 13-FACULTY SUPPORT] NOT row-aligned predictions / NOT bootstrap-comparable')
        for model,g in c.groupby('model'):
            note('COUNCIL_HEAD_TO_HEAD',model=model,cells=len(g),faculty_means=g.faculty.nunique(),
                candidate_ap=float(g.ap.mean()),council_ap=float(g.council_ap.mean()),
                mean_delta_ap=float(g.delta_ap.mean()),wins=int((g.delta_ap>0).sum()))
    op=operational(pred)
    if len(op):
        atomic_csv(op,out/'reports/operational_frozen_dev_threshold.csv')
        print('\n[OPERATIONAL] Frozen F1-F5 selected F2 thresholds on F6,F7; model/faculty specific')
        print(op.groupby('model')[['precision','recall','f1','f2','flag_rate']].mean().round(5).to_string())
    edge_files=sorted((out/'selection').glob('fold_*_edges.csv'))
    edge_parts=[]
    for file in edge_files:
        e=pd.read_csv(file)
        if len(e):edge_parts.append(e)
    if edge_parts:
        ev=pd.concat(edge_parts,ignore_index=True)
        edges=ev.groupby(['fold','target','source'],as_index=False).agg(
          evaluated_origins=('origin','nunique'),mean_delta_ap=('delta_ap','mean'),
          min_delta_ap=('delta_ap','min'),max_delta_ap=('delta_ap','max'),
          win_rate=('delta_ap',lambda x:float((x>0).mean())),
          mean_donor_failures=('donor_failures','mean'))
        atomic_csv(edges,out/'reports/directed_transfer_matrix_edges.csv')
        print('\n[INSIGHT TRANSFER] Strongest and most harmful observed historical directed relationships')
        print(edges.sort_values('mean_delta_ap',ascending=False).head(12).to_string(index=False,float_format=lambda x:f'{x:+.6f}'))
        print(edges.sort_values('mean_delta_ap').head(12).to_string(index=False,float_format=lambda x:f'{x:+.6f}'))
        note('INSIGHT_TRANSFER',edges_tested=len(edges),positive_mean_edges=int((edges.mean_delta_ap>0).sum()),
             stable_positive_edges=int(((edges.mean_delta_ap>0)&(edges.win_rate>=.75)).sum()),
             note='historical_screen_not_prospective_evidence')
    decision_files=sorted((out/'selection').glob('fold_*_decision.json'))
    if decision_files:
        decisions=pd.DataFrame([json.loads(f.read_text()) for f in decision_files])
        decisions['borrow_admitted']=decisions.donors.map(bool)
        atomic_csv(decisions,out/'reports/policy_admission.csv')
        print('\n[INSIGHT ADMISSION] Chosen donor-policy admission / local fallback per faculty and fold')
        print(decisions[['fold','faculty','borrow_admitted','donors','reason','historical_replay_delta']].to_string(index=False))
        note('INSIGHT_ADMISSION',target_fold_cells=len(decisions),admitted=int(decisions.borrow_admitted.sum()),
             rejected=int((~decisions.borrow_admitted).sum()))
    training_files=sorted((out/'insights').glob('fold_*_training.csv'))
    if training_files:
        tr=pd.concat([pd.read_csv(f) for f in training_files],ignore_index=True)
        group=tr.groupby('model',as_index=False).agg(fits=('fold','count'),mean_seconds=('seconds','mean'),
                mean_ess=('weight_ess','mean'),mean_positive_ess=('positive_weight_ess','mean'),
                mean_donor_rows=('n_donor','mean'),mean_in_sample_loss=('training_weighted_bce','mean'),
                mean_local_in_sample_ap=('training_local_ap_IN_SAMPLE','mean'))
        atomic_csv(group,out/'reports/training_resource_summary.csv')
        print('\n[INSIGHT TRAINING] Training cost, mass concentration and effective support')
        print(group.to_string(index=False,float_format=lambda x:f'{x:.3f}'))
    # Historical headline values are NOT row-aligned and MUST NOT be represented as a tested metric pair.
    headlines={'suite25_blend_historical_14faculty_ap':.369526,
        'suite22_blend_historical_14faculty_ap':.369159,
        'suite31_hgb_full_historical_14faculty_ap':.353807,
        'committee_historical_14faculty_ap':.350152,
        'council_historical_13faculty_ap':.364204}
    save_json({'scope':'descriptive historical headline references; no row-level alignment',
               'values':headlines},out/'reports/historical_references.json')
    notes=[]
    for facname in sorted(metrics.faculty.unique()):
        subset=metrics[(metrics.faculty==facname)&metrics.fold.isin([6,7])]
        z=subset.pivot(index='fold',columns='model',values='ap')
        if all(n in z for n in ('suite32_selective_native','hgb_local_common','hgb_local_native')):
            notes.append({'faculty':facname,'native_minus_common':float((z.hgb_local_native-z.hgb_local_common).mean()),
             'borrowed_minus_native':float((z.suite32_selective_native-z.hgb_local_native).mean()),
             'total_minus_common':float((z.suite32_selective_native-z.hgb_local_common).mean()),
             'worse_than_local_native_folds':int((z.suite32_selective_native<z.hgb_local_native).sum())})
    if notes:
        ins=pd.DataFrame(notes);atomic_csv(ins,out/'reports/native_vs_borrowing_decomposition.csv')
        print('\n[INSIGHT DECOMPOSITION] Native information vs incremental donor-policy benefit')
        print(ins.to_string(index=False,float_format=lambda x:f'{x:+.6f}'))
    print('\n[INSIGHT NOTES] Missingness masks from cached NaNs are NOT database-provenance masks;')
    print('[INSIGHT NOTES] F6/F7 were repeatedly inspected historically; retrospective not untouched prospective tests.')
    print('[INSIGHT NOTES] Historic Council/Suite25 aggregate references use non-identical protocols until row-keys verified.')
    return metrics

def main():
    a=paths(parse())
    if not 0<a.rho<1:raise ValueError('rho must be in (0,1)')
    if a.release_gap<0 or a.max_selected<1 or a.donor_budget<1 or a.selection_origins<3:raise ValueError('Invalid config')
    folds=selected_folds(a);out=Path(a.output_dir)
    if a.stage=='report':report(out);return
    from s32_data import names_in
    faculties=names_in(a.cache_dir,folds[0])
    if a.faculties not in ('all','auto',''):
        requested=[x.strip() for x in a.faculties.split(',') if x.strip()]
        missing=set(requested)-set(faculties)
        if missing:raise ValueError(f'Unknown faculties {missing}')
        faculties=requested
    targets=faculties[:a.max_targets] if a.max_targets else faculties
    if a.max_donor_candidates:
        raise ValueError('The directed source grid must not be silently truncated. Use --max-targets for a pilot.')
    note('ENV',python=platform.python_version(),numpy=np.__version__,pandas=pd.__version__,
         sklearn=__import__('sklearn').__version__,root=a.root,cache=a.cache_dir,
         structured=a.structured_dir,folds=folds,targets=len(targets),donors=len(faculties),seed=a.seed)
    note('SCIENTIFIC_CONTRACT',labels='grade10_lt5',prediction='semester_origin',
         selection='historical_only',evaluation='faculty_fold_macro_AP',
         council='13_faculty_archived_metrics_only',holdout='retrospective_not_fresh',
         masks='cache_nan_is_not_true_observation_provenance',borrow_mass=a.rho)
    expected_common=None
    if a.stage!='preflight':
        out.mkdir(parents=True,exist_ok=True)
        manifest={**vars(a),'effective_folds':folds,'available_faculties':faculties,
                 'target_faculties':targets,'python':platform.python_version(),
                 'numpy':np.__version__,'sklearn':__import__('sklearn').__version__}
        signature=hashlib.sha256(json.dumps(manifest,sort_keys=True,default=str).encode()).hexdigest()
        manifest['signature']=signature
        existed=out/'run_manifest.json'
        if existed.exists() and not a.force:
            prior=json.loads(existed.read_text())
            # Cross-stage continuation is permitted only with same scientific hyperparameters.
            exclude={'stage','folds','effective_folds','signature','force'}
            changed=[k for k in manifest if k not in exclude and manifest.get(k)!=prior.get(k)]
            if changed:raise RuntimeError(f'Output exists with changed configuration {changed}; use a NEW output-dir')
        save_json(manifest,existed)
    for fold in folds:
        note('FOLD_START',fold=fold,stage=a.stage)
        data=load_fold(a.cache_dir,a.structured_dir,faculties,fold,expected_common,a.release_gap)
        expected_common=data.common_names
        note('DATA',fold=fold,common_raw=len(data.common_names),course_features=17,
             common_total=len(data.common_names)+17,faculties=len(faculties),
             history_rows=sum(len(z) for z in data.history.values()),target_rows=sum(len(z) for z in data.target.values()),
             target_failures=sum(int(z.y.sum()) for z in data.target.values()),
             history_last=max(int(z.semester.max()) for z in data.history.values()),
             target_first=min(int(z.semester.min()) for z in data.target.values()))
        audits=pd.DataFrame([status_audit(data,f) for f in faculties])
        for r in audits.to_dict('records'):note('OBSERVATION_AUDIT',**r)
        if a.stage=='preflight':
            note('PREFLIGHT_OK',fold=fold,all_faculties=len(faculties),total_features=len(data.common_names)+17,
                note='data_clock_is_semester_proxy')
            continue
        atomic_csv(audits,out/f'audits/data_fold_{fold}.csv')
        atomic_csv(feature_ledger(data),out/f'audits/feature_ledger_fold_{fold}.csv')
        atomic_csv(student_overlap_ledger(data),out/f'audits/student_overlap_fold_{fold}.csv')
        save_json({'fold':fold,'common_names':data.common_names,
                   'feature_counts':{f:len(data.history[f].raw_names) for f in faculties},
                   'course_names':__import__('s32_data').COURSE_FEATURES},out/f'audits/schema_fold_{fold}.json')
        for target in targets:
            slug=f'fold_{fold}__{target}'
            dest=out/'predictions'/f'{slug}.csv'
            if dest.exists() and not a.force:
                note('RESUME_SKIP',fold=fold,faculty=target,path=str(dest));continue
            p,edges,replay,training,decisions=one_faculty(data,target,a)
            atomic_csv(edges,out/'selection'/f'{slug}_edges.csv')
            atomic_csv(replay,out/'selection'/f'{slug}_replay.csv')
            atomic_csv(training,out/'insights'/f'{slug}_training.csv')
            save_json(decisions,out/'selection'/f'{slug}_decision.json')
            # Write predictions LAST: resume means prior diagnostics are complete.
            atomic_csv(p,dest)
            note('FACULTY_COMPLETE',fold=fold,faculty=target,models=p.model.nunique(),
                 positive_rows=int(p[p.model=='hgb_local_common'].y.sum()),
                 num_edges=len(edges),num_replays=len(replay),chosen=decisions['donors'])
    if a.stage!='preflight':
        note('REPORT',source='saved_row_keyed_predictions',output=str(out))
        report(out)
        note('COMPLETE',stage=a.stage,folder=str(out),scope='development_and_or_retrospective')

if __name__=='__main__':
    try:main()
    except Exception as err:
        print('[FATAL]',type(err).__name__,str(err),flush=True)
        traceback.print_exc()
        sys.exit(1)
