#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

import joblib
import numpy as np
import pandas as pd
import torch

from suite22_data import (
    Suite22Preprocessor, current_feature_names, discover_faculties, evidence_support_table,
    latest_block_split, load_fold_bundle, parse_faculties,
)
from suite22_models import Suite22Model, VARIANTS
from suite22_training import (
    blend_logits, choose_device, predict_model, seed_everything, select_safe_blend_alpha,
    train_fixed_epochs, train_select_epochs,
)
from suite22_reporting import (
    build_operational_evaluation, feature_drift_rows, head_to_head, macro_summary,
    mean_fold_summary, metric_row, operational_macro, overall_pooled_confusions,
    paired_bootstrap_ap, pooled_confusions, print_confusion, print_table,
    reference_scores, relational_support_uplift, uncertainty_diagnostics,
)


def parse_int_list(x): return [int(v.strip()) for v in str(x).split(',') if v.strip()]
def parse_float_list(x): return [float(v.strip()) for v in str(x).split(',') if v.strip()]

def variant_name(v): return f"suite22_{v}"


def verify_chronology(hist, train_mask, val_mask, faculties, fold):
    for fi, faculty in enumerate(faculties):
        tm=train_mask&(hist.faculty_idx==fi); vm=val_mask&(hist.faculty_idx==fi)
        if tm.any() and vm.any() and not np.max(hist.blocks[tm]) < np.min(hist.blocks[vm]):
            raise RuntimeError(f"Chronology violation faculty={faculty} fold={fold}")


def build_model(args, variant, bundle, prep):
    max_type=lambda *xs: max([int(np.max(x)) if np.size(x) else 0 for x in xs]+[1])+2
    return Suite22Model(
        variant=variant, knots=prep.current_knots, knot_mask=prep.current_knot_mask,
        n_faculties=len(bundle.faculty_names), student_dim=bundle.student_seq.shape[-1],
        course_dim=bundle.course_target.shape[-1], relation_dim=bundle.relation_events.shape[-1],
        component_dim=bundle.component_events.shape[-1],
        relation_types=max_type(bundle.relation_type), component_types=max_type(bundle.component_type),
        course_relation_types=max_type(bundle.course_relation_type),
        token_dim=args.token_dim, faculty_dim=args.faculty_dim, time_dim=args.time_dim,
        modulation_rank=args.modulation_rank, graph_neighbors=args.graph_neighbors,
        relation_out_dim=args.relation_dim, ensemble_members=args.ensemble_members,
        tabm_blocks=args.tabm_blocks, tabm_width=args.tabm_width, dropout=args.dropout,
        backend=args.tabm_backend,
    )


def aggregate_modulation(model, bundle, faculty_idx, batch_size, device):
    idx=np.where(bundle.faculty_idx==faculty_idx)[0]
    sums={c:{k:np.zeros(model.n_current,float) for k in ['gamma_abs','beta_abs','lambda_abs']} for c in ['faculty','time','interaction','full']}
    seen=0
    model.eval()
    with torch.no_grad():
        for s in range(0,len(idx),batch_size):
            ii=idx[s:s+batch_size]
            fi=torch.as_tensor(bundle.faculty_idx[ii],dtype=torch.long,device=device)
            so=torch.as_tensor(bundle.semester_ordinal[ii],dtype=torch.float32,device=device)
            sz=torch.as_tensor(bundle.semester_z[ii],dtype=torch.float32,device=device)
            out=model.modulation_magnitudes(fi,so,sz)
            for c in sums:
                for k in sums[c]: sums[c][k]+=out[c][k].sum(0).cpu().numpy()
            seen+=len(ii)
    if seen:
        for c in sums:
            for k in sums[c]: sums[c][k]/=seen
    return sums


def run_fold(args, fold, faculties, common_names, out_dir, device):
    print('\n'+'='*170)
    print(f"SUITE22 — SHARED FACULTY-TIME RELATIONAL TABM | TARGET FOLD {fold}")
    print('='*170)
    hist_raw,target_raw,common_names=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold,common_names)
    feat_names=current_feature_names(common_names)
    print(f"[DATA] faculties={len(faculties)} common_raw={len(common_names)} current_features={len(feat_names)} "
          f"history={len(hist_raw):,} fail={hist_raw.y.sum():,} ({100*hist_raw.y.mean():.2f}%) "
          f"target={len(target_raw):,} fail={target_raw.y.sum():,} ({100*target_raw.y.mean():.2f}%)")
    support=pd.concat([evidence_support_table(hist_raw,fold,'history'),evidence_support_table(target_raw,fold,'target')],ignore_index=True)
    print_table('STRUCTURED EVIDENCE COVERAGE — TARGET',support[support.split=='target'],[
        'faculty','rows','prevalence','student_semesters_mean','course_neighbors_mean','relation_events_mean','component_events_mean'])

    train_mask,val_mask=latest_block_split(hist_raw.faculty_idx,hist_raw.blocks,args.val_blocks)
    if not train_mask.any() or not val_mask.any(): raise RuntimeError(f"Fold {fold}: invalid chronological split")
    verify_chronology(hist_raw,train_mask,val_mask,faculties,fold)
    print(f"[SPLIT] train={train_mask.sum():,} validation={val_mask.sum():,} latest_blocks={args.val_blocks}")

    fold_dir=out_dir/f"fold_{fold}"; fold_dir.mkdir(parents=True,exist_ok=True)
    support.to_csv(fold_dir/'structured_evidence_coverage.csv',index=False)
    sel_prep=Suite22Preprocessor.fit(hist_raw,train_mask,args.piecewise_knots)
    hist=sel_prep.transform(hist_raw); joblib.dump(sel_prep,fold_dir/'selection_preprocessor.joblib',compress=3)
    prod_prep=Suite22Preprocessor.fit(hist_raw,np.ones(len(hist_raw),bool),args.piecewise_knots)
    prod_hist=prod_prep.transform(hist_raw); prod_target=prod_prep.transform(target_raw)
    joblib.dump(prod_prep,fold_dir/'production_preprocessor.joblib',compress=3)

    variants=[v.strip() for v in args.variants.split(',') if v.strip()]
    unknown=[v for v in variants if v not in VARIANTS]
    if unknown: raise ValueError(f"Unknown Suite22 variants: {unknown}")
    selection_rows=[]; predictions={}; uncertainties={}; models={}; selected_models={}; safe_alpha=0.0

    for vi,variant in enumerate(variants):
        print('\n'+'-'*135); print(f"[VARIANT {variant.upper()}] direct shared TabM training")
        seed_everything(args.seed+100*fold)
        model=build_model(args,variant,hist,sel_prep).to(device)
        ep,stats,curve=train_select_epochs(
            model,hist,train_mask,val_mask,device,args.epochs,args.min_epochs,args.patience,
            args.lr,args.weight_decay,args.batch_size,args.predict_batch_size,args.samples_per_epoch,
            args.ranking_weight,args.seed+100*fold,args.threshold_beta)
        curve.to_csv(fold_dir/f'{variant}_selection_history.csv',index=False)
        selection_rows.append({'fold':fold,'variant':variant,'selected_epochs':ep,**{f'validation_{k}':v for k,v in stats.items()}})
        print(f"[VARIANT {variant.upper()}] selected_epochs={ep} macroAP={stats['macro_ap']:.4f} F2@20={stats['macro_f2']:.4f} Brier={stats['macro_brier']:.4f}")
        selected_models[variant]=model

        if variant==args.primary_variant:
            val_pred,_=predict_model(model,hist,val_mask,args.predict_batch_size,device,seed=args.seed+991+fold)
            safe_alpha,blend_table=select_safe_blend_alpha(hist,val_mask,val_pred,args.blend_alpha_grid,args.threshold_beta)
            blend_table.to_csv(fold_dir/'safe_blend_selection.csv',index=False)
            print(f"[SAFE BLEND] selected alpha={safe_alpha:.2f} where 0=Committee, 1=Suite22 primary")

        seed_everything(args.seed+10000+100*fold)
        pm=build_model(args,variant,prod_hist,prod_prep).to(device)
        train_fixed_epochs(pm,prod_hist,np.ones(len(prod_hist),bool),device,ep,args.lr,args.weight_decay,
                           args.batch_size,args.samples_per_epoch,args.ranking_weight,args.seed+10000+100*fold)
        score,unc=predict_model(pm,prod_target,np.ones(len(prod_target),bool),args.predict_batch_size,device,seed=args.seed+fold)
        predictions[variant]=score; uncertainties[variant]=unc; models[variant]=pm
        torch.save({'state_dict':pm.state_dict(),'variant':variant,'common_features':common_names,
                    'current_feature_names':feat_names,'selected_epochs':ep,'args':vars(args)},fold_dir/f'{variant}_model.pt')

    primary=predictions[args.primary_variant]
    predictions['safe_blend']=blend_logits(target_raw.committee,primary,safe_alpha)

    metric_rows=[];score_frames=[];unc_rows=[];support_rows=[]
    for fi,faculty in enumerate(faculties):
        m=target_raw.faculty_idx==fi; y=target_raw.y[m]; committee=target_raw.committee[m]
        refs=reference_scores(args,fold,faculty,y,committee)
        for rn,rs in refs.items(): metric_rows.append(metric_row(fold,faculty,rn,y,rs,args.threshold_beta))
        frame=pd.DataFrame({'fold':fold,'faculty':faculty,'actual':y,'student_id':target_raw.student_id[m],
                            'course_id':target_raw.course_id[m],'target_semester':target_raw.semester_ordinal[m],
                            'history_semesters':target_raw.student_mask[m].sum(1)})
        for rn,rs in refs.items(): frame[rn]=rs
        for variant,score_all in predictions.items():
            name='suite22_safe_blend' if variant=='safe_blend' else variant_name(variant)
            s=score_all[m]; frame[name]=s
            metric_rows.append(metric_row(fold,faculty,name,y,s,args.threshold_beta))
            if variant in uncertainties:
                frame[f'{name}_uncertainty']=uncertainties[variant][m]
                ud=uncertainty_diagnostics(fold,faculty,y,s,uncertainties[variant][m])
                if len(ud): ud['model']=name; unc_rows.append(ud)
        # Relational support: C -> D isolates structured student-course evidence.
        if 'faculty_time_tabm' in predictions and 'relational_ft_tabm' in predictions:
            for source,count in {
                'student_semesters':target_raw.student_mask[m].sum(1),
                'course_neighbors':target_raw.course_neighbor_mask[m].sum(1),
                'relation_events':target_raw.relation_mask[m].sum(1),
                'component_events':target_raw.component_mask[m].sum(1),
            }.items():
                z=relational_support_uplift(fold,faculty,y,count,predictions['faculty_time_tabm'][m],predictions['relational_ft_tabm'][m],source)
                if len(z): support_rows.append(z)
        frame.to_csv(fold_dir/f'{faculty}_scores.csv',index=False);score_frames.append(frame)

    metrics=pd.DataFrame(metric_rows);metrics.to_csv(fold_dir/'metrics.csv',index=False)
    pd.DataFrame(selection_rows).to_csv(fold_dir/'selected_epochs.csv',index=False)
    if unc_rows: pd.concat(unc_rows,ignore_index=True).to_csv(fold_dir/'uncertainty_diagnostics.csv',index=False)
    if support_rows: pd.concat(support_rows,ignore_index=True).to_csv(fold_dir/'relational_support_uplift.csv',index=False)

    # Post-hoc mechanism diagnostics on holdout by default. These never select the model.
    ablation_rows=[]; time_rows=[]; modulation_rows=[]; drift_rows=[]; graph_rows=[]
    do_diag = fold in args.holdout_folds or args.diagnostics_all_folds
    if do_diag:
        pm=models[args.primary_variant]; base=predictions[args.primary_variant]
        ablations=['no_faculty_mod','no_time_mod','no_interaction_mod','identity_mod','graph_off','zero_relations','shuffle_relations']
        for kind in ablations:
            pp,_=predict_model(pm,prod_target,np.ones(len(prod_target),bool),args.predict_batch_size,device,
                               ablation=kind,seed=args.seed+7000+fold)
            for fi,faculty in enumerate(faculties):
                m=target_raw.faculty_idx==fi; y=target_raw.y[m]
                bap=metric_row(fold,faculty,'base',y,base[m])['ap']; pap=metric_row(fold,faculty,kind,y,pp[m])['ap']
                ablation_rows.append({'fold':fold,'faculty':faculty,'ablation':kind,'base_ap':bap,'ablated_ap':pap,
                                      'delta_ap':pap-bap,'mean_abs_probability_change':float(np.mean(np.abs(pp[m]-base[m])))})
        for shift,label in [(-1.0,'t_minus_1'),(1.0,'t_plus_1')]:
            pp,_=predict_model(pm,prod_target,np.ones(len(prod_target),bool),args.predict_batch_size,device,
                               seed=args.seed+8000+fold,semester_shift=shift,semester_scale=prod_prep.semester_scale)
            for fi,faculty in enumerate(faculties):
                m=target_raw.faculty_idx==fi
                d=np.abs(pp[m]-base[m])
                time_rows.append({'fold':fold,'faculty':faculty,'counterfactual':label,'mean_abs_change':float(d.mean()),
                                  'median_abs_change':float(np.median(d)),'p95_abs_change':float(np.quantile(d,.95))})

        # Drift vs learned modulation, feature by feature.
        for fi,faculty in enumerate(faculties):
            rm=val_mask&(hist_raw.faculty_idx==fi); tm=target_raw.faculty_idx==fi
            d=feature_drift_rows(fold,faculty,feat_names,hist_raw.current_features[rm],target_raw.current_features[tm])
            mods=aggregate_modulation(pm,prod_target,fi,args.predict_batch_size,device)
            for comp in mods:
                mag=mods[comp]['gamma_abs']+mods[comp]['beta_abs']+mods[comp]['lambda_abs']
                z=d.copy();z['component']=comp;z['modulation_magnitude']=mag
                modulation_rows.append(z)
            drift_rows.append(d)

        # Learned sparse graph edges.
        idx,w,rho=pm.feature_graph_edges()
        for src in range(len(feat_names)):
            for rank,(dst,weight) in enumerate(zip(idx[src],w[src]),1):
                graph_rows.append({'fold':fold,'source_feature':feat_names[src],'target_feature':feat_names[int(dst)],
                                   'neighbor_rank':rank,'attention_weight':float(weight),'graph_rho':rho})

    for name,rows in [('primary_ablation_diagnostics',ablation_rows),('time_counterfactual_diagnostics',time_rows),
                      ('feature_drift',drift_rows),('modulation_vs_drift',modulation_rows),('feature_graph_edges',graph_rows)]:
        if rows:
            df=pd.concat(rows,ignore_index=True) if isinstance(rows[0],pd.DataFrame) else pd.DataFrame(rows)
            df.to_csv(fold_dir/f'{name}.csv',index=False)

    # Component uplift table for direct A->B->C->D->E interpretation.
    uplift=[]
    chain=['shared_tabm','faculty_mod_tabm','faculty_time_tabm','relational_ft_tabm','sftr_tabm']
    for fi,faculty in enumerate(faculties):
        m=target_raw.faculty_idx==fi; y=target_raw.y[m]
        vals={v:metric_row(fold,faculty,v,y,predictions[v][m])['ap'] for v in chain if v in predictions}
        row={'fold':fold,'faculty':faculty,**{f'ap_{k}':v for k,v in vals.items()}}
        for a,b,label in [('shared_tabm','faculty_mod_tabm','faculty_gain'),('faculty_mod_tabm','faculty_time_tabm','time_gain'),
                          ('faculty_time_tabm','relational_ft_tabm','relation_gain'),('relational_ft_tabm','sftr_tabm','graph_gain')]:
            if a in vals and b in vals: row[label]=vals[b]-vals[a]
        uplift.append(row)
    uplift=pd.DataFrame(uplift);uplift.to_csv(fold_dir/'component_uplift.csv',index=False)

    new_names=[variant_name(v) for v in variants]+['suite22_safe_blend']
    print_table(f'SUITE22 TARGET FOLD {fold} — NEW MODEL RESULTS',metrics[metrics.model.isin(['committee']+new_names)],
                ['faculty','model','rows','prevalence','ap','auc','brier','ece10','precision_at_20','recall_at_20','fbeta_at_20'])
    print_table(f'SUITE22 TARGET FOLD {fold} — COMPONENT UPLIFT',uplift,
                ['faculty','faculty_gain','time_gain','relation_gain','graph_gain'])

    return {'metrics':metrics,'scores':pd.concat(score_frames,ignore_index=True),'selection':pd.DataFrame(selection_rows),
            'support':support,'support_uplift':pd.concat(support_rows,ignore_index=True) if support_rows else pd.DataFrame(),
            'ablation':pd.DataFrame(ablation_rows),'time_cf':pd.DataFrame(time_rows),
            'modulation':pd.concat(modulation_rows,ignore_index=True) if modulation_rows else pd.DataFrame(),
            'graph':pd.DataFrame(graph_rows),'uncertainty':pd.concat(unc_rows,ignore_index=True) if unc_rows else pd.DataFrame(),
            'uplift':uplift,'common_names':common_names,'safe_alpha':safe_alpha}


def build_parser():
    root='/root/disk_3tb/earlywarning/30072026'
    p=argparse.ArgumentParser(description='Suite22: Shared Faculty-Time Relational TabM (SFTR-TabM)')
    p.add_argument('--cache_dir',default=f'{root}/results/cross_faculty_residual_moe/cache')
    p.add_argument('--structured_cache_dir',default=f'{root}/results/suite12_structured_cache')
    p.add_argument('--out_dir',default=f'{root}/results/suite22_sftr_tabm')
    p.add_argument('--faculties',default='all');p.add_argument('--folds',default='1,2,3,4,5,6,7')
    p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7')
    p.add_argument('--variants',default='shared_tabm,faculty_mod_tabm,faculty_time_tabm,relational_ft_tabm,sftr_tabm')
    p.add_argument('--primary_variant',default='sftr_tabm');p.add_argument('--primary_faculty',default='khoa_cong_nghe_thong_tin')
    p.add_argument('--tabm_backend',choices=['official','internal_smoke'],default='official')
    p.add_argument('--ensemble_members',type=int,default=32);p.add_argument('--tabm_blocks',type=int,default=3);p.add_argument('--tabm_width',type=int,default=512)
    p.add_argument('--token_dim',type=int,default=4);p.add_argument('--piecewise_knots',type=int,default=5)
    p.add_argument('--faculty_dim',type=int,default=16);p.add_argument('--time_dim',type=int,default=16);p.add_argument('--modulation_rank',type=int,default=8)
    p.add_argument('--graph_neighbors',type=int,default=8);p.add_argument('--relation_dim',type=int,default=64);p.add_argument('--dropout',type=float,default=.10)
    p.add_argument('--epochs',type=int,default=15);p.add_argument('--min_epochs',type=int,default=3);p.add_argument('--patience',type=int,default=3)
    p.add_argument('--lr',type=float,default=2e-3);p.add_argument('--weight_decay',type=float,default=3e-4)
    p.add_argument('--batch_size',type=int,default=512);p.add_argument('--predict_batch_size',type=int,default=2048);p.add_argument('--samples_per_epoch',type=int,default=250000)
    p.add_argument('--ranking_weight',type=float,default=.10);p.add_argument('--val_blocks',type=int,default=3);p.add_argument('--threshold_beta',type=float,default=2.0)
    p.add_argument('--blend_alpha_grid',default='0,0.25,0.5,0.75,1.0');p.add_argument('--seed',type=int,default=42)
    p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto');p.add_argument('--diagnostics_all_folds',action='store_true')
    p.add_argument('--bootstrap_reps',type=int,default=200)

    p.add_argument('--shared_results_dir',default=f'{root}/results/cross_faculty_residual_moe')
    p.add_argument('--suite7_results_dir',default=f'{root}/results/suite7_shared_private_adapter')
    p.add_argument('--suite8_results_dir',default=f'{root}/results/suite8_true_specialist')
    p.add_argument('--suite9_results_dir',default=f'{root}/results/suite9_competitive_responsibility')
    p.add_argument('--suite10_results_dir',default=f'{root}/results/suite10_semantic_specialists')
    p.add_argument('--suite11_results_dir',default=f'{root}/results/suite11_opportunity_latent_domains')
    p.add_argument('--suite12_results_dir',default=f'{root}/results/suite12_collaborative_evidence_moe')
    p.add_argument('--suite13_results_dir',default=f'{root}/results/suite13_committee_preserving_collaborative_moe')
    p.add_argument('--suite14_results_dir',default=f'{root}/results/suite14_authority_safe_collaborative_moe')
    p.add_argument('--suite16_results_dir',default=f'{root}/results/suite16_conditional_tree_boosting')
    p.add_argument('--suite17_results_dir',default=f'{root}/results/suite17_robust_forward_specialist_boosting')
    p.add_argument('--suite18_results_dir',default=f'{root}/results/suite18_fmm_forward_specialist_boosting')
    p.add_argument('--suite20_results_dir',default=f'{root}/results/suite20_semester_target_transformer')
    p.add_argument('--suite21_results_dir',default=f'{root}/results/suite21_guarded_temporal_residual')
    p.add_argument('--v2_results_dir',default=f'{root}/results/final_train_v2')
    p.add_argument('--council_results_dir',default=f'{root}/results/suite4/2_competence_council')
    p.add_argument('--council_static_csv',default=str(Path(__file__).with_name('reference_council_13_faculties.csv')))
    return p


def main():
    args=build_parser().parse_args();args.folds=parse_int_list(args.folds);args.dev_folds=parse_int_list(args.dev_folds);args.holdout_folds=parse_int_list(args.holdout_folds);args.blend_alpha_grid=parse_float_list(args.blend_alpha_grid)
    device=choose_device(args.device);out=Path(args.out_dir);out.mkdir(parents=True,exist_ok=True)
    faculties=parse_faculties(args.faculties,discover_faculties(args.cache_dir,args.folds[0]))
    print(f"[ENV] python={sys.version.split()[0]} torch={torch.__version__} device={device} backend={args.tabm_backend}")
    print(f"[ENV] faculties={len(faculties)} folds={args.folds} dev={args.dev_folds} holdout={args.holdout_folds}")
    print('[SCIENTIFIC CONTRACT] Direct shared classifier; no neural OOF; faculty-balanced but NOT class-balanced sampling; F6/F7 never select architecture/epochs/blend.')
    print('[ARCHITECTURE] A=shared TabM; B=faculty modulation; C=faculty+time modulation; D=student-course relational evidence; E=full sparse feature graph SFTR-TabM.')
    (out/'run_config.json').write_text(json.dumps({**vars(args),'faculties_resolved':faculties},indent=2,default=str))
    start=time.time();common_names=None;results=[]
    for fold in args.folds:
        r=run_fold(args,fold,faculties,common_names,out,device);common_names=r['common_names'];results.append(r)

    metrics=pd.concat([r['metrics'] for r in results],ignore_index=True);scores=pd.concat([r['scores'] for r in results],ignore_index=True)
    selected=pd.concat([r['selection'] for r in results],ignore_index=True);uplift=pd.concat([r['uplift'] for r in results],ignore_index=True)
    metrics.to_csv(out/'all_metrics.csv',index=False);scores.to_csv(out/'all_scores_for_operational_evaluation.csv',index=False);selected.to_csv(out/'selected_epochs.csv',index=False);uplift.to_csv(out/'all_component_uplift.csv',index=False)
    safe_df=pd.DataFrame([{'fold':args.folds[i],'selected_alpha':results[i]['safe_alpha']} for i in range(len(results))]);safe_df.to_csv(out/'safe_blend_alpha_by_fold.csv',index=False)
    for key,fn in [('support','all_structured_evidence_coverage.csv'),('support_uplift','all_relational_support_uplift.csv'),('ablation','all_primary_ablation_diagnostics.csv'),('time_cf','all_time_counterfactual_diagnostics.csv'),('modulation','all_modulation_vs_drift.csv'),('graph','all_feature_graph_edges.csv'),('uncertainty','all_uncertainty_diagnostics.csv')]:
        parts=[r[key] for r in results if len(r[key])]
        if parts: pd.concat(parts,ignore_index=True).to_csv(out/fn,index=False)

    all_mean=mean_fold_summary(metrics,args.folds);all_macro=macro_summary(all_mean);hold_mean=mean_fold_summary(metrics,args.holdout_folds);hold_macro=macro_summary(hold_mean)
    all_mean.to_csv(out/'all_fold_mean_by_faculty.csv',index=False);all_macro.to_csv(out/'all_fold_macro_summary.csv',index=False);hold_mean.to_csv(out/'holdout_mean_by_faculty.csv',index=False);hold_macro.to_csv(out/'holdout_macro_summary.csv',index=False)
    fold_macro=metrics.groupby(['fold','model'],as_index=False).agg(faculties=('faculty','nunique'),macro_ap=('ap','mean'),macro_auc=('auc','mean'),macro_brier=('brier','mean'),macro_ece10=('ece10','mean'),macro_f2_at_20=('fbeta_at_20','mean'))
    fold_macro.to_csv(out/'fold_macro_summary.csv',index=False)
    primary=variant_name(args.primary_variant);h2h=head_to_head(hold_mean,primary);h2h.to_csv(out/'primary_holdout_head_to_head.csv',index=False)

    policies,operational,decisions=build_operational_evaluation(scores,args.dev_folds,args.holdout_folds,args.threshold_beta)
    policies.to_csv(out/'operational_policies_from_dev_f1_f5.csv',index=False);operational.to_csv(out/'operational_holdout_metrics.csv',index=False)
    if len(decisions): decisions.to_csv(out/'operational_holdout_decisions.csv',index=False)
    op_macro=operational_macro(operational);op_macro.to_csv(out/'operational_holdout_macro_summary.csv',index=False)
    conf=pooled_confusions(decisions);overall=overall_pooled_confusions(decisions);conf.to_csv(out/'operational_pooled_confusions_by_faculty.csv',index=False);overall.to_csv(out/'operational_overall_confusions.csv',index=False)

    print_table('SUITE22 FOLD-BY-FOLD MACRO SUMMARY — ALL FACULTIES',fold_macro[fold_macro.model.isin([primary,'suite22_safe_blend','suite22_shared_tabm','suite22_faculty_mod_tabm','suite22_faculty_time_tabm','suite22_relational_ft_tabm','committee','suite18','suite21_guarded'])].sort_values(['fold','model']))
    print_table('SUITE22 ALL-FOLD MACRO SUMMARY — F1..F7',all_macro)
    print_table('SUITE22 HOLDOUT MACRO SUMMARY — F6/F7',hold_macro)
    print_table('SUITE22 PRIMARY vs ALL AVAILABLE PREVIOUS VERSIONS — HOLDOUT',h2h)
    print_table('SUITE22 OPERATIONAL HOLDOUT SUMMARY — THRESHOLDS FROZEN ON F1..F5',op_macro)
    print_table('SUITE22 SELECTED EPOCHS',selected)
    print_table('SUITE22 SAFE-BLEND ALPHA BY FOLD',safe_df)
    print_table('SUITE22 COMPONENT UPLIFT — HOLDOUT',uplift[uplift.fold.isin(args.holdout_folds)])
    sup_parts=[r['support_uplift'] for r in results if len(r['support_uplift'])]
    if sup_parts:
        su=pd.concat(sup_parts,ignore_index=True); hs=su[su.fold.isin(args.holdout_folds)]
        if len(hs):
            ss=hs.groupby(['source','support_bucket'],as_index=False).agg(cells=('delta_ap','size'),mean_delta_ap=('delta_ap','mean'),median_delta_ap=('delta_ap','median'),rows=('rows','sum'))
            ss.to_csv(out/'relational_support_uplift_summary.csv',index=False)
            print_table('SUITE22 RELATIONAL UPLIFT BY EVIDENCE SUPPORT — HOLDOUT',ss)

    # Mechanism ablations and their compact summaries.
    abparts=[r['ablation'] for r in results if len(r['ablation'])]
    if abparts:
        ab=pd.concat(abparts,ignore_index=True);print_table('SUITE22 PRIMARY ABLATIONS — HOLDOUT',ab)
        print('\n[ABLATION SUMMARY]');print(ab.groupby('ablation').agg(delta_ap_mean=('delta_ap','mean'),delta_ap_median=('delta_ap','median'),harmed_cells=('delta_ap',lambda s:int((s<0).sum())),cells=('delta_ap','size'),mean_abs_change=('mean_abs_probability_change','mean')).to_string(float_format=lambda x:f'{x:.4f}'))
    tparts=[r['time_cf'] for r in results if len(r['time_cf'])]
    if tparts:
        td=pd.concat(tparts,ignore_index=True);print_table('SUITE22 TIME COUNTERFACTUAL STABILITY — HOLDOUT',td)
    mparts=[r['modulation'] for r in results if len(r['modulation'])]
    if mparts:
        md=pd.concat(mparts,ignore_index=True)
        corr=[]
        for (fold,faculty,comp),g in md.groupby(['fold','faculty','component']):
            x=g.mean_shift_std.to_numpy(float);y=g.modulation_magnitude.to_numpy(float)
            c=float(np.corrcoef(x,y)[0,1]) if len(x)>2 and np.std(x)>0 and np.std(y)>0 else np.nan
            corr.append({'fold':fold,'faculty':faculty,'component':comp,'drift_modulation_corr':c,'mean_drift':float(np.mean(x)),'mean_modulation':float(np.mean(y))})
        corr=pd.DataFrame(corr);corr.to_csv(out/'drift_modulation_correlations.csv',index=False);print_table('SUITE22 DRIFT ↔ MODULATION CORRELATION — HOLDOUT',corr)
    gparts=[r['graph'] for r in results if len(r['graph'])]
    if gparts:
        gd=pd.concat(gparts,ignore_index=True);top=gd.sort_values('attention_weight',ascending=False).head(40);print_table('SUITE22 STRONGEST LEARNED FEATURE-GRAPH EDGES — HOLDOUT',top)
        stability=[]
        folds=sorted(gd.fold.unique())
        for a,b in zip(folds[:-1],folds[1:]):
            ga=gd[gd.fold==a].copy();gb=gd[gd.fold==b].copy()
            ga['edge']=ga.source_feature.astype(str)+' -> '+ga.target_feature.astype(str);gb['edge']=gb.source_feature.astype(str)+' -> '+gb.target_feature.astype(str)
            ta=set(ga.nlargest(min(100,len(ga)),'attention_weight').edge);tb=set(gb.nlargest(min(100,len(gb)),'attention_weight').edge)
            jac=len(ta&tb)/max(len(ta|tb),1)
            ma=ga[['edge','attention_weight']].rename(columns={'attention_weight':'wa'});mb=gb[['edge','attention_weight']].rename(columns={'attention_weight':'wb'})
            z=ma.merge(mb,on='edge');rho=float(z.wa.rank().corr(z.wb.rank())) if len(z)>2 else np.nan
            stability.append({'fold_a':a,'fold_b':b,'top100_jaccard':jac,'common_edges':len(z),'edge_weight_rank_corr':rho})
        st=pd.DataFrame(stability);st.to_csv(out/'feature_graph_stability.csv',index=False);print_table('SUITE22 FEATURE-GRAPH STABILITY',st)

    # Is time modulation most helpful where distribution drift is strongest?
    if mparts:
        md=pd.concat(mparts,ignore_index=True);dr=md[md.component=='time'].groupby(['fold','faculty'],as_index=False).agg(mean_drift=('mean_shift_std','mean'))
        tu=uplift[uplift.fold.isin(args.holdout_folds)][['fold','faculty','time_gain']].merge(dr,on=['fold','faculty'])
        if len(tu)>2:
            rho=float(tu.time_gain.corr(tu.mean_drift,method='spearman'))
            tu.to_csv(out/'time_gain_vs_drift.csv',index=False)
            print(f"\n[TIME MODULATION] Spearman corr(time_gain, mean_feature_drift)={rho:+.4f} over {len(tu)} faculty-fold cells")

    # Primary ensemble uncertainty: separate error correlation from score-magnitude correlation.
    ucol=primary+'_uncertainty'
    if ucol in scores.columns:
        ur=[]
        hs=scores[scores.fold.isin(args.holdout_folds)]
        for (fold,faculty),z in hs.groupby(['fold','faculty']):
            u=z[ucol].to_numpy(float);p0=z[primary].to_numpy(float);err=np.abs(p0-z.actual.to_numpy(float))
            def corr(a,b): return float(np.corrcoef(a,b)[0,1]) if len(a)>2 and np.std(a)>0 and np.std(b)>0 else np.nan
            ce=corr(u,err);cp=corr(u,p0)
            X=np.column_stack([np.ones(len(p0)),p0]);bu=np.linalg.lstsq(X,u,rcond=None)[0];be=np.linalg.lstsq(X,err,rcond=None)[0]
            cr=corr(u-X@bu,err-X@be)
            ur.append({'fold':fold,'faculty':faculty,'uncertainty_error_corr':ce,'uncertainty_score_corr':cp,'residualized_uncertainty_error_corr':cr})
        ur=pd.DataFrame(ur);ur.to_csv(out/'primary_uncertainty_correlations.csv',index=False);print_table('SUITE22 PRIMARY UNCERTAINTY CORRELATIONS — HOLDOUT',ur)

    # Bootstrap AP CIs: final primary vs the strongest established references.
    boot=[]
    for faculty in faculties:
        z=scores[(scores.faculty==faculty)&scores.fold.isin(args.holdout_folds)]
        if primary not in z: continue
        for ref in ['committee','suite18','suite21_guarded']:
            if ref not in z or z[ref].isna().any(): continue
            stat=paired_bootstrap_ap(z.actual.to_numpy(int),z[primary].to_numpy(float),z[ref].to_numpy(float),args.bootstrap_reps,args.seed+hash(faculty+ref)%10000)
            boot.append({'faculty':faculty,'reference':ref,'rows':len(z),**stat})
    boot=pd.DataFrame(boot)
    if len(boot): boot.to_csv(out/'holdout_paired_bootstrap_ap.csv',index=False);print_table('SUITE22 HOLDOUT PAIRED BOOTSTRAP ΔAP',boot)

    # Historical 13-faculty Council comparison kept separate from aligned row-score leaderboard.
    cp=Path(args.council_static_csv)
    if cp.exists():
        cs=pd.read_csv(cp);overlap=sorted(set(cs.faculty)&set(faculties));c=metrics[(metrics.model==primary)&metrics.faculty.isin(overlap)&metrics.fold.isin(args.holdout_folds)];r=cs[cs.faculty.isin(overlap)&cs.fold.isin(args.holdout_folds)]
        if len(c) and len(r):
            rows=[{'source':'suite22_primary','faculties':len(overlap),'macro_ap':float(c.groupby('faculty').ap.mean().mean()),'macro_auc':float(c.groupby('faculty').auc.mean().mean())},{'source':'council_historical_13_faculty','faculties':len(overlap),'macro_ap':float(r.groupby('faculty').ap.mean().mean()),'macro_auc':float(r.groupby('faculty').auc.mean().mean())}]
            cdf=pd.DataFrame(rows);cdf.to_csv(out/'historical_council_13_faculty_holdout_comparison.csv',index=False);print_table('HISTORICAL COUNCIL REFERENCE — 13 FACULTIES ONLY',cdf)

    key_models=[primary,'suite22_safe_blend','committee','suite18','suite21_guarded','v2','council']
    print('\n'+'='*110);print(f'OPERATIONAL CONFUSION MATRICES — {args.primary_faculty} — POOLED F6/F7');print('='*110)
    for model in key_models:
        for policy in ['fixed','drift']:
            z=conf[(conf.faculty==args.primary_faculty)&(conf.model==model)&(conf.policy==policy)]
            if len(z): print_confusion(f'{model} — {policy}',z.iloc[0])
    print('\n'+'='*110);print('OVERALL 14-FACULTY POOLED CONFUSIONS — F6/F7');print('='*110)
    for model in [primary,'suite22_safe_blend','committee','suite18','suite21_guarded']:
        for policy in ['fixed','drift']:
            z=overall[(overall.model==model)&(overall.policy==policy)]
            if len(z): print_confusion(f'{model} — {policy}',z.iloc[0])

    elapsed=time.time()-start;print(f"\n[DONE] Suite22 complete in {elapsed/3600:.2f} hours. Results: {out}")

if __name__=='__main__': main()
