# -*- coding: utf-8 -*-
"""Suite4 Experiment 2 — Council of specialists with learned competence.

The competence model predicts which specialist is expected to reduce committee
log-loss for each registration; it does not directly predict the failure label.
"""
from __future__ import annotations
import argparse
from pathlib import Path
import numpy as np,pandas as pd
from sklearn.metrics import mean_squared_error
import train_v3 as core
from suite2_common import fit_hgb_classifier,predict_positive,effective_weights,support_matrix,v2_meta_features
from relational_helpers import fit_relational_predict
from suite4_common import *

def fit_meta_hgb(Z,y,args,seed,idx):
    idx=np.asarray(idx,int)
    if len(idx)<args.min_train_rows or np.unique(y[idx]).size<2:return None
    w,_=effective_weights(y[idx],args.effective_beta)
    return fit_hgb_classifier(Z[idx],y[idx],seed=seed,max_iter=args.meta_max_iter,max_leaf_nodes=args.meta_leaf_nodes,min_samples_leaf=args.meta_min_leaf,learning_rate=args.meta_lr,l2_regularization=args.meta_l2,sample_weight=w)

def council_features(X,expert_scores,support):
    p=np.asarray(expert_scores,float);meta,mn=v2_meta_features(p);sup,sn=support_matrix(support)
    q=p[:,0];g=p[:,1:]
    gaps=np.column_stack([g[:,j]-q for j in range(g.shape[1])]+[np.abs(g[:,j]-q) for j in range(g.shape[1])])
    gn=[f'expert_{j+1}_minus_committee' for j in range(g.shape[1])]+[f'abs_expert_{j+1}_gap' for j in range(g.shape[1])]
    return np.column_stack([core.clean_X(X),meta,sup,gaps]),[f'raw_{i}' for i in range(np.asarray(X).shape[1])]+mn+sn+gn

def make_forward_experts(d,args,fold):
    m=d['cf_covered_mask'];idx=np.where(m)[0];Xo=d['Xtr'][idx];yo=d['ytr'][idx];po=d['cf_probabilities'][idx];q=core.committee_statistics(po)['committee_mean'];bo=numeric_blocks(d['cf_block_ids'][idx]);support=d['support_hist'].iloc[idx].reset_index(drop=True)
    Z,_=full_meta_features(Xo,po,support,d['feature_names']);groups=d['train_groups'];blocks=eligible_numeric_blocks(bo,yo,args.min_block_rows,args.min_block_failures)
    rows=[]
    print(f'[FORWARD EXPERTS] blocks={blocks}')
    for j,b in enumerate(blocks):
        va=np.where(bo==b)[0];tr=np.where(bo<b)[0]
        if len(tr)<args.min_train_rows:continue
        mg=fit_meta_hgb(Z,yo,args,args.seed+fold*10000+j*10+1,tr)
        recent_blocks=sorted(pd.unique(bo[tr]))[-args.recent_blocks:];trr=tr[np.isin(bo[tr],recent_blocks)]
        mr=fit_meta_hgb(Z,yo,args,args.seed+fold*10000+j*10+2,trr)
        pg=q[va] if mg is None else predict_positive(mg,Z[va]);pr=q[va] if mr is None else predict_positive(mr,Z[va])
        orig_va=idx[va];rel_tr=np.where(np.isfinite(groups)&(groups<b))[0]
        if len(rel_tr)>=args.rel_min_train_rows and np.unique(d['ytr'][rel_tr]).size==2:
            prel,ri=fit_relational_predict(d['Xtr'][rel_tr],d['ytr'][rel_tr],d['train_frame'].iloc[rel_tr].reset_index(drop=True),d['Xtr'][orig_va],d['train_frame'].iloc[orig_va].reset_index(drop=True),d['feature_names'],args,args.seed+fold*20000+j)
        else:prel=q[va].copy();ri={'fallback':True}
        experts=np.column_stack([q[va],pg,pr,prel]);F,fn=council_features(Xo[va],experts,support.iloc[va].reset_index(drop=True))
        rows.append({'block':b,'indices':va,'y':yo[va],'experts':experts,'features':F,'feature_names':fn,'support':support.iloc[va].reset_index(drop=True)})
        aps=[safe_ap(yo[va],experts[:,k]) for k in range(4)];print(f'  block={b:g} n={len(va):,} AP q/global/recent/rel={aps[0]:.4f}/{aps[1]:.4f}/{aps[2]:.4f}/{aps[3]:.4f}')
    return {'idx':idx,'Xo':Xo,'yo':yo,'po':po,'q':q,'bo':bo,'support':support,'Z':Z,'rows':rows}

def fit_competence(rows,args,seed):
    if not rows:return None
    X=np.concatenate([r['features'] for r in rows]);Y=np.concatenate([r['y'] for r in rows]);E=np.concatenate([r['experts'] for r in rows])
    lq=binary_logloss_each(Y,E[:,0]);reg=np.column_stack([lq-binary_logloss_each(Y,E[:,j]) for j in range(1,4)])
    models=[]
    for j in range(3):models.append(fit_hgb_regressor(X,np.clip(reg[:,j],-args.regret_cap,args.regret_cap),seed+j*17,max_iter=args.competence_max_iter,max_leaf_nodes=args.competence_leaf_nodes,min_samples_leaf=args.competence_min_leaf,learning_rate=args.competence_lr,l2_regularization=args.competence_l2))
    return models,X,Y,E,reg

def predict_regrets(models,X):return np.column_stack([m.predict(core.clean_X(X)) for m in models])
def council_score(E,R,alpha,temp):
    w=competence_weights(R,temp);mix=(w*E).sum(axis=1);return clip_prob((1-alpha)*E[:,0]+alpha*mix),w

def select_policy(rows,args,fold):
    if len(rows)<2:return 0.0,1.0,[]
    val_rows=rows[-min(args.competence_val_blocks,len(rows)-1):];allrows=rows
    candidates=[]
    for alpha in args.alpha_grid:
      for temp in args.temperature_grid:
        deltas=[];aps=[];accs=[]
        for vr in val_rows:
            train=[r for r in allrows if r['block']<vr['block']]
            if not train:continue
            fitted=fit_competence(train,args,args.seed+fold*30000+int(vr['block']));
            if fitted is None:continue
            models,_,_,_,_=fitted;R=predict_regrets(models,vr['features']);p,w=council_score(vr['experts'],R,alpha,temp)
            base=safe_ap(vr['y'],vr['experts'][:,0]);deltas.append(safe_ap(vr['y'],p)-base);aps.append(safe_ap(vr['y'],p))
            true_best=np.argmin(np.column_stack([binary_logloss_each(vr['y'],vr['experts'][:,j]) for j in range(4)]),axis=1);pred_best=np.argmax(w,axis=1);accs.append(float((true_best==pred_best).mean()))
        if deltas:candidates.append({'alpha':alpha,'temperature':temp,'median_delta_ap':float(np.median(deltas)),'mean_delta_ap':float(np.mean(deltas)),'worst_delta_ap':float(np.min(deltas)),'win_rate':float((np.asarray(deltas)>0).mean()),'competence_top1_accuracy':float(np.mean(accs))})
    if not candidates:return 0.0,1.0,candidates
    good=[r for r in candidates if r['alpha']==0 or (r['win_rate']>=args.min_win_rate and r['worst_delta_ap']>=-args.max_worst_drop)]
    best=max(good,key=lambda r:(r['median_delta_ap'],r['mean_delta_ap'],r['competence_top1_accuracy'],-r['alpha']))
    if best['median_delta_ap']<args.min_median_gain:best=min(candidates,key=lambda r:(r['alpha'],abs(r['temperature']-1)))
    return float(best['alpha']),float(best['temperature']),candidates

def production_experts(d,h,args,fold):
    q=d['qte'];Zt,_=full_meta_features(d['Xte'],d['pte'],d['support_target'],d['feature_names']);yo=h['yo'];bo=h['bo'];Z=h['Z']
    allidx=np.arange(len(yo));mg=fit_meta_hgb(Z,yo,args,args.seed+fold*70001,allidx);recent=sorted(pd.unique(bo))[-args.recent_blocks:];ri=np.where(np.isin(bo,recent))[0];mr=fit_meta_hgb(Z,yo,args,args.seed+fold*70002,ri)
    pg=q.copy() if mg is None else predict_positive(mg,Zt);pr=q.copy() if mr is None else predict_positive(mr,Zt)
    prel,info=fit_relational_predict(d['Xtr'],d['ytr'],d['train_frame'],d['Xte'],d['test_frame'],d['feature_names'],args,args.seed+fold*70003)
    E=np.column_stack([q,pg,pr,prel]);F,fn=council_features(d['Xte'],E,d['support_target']);return E,F,fn,info

def run_fold(args,fold,out):
    d=load_cache(args.cache_dir,args.faculty,fold);print('\n'+'='*120);print(f'EXP2 COMPETENCE COUNCIL | fold={fold}')
    h=make_forward_experts(d,args,fold);alpha,temp,grid=select_policy(h['rows'],args,fold);print(f'[POLICY] alpha={alpha:.2f} temperature={temp:.2f}')
    fitted=fit_competence(h['rows'],args,args.seed+fold*80000);E,F,fn,ri=production_experts(d,h,args,fold)
    if fitted is None:final=d['qte'];W=np.column_stack([np.ones(len(final)),np.zeros((len(final),3))]);R=np.zeros((len(final),3))
    else:
        models,Xc,Yc,Ec,reg=fitted;R=predict_regrets(models,F);final,W=council_score(E,R,alpha,temp)
    y=d['yte'];q=d['qte'];print_metrics('committee',y,q);print_metrics('competence_council',y,final,q)
    # Diagnostics: actual best expert is target-label diagnostic only.
    losses=np.column_stack([binary_logloss_each(y,E[:,j]) for j in range(4)]);actual=np.argmin(losses,axis=1);pred=np.argmax(W,axis=1);oracle=E[np.arange(len(y)),actual]
    print(f'[COMPETENCE] target top1 accuracy={float((actual==pred).mean()):.3f} | NON_DEPLOYABLE best-expert oracle AP={safe_ap(y,oracle):.4f}')
    regs=history_regimes(d['support_target']);wr=[]
    for g in ['history0','history1_2','history3plus']:
        m=regs==g
        if not m.any():continue
        row={'fold':fold,'regime':g,'rows':int(m.sum()),'failures':int(y[m].sum()),'committee_ap':safe_ap(y[m],q[m]),'council_ap':safe_ap(y[m],final[m]),'top1_accuracy':float((actual[m]==pred[m]).mean())}
        for j,n in enumerate(['committee','global_hgb','recent_hgb','relational']):row[f'mean_weight_{n}']=float(W[m,j].mean())
        wr.append(row);print(f"  {g:12s} n={m.sum():,} AP {row['committee_ap']:.4f}->{row['council_ap']:.4f} weights={W[m].mean(axis=0).round(3)}")
    tag=f'{args.faculty}_fold_{fold}';save_csv([{'fold':fold,**r} for r in grid],out/f'{tag}_policy_grid.csv');save_csv(wr,out/f'{tag}_competence_by_regime.csv')
    pd.DataFrame({'actual':y,'committee':E[:,0],'global_hgb':E[:,1],'recent_hgb':E[:,2],'relational':E[:,3],'final':final,'predicted_best':pred,'actual_best_DIAGNOSTIC':actual,**{f'weight_{j}':W[:,j] for j in range(4)},**{f'pred_regret_{j+1}':R[:,j] for j in range(3)}}).to_csv(out/f'{tag}_scores.csv',index=False)
    save_csv([metric_row(fold,'committee',y,q),metric_row(fold,'competence_council',y,final,q),metric_row(fold,'NON_DEPLOYABLE_best_expert_oracle',y,oracle,q)],out/f'{tag}_metrics.csv')
    return {'fold':fold,'y':y,'scores':{'committee':q,'competence_council':final,'NON_DEPLOYABLE_best_expert_oracle':oracle}}



# ============================================================================
# V2-style operational evaluation (reporting only; does NOT change the Council)
# ============================================================================

def _operational_score(record, model):
    """Return the requested deployable score from one fold record."""
    if model == 'competence_council':
        return np.asarray(record['scores']['competence_council'], float)
    if model == 'committee':
        return np.asarray(record['scores']['committee'], float)
    raise ValueError(f'Unsupported operational model: {model}')


def print_confusion_matrix_v2(title, metrics, boundary_label='5'):
    """Print a confusion matrix in the same layout as original V2."""
    tn, fp, fn, tp = (
        int(metrics['tn']), int(metrics['fp']), int(metrics['fn']), int(metrics['tp'])
    )
    normal_total = tn + fp
    failure_total = fn + tp
    normal_correct = 100.0 * tn / normal_total if normal_total else float('nan')
    failure_detected = 100.0 * tp / failure_total if failure_total else float('nan')

    print(f"\n  Confusion matrix — {title}")
    print('  ' + '-' * 66)
    print(
        f"  {'Actual / Predicted':<24}"
        f"{f'Normal >={boundary_label}':>16}"
        f"{f'Failure <{boundary_label}':>16}"
    )
    print('  ' + '-' * 66)
    print(f"  {f'Normal >={boundary_label}':<24}{tn:>16,d}{fp:>16,d}")
    print(f"  {f'Failure <{boundary_label}':<24}{fn:>16,d}{tp:>16,d}")
    print('  ' + '-' * 66)
    print(f"  Normal correctly rejected: {normal_correct:.2f}%")
    print(f"  Failures successfully detected: {failure_detected:.2f}%")


def tune_operational_policies(dev_records, args):
    """Tune F-beta thresholds ONLY on pooled development predictions."""
    if not dev_records:
        raise ValueError('No development records available for threshold tuning.')

    y_dev = np.concatenate([np.asarray(r['y'], int) for r in dev_records])
    policies = {}
    print('\n' + '=' * 120)
    print(f'DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta={args.threshold_beta:g}]')
    print('Thresholds and warning ratios below are frozen before holdout evaluation.')
    print('=' * 120)

    for model in ['competence_council', 'committee']:
        score_dev = np.concatenate([_operational_score(r, model) for r in dev_records])
        policy = core.tune_threshold(
            y_dev,
            score_dev,
            None,
            None,
            args.threshold_beta,
            args.min_precision,
            args.max_warning_ratio,
            args.min_flags,
            False,
        )
        policies[model] = policy
        print(
            f"  {model:20s}: AP={policy.development_ap:.4f} | AUC={policy.development_auc:.4f} | "
            f"F{policy.beta:g}-opt tau={policy.threshold:.6f} | "
            f"P={policy.precision:.4f} R={policy.recall:.4f} F1={policy.f1:.4f} "
            f"F{policy.beta:g}={policy.fbeta:.4f} | "
            f"flag={100.0*policy.flag_rate:.2f}% | ratio={policy.warning_ratio:.3f}"
        )
        if policy.constraints_relaxed != 'none':
            print(f"    threshold fallback relaxed: {policy.constraints_relaxed}")
    return policies


def report_policy_block_v2(
    title,
    records,
    policy,
    model,
    prevalence_map,
    default_previous_prevalence,
    boundary_label='5',
    show_confusion=False,
):
    """Print V2-style fold and pooled metrics for one frozen production policy."""
    print(f"\n### {title} [model={model}, beta={policy.beta:g}]")
    print(
        f"{'fold':>6} {'test':>7} {'base%':>7} | {'AP':>7} {'AUC':>7} | "
        f"{'P@t':>7} {'R@t':>7} {'F1@t':>7} {'Fβ@t':>7} {'rate@t':>8} | "
        f"{'P@r':>7} {'R@r':>7} {'F1@r':>7} {'Fβ@r':>7} {'rate@r':>8}"
    )

    all_y, all_score, all_fixed, all_drift = [], [], [], []
    rows = []
    for record in records:
        y = np.asarray(record['y'], int)
        score = _operational_score(record, model)
        prevalence = float(np.mean(y))
        previous_prevalence = float(
            prevalence_map.get(record['fold'] - 1, default_previous_prevalence)
        )
        drift_rate = float(
            np.clip(policy.warning_ratio * previous_prevalence, 1e-4, 0.99)
        )

        fixed_warning = core.apply_decision_rule(score, policy.threshold)
        drift_warning = core.warn_by_rate(score, drift_rate)
        fixed = core.report(y, score, fixed_warning, policy.beta)
        drift = core.report(y, score, drift_warning, policy.beta)

        print(
            f"{record['fold']:>6} {len(y):>7} {100.0*prevalence:6.2f}% | "
            f"{fixed['ap']:7.4f} {fixed['auc']:7.4f} | "
            f"{fixed['precision']:7.4f} {fixed['recall']:7.4f} "
            f"{fixed['f1']:7.4f} {fixed['fbeta']:7.4f} "
            f"{100.0*fixed['flag_rate']:7.2f}% | "
            f"{drift['precision']:7.4f} {drift['recall']:7.4f} "
            f"{drift['f1']:7.4f} {drift['fbeta']:7.4f} "
            f"{100.0*drift['flag_rate']:7.2f}%"
        )

        if show_confusion:
            print_confusion_matrix_v2(
                f"fold {record['fold']} — fixed development threshold {policy.threshold:.6f}",
                fixed,
                boundary_label,
            )
            print_confusion_matrix_v2(
                f"fold {record['fold']} — drift-aware rate {drift_rate:.3%}",
                drift,
                boundary_label,
            )

        base = {
            'split': title,
            'fold': record['fold'],
            'model': model,
            'threshold': float(policy.threshold),
            'warning_ratio': float(policy.warning_ratio),
            'drift_rate': drift_rate,
        }
        rows.append({**base, 'policy': 'fixed_development_threshold', **{f'fixed_{k}': v for k, v in fixed.items()}})
        rows.append({**base, 'policy': 'drift_aware_rate', **{f'drift_{k}': v for k, v in drift.items()}})

        all_y.append(y)
        all_score.append(score)
        all_fixed.append(fixed_warning)
        all_drift.append(drift_warning)

    if not all_y:
        return rows

    y = np.concatenate(all_y)
    score = np.concatenate(all_score)
    fixed_warning = np.concatenate(all_fixed)
    drift_warning = np.concatenate(all_drift)
    fixed = core.report(y, score, fixed_warning, policy.beta)
    drift = core.report(y, score, drift_warning, policy.beta)

    print('-' * 132)
    print(
        f"{'POOLED':>6} {len(y):>7} {100.0*np.mean(y):6.2f}% | "
        f"{fixed['ap']:7.4f} {fixed['auc']:7.4f} | "
        f"{fixed['precision']:7.4f} {fixed['recall']:7.4f} "
        f"{fixed['f1']:7.4f} {fixed['fbeta']:7.4f} "
        f"{100.0*fixed['flag_rate']:7.2f}% | "
        f"{drift['precision']:7.4f} {drift['recall']:7.4f} "
        f"{drift['f1']:7.4f} {drift['fbeta']:7.4f} "
        f"{100.0*drift['flag_rate']:7.2f}%"
    )

    if show_confusion:
        print_confusion_matrix_v2(
            'POOLED — fixed development threshold', fixed, boundary_label
        )
        print_confusion_matrix_v2(
            'POOLED — drift-aware warning policy', drift, boundary_label
        )

    base = {
        'split': title,
        'fold': 'POOLED',
        'model': model,
        'threshold': float(policy.threshold),
        'warning_ratio': float(policy.warning_ratio),
        'drift_rate': np.nan,
    }
    rows.append({**base, 'policy': 'fixed_development_threshold', **{f'fixed_{k}': v for k, v in fixed.items()}})
    rows.append({**base, 'policy': 'drift_aware_rate', **{f'drift_{k}': v for k, v in drift.items()}})
    return rows


def run_operational_evaluation(dev_records, holdout_records, policies, out, faculty):
    """Run complete V2-style development and holdout operational reporting."""
    all_records = list(dev_records) + list(holdout_records)
    prevalence_map = {
        int(r['fold']): float(np.mean(np.asarray(r['y'], int))) for r in all_records
    }
    dev_prevalence = float(
        np.mean(np.concatenate([np.asarray(r['y'], int) for r in dev_records]))
    )

    operational_rows = []
    for model in ['competence_council', 'committee']:
        policy = policies[model]
        operational_rows += report_policy_block_v2(
            'DEVELOPMENT',
            dev_records,
            policy,
            model,
            prevalence_map,
            dev_prevalence,
            boundary_label='5',
            show_confusion=False,
        )

        print('\n' + '!' * 104)
        print(
            'HOLDOUT: pooled development threshold and warning ratio are frozen. '
            'No holdout label is used for threshold tuning.'
        )
        print('!' * 104)
        operational_rows += report_policy_block_v2(
            'HOLDOUT (test)',
            holdout_records,
            policy,
            model,
            prevalence_map,
            dev_prevalence,
            boundary_label='5',
            show_confusion=True,
        )

    pd.DataFrame(operational_rows).to_csv(
        out / f'{faculty}_operational_warning_metrics.csv', index=False
    )
    policy_rows = []
    for model, p in policies.items():
        policy_rows.append({'model': model, **p.__dict__})
    pd.DataFrame(policy_rows).to_csv(
        out / f'{faculty}_operational_threshold_policy.csv', index=False
    )


def parse_args():
    p=argparse.ArgumentParser();p.add_argument('--cache_dir',required=True);p.add_argument('--faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--models_dir',required=True);p.add_argument('--seed',type=int,default=42);p.add_argument('--effective_beta',type=float,default=.9999);p.add_argument('--recent_blocks',type=int,default=4);p.add_argument('--min_train_rows',type=int,default=800);p.add_argument('--min_block_rows',type=int,default=200);p.add_argument('--min_block_failures',type=int,default=5);p.add_argument('--meta_max_iter',type=int,default=250);p.add_argument('--meta_leaf_nodes',type=int,default=31);p.add_argument('--meta_min_leaf',type=int,default=20);p.add_argument('--meta_lr',type=float,default=.05);p.add_argument('--meta_l2',type=float,default=1.0);p.add_argument('--regret_cap',type=float,default=2.5);p.add_argument('--competence_max_iter',type=int,default=250);p.add_argument('--competence_leaf_nodes',type=int,default=15);p.add_argument('--competence_min_leaf',type=int,default=30);p.add_argument('--competence_lr',type=float,default=.04);p.add_argument('--competence_l2',type=float,default=2.0);p.add_argument('--competence_val_blocks',type=int,default=2);p.add_argument('--alpha_grid',type=lambda s:[float(x) for x in s.split(',')],default=[0,.25,.5,.75,1]);p.add_argument('--temperature_grid',type=lambda s:[float(x) for x in s.split(',')],default=[.5,1,2]);p.add_argument('--min_win_rate',type=float,default=.5);p.add_argument('--max_worst_drop',type=float,default=.01);p.add_argument('--min_median_gain',type=float,default=0.0)
    p.add_argument('--top_raw_features',type=int,default=48);p.add_argument('--student_dim',type=int,default=16);p.add_argument('--course_dim',type=int,default=12);p.add_argument('--section_dim',type=int,default=4);p.add_argument('--rel_epochs',type=int,default=60);p.add_argument('--rel_patience',type=int,default=8);p.add_argument('--rel_lr',type=float,default=1e-3);p.add_argument('--rel_batch_size',type=int,default=1024);p.add_argument('--rel_predict_batch',type=int,default=4096);p.add_argument('--rel_min_train_rows',type=int,default=1200);p.add_argument('--rel_val_min_rows',type=int,default=300);p.add_argument('--rel_val_min_failures',type=int,default=15);p.add_argument('--device',choices=['auto','cpu'],default='auto')
    # V2-compatible operational evaluation. These defaults match original V2.
    p.add_argument('--threshold_beta',type=float,default=2.0)
    p.add_argument('--min_precision',type=float,default=.15)
    p.add_argument('--max_warning_ratio',type=float,default=2.5)
    p.add_argument('--min_flags',type=int,default=5)
    return p.parse_args()

def main():
    a=parse_args();out=Path(a.models_dir);out.mkdir(parents=True,exist_ok=True)
    print('='*120)
    print('SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL')
    print('V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED')
    print('='*120)

    rec={'development':[],'holdout':[]}

    # 1) Build development predictions first.
    for f in parse_folds(a.dev_folds):
        rec['development'].append(run_fold(a,f,out))

    # 2) Select F-beta threshold / warning ratio strictly from pooled development.
    policies=tune_operational_policies(rec['development'],a)

    # 3) Holdout model predictions. No holdout labels affect the frozen policies.
    for f in parse_folds(a.holdout_folds):
        rec['holdout'].append(run_fold(a,f,out))

    # Existing Suite4 ranking/equal-budget summaries remain unchanged.
    rows=summarize_records(rec,out,a.faculty)
    print('\n'+'='*120)
    print('SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY')
    print('='*120)
    for r in rows:
        print(f"{r['split']:11s} {r['model']:34s} pooled AP={r['ap']:.4f} mean-fold AP={r['mean_fold_ap']:.4f} F2@20={r['mean_fold_f2']:.4f}")

    # New V2-style threshold-transfer and confusion-matrix reporting.
    run_operational_evaluation(
        rec['development'],rec['holdout'],policies,out,a.faculty
    )

if __name__=='__main__':main()
