# -*- coding: utf-8 -*-
"""Suite11 — opportunity-aware latent-domain residual MoE.

Research question
-----------------
Can the noisy hindsight opportunity structure exposed by prior OOF committee/base-expert
behavior be compressed into a small number of latent correction domains that are both:
  (a) useful for genuinely different residual specialists, and
  (b) prospectively predictable from pre-outcome observable features?

Chronology
----------
Only historical cross-fitted (OOF) committee/base-expert predictions are used to discover
latent domains. Target outcomes are used only after target scoring for diagnostics/reporting.
Suite11 deploys only when a newest historical selection block beats the exact Suite7 baseline.
"""
from __future__ import annotations

import argparse
import copy
from pathlib import Path

import joblib
import numpy as np
import pandas as pd
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader,TensorDataset,WeightedRandomSampler

from suite2_common import safe_ap,safe_auc,equal_budget_metrics
from suite7_common import (
    FeatureStandardizer,load_fold_caches,build_historical_bundle,build_target_bundle,
    faculty_class_sampling_weights,latest_block_split,per_faculty_validation_rows,
    model_selection_score,parse_faculties,save_json,
)
from suite7_reporting import load_shared_feature_schema,load_shared_fold_frame
from suite7_eval import evaluate_faculty
from suite7_shared_private_moe import SharedPrivateResidualMoE,residual_moe_loss,seed_everything,choose_device
from suite11_opportunity_domains import (
    extract_base_probabilities,build_opportunity_signature,OpportunityDomainDiscoverer,
)
from suite11_latent_moe import (
    OpportunityRouter,LatentOpportunityResidualMoE,load_suite7_initialization,
    freeze_for_latent_specialists,latent_specialist_loss,authority_adjusted_weights,fuse_numpy,
)
from suite11_diagnostics import router_quality,mean_pairwise_residual_correlation,top_budget_transitions

COMMITTEE='committee'
SHARED='residual_moe_shared'
ADAPTER='residual_moe_shared_private'
SUITE8='residual_moe_true_specialist'
SUITE9C='residual_moe_competitive_candidate'
SUITE9F='residual_moe_competitive_final'
SUITE10C='semantic_specialist_candidate'
SUITE10F='semantic_specialist_final'
V2='v2_classifier'
COUNCIL='suite4_competence_council'
CANDIDATE='opportunity_latent_candidate'
FINAL='opportunity_latent_final'
BUDGETS=(.10,.15,.20,.25)
EPS=1e-7


def _base_cfg(checkpoint,args):
    c=dict(checkpoint.get('model_config',{}));g=lambda n,d:c.get(n,getattr(args,n,d))
    return {'n_experts':int(g('n_experts',args.n_domains)),'faculty_dim':int(g('faculty_dim',8)),'encoder_hidden':int(g('encoder_hidden',256)),
            'representation_dim':int(g('representation_dim',128)),'expert_hidden':int(g('expert_hidden',64)),'delta_max':float(g('delta_max',2.0)),
            'dropout':float(g('dropout',.1)),'router_temperature':float(g('router_temperature',1.0)),'adapter_dim':int(g('adapter_dim',16)),
            'adapter_dropout':float(g('adapter_dropout',0.0)),'adapter_scale':float(g('adapter_scale',1.0))}


def _make_suite7(input_dim,nfac,cfg):return SharedPrivateResidualMoE(input_dim=input_dim,n_faculties=nfac,**cfg)
def _make_latent(input_dim,nfac,cfg):return LatentOpportunityResidualMoE(input_dim=input_dim,n_faculties=nfac,**cfg)


def _tensor_loader(bundle,scaler,mask,args,seed,sampled=True,responsibilities=None):
    mask=np.asarray(mask,bool);X=scaler.transform(bundle.X[mask]);f=bundle.faculty_idx[mask].astype(np.int64);q=bundle.q[mask].astype(np.float32);y=bundle.y[mask].astype(np.float32)
    fields=[torch.from_numpy(X),torch.from_numpy(f),torch.from_numpy(q),torch.from_numpy(y)]
    if responsibilities is not None:fields.append(torch.from_numpy(np.asarray(responsibilities,float)[mask].astype(np.float32)))
    ds=TensorDataset(*fields)
    if not sampled:return DataLoader(ds,batch_size=args.predict_batch_size,shuffle=False,num_workers=0)
    w,_=faculty_class_sampling_weights(bundle.y[mask],bundle.faculty_idx[mask],bundle.faculty_names,args.sampling_beta)
    gen=torch.Generator();gen.manual_seed(int(seed));n=len(ds) if args.samples_per_epoch<=0 else int(args.samples_per_epoch)
    sampler=WeightedRandomSampler(torch.as_tensor(w,dtype=torch.double),num_samples=n,replacement=True,generator=gen)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0)


def _router_loader(bundle,scaler,mask,labels,args,seed,sampled=True):
    mask=np.asarray(mask,bool);X=scaler.transform(bundle.X[mask]);y=np.asarray(labels,int)[mask];f=bundle.faculty_idx[mask]
    ds=TensorDataset(torch.from_numpy(X),torch.from_numpy(y.astype(np.int64)))
    if not sampled:return DataLoader(ds,batch_size=args.predict_batch_size,shuffle=False,num_workers=0)
    counts=np.bincount(y,minlength=args.n_domains+1).astype(float);fw=np.bincount(f,minlength=len(bundle.faculty_names)).astype(float)
    cw=1.0/np.sqrt(np.maximum(counts,1.0));fww=1.0/np.sqrt(np.maximum(fw,1.0));w=cw[y]*fww[f];w*=len(w)/max(w.sum(),EPS)
    gen=torch.Generator();gen.manual_seed(int(seed));sampler=WeightedRandomSampler(torch.as_tensor(w,dtype=torch.double),num_samples=len(ds),replacement=True,generator=gen)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0)


def _train_suite7_epoch(model,loader,opt,device,args):
    model.train();tot=0.;n=0
    for X,f,q,y in loader:
        X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device);opt.zero_grad(set_to_none=True);out=model(X,f,q)
        ls=residual_moe_loss(out,y,q,router_lambda=args.baseline_router_lambda,specialization_lambda=args.baseline_specialization_lambda,
                             magnitude_lambda=args.baseline_magnitude_lambda,utility_temperature=args.baseline_utility_temperature)
        ls['total'].backward();torch.nn.utils.clip_grad_norm_(model.parameters(),args.grad_clip);opt.step();tot+=float(ls['total'].detach().cpu())*len(y);n+=len(y)
    return tot/max(n,1)


def _reproduce_suite7(bundle,tr,scaler,args,seed,epochs,cfg,device,label):
    seed_everything(seed);m=_make_suite7(bundle.X.shape[1],len(bundle.faculty_names),cfg).to(device);opt=torch.optim.AdamW(m.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    for ep in range(1,max(int(epochs),1)+1):
        loss=_train_suite7_epoch(m,_tensor_loader(bundle,scaler,tr,args,seed+ep*997,True),opt,device,args)
        if ep==1 or ep==epochs:print(f'[{label}-base] reproduced Suite7 epoch={ep}/{epochs} loss={loss:.5f}')
    return m


def _split_rel_selection(bundle,val_mask):
    val_mask=np.asarray(val_mask,bool);rel=np.zeros(len(val_mask),bool);sel=np.zeros(len(val_mask),bool);ind=True
    for fi in np.unique(bundle.faculty_idx[val_mask]):
        m=val_mask&(bundle.faculty_idx==fi)&np.isfinite(bundle.blocks);bs=sorted(pd.unique(bundle.blocks[m]))
        if len(bs)>=2:sel[m&(bundle.blocks==bs[-1])]=True;rel[m&(bundle.blocks!=bs[-1])]=True
        elif len(bs)==1:rel[m]=True;sel[m]=True;ind=False
    return rel,sel,ind


def _opportunity_signature(bundle):
    base=extract_base_probabilities(bundle.X,bundle.feature_names)
    return build_opportunity_signature(bundle.y,bundle.q,base,bundle.faculty_idx,bundle.blocks)


def _train_router(router,bundle,scaler,tr,rel,labels,args,seed,device,label,epochs=None):
    seed_everything(seed+11011);router.to(device);opt=torch.optim.AdamW(router.parameters(),lr=args.router_lr,weight_decay=args.weight_decay)
    maxep=int(args.router_epochs if epochs is None else epochs);best=-np.inf;best_ep=1;best_state=copy.deepcopy(router.state_dict());pat=0;history=[]
    for ep in range(1,maxep+1):
        router.train();tot=0.;n=0
        for X,y in _router_loader(bundle,scaler,tr,labels,args,seed+ep*101,True):
            X=X.to(device);y=y.to(device);opt.zero_grad(set_to_none=True);logits=router(X);loss=F.cross_entropy(logits,y,label_smoothing=args.router_label_smoothing);loss.backward();torch.nn.utils.clip_grad_norm_(router.parameters(),args.grad_clip);opt.step();tot+=float(loss.detach().cpu())*len(y);n+=len(y)
        probs=_router_predict(router,bundle,scaler,rel,args,device);d=router_quality(np.asarray(labels)[rel],probs,args.n_domains+1);score=d['macro_f1']+.5*d['balanced_accuracy']+.25*d['nmi'];history.append({'epoch':ep,'loss':tot/max(n,1),'score':score,**{k:d[k] for k in ['accuracy','balanced_accuracy','macro_f1','nmi']}})
        if ep==1 or ep%args.print_every==0:print(f'[{label} ROUTER] ep={ep:02d} loss={tot/max(n,1):.5f} balacc={d["balanced_accuracy"]:.3f} macroF1={d["macro_f1"]:.3f} NMI={d["nmi"]:.3f}')
        if score>best+args.router_min_improvement:best=score;best_ep=ep;best_state=copy.deepcopy(router.state_dict());pat=0
        else:pat+=1
        if epochs is None and ep>=args.router_min_epochs and pat>=args.router_patience:break
    router.load_state_dict(best_state);return router,best_ep,pd.DataFrame(history)


@torch.no_grad()
def _router_predict(router,bundle,scaler,mask,args,device):
    router.eval();out=[]
    for X,y in _router_loader(bundle,scaler,mask,np.zeros(len(bundle.y),int),args,0,False):out.append(torch.softmax(router(X.to(device)),dim=1).cpu().numpy())
    return np.concatenate(out) if out else np.zeros((0,args.n_domains+1))


@torch.no_grad()
def _predict_latent(model,bundle,scaler,mask,args,device):
    model.eval();res=[];cf=[]
    for X,f,q,y in _tensor_loader(bundle,scaler,mask,args,0,False):
        o=model(X.to(device),f.to(device),q.to(device));res.append(o['residuals'].cpu().numpy());cf.append(o['counterfactual_prob'].cpu().numpy())
    return {'residuals':np.concatenate(res) if res else np.zeros((0,args.n_domains)),'counterfactual_prob':np.concatenate(cf) if cf else np.zeros((0,args.n_domains))}


@torch.no_grad()
def _predict_suite7(model,bundle,scaler,mask,args,device):
    model.eval();ps=[]
    for X,f,q,y in _tensor_loader(bundle,scaler,mask,args,0,False):ps.append(model(X.to(device),f.to(device),q.to(device))['final_prob'].cpu().numpy())
    return np.concatenate(ps) if ps else np.array([])


def _train_specialist_epoch(model,bundle,scaler,tr,resp,args,seed,device,epoch):
    model.eval();tot=0.;n=0;mass=np.zeros(args.n_domains,float)
    for X,f,q,y,r in _tensor_loader(bundle,scaler,tr,args,seed+epoch*211,True,resp):
        X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device);r=r.to(device);opt=args._expert_optimizer;opt.zero_grad(set_to_none=True);o=model(X,f,q)
        ls=latent_specialist_loss(o,y,r,magnitude_lambda=args.magnitude_lambda,domain_power=args.domain_power,min_domain_weight=args.domain_train_floor)
        if ls['total'].requires_grad:ls['total'].backward();torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad],args.grad_clip);opt.step()
        tot+=float(ls['total'].detach().cpu())*len(y);n+=len(y);mass+=ls['domain_mass'].cpu().numpy()
    return tot/max(n,1),mass


def _selection_vs_suite7(y,base,candidate,faculty_idx,names,args):
    df=per_faculty_validation_rows(y,base,candidate,faculty_idx,names,args.threshold_beta,.20);score,macro=model_selection_score(df,args.macro_f2_weight);return score,macro,df


def _candidate_for_alpha(q,residuals,router_prob,alpha):
    w=authority_adjusted_weights(router_prob,alpha);p,c=fuse_numpy(q,residuals,w);return p,w,c


def _fit_validation(hist,tr,rel,sel,args,seed,cfg,suite7_epochs,device,label):
    scaler=FeatureStandardizer.fit(hist.X[tr]);base=_reproduce_suite7(hist,tr,scaler,args,seed,suite7_epochs,cfg,device,label)
    base_rel=_predict_suite7(base,hist,scaler,rel,args,device);base_sel=_predict_suite7(base,hist,scaler,sel,args,device)
    sig=_opportunity_signature(hist);disc=OpportunityDomainDiscoverer(args.n_domains,args.opportunity_quantile,args.cluster_temperature,seed,args.max_cluster_rows,args.cluster_batch_size);disc.fit(sig, hist.faculty_idx[tr], hist.blocks[tr]) if False else None
    # Fit only on train rows: use sliced signatures so labels from reliability/selection cannot affect centers or NULL threshold.
    from suite11_opportunity_domains import OpportunitySignature
    sig_tr=OpportunitySignature(sig.features[tr],sig.feature_names,sig.opportunity_score[tr],sig.committee_rank[tr]);disc.fit(sig_tr,hist.faculty_idx[tr],hist.blocks[tr])
    resp=np.zeros((len(hist.y),args.n_domains+1),np.float32)
    for m in [tr,rel,sel]:
        ss=OpportunitySignature(sig.features[m],sig.feature_names,sig.opportunity_score[m],sig.committee_rank[m]);resp[m]=disc.transform(ss,hist.faculty_idx[m],hist.blocks[m])
    labels=np.argmax(resp,axis=1)
    router=OpportunityRouter(hist.X.shape[1],args.n_domains,args.router_hidden,args.router_dropout)
    router,router_ep,router_hist=_train_router(router,hist,scaler,tr,rel,labels,args,seed,device,label)
    rp_rel=_router_predict(router,hist,scaler,rel,args,device);rq_rel=router_quality(labels[rel],rp_rel,args.n_domains+1)
    rp_sel=_router_predict(router,hist,scaler,sel,args,device);rq_sel=router_quality(labels[sel],rp_sel,args.n_domains+1)
    model=_make_latent(hist.X.shape[1],len(hist.faculty_names),cfg).to(device);model.load_state_dict(base.state_dict(),strict=True);params=freeze_for_latent_specialists(model);args._expert_optimizer=torch.optim.AdamW(params,lr=args.expert_lr,weight_decay=args.weight_decay)
    best=-np.inf;best_epoch=1;best_state=copy.deepcopy(model.state_dict());best_alpha=0.;best_macro=None;pat=0;history=[]
    alpha_grid=[float(x) for x in args.alpha_grid.split(',') if str(x).strip()]
    for ep in range(1,args.expert_epochs+1):
        loss,mass=_train_specialist_epoch(model,hist,scaler,tr,resp,args,seed,device,ep);pred=_predict_latent(model,hist,scaler,rel,args,device);ep_best=-np.inf;ep_alpha=0.;ep_macro=None
        for alpha in alpha_grid:
            cand,_,_=_candidate_for_alpha(hist.q[rel],pred['residuals'],rp_rel,alpha);score,macro,_=_selection_vs_suite7(hist.y[rel],base_rel,cand,hist.faculty_idx[rel],hist.faculty_names,args)
            if score>ep_best:ep_best=score;ep_alpha=alpha;ep_macro=macro
        rc=mean_pairwise_residual_correlation(pred['residuals']);history.append({'epoch':ep,'loss':loss,'selection_score':ep_best,'alpha':ep_alpha,'residual_correlation':rc,**(ep_macro or {})})
        if ep==1 or ep%args.print_every==0:print(f'[{label} EXPERT] ep={ep:02d} loss={loss:.5f} score={ep_best:+.5f} alpha={ep_alpha:.2f} Rcorr={rc:.3f}')
        if ep_best>best+args.expert_min_improvement:best=ep_best;best_epoch=ep;best_state=copy.deepcopy(model.state_dict());best_alpha=ep_alpha;best_macro=ep_macro;pat=0
        else:pat+=1
        if ep>=args.expert_min_epochs and pat>=args.expert_patience:break
    model.load_state_dict(best_state);pred_sel=_predict_latent(model,hist,scaler,sel,args,device);cand_sel,_,_=_candidate_for_alpha(hist.q[sel],pred_sel['residuals'],rp_sel,best_alpha);deploy_score,deploy_macro,sel_rows=_selection_vs_suite7(hist.y[sel],base_sel,cand_sel,hist.faculty_idx[sel],hist.faculty_names,args)
    deploy=bool(np.isfinite(deploy_score) and deploy_score>=args.min_selection_improvement)
    print(f'[{label}] router_ep={router_ep} expert_ep={best_epoch} alpha={best_alpha:.2f} router(rel) F1={rq_rel["macro_f1"]:.3f} NMI={rq_rel["nmi"]:.3f} router(sel) F1={rq_sel["macro_f1"]:.3f} | deploy_sel={deploy_score:+.5f} medianΔAP={deploy_macro["median_delta_ap"]:+.4f} medianΔF2={deploy_macro["median_delta_f2_20"]:+.4f} -> {"SUITE11" if deploy else "EXACT SUITE7 FALLBACK"}')
    delattr(args,'_expert_optimizer')
    return {'scaler':scaler,'discoverer':disc,'router_state':copy.deepcopy(router.state_dict()),'router_epoch':router_ep,'expert_state':best_state,'expert_epoch':best_epoch,'alpha':best_alpha,
            'deploy':deploy,'deploy_score':deploy_score,'deploy_macro':deploy_macro,'selection_rows':sel_rows,'router_history':router_hist,'expert_history':pd.DataFrame(history),
            'router_rel':rq_rel,'router_sel':rq_sel,'responsibilities':resp,'labels':labels,'independent_selection':not np.array_equal(rel,sel)}


def _full_refit(hist,target,args,seed,cfg,suite7_ck,selected,device,label):
    scaler=FeatureStandardizer.fit(hist.X);sig=_opportunity_signature(hist);disc=OpportunityDomainDiscoverer(args.n_domains,args.opportunity_quantile,args.cluster_temperature,seed,args.max_cluster_rows,args.cluster_batch_size);resp=disc.fit_transform(sig,hist.faculty_idx,hist.blocks);labels=np.argmax(resp,axis=1)
    router=OpportunityRouter(hist.X.shape[1],args.n_domains,args.router_hidden,args.router_dropout);router,_,_= _train_router(router,hist,scaler,np.ones(len(hist.y),bool),np.ones(len(hist.y),bool),labels,args,seed,device,label+' REFIT-R',epochs=selected['router_epoch'])
    model=_make_latent(hist.X.shape[1],len(hist.faculty_names),cfg).to(device);load_suite7_initialization(model,suite7_ck,hist.faculty_names);params=freeze_for_latent_specialists(model);args._expert_optimizer=torch.optim.AdamW(params,lr=args.expert_lr,weight_decay=args.weight_decay)
    allm=np.ones(len(hist.y),bool)
    for ep in range(1,selected['expert_epoch']+1):
        loss,mass=_train_specialist_epoch(model,hist,scaler,allm,resp,args,seed,device,ep)
        if ep==1 or ep==selected['expert_epoch']:print(f'[{label} REFIT-E] ep={ep}/{selected["expert_epoch"]} loss={loss:.5f}')
    delattr(args,'_expert_optimizer')
    tmask=np.ones(len(target.y),bool);pred=_predict_latent(model,target,scaler,tmask,args,device);rp=_router_predict(router,target,scaler,tmask,args,device);candidate,weights,corr=_candidate_for_alpha(target.q,pred['residuals'],rp,selected['alpha'])
    return model,router,scaler,disc,resp,labels,pred,rp,weights,candidate,corr


def _load_adapter_score(root,fold,faculty,y,q,atol):
    p=Path(root)/f'fold_{fold}'/f'{faculty}_scores.csv';df=pd.read_csv(p)
    if len(df)!=len(y) or not np.array_equal(df.actual.to_numpy(int),np.asarray(y,int)):raise ValueError(f'Suite7 alignment mismatch {faculty} F{fold}')
    if 'committee' in df and not np.allclose(df.committee.to_numpy(float),q,atol=atol,rtol=1e-5):raise ValueError(f'Suite7 committee mismatch {faculty} F{fold}')
    if ADAPTER not in df:raise ValueError(f'{ADAPTER} missing in {p}')
    return df[ADAPTER].to_numpy(float)


def _optional_score(root,fold,faculty,column,y,q,atol):
    if not root:return None
    p=Path(root)/f'fold_{fold}'/f'{faculty}_scores.csv'
    if not p.exists():return None
    df=pd.read_csv(p)
    if column not in df:return None
    if len(df)!=len(y) or not np.array_equal(df.actual.to_numpy(int),np.asarray(y,int)):raise ValueError(f'optional score alignment mismatch {p}')
    if 'committee' in df and not np.allclose(df.committee.to_numpy(float),q,atol=atol,rtol=1e-5):raise ValueError(f'optional committee mismatch {p}')
    return df[column].to_numpy(float)


def _v2_score(root,fold,faculty,y):
    if not root:return None
    p=Path(root)/f'{faculty}_fold_{fold}_test_results.csv'
    if not p.exists():return None
    df=pd.read_csv(p)
    if 'actual_failure_lt5' not in df or 'failure_probability_lt5' not in df:return None
    if len(df)!=len(y) or not np.array_equal(df.actual_failure_lt5.to_numpy(int),np.asarray(y,int)):raise ValueError(f'V2 label alignment mismatch {p}')
    return df.failure_probability_lt5.to_numpy(float)


def _council_score(root,fold,faculty,y,q,atol):
    if not root:return None
    candidates=[Path(root)/f'{faculty}_fold_{fold}_scores.csv',Path(root)/f'fold_{fold}'/f'{faculty}_scores.csv']
    p=next((x for x in candidates if x.exists()),None)
    if p is None:return None
    df=pd.read_csv(p)
    if 'actual' not in df or 'final' not in df:return None
    if len(df)!=len(y) or not np.array_equal(df.actual.to_numpy(int),np.asarray(y,int)):raise ValueError(f'Council label alignment mismatch {p}')
    # Council committee is sometimes calibrated differently from Suite5 committee, so only labels are required for fair score evaluation.
    return df.final.to_numpy(float)


def _global_reference_scores(args,fold,target):
    scores={COMMITTEE:target.q};parts={SHARED:[],ADAPTER:[],SUITE8:[],SUITE9C:[],SUITE9F:[],SUITE10C:[],SUITE10F:[]};has={k:True for k in parts}
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m];q=target.q[m];sf=load_shared_fold_frame(args.shared_results_dir,fold,faculty,y,q,args.reference_alignment_atol);parts[SHARED].append(sf[SHARED].to_numpy(float));parts[ADAPTER].append(_load_adapter_score(args.suite7_results_dir,fold,faculty,y,q,args.reference_alignment_atol))
        for key,root,col in [(SUITE8,args.suite8_results_dir,SUITE8),(SUITE9C,args.suite9_results_dir,SUITE9C),(SUITE9F,args.suite9_results_dir,SUITE9F),(SUITE10C,args.suite10_results_dir,SUITE10C),(SUITE10F,args.suite10_results_dir,SUITE10F)]:
            z=_optional_score(root,fold,faculty,col,y,q,args.reference_alignment_atol)
            if z is None:has[key]=False
            else:parts[key].append(z)
    scores[SHARED]=np.concatenate(parts[SHARED]);scores[ADAPTER]=np.concatenate(parts[ADAPTER])
    for k in [SUITE8,SUITE9C,SUITE9F,SUITE10C,SUITE10F]:
        if has[k] and len(parts[k])==len(target.faculty_names):scores[k]=np.concatenate(parts[k])
    return scores


def _metric_row(fold,faculty,model,y,s,args):
    r={'fold':fold,'faculty':faculty,'model':model,'rows':len(y),'failures':int(np.sum(y)),'prevalence':float(np.mean(y)),'ap':safe_ap(y,s),'auc':safe_auc(y,s)}
    for b in BUDGETS:
        z=equal_budget_metrics(y,s,b,args.threshold_beta);tag=int(round(100*b));r[f'precision_at_{tag}']=z['precision'];r[f'recall_at_{tag}']=z['recall'];r[f'f1_at_{tag}']=z['f1'];r[f'f2_at_{tag}']=z['fbeta']
    return r


def _mean_fold_summary(metrics,holdout_folds):
    d=pd.DataFrame(metrics);d=d[d.fold.isin(holdout_folds)];rows=[]
    for (faculty,model),g in d.groupby(['faculty','model']):
        r={'faculty':faculty,'model':model,'holdout_folds':len(g)}
        for c in ['ap','auc','precision_at_20','recall_at_20','f1_at_20','f2_at_10','f2_at_15','f2_at_20','f2_at_25']:r[f'mean_fold_{c}']=float(g[c].mean())
        rows.append(r)
    return pd.DataFrame(rows)


def _pooled_rows(records,holdout_folds,args):
    rows=[]
    for faculty in sorted({r['faculty'] for r in records}):
        use=[r for r in records if r['faculty']==faculty and r['fold'] in holdout_folds]
        if not use:continue
        models=sorted(set.intersection(*[set(r['scores']) for r in use]));y=np.concatenate([r['y'] for r in use])
        for model in models:
            s=np.concatenate([r['scores'][model] for r in use]);z=equal_budget_metrics(y,s,.20,args.threshold_beta);rows.append({'faculty':faculty,'model':model,'rows':len(y),'failures':int(y.sum()),'ap':safe_ap(y,s),'auc':safe_auc(y,s),'precision_at_20':z['precision'],'recall_at_20':z['recall'],'f1_at_20':z['f1'],'f2_at_20':z['fbeta']})
    return pd.DataFrame(rows)


def _operational(records,faculties,args,out):
    rows=[];all_models=sorted(set().union(*[set(r['scores']) for r in records])) if records else []
    primary={COMMITTEE,ADAPTER,SUITE9F,SUITE10C,SUITE10F,V2,COUNCIL,CANDIDATE,FINAL}
    for model in all_models:
        for faculty in faculties:
            use=[r for r in records if r['faculty']==faculty and r['fold'] in args.dev_folds+args.holdout_folds]
            if not use or not all(model in r['scores'] for r in use):continue
            verbose=bool(args.verbose_operational or (faculty==args.primary_faculty and model in primary))
            try:_,rr=evaluate_faculty(records,faculty,model,args.dev_folds,args.holdout_folds,args.threshold_beta,args.min_precision,args.max_warning_ratio,args.min_flags,verbose=verbose);rows.extend(rr)
            except Exception as e:print(f'[WARN] operational evaluation failed faculty={faculty} model={model}: {e}')
    df=pd.DataFrame(rows);df.to_csv(out/'suite11_warning_metrics.csv',index=False);return df


def _macro_model_summary(mean_fold,warning):
    rows=[];po=warning[warning.fold.astype(str)=='POOLED'] if len(warning) else pd.DataFrame()
    for model in sorted(mean_fold.model.unique()):
        mf=mean_fold[mean_fold.model==model];r={'model':model,'faculties':int(mf.faculty.nunique())}
        for c in ['mean_fold_ap','mean_fold_auc','mean_fold_precision_at_20','mean_fold_recall_at_20','mean_fold_f1_at_20','mean_fold_f2_at_20']:r[c]=float(mf[c].mean()) if c in mf else np.nan
        for policy in ['fixed','drift']:
            d=po[(po.model==model)&(po.policy==policy)] if len(po) else pd.DataFrame()
            for c in ['precision','recall','f1','fbeta','flag_rate']:
                r[f'{policy}_{c}']=float(d[c].mean()) if len(d) else np.nan
            for c in ['tp','fn','fp','tn']:r[f'{policy}_{c}_total']=int(d[c].sum()) if len(d) else 0
        rows.append(r)
    return pd.DataFrame(rows).sort_values(['mean_fold_ap','drift_fbeta'],ascending=False,na_position='last')



def _print_win_rates(mean_fold,warning):
    po=warning[warning.fold.astype(str)=='POOLED'] if len(warning) else pd.DataFrame()
    def overlap_metric(a,b,col):
        aa=mean_fold[mean_fold.model==a][['faculty',col]].rename(columns={col:'a'});bb=mean_fold[mean_fold.model==b][['faculty',col]].rename(columns={col:'b'});d=aa.merge(bb,on='faculty')
        if d.empty:return None
        delta=d.a-d.b
        return len(d),int((delta>0).sum()),float(delta.mean()),float(delta.median())
    def overlap_op(a,b,policy,col):
        aa=po[(po.model==a)&(po.policy==policy)][['faculty',col]].rename(columns={col:'a'});bb=po[(po.model==b)&(po.policy==policy)][['faculty',col]].rename(columns={col:'b'});d=aa.merge(bb,on='faculty')
        if d.empty:return None
        delta=d.a-d.b
        return len(d),int((delta>0).sum()),float(delta.mean()),float(delta.median())
    print('\nSUITE11 CANDIDATE WIN-RATE AUDIT (overlapping faculties only)')
    for ref in [COMMITTEE,ADAPTER,SUITE9C,SUITE9F,SUITE10C,SUITE10F,COUNCIL,V2]:
        vals=[]
        for label,col in [('AP','mean_fold_ap'),('F2@20','mean_fold_f2_at_20')]:
            z=overlap_metric(CANDIDATE,ref,col)
            if z:vals.append(f'{label}: {z[1]}/{z[0]} wins meanΔ={z[2]:+.4f} medΔ={z[3]:+.4f}')
        z=overlap_op(CANDIDATE,ref,'drift','fbeta')
        if z:vals.append(f'driftF2: {z[1]}/{z[0]} wins meanΔ={z[2]:+.4f} medΔ={z[3]:+.4f}')
        if vals:print(f'  vs {ref}: '+' | '.join(vals))

def _hindsight_oracle(y,q,cf):
    y=np.asarray(y,int);cand=np.column_stack([q,cf]);loss=-(y[:,None]*np.log(np.clip(cand,EPS,1-EPS))+(1-y[:,None])*np.log(np.clip(1-cand,EPS,1-EPS)));idx=np.argmin(loss,axis=1);return cand[np.arange(len(y)),idx]


def run_fold(args,faculties,fold,out):
    print('\n'+'='*180);print(f'SUITE11 OPPORTUNITY-AWARE LATENT DOMAINS | TARGET FOLD {fold}');print('='*180)
    schema=load_shared_feature_schema(args.shared_results_dir,fold);common=list(schema['common_raw_features']);caches=load_fold_caches(args.cache_dir,faculties,fold);hist=build_historical_bundle(caches,faculties,common);target=build_target_bundle(caches,faculties,common)
    if list(schema.get('shared_feature_names',hist.feature_names))!=list(hist.feature_names):raise ValueError(f'F{fold}: shared feature schema mismatch')
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True);suite7_ck=Path(args.suite7_results_dir)/f'fold_{fold}'/'shared_private_model.pt';ck=torch.load(suite7_ck,map_location='cpu',weights_only=False);cfg=_base_cfg(ck,args)
    if cfg['n_experts']!=args.n_domains:raise ValueError(f'Suite11 n_domains={args.n_domains} must equal Suite7 n_experts={cfg["n_experts"]} for controlled checkpoint reuse')
    suite7_epochs=int(ck.get('selected_epoch') or 1);seed=args.seed+fold*100003;device=choose_device(args.device);tr,va=latest_block_split(hist.faculty_idx,hist.blocks,args.val_blocks);rel,sel,independent=_split_rel_selection(hist,va)
    print(f'[DATA] common={len(common)} inputs={hist.X.shape[1]} OOF={len(hist.y):,} failures={hist.y.sum():,} ({hist.y.mean():.2%}) target={len(target.y):,} | train={tr.sum():,} reliability={rel.sum():,} selection={sel.sum():,} independent_rel_selection={independent}')
    selected=_fit_validation(hist,tr,rel,sel,args,seed,cfg,suite7_epochs,device,f'F{fold}')
    refs=_global_reference_scores(args,fold,target);model,router,scaler,disc,hresp,hlabels,pred,rp,weights,candidate,corr=_full_refit(hist,target,args,seed,cfg,suite7_ck,selected,device,f'F{fold}');adapter=refs[ADAPTER];final=candidate.copy() if selected['deploy'] else adapter.copy();global_scores={**refs,CANDIDATE:candidate,FINAL:final}
    torch.save({'state_dict':model.state_dict(),'router_state_dict':router.state_dict(),'router_epoch':selected['router_epoch'],'expert_epoch':selected['expert_epoch'],'alpha':selected['alpha'],'deploy':selected['deploy'],'deployment_selection_score':selected['deploy_score'],'model_config':cfg,'faculties':faculties,'n_domains':args.n_domains},fold_dir/'opportunity_latent_model.pt');scaler.save(fold_dir/'suite11_scaler.npz');disc.save(fold_dir/'opportunity_discoverer.json');joblib.dump(disc,fold_dir/'opportunity_discoverer.joblib');disc.center_frame().to_csv(fold_dir/'opportunity_domain_centers.csv',index=False);selected['router_history'].to_csv(fold_dir/'router_training_history.csv',index=False);selected['expert_history'].to_csv(fold_dir/'expert_training_history.csv',index=False);selected['selection_rows'].to_csv(fold_dir/'deployment_selection_by_faculty.csv',index=False)
    # Target labels enter only from this point onward, for diagnostics/reporting.
    tsig=_opportunity_signature(target);tresp=disc.transform(tsig,target.faculty_idx,target.blocks);tlabels=np.argmax(tresp,axis=1);tq=router_quality(tlabels,rp,args.n_domains+1);oracle=_hindsight_oracle(target.y,target.q,pred['counterfactual_prob'])
    print(f'[F{fold} TARGET-DIAGNOSTIC] router hindsight balacc={tq["balanced_accuracy"]:.3f} macroF1={tq["macro_f1"]:.3f} NMI={tq["nmi"]:.3f} | oracle AP={safe_ap(target.y,oracle):.4f} candidate AP={safe_ap(target.y,candidate):.4f} | mean authority={weights[:,1:].sum(axis=1).mean():.3f}')
    qconf=pd.DataFrame(tq['confusion']);qconf.to_csv(fold_dir/'target_router_hindsight_confusion.csv',index=False)
    domain_rows=[];corr_rows=[];rescue_rows=[];metrics=[];records=[]
    centers=disc.center_frame();keycols=[c for c in ['domain','missed_failure_severity','false_alarm_severity','committee_probability','base_disagreement_spread','best_base_utility'] if c in centers]
    print('[LATENT CENTERS]\n'+centers[keycols].round(4).to_string(index=False))
    for fi,name in enumerate(faculties):
        m=target.faculty_idx==fi;y=target.y[m];qs=target.q[m];local={k:np.asarray(v)[m] for k,v in global_scores.items()}
        v2=_v2_score(args.v2_results_dir,fold,name,y);coun=_council_score(args.council_results_dir,fold,name,y,qs,args.reference_alignment_atol)
        if v2 is not None:local[V2]=v2
        if coun is not None:local[COUNCIL]=coun
        for mod,s in local.items():metrics.append(_metric_row(fold,name,mod,y,s,args))
        frame=pd.DataFrame({'actual':y,'committee':qs,CANDIDATE:candidate[m],FINAL:final[m],'suite11_deployed':int(selected['deploy']),'router_null':rp[m,0],'specialist_authority':weights[m,1:].sum(axis=1),'correction_logit':corr[m],'hindsight_domain':tlabels[m],'router_domain':np.argmax(rp[m],axis=1)})
        for mod,s in local.items():
            if mod not in frame:frame[mod]=s
        for k in range(args.n_domains):frame[f'router_domain_{k+1}']=rp[m,k+1];frame[f'weight_expert_{k+1}']=weights[m,k+1];frame[f'residual_expert_{k+1}']=pred['residuals'][m,k];frame[f'counterfactual_expert_{k+1}']=pred['counterfactual_prob'][m,k]
        frame.to_csv(fold_dir/f'{name}_scores.csv',index=False);records.append({'faculty':name,'fold':fold,'y':y,'scores':local})
        rc=mean_pairwise_residual_correlation(pred['residuals'][m]);corr_rows.append({'fold':fold,'faculty':name,'mean_pairwise_residual_correlation':rc});res=top_budget_transitions(y,qs,candidate[m],.20);res.update({'fold':fold,'faculty':name});rescue_rows.append(res)
        print(f'[TARGET {name}] AP C/A/11c/11f={safe_ap(y,qs):.4f}/{safe_ap(y,local[ADAPTER]):.4f}/{safe_ap(y,candidate[m]):.4f}/{safe_ap(y,final[m]):.4f} | F2@20 A/11c/11f={equal_budget_metrics(y,local[ADAPTER],.20,args.threshold_beta)["fbeta"]:.4f}/{equal_budget_metrics(y,candidate[m],.20,args.threshold_beta)["fbeta"]:.4f}/{equal_budget_metrics(y,final[m],.20,args.threshold_beta)["fbeta"]:.4f} | Rcorr={rc:.3f} authority={weights[m,1:].sum(axis=1).mean():.3f} deploy={selected["deploy"]}')
        for d in range(args.n_domains+1):
            md=tlabels[m]==d;n=int(md.sum());fail=int(y[md].sum()) if n else 0;rr={'fold':fold,'faculty':name,'domain':d,'domain_name':'NULL' if d==0 else f'latent_{d}','rows':n,'failures':fail,'prevalence':float(y[md].mean()) if n else np.nan,'router_recall':float(np.mean(np.argmax(rp[m][md],axis=1)==d)) if n else np.nan}
            if d>0 and n>=args.domain_min_rows and fail>=args.domain_min_failures and n-fail>=args.domain_min_failures:
                rr['committee_ap']=safe_ap(y[md],qs[md]);rr['expert_ap']=safe_ap(y[md],pred['counterfactual_prob'][m][md,d-1]);rr['delta_ap']=rr['expert_ap']-rr['committee_ap']
            else:rr.update({'committee_ap':np.nan,'expert_ap':np.nan,'delta_ap':np.nan})
            domain_rows.append(rr)
    pd.DataFrame(metrics).to_csv(fold_dir/'fold_metrics.csv',index=False);pd.DataFrame(domain_rows).to_csv(fold_dir/'latent_domain_diagnostics.csv',index=False);pd.DataFrame(corr_rows).to_csv(fold_dir/'residual_correlations.csv',index=False);pd.DataFrame(rescue_rows).to_csv(fold_dir/'top20_transitions.csv',index=False)
    save_json({'fold':fold,'selected':{'router_epoch':selected['router_epoch'],'expert_epoch':selected['expert_epoch'],'alpha':selected['alpha'],'deploy':selected['deploy'],'deployment_selection_score':selected['deploy_score']},'router_rel':{k:v for k,v in selected['router_rel'].items() if k!='confusion'},'router_sel':{k:v for k,v in selected['router_sel'].items() if k!='confusion'},'target_hindsight_router':{k:v for k,v in tq.items() if k!='confusion'},'oracle_ap':safe_ap(target.y,oracle),'oracle_auc':safe_auc(target.y,oracle)},fold_dir/'suite11_fold_summary.json')
    return records,metrics,domain_rows,corr_rows,rescue_rows,{'fold':fold,'router_epoch':selected['router_epoch'],'expert_epoch':selected['expert_epoch'],'alpha':selected['alpha'],'deploy':selected['deploy'],'selection_score':selected['deploy_score'],'router_rel_macro_f1':selected['router_rel']['macro_f1'],'router_rel_nmi':selected['router_rel']['nmi'],'router_sel_macro_f1':selected['router_sel']['macro_f1'],'router_sel_nmi':selected['router_sel']['nmi']}


def _print_primary_comparison(mean_fold,warning,args):
    macro=_macro_model_summary(mean_fold,warning);print('\nSUITE11 PRIMARY HOLDOUT COMPARISON — SAME EVALUATOR FOR ALL AVAILABLE MODELS');cols=['model','faculties','mean_fold_ap','mean_fold_auc','mean_fold_precision_at_20','mean_fold_recall_at_20','mean_fold_f1_at_20','mean_fold_f2_at_20','fixed_precision','fixed_recall','fixed_f1','fixed_fbeta','drift_precision','drift_recall','drift_f1','drift_fbeta','drift_tp_total','drift_fn_total','drift_fp_total','drift_tn_total'];print(macro[[c for c in cols if c in macro]].to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    pf=args.primary_faculty;mf=mean_fold[mean_fold.faculty==pf];po=warning[(warning.faculty==pf)&(warning.fold.astype(str)=='POOLED')] if len(warning) else pd.DataFrame()
    if len(mf):
        rows=[]
        for model in sorted(mf.model.unique()):
            r=mf[mf.model==model].iloc[0];z={'model':model,'AP':r.get('mean_fold_ap',np.nan),'AUC':r.get('mean_fold_auc',np.nan),'P@20':r.get('mean_fold_precision_at_20',np.nan),'R@20':r.get('mean_fold_recall_at_20',np.nan),'F1@20':r.get('mean_fold_f1_at_20',np.nan),'F2@20':r.get('mean_fold_f2_at_20',np.nan)}
            for pol in ['fixed','drift']:
                d=po[(po.model==model)&(po.policy==pol)]
                if len(d):
                    x=d.iloc[0]
                    for c in ['precision','recall','f1','fbeta','tp','fn','fp','tn']:z[f'{pol}_{c}']=x[c]
            rows.append(z)
        print(f'\nPRIMARY FACULTY COMPARISON — {pf}');print(pd.DataFrame(rows).to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    return macro


def parse_args():
    p=argparse.ArgumentParser(description='Suite11 opportunity-aware latent-domain specialists')
    p.add_argument('--cache_dir',required=True);p.add_argument('--shared_results_dir',required=True);p.add_argument('--suite7_results_dir',required=True);p.add_argument('--suite8_results_dir',default=None);p.add_argument('--suite9_results_dir',default=None);p.add_argument('--suite10_results_dir',default=None);p.add_argument('--v2_results_dir',default=None);p.add_argument('--council_results_dir',default=None);p.add_argument('--models_dir',required=True);p.add_argument('--faculties',default='all');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--primary_faculty',default='khoa_cong_nghe_thong_tin')
    p.add_argument('--seed',type=int,default=42);p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto');p.add_argument('--n_domains',type=int,default=4);p.add_argument('--opportunity_quantile',type=float,default=.65);p.add_argument('--cluster_temperature',type=float,default=.75);p.add_argument('--max_cluster_rows',type=int,default=150000);p.add_argument('--cluster_batch_size',type=int,default=4096)
    p.add_argument('--router_hidden',type=int,default=128);p.add_argument('--router_dropout',type=float,default=.10);p.add_argument('--router_epochs',type=int,default=12);p.add_argument('--router_min_epochs',type=int,default=3);p.add_argument('--router_patience',type=int,default=4);p.add_argument('--router_lr',type=float,default=1e-3);p.add_argument('--router_label_smoothing',type=float,default=.02);p.add_argument('--router_min_improvement',type=float,default=1e-4)
    p.add_argument('--expert_epochs',type=int,default=8);p.add_argument('--expert_min_epochs',type=int,default=2);p.add_argument('--expert_patience',type=int,default=4);p.add_argument('--expert_lr',type=float,default=5e-4);p.add_argument('--expert_min_improvement',type=float,default=1e-5);p.add_argument('--alpha_grid',default='0,0.25,0.5,0.75,1.0');p.add_argument('--domain_power',type=float,default=1.0);p.add_argument('--domain_train_floor',type=float,default=.02);p.add_argument('--domain_min_rows',type=int,default=50);p.add_argument('--domain_min_failures',type=int,default=5)
    p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--encoder_hidden',type=int,default=256);p.add_argument('--representation_dim',type=int,default=128);p.add_argument('--expert_hidden',type=int,default=64);p.add_argument('--delta_max',type=float,default=2.0);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--router_temperature',type=float,default=1.0);p.add_argument('--adapter_dim',type=int,default=16);p.add_argument('--adapter_dropout',type=float,default=0.0);p.add_argument('--adapter_scale',type=float,default=1.0)
    p.add_argument('--val_blocks',type=int,default=2);p.add_argument('--batch_size',type=int,default=1024);p.add_argument('--predict_batch_size',type=int,default=4096);p.add_argument('--samples_per_epoch',type=int,default=0);p.add_argument('--lr',type=float,default=1e-3);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--grad_clip',type=float,default=5.0);p.add_argument('--print_every',type=int,default=1);p.add_argument('--magnitude_lambda',type=float,default=.01);p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--macro_f2_weight',type=float,default=1.0);p.add_argument('--min_selection_improvement',type=float,default=.0001)
    p.add_argument('--baseline_router_lambda',type=float,default=.5);p.add_argument('--baseline_specialization_lambda',type=float,default=.01);p.add_argument('--baseline_magnitude_lambda',type=float,default=.01);p.add_argument('--baseline_utility_temperature',type=float,default=.25)
    p.add_argument('--threshold_beta',type=float,default=2.0);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--reference_alignment_atol',type=float,default=1e-6);p.add_argument('--verbose_operational',action='store_true')
    a=p.parse_args();a.dev_folds=[int(x) for x in a.dev_folds.split(',') if x.strip()];a.holdout_folds=[int(x) for x in a.holdout_folds.split(',') if x.strip()];return a


def main():
    args=parse_args();out=Path(args.models_dir);out.mkdir(parents=True,exist_ok=True);first=min(args.dev_folds+args.holdout_folds);schema=load_shared_feature_schema(args.shared_results_dir,first);available=list(schema.get('faculties',[]))
    if not available:
        suffix=f'_fold_{first}.joblib';available=sorted([p.name[:-len(suffix)] for p in Path(args.cache_dir).glob(f'*{suffix}')])
    faculties=parse_faculties(args.faculties,available)
    print('='*180);print('SUITE11 — OPPORTUNITY-AWARE LATENT-DOMAIN SPECIALISTS');print('OOF committee/base-expert opportunity signatures -> latent domains -> prospective observable router -> dedicated residual experts');print('='*180);print(f'faculties={len(faculties)} folds={args.dev_folds+args.holdout_folds} domains={args.n_domains} opportunity_top={1-args.opportunity_quantile:.0%} router_epochs={args.router_epochs} expert_epochs={args.expert_epochs} alpha_grid={args.alpha_grid}')
    save_json({'faculties':faculties,'args':vars(args),'responsibility':'historical OOF opportunity-aware latent domains','router':'prospective MLP using observable X only','fallback':'exact Suite7 if newest historical selection block does not improve','target_label_role':'post-scoring diagnostics only'},out/'suite11_run_config.json')
    records=[];metrics=[];domains=[];corr=[];rescues=[];training=[]
    for fold in args.dev_folds+args.holdout_folds:
        rr,mm,dd,cc,re,ss=run_fold(args,faculties,fold,out);records.extend(rr);metrics.extend(mm);domains.extend(dd);corr.extend(cc);rescues.extend(re);training.append(ss)
    pd.DataFrame(metrics).to_csv(out/'suite11_fold_metrics.csv',index=False);pd.DataFrame(domains).to_csv(out/'suite11_latent_domain_diagnostics.csv',index=False);pd.DataFrame(corr).to_csv(out/'suite11_residual_correlations.csv',index=False);pd.DataFrame(rescues).to_csv(out/'suite11_top20_transitions.csv',index=False);pd.DataFrame(training).to_csv(out/'suite11_training_summary.csv',index=False)
    meanf=_mean_fold_summary(metrics,args.holdout_folds);meanf.to_csv(out/'suite11_holdout_mean_fold.csv',index=False);pooled=_pooled_rows(records,args.holdout_folds,args);pooled.to_csv(out/'suite11_holdout_pooled_raw.csv',index=False);warning=_operational(records,faculties,args,out);macro=_print_primary_comparison(meanf,warning,args);_print_win_rates(meanf,warning);macro.to_csv(out/'suite11_macro_model_comparison.csv',index=False)
    print('\nNOTE: mean-within-fold AP/AUC/budget metrics are primary for ranking comparisons when fallback/candidate score scales differ. Fixed/drift operational metrics are recomputed for every available model using the same evaluator and development folds.')
    print(f'\nCompleted. Results: {out}')


if __name__=='__main__':main()
