# -*- coding: utf-8 -*-
"""Suite10 — semantic academic-mechanism specialists.

Four observable pre-outcome domains define expert responsibility:
  E1 student-sparse/cold-start
  E2 course-sparse/curriculum-novel
  E3 temporal deterioration
  E4 history-rich relational

No learned gate is trained. Semantic membership chooses eligible experts; Committee
uncertainty only caps total specialist authority. The expensive Suite5 Committee
cache and Suite7 checkpoints are reused.
"""
from __future__ import annotations

import argparse, copy, json
from pathlib import Path
import numpy as np
import pandas as pd
import torch
from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler

from suite2_common import safe_ap,safe_auc,equal_budget_metrics
from suite7_common import (
    FeatureStandardizer,load_fold_caches,build_historical_bundle,build_target_bundle,
    faculty_class_sampling_weights,latest_block_split,per_faculty_validation_rows,
    model_selection_score,parse_faculties,save_json,
)
from suite7_reporting import load_shared_feature_schema,load_shared_fold_frame,merge_external_references
from suite7_eval import evaluate_faculty
from suite7_shared_private_moe import SharedPrivateResidualMoE,residual_moe_loss,seed_everything,choose_device
from suite10_semantic_domains import SemanticDomainCalibrator,DOMAIN_NAMES
from suite10_semantic_moe import (
    SemanticSpecialistResidualMoE,semantic_specialist_loss,structured_semantic_weights,
    freeze_for_semantic_discovery,load_suite7_initialization,
)
from suite10_diagnostics import (
    domain_viability_rows,domain_overlap_rows,expert_contract_rows,estimate_reliability,
    authority_summary,mean_pairwise_residual_correlation,domain_drift_rows,
)

COMMITTEE='committee';SHARED='residual_moe_shared';ADAPTER='residual_moe_shared_private'
SUITE8='residual_moe_true_specialist';SUITE9C='residual_moe_competitive_candidate';SUITE9F='residual_moe_competitive_final'
CANDIDATE='semantic_specialist_candidate';FINAL='semantic_specialist_final'
BUDGETS=(.10,.15,.20,.25)
EPS=1e-7


def _tensor_loader(bundle,scaler,mask,args,seed,sampled=True,domain_scores=None):
    mask=np.asarray(mask,bool);X=scaler.transform(bundle.X[mask]);f=bundle.faculty_idx[mask].astype(np.int64);q=bundle.q[mask].astype(np.float32);y=bundle.y[mask].astype(np.float32)
    fields=[torch.from_numpy(X),torch.from_numpy(f),torch.from_numpy(q),torch.from_numpy(y)]
    if domain_scores is not None:fields.append(torch.from_numpy(np.asarray(domain_scores,float)[mask].astype(np.float32)))
    ds=TensorDataset(*fields)
    if not sampled:return DataLoader(ds,batch_size=args.predict_batch_size,shuffle=False,num_workers=0)
    w,_=faculty_class_sampling_weights(bundle.y[mask],bundle.faculty_idx[mask],bundle.faculty_names,args.sampling_beta)
    gen=torch.Generator();gen.manual_seed(int(seed));n=len(ds) if args.samples_per_epoch<=0 else int(args.samples_per_epoch)
    sampler=WeightedRandomSampler(torch.as_tensor(w,dtype=torch.double),num_samples=n,replacement=True,generator=gen)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0)


def _base_cfg(checkpoint,args):
    c=dict(checkpoint.get('model_config',{}));g=lambda n,d:c.get(n,getattr(args,n,d))
    return {'n_experts':int(g('n_experts',4)),'faculty_dim':int(g('faculty_dim',8)),'encoder_hidden':int(g('encoder_hidden',256)),'representation_dim':int(g('representation_dim',128)),'expert_hidden':int(g('expert_hidden',64)),'delta_max':float(g('delta_max',2.0)),'dropout':float(g('dropout',.1)),'router_temperature':float(g('router_temperature',1.0)),'adapter_dim':int(g('adapter_dim',16)),'adapter_dropout':float(g('adapter_dropout',0.0)),'adapter_scale':float(g('adapter_scale',1.0))}


def _make_suite7(input_dim,nfac,cfg):return SharedPrivateResidualMoE(input_dim=input_dim,n_faculties=nfac,**cfg)
def _make_semantic(input_dim,nfac,cfg):return SemanticSpecialistResidualMoE(input_dim=input_dim,n_faculties=nfac,**cfg)


def _train_suite7_epoch(model,loader,opt,device,args):
    model.train();tot=0.;n=0
    for X,f,q,y in loader:
        X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device);opt.zero_grad(set_to_none=True);out=model(X,f,q)
        ls=residual_moe_loss(out,y,q,router_lambda=args.baseline_router_lambda,specialization_lambda=args.baseline_specialization_lambda,magnitude_lambda=args.baseline_magnitude_lambda,utility_temperature=args.baseline_utility_temperature)
        ls['total'].backward();torch.nn.utils.clip_grad_norm_(model.parameters(),args.grad_clip);opt.step();tot+=float(ls['total'].detach().cpu())*len(y);n+=len(y)
    return tot/max(n,1)


def _reproduce_suite7(bundle,tr,scaler,args,seed,epochs,cfg,device,label):
    seed_everything(seed);model=_make_suite7(bundle.X.shape[1],len(bundle.faculty_names),cfg).to(device);opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    for ep in range(1,max(int(epochs),1)+1):
        loader=_tensor_loader(bundle,scaler,tr,args,seed+ep*997,True);loss=_train_suite7_epoch(model,loader,opt,device,args)
        if ep==1 or ep==epochs:print(f'[{label}] reproduced Suite7 epoch={ep}/{epochs} loss={loss:.5f}')
    return model


def _semantic_from_base(base,bundle,cfg,device):
    m=_make_semantic(bundle.X.shape[1],len(bundle.faculty_names),cfg).to(device);m.load_state_dict(base.state_dict(),strict=True);return m


def _split_rel_selection(bundle,val_mask):
    val_mask=np.asarray(val_mask,bool);rel=np.zeros(len(val_mask),bool);sel=np.zeros(len(val_mask),bool);independent=True
    for fi in np.unique(bundle.faculty_idx[val_mask]):
        m=val_mask&(bundle.faculty_idx==fi)&np.isfinite(bundle.blocks);bs=sorted(pd.unique(bundle.blocks[m]))
        if len(bs)>=2:
            sel[m&(bundle.blocks==bs[-1])]=True;rel[m&(bundle.blocks!=bs[-1])]=True
        elif len(bs)==1:
            rel[m]=True;sel[m]=True;independent=False
    # Rows without finite blocks are not used for temporal selection.
    return rel,sel,independent


@torch.no_grad()
def _predict(model,bundle,scaler,mask,device,args):
    model.eval();loader=_tensor_loader(bundle,scaler,mask,args,0,False);res=[];cf=[];base=[]
    for X,f,q,y in loader:
        out=model(X.to(device),f.to(device),q.to(device));res.append(out['residuals'].cpu().numpy());cf.append(out['counterfactual_prob'].cpu().numpy());base.append(out['final_prob'].cpu().numpy())
    return {'residuals':np.concatenate(res) if res else np.zeros((0,4)),'counterfactual_prob':np.concatenate(cf) if cf else np.zeros((0,4)),'suite7_router_prob':np.concatenate(base) if base else np.array([])}


@torch.no_grad()
def _predict_suite7(model,bundle,scaler,mask,device,args):
    model.eval();loader=_tensor_loader(bundle,scaler,mask,args,0,False);outp=[]
    for X,f,q,y in loader:outp.append(model(X.to(device),f.to(device),q.to(device))['final_prob'].cpu().numpy())
    return np.concatenate(outp) if outp else np.array([])


def _semantic_train_epoch(model,loader,opt,device,args,active_epochs,epoch):
    model.eval();acc=0.;n=0;mass=np.zeros(4,float)
    for batch in loader:
        X,f,q,y,d=batch;X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device);d=d.to(device)
        # Experts that have already reached their selected refit epoch receive zero responsibility.
        if active_epochs is not None:
            gate=torch.as_tensor([1.0 if epoch<=int(e) and int(e)>0 else 0.0 for e in active_epochs],device=device,dtype=d.dtype);d=d*gate[None,:]
        opt.zero_grad(set_to_none=True);out=model(X,f,q);ls=semantic_specialist_loss(out,y,d,magnitude_lambda=args.magnitude_lambda,domain_power=args.domain_power,min_domain_weight=args.domain_train_floor)
        if ls['total'].requires_grad:
            ls['total'].backward();torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad],args.grad_clip);opt.step()
        acc+=float(ls['total'].detach().cpu())*len(y);n+=len(y);mass+=ls['domain_mass'].cpu().numpy()
    return acc/max(n,1),mass


def _expert_scores(contracts,args):
    scores=np.full(4,-np.inf,float);rows=[]
    for k in range(4):
        use=[r for r in contracts if r['expert']==k+1 and r['faculty']!='__ALL__' and np.isfinite(r.get('in_domain_delta_ap',np.nan)) and np.isfinite(r.get('in_domain_delta_f2_20',np.nan))]
        if use:
            vals=np.array([r['in_domain_delta_ap']+args.macro_f2_weight*r['in_domain_delta_f2_20'] for r in use],float);score=float(np.median(vals));med_ap=float(np.median([r['in_domain_delta_ap'] for r in use]));med_f2=float(np.median([r['in_domain_delta_f2_20'] for r in use]));win=float(np.mean(vals>0))
        else:
            pooled=next((r for r in contracts if r['expert']==k+1 and r['faculty']=='__ALL__'),None)
            if pooled and np.isfinite(pooled.get('in_domain_delta_ap',np.nan)) and np.isfinite(pooled.get('in_domain_delta_f2_20',np.nan)):
                score=float(pooled['in_domain_delta_ap']+args.macro_f2_weight*pooled['in_domain_delta_f2_20']);med_ap=pooled['in_domain_delta_ap'];med_f2=pooled['in_domain_delta_f2_20'];win=float(score>0)
            else:score=-np.inf;med_ap=med_f2=win=np.nan
        scores[k]=score;rows.append({'expert':k+1,'domain':DOMAIN_NAMES[k],'selection_score':score,'median_delta_ap':med_ap,'median_delta_f2_20':med_f2,'faculty_win_rate':win,'valid_faculties':len(use)})
    return scores,rows


def _fuse_np(q,residuals,weights):
    q=np.clip(np.asarray(q,float),EPS,1-EPS);corr=np.sum(np.asarray(weights,float)[:,1:]*np.asarray(residuals,float),axis=1);z=np.log(q/(1-q))+corr;z=np.clip(z,-30,30);return 1/(1+np.exp(-z)),corr


def _selection_vs_suite7(y,base,candidate,faculty_idx,names,args):
    df=per_faculty_validation_rows(y,base,candidate,faculty_idx,names,args.threshold_beta,.20);score,macro=model_selection_score(df,args.macro_f2_weight);return score,macro,df


def _fit_validation(bundle,tr,rel,sel,args,seed,cfg,suite7_epochs,device,label):
    scaler=FeatureStandardizer.fit(bundle.X[tr]);base=_reproduce_suite7(bundle,tr,scaler,args,seed,suite7_epochs,cfg,device,label+'-base');model=_semantic_from_base(base,bundle,cfg,device);freeze_for_semantic_discovery(model)
    cal=SemanticDomainCalibrator.fit(bundle.X[tr],bundle.feature_names,bundle.faculty_idx[tr],range(len(bundle.faculty_names)));dom_all=cal.transform(bundle.X,bundle.faculty_idx)
    audit=domain_viability_rows(bundle.y[tr],bundle.q[tr],dom_all.scores[tr],bundle.faculty_idx[tr],bundle.faculty_names,fold=-1,split='train',core_threshold=args.domain_core_threshold,min_rows=args.domain_min_rows,min_failures=args.domain_min_failures,beta=args.threshold_beta)
    pooled_a={r['domain']:r for r in audit if r['faculty']=='__ALL__'}
    print('['+label+' DOMAIN-AUDIT] '+' | '.join(f"{dn}:core={pooled_a[dn]['core_rate']:.1%} n={pooled_a[dn]['core_rows']} fail={pooled_a[dn]['core_failures']} prev={pooled_a[dn]['core_prevalence'] if np.isfinite(pooled_a[dn]['core_prevalence']) else float('nan'):.2%}" for dn in DOMAIN_NAMES))
    base_rel=_predict_suite7(base,bundle,scaler,rel,device,args);base_sel=_predict_suite7(base,bundle,scaler,sel,device,args)
    best_score=np.full(4,-np.inf);best_epoch=np.zeros(4,int);best_state=[copy.deepcopy(model.experts[k].state_dict()) for k in range(4)];history=[];pat=0
    opt=torch.optim.AdamW([p for p in model.parameters() if p.requires_grad],lr=args.expert_lr,weight_decay=args.weight_decay)
    # epoch 0 contract
    for ep in range(0,args.epochs+1):
        if ep>0:
            loader=_tensor_loader(bundle,scaler,tr,args,seed+20000+ep*977,True,dom_all.scores);loss,mass=_semantic_train_epoch(model,loader,opt,device,args,None,ep)
        else:loss=np.nan;mass=np.zeros(4)
        pred=_predict(model,bundle,scaler,rel,device,args);contracts=expert_contract_rows(bundle.y[rel],bundle.q[rel],pred['counterfactual_prob'],dom_all.scores[rel],bundle.faculty_idx[rel],bundle.faculty_names,fold=-1,split='reliability',core_threshold=args.domain_core_threshold,min_rows=args.domain_min_rows,min_failures=args.domain_min_failures,beta=args.threshold_beta);scores,srows=_expert_scores(contracts,args)
        improved=False
        for k in range(4):
            if np.isfinite(scores[k]) and (best_score[k]==-np.inf or scores[k]>best_score[k]+args.expert_min_improvement):
                best_score[k]=scores[k];best_epoch[k]=ep;best_state[k]=copy.deepcopy(model.experts[k].state_dict());improved=True
        rc=mean_pairwise_residual_correlation(pred['residuals']);history.extend([{'epoch':ep,'loss':loss,'residual_correlation':rc,'domain_mass':mass[k] if ep>0 else np.nan,**r} for k,r in enumerate(srows)])
        if ep==0 or ep==1 or ep%args.print_every==0:
            txt=' | '.join(f'E{k+1}:{DOMAIN_NAMES[k]} sel={scores[k]:+.4f} best={best_score[k]:+.4f}@{best_epoch[k]}' for k in range(4));print(f'[{label} EXPERT] ep={ep:02d} loss={loss if np.isfinite(loss) else float("nan"):.5f} Rcorr={rc:.3f} | {txt}')
        pat=0 if improved else pat+1
        if ep>=args.min_epochs and pat>=args.patience:break
    for k in range(4):model.experts[k].load_state_dict(best_state[k])
    enabled=(best_score>args.min_expert_selection_improvement)&(best_epoch>0);selected_epochs=np.where(enabled,best_epoch,0)
    pred_rel=_predict(model,bundle,scaler,rel,device,args)
    rel_est=estimate_reliability(bundle.y[rel],bundle.q[rel],pred_rel['counterfactual_prob'],dom_all.scores[rel],bundle.faculty_idx[rel],bundle.blocks[rel],bundle.faculty_names,core_threshold=args.domain_core_threshold,min_rows=args.domain_min_rows,min_failures=args.domain_min_failures,reliability_gain=args.reliability_gain,shrink_rows=args.reliability_shrink_rows,beta=args.threshold_beta,failure_equiv_rows=args.reliability_failure_equiv_rows)
    rel_matrix=rel_est['reliability']*enabled[None,:]
    pred_sel=_predict(model,bundle,scaler,sel,device,args);w_sel,route_sel=structured_semantic_weights(dom_all.scores[sel],bundle.faculty_idx[sel],rel_matrix,dom_all.uncertainty[sel],args.authority_min,args.authority_max,args.domain_route_floor);p_sel,_=_fuse_np(bundle.q[sel],pred_sel['residuals'],w_sel)
    deploy_score,deploy_macro,deploy_rows=_selection_vs_suite7(bundle.y[sel],base_sel,p_sel,bundle.faculty_idx[sel],bundle.faculty_names,args)
    deploy=bool(np.isfinite(deploy_score) and deploy_score>args.min_selection_improvement)
    print(f'[{label}] selected expert epochs={selected_epochs.tolist()} enabled={enabled.astype(int).tolist()} deploy_sel={deploy_score:+.5f} medianΔAP={deploy_macro["median_delta_ap"]:+.4f} medianΔF2={deploy_macro["median_delta_f2_20"]:+.4f} -> {"SUITE10" if deploy else "EXACT SUITE7 FALLBACK"}')
    return {'scaler':scaler,'calibrator':cal,'selected_epochs':selected_epochs,'expert_best_scores':best_score,'expert_enabled':enabled,'reliability':rel_matrix,'reliability_detail':rel_est['detail_rows'],'history':pd.DataFrame(history),'deploy_score':deploy_score,'deploy_macro':deploy_macro,'deploy':deploy,'selection_rows':deploy_rows,'selection_independent':not np.array_equal(rel,sel)}


def _full_refit(bundle,target,args,seed,cfg,suite7_checkpoint,selected,device,label):
    scaler=FeatureStandardizer.fit(bundle.X);cal=SemanticDomainCalibrator.fit(bundle.X,bundle.feature_names,bundle.faculty_idx,range(len(bundle.faculty_names)));dom_hist=cal.transform(bundle.X,bundle.faculty_idx);dom_target=cal.transform(target.X,target.faculty_idx)
    model=_make_semantic(bundle.X.shape[1],len(bundle.faculty_names),cfg).to(device);load_suite7_initialization(model,suite7_checkpoint,bundle.faculty_names);freeze_for_semantic_discovery(model);opt=torch.optim.AdamW([p for p in model.parameters() if p.requires_grad],lr=args.expert_lr,weight_decay=args.weight_decay)
    maxep=int(np.max(selected['selected_epochs'])) if len(selected['selected_epochs']) else 0
    mask=np.ones(len(bundle.y),bool)
    for ep in range(1,maxep+1):
        loader=_tensor_loader(bundle,scaler,mask,args,seed+20000+ep*977,True,dom_hist.scores);loss,mass=_semantic_train_epoch(model,loader,opt,device,args,selected['selected_epochs'],ep)
        if ep==1 or ep==maxep or ep%args.print_every==0:print(f'[{label} REFIT] ep={ep}/{maxep} loss={loss:.5f} active={[int(ep<=e and e>0) for e in selected["selected_epochs"]]}')
    pred=_predict(model,target,scaler,np.ones(len(target.y),bool),device,args);weights,route=structured_semantic_weights(dom_target.scores,target.faculty_idx,selected['reliability'],dom_target.uncertainty,args.authority_min,args.authority_max,args.domain_route_floor);candidate,corr=_fuse_np(target.q,pred['residuals'],weights)
    return model,scaler,cal,dom_hist,dom_target,pred,weights,route,candidate,corr


def _load_adapter_score(root,fold,faculty,y,q,atol):
    p=Path(root)/f'fold_{fold}'/f'{faculty}_scores.csv';df=pd.read_csv(p)
    if len(df)!=len(y) or not np.array_equal(df.actual.to_numpy(int),np.asarray(y,int)):raise ValueError(f'Suite7 alignment mismatch {faculty} F{fold}')
    if 'committee' in df and not np.allclose(df.committee.to_numpy(float),q,atol=atol,rtol=1e-5):raise ValueError(f'Suite7 committee mismatch {faculty} F{fold}')
    if ADAPTER not in df:raise ValueError(f'{ADAPTER} missing in {p}')
    return df[ADAPTER].to_numpy(float)


def _optional_score(root,fold,faculty,column,y,q,atol):
    if not root:return None
    p=Path(root)/f'fold_{fold}'/f'{faculty}_scores.csv'
    if not p.exists():return None
    df=pd.read_csv(p)
    if column not in df:return None
    if len(df)!=len(y) or not np.array_equal(df.actual.to_numpy(int),np.asarray(y,int)):raise ValueError(f'optional score alignment mismatch {p}')
    if 'committee' in df and not np.allclose(df.committee.to_numpy(float),q,atol=atol,rtol=1e-5):raise ValueError(f'optional committee mismatch {p}')
    return df[column].to_numpy(float)


def _reference_scores(args,fold,target):
    scores={COMMITTEE:target.q};parts={SHARED:[],ADAPTER:[],SUITE8:[],SUITE9C:[],SUITE9F:[]};has={k:True for k in parts}
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m];q=target.q[m]
        sf=load_shared_fold_frame(args.shared_results_dir,fold,faculty,y,q,args.reference_alignment_atol);parts[SHARED].append(sf[SHARED].to_numpy(float));parts[ADAPTER].append(_load_adapter_score(args.suite7_results_dir,fold,faculty,y,q,args.reference_alignment_atol))
        for key,root,col in [(SUITE8,args.suite8_results_dir,SUITE8),(SUITE9C,args.suite9_results_dir,SUITE9C),(SUITE9F,args.suite9_results_dir,SUITE9F)]:
            z=_optional_score(root,fold,faculty,col,y,q,args.reference_alignment_atol)
            if z is None:has[key]=False
            else:parts[key].append(z)
    scores[SHARED]=np.concatenate(parts[SHARED]);scores[ADAPTER]=np.concatenate(parts[ADAPTER])
    for k in [SUITE8,SUITE9C,SUITE9F]:
        if has[k] and len(parts[k])==len(target.faculty_names):scores[k]=np.concatenate(parts[k])
    return scores


def _metric_rows(fold,target,scores,args):
    rows=[]
    for fi,name in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m]
        for model,s in scores.items():
            ss=np.asarray(s)[m];r={'fold':fold,'faculty':name,'model':model,'rows':int(m.sum()),'failures':int(y.sum()),'prevalence':float(y.mean()),'ap':safe_ap(y,ss),'auc':safe_auc(y,ss)}
            for b in BUDGETS:
                z=equal_budget_metrics(y,ss,b,args.threshold_beta);tag=int(round(100*b));r[f'precision_at_{tag}']=z['precision'];r[f'recall_at_{tag}']=z['recall'];r[f'f1_at_{tag}']=z['f1'];r[f'f2_at_{tag}']=z['fbeta']
            rows.append(r)
    return rows


def _mean_fold_summary(metrics,holdout_folds):
    d=pd.DataFrame(metrics);d=d[d.fold.isin(holdout_folds)];rows=[]
    for (faculty,model),g in d.groupby(['faculty','model']):
        r={'faculty':faculty,'model':model,'holdout_folds':len(g)}
        for c in ['ap','auc','f2_at_10','f2_at_15','f2_at_20','f2_at_25']:r[f'mean_fold_{c}']=float(g[c].mean())
        rows.append(r)
    return pd.DataFrame(rows)


def _pooled_rows(records,holdout_folds,args):
    rows=[]
    for faculty in sorted({r['faculty'] for r in records}):
        use=[r for r in records if r['faculty']==faculty and r['fold'] in holdout_folds];models=sorted(set.intersection(*[set(r['scores']) for r in use])) if use else []
        y=np.concatenate([r['y'] for r in use])
        for model in models:
            s=np.concatenate([r['scores'][model] for r in use]);z=equal_budget_metrics(y,s,.20,args.threshold_beta);rows.append({'faculty':faculty,'model':model,'rows':len(y),'failures':int(y.sum()),'ap':safe_ap(y,s),'auc':safe_auc(y,s),'f2_at_20':z['fbeta']})
    return pd.DataFrame(rows)


def _operational(records,faculties,args,out):
    rows=[];models=[COMMITTEE,SHARED,ADAPTER,CANDIDATE,FINAL]
    available=set.intersection(*[set(r['scores']) for r in records]) if records else set()
    for model in models:
        if model not in available:continue
        for faculty in faculties:
            try:_,rr=evaluate_faculty(records,faculty,model,args.dev_folds,args.holdout_folds,args.threshold_beta,args.min_precision,args.max_warning_ratio,args.min_flags,verbose=args.verbose_operational);rows.extend(rr)
            except Exception as e:print(f'[WARN] operational evaluation failed faculty={faculty} model={model}: {e}')
    df=pd.DataFrame(rows);df.to_csv(out/'suite10_warning_metrics.csv',index=False);return df


def _comparison(mean_fold,pooled,warning,faculties):
    rows=[];wf=warning[(warning.fold.astype(str)=='POOLED')] if len(warning) else pd.DataFrame()
    def mf(f,m,c):
        d=mean_fold[(mean_fold.faculty==f)&(mean_fold.model==m)];return float(d.iloc[0][c]) if len(d) else np.nan
    def pp(f,m,c):
        d=pooled[(pooled.faculty==f)&(pooled.model==m)];return float(d.iloc[0][c]) if len(d) else np.nan
    def op(f,m,pol,c):
        if wf.empty:return np.nan
        d=wf[(wf.faculty==f)&(wf.model==m)&(wf.policy==pol)];return float(d.iloc[0][c]) if len(d) else np.nan
    for f in faculties:
        r={'faculty':f}
        for m,short in [(COMMITTEE,'committee'),(SHARED,'shared'),(ADAPTER,'adapter'),(CANDIDATE,'suite10_candidate'),(FINAL,'suite10_final')]:
            for c in ['ap','auc','f2_at_20']:r[f'{short}_mean_fold_{c}']=mf(f,m,'mean_fold_'+c)
            for c in ['ap','auc','f2_at_20']:r[f'{short}_pooled_{c}']=pp(f,m,c)
            r[f'{short}_fixed_f2']=op(f,m,'fixed','fbeta');r[f'{short}_drift_f2']=op(f,m,'drift','fbeta')
        r['candidate_minus_adapter_mean_fold_ap']=r['suite10_candidate_mean_fold_ap']-r['adapter_mean_fold_ap'];r['candidate_minus_adapter_mean_fold_f2_20']=r['suite10_candidate_mean_fold_f2_at_20']-r['adapter_mean_fold_f2_at_20'];r['final_minus_adapter_mean_fold_ap']=r['suite10_final_mean_fold_ap']-r['adapter_mean_fold_ap'];r['final_minus_adapter_drift_f2']=r['suite10_final_drift_f2']-r['adapter_drift_f2'];rows.append(r)
    return pd.DataFrame(rows)


def run_fold(args,faculties,fold,out):
    print('\n'+'='*180);print(f'SUITE10 SEMANTIC ACADEMIC-MECHANISM SPECIALISTS | TARGET FOLD {fold}');print('='*180)
    schema=load_shared_feature_schema(args.shared_results_dir,fold);common=list(schema['common_raw_features']);caches=load_fold_caches(args.cache_dir,faculties,fold);hist=build_historical_bundle(caches,faculties,common);target=build_target_bundle(caches,faculties,common)
    if list(schema.get('shared_feature_names',hist.feature_names))!=list(hist.feature_names):raise ValueError(f'F{fold}: shared feature schema mismatch')
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True);suite7_ck=Path(args.suite7_results_dir)/f'fold_{fold}'/'shared_private_model.pt';ck=torch.load(suite7_ck,map_location='cpu',weights_only=False);cfg=_base_cfg(ck,args);suite7_epochs=int(ck.get('selected_epoch') or 1);seed=args.seed+fold*100003;device=choose_device(args.device)
    tr,va=latest_block_split(hist.faculty_idx,hist.blocks,args.val_blocks);rel,sel,independent=_split_rel_selection(hist,va)
    if tr.sum()<args.min_train_rows or rel.sum()<args.min_val_rows or sel.sum()<args.min_val_rows:
        print(f'[WARN F{fold}] small split train={tr.sum()} reliability={rel.sum()} selection={sel.sum()} — using available chronology-safe rows')
    print(f'[DATA] common={len(common)} inputs={hist.X.shape[1]} OOF={len(hist.y):,} failures={hist.y.sum():,} ({hist.y.mean():.2%}) target={len(target.y):,} | train={tr.sum():,} reliability={rel.sum():,} selection={sel.sum():,} independent_rel_selection={independent}')
    selected=_fit_validation(hist,tr,rel,sel,args,seed,cfg,suite7_epochs,device,f'F{fold}')
    refs=_reference_scores(args,fold,target);model,scaler,cal,dom_hist,dom_target,pred,weights,route,candidate,corr=_full_refit(hist,target,args,seed,cfg,suite7_ck,selected,device,f'F{fold}');adapter=refs[ADAPTER];final=candidate.copy() if selected['deploy'] else adapter.copy();scores={**refs,CANDIDATE:candidate,FINAL:final}
    torch.save({'state_dict':model.state_dict(),'selected_expert_epochs':selected['selected_epochs'].tolist(),'expert_enabled':selected['expert_enabled'].astype(int).tolist(),'expert_best_scores':selected['expert_best_scores'].tolist(),'reliability':selected['reliability'],'deploy':selected['deploy'],'deployment_selection_score':selected['deploy_score'],'model_config':cfg,'faculties':faculties,'domain_names':DOMAIN_NAMES,'domain_design':'semantic pre-outcome academic mechanisms'},fold_dir/'semantic_specialist_model.pt');scaler.save(fold_dir/'semantic_scaler.npz');save_json(cal.to_dict(),fold_dir/'semantic_domain_calibrator.json');selected['history'].to_csv(fold_dir/'expert_training_history.csv',index=False);selected['selection_rows'].to_csv(fold_dir/'deployment_selection_by_faculty.csv',index=False)

    # Descriptive final-domain diagnostics (calibrator fit on all historical rows; target remains unseen).
    viability=[];overlap=[];contracts=[];drift=[];authority=[];corrrows=[]
    viability+=domain_viability_rows(hist.y,hist.q,dom_hist.scores,hist.faculty_idx,faculties,fold,'historical',args.domain_core_threshold,args.domain_min_rows,args.domain_min_failures,args.threshold_beta)
    viability+=domain_viability_rows(target.y,target.q,dom_target.scores,target.faculty_idx,faculties,fold,'target',args.domain_core_threshold,args.domain_min_rows,args.domain_min_failures,args.threshold_beta)
    overlap+=domain_overlap_rows(dom_hist.scores,hist.faculty_idx,faculties,fold,'historical',args.domain_core_threshold);overlap+=domain_overlap_rows(dom_target.scores,target.faculty_idx,faculties,fold,'target',args.domain_core_threshold)
    contracts+=expert_contract_rows(target.y,target.q,pred['counterfactual_prob'],dom_target.scores,target.faculty_idx,faculties,fold,'target',args.domain_core_threshold,args.domain_min_rows,args.domain_min_failures,args.threshold_beta)
    # target-vs-historical domain drift, using the same all-history calibrator
    hist_recent=np.zeros(len(hist.y),bool)
    for fi in np.unique(hist.faculty_idx):
        m=(hist.faculty_idx==fi)&np.isfinite(hist.blocks);ub=sorted(pd.unique(hist.blocks[m]));
        if ub:hist_recent[m&(hist.blocks==ub[-1])]=True
    drift+=domain_drift_rows(dom_hist.scores,dom_hist.scores[hist_recent],dom_target.scores,hist.faculty_idx,hist.faculty_idx[hist_recent],target.faculty_idx,faculties,fold,args.domain_core_threshold)
    hist_core=[np.mean(dom_hist.scores[:,k]>=args.domain_core_threshold) for k in range(4)];tgt_core=[np.mean(dom_target.scores[:,k]>=args.domain_core_threshold) for k in range(4)]
    print(f'[F{fold} DOMAIN-DRIFT] '+' | '.join(f"E{k+1}:{DOMAIN_NAMES[k]} hist={hist_core[k]:.1%} target={tgt_core[k]:.1%} Δ={tgt_core[k]-hist_core[k]:+.1%}" for k in range(4)))
    for fi,name in enumerate(faculties):
        m=target.faculty_idx==fi;a=authority_summary(weights[m],dom_target.uncertainty[m],dom_target.scores[m]);a.update({'fold':fold,'faculty':name,'residual_correlation':mean_pairwise_residual_correlation(pred['residuals'][m]),'deploy':selected['deploy']});authority.append(a)
        corrrows.append({'fold':fold,'faculty':name,'mean_pairwise_residual_correlation':a['residual_correlation']})
    relrows=[]
    for r in selected['reliability_detail']:
        rr=dict(r);rr.update({'fold':fold,'deploy':selected['deploy']});relrows.append(rr)

    metrics=_metric_rows(fold,target,scores,args);pd.DataFrame(metrics).to_csv(fold_dir/'fold_metrics.csv',index=False);pd.DataFrame(viability).to_csv(fold_dir/'domain_viability.csv',index=False);pd.DataFrame(overlap).to_csv(fold_dir/'domain_overlap.csv',index=False);pd.DataFrame(contracts).to_csv(fold_dir/'expert_contracts.csv',index=False);pd.DataFrame(relrows).to_csv(fold_dir/'reliability.csv',index=False);pd.DataFrame(authority).to_csv(fold_dir/'authority_diagnostics.csv',index=False);pd.DataFrame(drift).to_csv(fold_dir/'domain_drift.csv',index=False)

    records=[]
    for fi,name in enumerate(faculties):
        m=target.faculty_idx==fi;y=target.y[m];frame=pd.DataFrame({'actual':y,'committee':target.q[m],CANDIDATE:candidate[m],FINAL:final[m],'suite10_deployed':int(selected['deploy']),'uncertainty':dom_target.uncertainty[m],'authority':route['authority'][m],'correction_logit':corr[m],'history_semesters':target.support_history_semesters[m]})
        for key in [SHARED,ADAPTER,SUITE8,SUITE9C,SUITE9F]:
            if key in scores:frame[key]=scores[key][m]
        for k,dn in enumerate(DOMAIN_NAMES):frame[f'domain_{dn}']=dom_target.scores[m,k];frame[f'weight_expert_{k+1}']=weights[m,k+1];frame[f'residual_expert_{k+1}']=pred['residuals'][m,k];frame[f'counterfactual_expert_{k+1}']=pred['counterfactual_prob'][m,k]
        frame['weight_null']=weights[m,0];frame.to_csv(fold_dir/f'{name}_scores.csv',index=False)
        sm={k:np.asarray(v)[m] for k,v in scores.items()};records.append({'faculty':name,'fold':fold,'y':y,'scores':sm})
        mm={r['model']:r for r in metrics if r['faculty']==name};au=next(r for r in authority if r['faculty']==name)
        print(f'[TARGET {name}] AP C/S/A/10c/10f={mm[COMMITTEE]["ap"]:.4f}/{mm[SHARED]["ap"]:.4f}/{mm[ADAPTER]["ap"]:.4f}/{mm[CANDIDATE]["ap"]:.4f}/{mm[FINAL]["ap"]:.4f} | F2@20 A/10c/10f={mm[ADAPTER]["f2_at_20"]:.4f}/{mm[CANDIDATE]["f2_at_20"]:.4f}/{mm[FINAL]["f2_at_20"]:.4f} | auth={au["mean_authority"]:.3f} NULL={au["mean_null_weight"]:.3f} Rcorr={au["residual_correlation"]:.3f} deploy={selected["deploy"]}')
        for k,dn in enumerate(DOMAIN_NAMES):
            vr=next(r for r in viability if r['split']=='target' and r['faculty']==name and r['domain']==dn);cr=next(r for r in contracts if r['faculty']==name and r['expert']==k+1);rho=selected['reliability'][fi,k]
            print(f'  [E{k+1} {dn}] core={vr["core_rate"]:.1%} n={vr["core_rows"]} fail={vr["core_failures"]} prev={vr["core_prevalence"] if np.isfinite(vr["core_prevalence"]) else float("nan"):.2%} rho={rho:.3f} ΔAP_in={cr["in_domain_delta_ap"] if np.isfinite(cr["in_domain_delta_ap"]) else float("nan"):+.4f} contrast={cr["specialization_contrast_ap"] if np.isfinite(cr["specialization_contrast_ap"]) else float("nan"):+.4f}')
    return records,metrics,viability,overlap,contracts,relrows,authority,drift,corrrows,{'fold':fold,'selected_expert_epochs':'|'.join(map(str,selected['selected_epochs'])),'enabled':'|'.join(map(str,selected['expert_enabled'].astype(int))),'deploy':selected['deploy'],'selection_score':selected['deploy_score'],'selection_independent':selected['selection_independent']}


def parse_args():
    p=argparse.ArgumentParser(description='Suite10 semantic academic-mechanism specialists')
    p.add_argument('--cache_dir',required=True);p.add_argument('--shared_results_dir',required=True);p.add_argument('--suite7_results_dir',required=True);p.add_argument('--suite8_results_dir',default=None);p.add_argument('--suite9_results_dir',default=None);p.add_argument('--models_dir',required=True);p.add_argument('--faculties',default='all');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--reference_metrics',default=None)
    p.add_argument('--seed',type=int,default=42);p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto');p.add_argument('--n_experts',type=int,default=4);p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--encoder_hidden',type=int,default=256);p.add_argument('--representation_dim',type=int,default=128);p.add_argument('--expert_hidden',type=int,default=64);p.add_argument('--delta_max',type=float,default=2.0);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--router_temperature',type=float,default=1.0);p.add_argument('--adapter_dim',type=int,default=16);p.add_argument('--adapter_dropout',type=float,default=0.0);p.add_argument('--adapter_scale',type=float,default=1.0)
    p.add_argument('--epochs',type=int,default=10);p.add_argument('--min_epochs',type=int,default=3);p.add_argument('--patience',type=int,default=4);p.add_argument('--expert_lr',type=float,default=5e-4);p.add_argument('--expert_min_improvement',type=float,default=1e-5);p.add_argument('--min_expert_selection_improvement',type=float,default=0.0);p.add_argument('--domain_power',type=float,default=2.0);p.add_argument('--domain_train_floor',type=float,default=.05);p.add_argument('--domain_route_floor',type=float,default=.10);p.add_argument('--domain_core_threshold',type=float,default=.60);p.add_argument('--domain_min_rows',type=int,default=50);p.add_argument('--domain_min_failures',type=int,default=5)
    p.add_argument('--reliability_gain',type=float,default=.03);p.add_argument('--reliability_shrink_rows',type=float,default=200.0);p.add_argument('--reliability_failure_equiv_rows',type=float,default=20.0);p.add_argument('--authority_min',type=float,default=.10);p.add_argument('--authority_max',type=float,default=.70)
    p.add_argument('--val_blocks',type=int,default=2);p.add_argument('--min_train_rows',type=int,default=800);p.add_argument('--min_val_rows',type=int,default=200);p.add_argument('--batch_size',type=int,default=1024);p.add_argument('--predict_batch_size',type=int,default=4096);p.add_argument('--samples_per_epoch',type=int,default=0);p.add_argument('--lr',type=float,default=1e-3);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--grad_clip',type=float,default=5.0);p.add_argument('--print_every',type=int,default=1);p.add_argument('--magnitude_lambda',type=float,default=.01);p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--macro_f2_weight',type=float,default=1.0);p.add_argument('--min_selection_improvement',type=float,default=.001)
    p.add_argument('--baseline_router_lambda',type=float,default=.5);p.add_argument('--baseline_specialization_lambda',type=float,default=.01);p.add_argument('--baseline_magnitude_lambda',type=float,default=.01);p.add_argument('--baseline_utility_temperature',type=float,default=.25)
    p.add_argument('--threshold_beta',type=float,default=2.0);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--reference_alignment_atol',type=float,default=1e-6);p.add_argument('--verbose_operational',action='store_true')
    a=p.parse_args();a.dev_folds=[int(x) for x in a.dev_folds.split(',') if x.strip()];a.holdout_folds=[int(x) for x in a.holdout_folds.split(',') if x.strip()];return a


def main():
    args=parse_args();out=Path(args.models_dir);out.mkdir(parents=True,exist_ok=True);first=min(args.dev_folds+args.holdout_folds);schema=load_shared_feature_schema(args.shared_results_dir,first);available=list(schema.get('faculties',[]))
    if not available:
        suffix=f'_fold_{first}.joblib';available=sorted([p.name[:-len(suffix)] for p in Path(args.cache_dir).glob(f'*{suffix}')])
    faculties=parse_faculties(args.faculties,available)
    print('='*180);print('SUITE10 — SEMANTIC ACADEMIC-MECHANISM SPECIALISTS');print('E1 student-sparse | E2 course-sparse | E3 temporal deterioration | E4 history-rich relational | uncertainty=authority only');print('='*180);print(f'faculties={len(faculties)} folds={args.dev_folds+args.holdout_folds} epochs={args.epochs} core={args.domain_core_threshold} authority=[{args.authority_min},{args.authority_max}]')
    save_json({'faculties':faculties,'args':vars(args),'domains':list(DOMAIN_NAMES),'responsibility':'observable pre-outcome semantic academic mechanisms','router':'deterministic structured semantic routing','uncertainty_role':'specialist authority cap only','fallback':'exact Suite7 if newest historical selection block does not improve'},out/'suite10_run_config.json')
    records=[];metrics=[];viability=[];overlap=[];contracts=[];reliability=[];authority=[];drift=[];corr=[];training=[]
    for fold in args.dev_folds+args.holdout_folds:
        rr,mm,vv,oo,cc,rel,aa,dd,co,ss=run_fold(args,faculties,fold,out);records.extend(rr);metrics.extend(mm);viability.extend(vv);overlap.extend(oo);contracts.extend(cc);reliability.extend(rel);authority.extend(aa);drift.extend(dd);corr.extend(co);training.append(ss)
    pd.DataFrame(metrics).to_csv(out/'suite10_fold_metrics.csv',index=False);pd.DataFrame(viability).to_csv(out/'suite10_domain_viability.csv',index=False);pd.DataFrame(overlap).to_csv(out/'suite10_domain_overlap.csv',index=False);pd.DataFrame(contracts).to_csv(out/'suite10_expert_contracts.csv',index=False);pd.DataFrame(reliability).to_csv(out/'suite10_reliability.csv',index=False);pd.DataFrame(authority).to_csv(out/'suite10_authority_diagnostics.csv',index=False);pd.DataFrame(drift).to_csv(out/'suite10_domain_drift.csv',index=False);pd.DataFrame(corr).to_csv(out/'suite10_residual_correlations.csv',index=False);pd.DataFrame(training).to_csv(out/'suite10_training_summary.csv',index=False)
    meanf=_mean_fold_summary(metrics,args.holdout_folds);meanf.to_csv(out/'suite10_holdout_mean_fold.csv',index=False);pooled=_pooled_rows(records,args.holdout_folds,args);pooled.to_csv(out/'suite10_holdout_pooled_raw.csv',index=False);warning=_operational(records,faculties,args,out);cmp=_comparison(meanf,pooled,warning,faculties);cmp.to_csv(out/'suite10_comparison.csv',index=False)
    if args.reference_metrics:
        official=merge_external_references(cmp,args.reference_metrics);official.to_csv(out/'suite10_official_comparison.csv',index=False)
    print('\nSUITE10 PRIMARY HOLDOUT COMPARISON — MEAN WITHIN-FOLD METRICS (score-scale robust)');print(cmp[['faculty','adapter_mean_fold_ap','suite10_candidate_mean_fold_ap','candidate_minus_adapter_mean_fold_ap','adapter_mean_fold_f2_at_20','suite10_candidate_mean_fold_f2_at_20','candidate_minus_adapter_mean_fold_f2_20','suite10_final_drift_f2','final_minus_adapter_drift_f2']].to_string(index=False))
    print('\nNOTE: suite10_holdout_pooled_raw.csv is retained for calibration diagnostics, but mean-within-fold AP/F2 is the primary ranking comparison because candidate/fallback score spaces can differ across folds.')
    print(f'\nCompleted. Results: {out}')

if __name__=='__main__':main()
