# -*- coding: utf-8 -*-
"""Suite7 — Cross-faculty shared Residual-MoE + faculty-private adapters.

Controlled Experiment B:
    Suite5: [x, faculty embedding] -> shared encoder h -> shared experts/router
    Suite7: [x, faculty embedding] -> shared encoder h -> h + A_f(h) -> shared experts/router

The completed Suite5 canonical Committee cache is reused. Suite5 target predictions
are read as a frozen comparison reference and are never retrained here.
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

import numpy as np
import pandas as pd
import torch
from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler

from suite2_common import safe_ap, safe_auc, equal_budget_metrics
from suite7_common import (
    FeatureStandardizer,
    load_fold_caches,
    build_historical_bundle,
    build_target_bundle,
    faculty_class_sampling_weights,
    latest_block_split,
    per_faculty_validation_rows,
    model_selection_score,
    parse_faculties,
    save_json,
)
from suite7_eval import evaluate_faculty
from suite7_reporting import (
    load_shared_feature_schema,
    load_shared_fold_frame,
    build_shared_private_comparison,
    merge_external_references,
    summarize_shared_router_frame,
)
from suite7_shared_private_moe import (
    SharedPrivateResidualMoE,
    residual_moe_loss,
    router_diagnostics,
    adapter_diagnostics,
    adapter_parameter_rows,
    seed_everything,
    choose_device,
)

BUDGETS=(0.10,0.15,0.20,0.25)
PRIVATE_MODEL='residual_moe_shared_private'
SHARED_MODEL='residual_moe_shared'
COMMITTEE_MODEL='committee'


def _tensor_loader(bundle,scaler,mask,args,seed,sampled=True):
    mask=np.asarray(mask,bool)
    X=scaler.transform(bundle.X[mask])
    y=bundle.y[mask].astype(np.float32)
    q=bundle.q[mask].astype(np.float32)
    f=bundle.faculty_idx[mask].astype(np.int64)
    ds=TensorDataset(torch.from_numpy(X),torch.from_numpy(f),torch.from_numpy(q),torch.from_numpy(y))
    if not sampled:
        return DataLoader(ds,batch_size=args.predict_batch_size,shuffle=False,num_workers=0)
    w,diag=faculty_class_sampling_weights(bundle.y[mask],bundle.faculty_idx[mask],bundle.faculty_names,args.sampling_beta)
    gen=torch.Generator();gen.manual_seed(int(seed))
    n=len(ds) if args.samples_per_epoch<=0 else int(args.samples_per_epoch)
    sampler=WeightedRandomSampler(torch.as_tensor(w,dtype=torch.double),num_samples=n,replacement=True,generator=gen)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0),diag


def _make_model(input_dim,n_faculties,args):
    return SharedPrivateResidualMoE(
        input_dim=input_dim,
        n_faculties=n_faculties,
        n_experts=args.n_experts,
        faculty_dim=args.faculty_dim,
        encoder_hidden=args.encoder_hidden,
        representation_dim=args.representation_dim,
        expert_hidden=args.expert_hidden,
        delta_max=args.delta_max,
        dropout=args.dropout,
        router_temperature=args.router_temperature,
        adapter_dim=args.adapter_dim,
        adapter_dropout=args.adapter_dropout,
        adapter_scale=args.adapter_scale,
    )


def train_epoch(model,loader,optimizer,device,args):
    model.train();acc={'total':0.,'prediction':0.,'router':0.,'specialization_mi':0.,'magnitude':0.};n=0
    for X,f,q,y in loader:
        X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device)
        optimizer.zero_grad(set_to_none=True)
        out=model(X,f,q)
        ls=residual_moe_loss(
            out,y,q,
            router_lambda=args.router_lambda,
            specialization_lambda=args.specialization_lambda,
            magnitude_lambda=args.magnitude_lambda,
            utility_temperature=args.utility_temperature,
        )
        ls['total'].backward();torch.nn.utils.clip_grad_norm_(model.parameters(),args.grad_clip);optimizer.step()
        b=len(y);n+=b
        for k in acc:acc[k]+=float(ls[k].detach().cpu())*b
    return {k:v/max(n,1) for k,v in acc.items()}


@torch.no_grad()
def predict_bundle(model,bundle,scaler,mask,device,args):
    model.eval();loader=_tensor_loader(bundle,scaler,mask,args,seed=0,sampled=False)
    ps=[];ws=[];rs=[];cs=[];an=[];hn=[];hfn=[];rel=[];cos=[];aabs=[]
    for X,f,q,_ in loader:
        out=model(X.to(device),f.to(device),q.to(device))
        p=out['final_prob'].cpu().numpy();w=out['weights'].cpu().numpy();r=out['residuals'].cpu().numpy();c=out['correction'].cpu().numpy()
        h=out['shared_representation'].cpu().numpy();a=out['adapter_output'].cpu().numpy();hf=out['adapted_representation'].cpu().numpy()
        ps.append(p);ws.append(w);rs.append(r);cs.append(c)
        aa=np.linalg.norm(a,axis=1);hh=np.linalg.norm(h,axis=1);ff=np.linalg.norm(hf,axis=1)
        an.append(aa);hn.append(hh);hfn.append(ff);rel.append(aa/np.maximum(hh,1e-12))
        cos.append(np.clip(np.sum(h*hf,axis=1)/np.maximum(hh*ff,1e-12),-1,1));aabs.append(np.mean(np.abs(a),axis=1))
    if not ps:
        return {'p':np.array([]),'weights':np.empty((0,args.n_experts+1)),'residuals':np.empty((0,args.n_experts)),'correction':np.array([])}
    return {
        'p':np.concatenate(ps),'weights':np.concatenate(ws),'residuals':np.concatenate(rs),'correction':np.concatenate(cs),
        'adapter_norm':np.concatenate(an),'shared_norm':np.concatenate(hn),'adapted_norm':np.concatenate(hfn),
        'relative_adapter_norm':np.concatenate(rel),'representation_cosine':np.concatenate(cos),'mean_abs_adapter_activation':np.concatenate(aabs),
    }


def _adapter_summary_from_pred(pred,mask=None):
    if mask is None:mask=np.ones(len(pred['p']),bool)
    mask=np.asarray(mask,bool)
    if not np.any(mask):
        return {'mean_adapter_norm':np.nan,'mean_shared_norm':np.nan,'mean_adapted_norm':np.nan,'mean_relative_adapter_norm':np.nan,'mean_representation_cosine':np.nan,'mean_abs_adapter_activation':np.nan}
    return {
        'mean_adapter_norm':float(np.mean(pred['adapter_norm'][mask])),
        'mean_shared_norm':float(np.mean(pred['shared_norm'][mask])),
        'mean_adapted_norm':float(np.mean(pred['adapted_norm'][mask])),
        'mean_relative_adapter_norm':float(np.mean(pred['relative_adapter_norm'][mask])),
        'mean_representation_cosine':float(np.mean(pred['representation_cosine'][mask])),
        'mean_abs_adapter_activation':float(np.mean(pred['mean_abs_adapter_activation'][mask])),
    }


def fit_with_temporal_validation(bundle,args,seed,label='shared-private'):
    tr,va=latest_block_split(bundle.faculty_idx,bundle.blocks,args.val_blocks)
    if tr.sum()<args.min_train_rows:raise RuntimeError(f'{label}: only {tr.sum()} chronological training rows')
    if va.sum()<args.min_val_rows:
        print(f'[{label}] validation rows={va.sum():,} < {args.min_val_rows}; using fixed {args.fallback_epochs} epochs')
        return int(args.fallback_epochs),pd.DataFrame(),pd.DataFrame([{'epoch':0,'selection_score':np.nan,'note':'no_validation'}])
    scaler=FeatureStandardizer.fit(bundle.X[tr]);seed_everything(seed);device=choose_device(args.device)
    model=_make_model(bundle.X.shape[1],len(bundle.faculty_names),args).to(device)
    opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    best_score=-np.inf;best_epoch=1;pat=0;history=[];best_rows=pd.DataFrame()
    print(f'[{label}] chronological train={tr.sum():,} val={va.sum():,} faculties={len(bundle.faculty_names)} input_dim={bundle.X.shape[1]} adapter={args.representation_dim}->{args.adapter_dim}->{args.representation_dim} device={device}')
    for epoch in range(1,args.epochs+1):
        loader,_=_tensor_loader(bundle,scaler,tr,args,seed+epoch*997,sampled=True);losses=train_epoch(model,loader,opt,device,args)
        pred=predict_bundle(model,bundle,scaler,va,device,args)
        rows=per_faculty_validation_rows(bundle.y[va],bundle.q[va],pred['p'],bundle.faculty_idx[va],bundle.faculty_names,args.threshold_beta,.20)
        score,macro=model_selection_score(rows,args.macro_f2_weight)
        rd=router_diagnostics(pred['weights'],pred['residuals']);ad=_adapter_summary_from_pred(pred)
        hist={'epoch':epoch,'selection_score':score,**losses,**macro,
              'null_weight':rd['mean_null_weight'],'router_entropy':rd['normalized_entropy'],'mean_max_weight':rd['mean_max_weight'],
              'mean_adapter_norm':ad['mean_adapter_norm'],'mean_relative_adapter_norm':ad['mean_relative_adapter_norm'],
              'mean_representation_cosine':ad['mean_representation_cosine'],'mean_abs_adapter_activation':ad['mean_abs_adapter_activation'],
              'mean_abs_correction':float(np.mean(np.abs(pred['correction'])))}
        history.append(hist)
        if epoch==1 or epoch%args.print_every==0:
            print(f'[{label}] epoch={epoch:03d} loss={losses["total"]:.5f} sel={score:+.5f} median ΔAP={macro["median_delta_ap"]:+.4f} median ΔF2@20={macro["median_delta_f2_20"]:+.4f} AP-win={macro["win_rate_ap"]:.1%} NULL={rd["mean_null_weight"]:.3f} H={rd["normalized_entropy"]:.3f} Arel={ad["mean_relative_adapter_norm"]:.3f} cos={ad["mean_representation_cosine"]:.4f}')
        if np.isfinite(score) and score>best_score+args.min_selection_improvement:
            best_score=score;best_epoch=epoch;best_rows=rows.copy();pat=0
        else:pat+=1
        if epoch>=args.min_epochs and pat>=args.patience:break
    print(f'[{label}] selected epoch={best_epoch} macro selection score={best_score:+.5f}')
    return best_epoch,best_rows,pd.DataFrame(history)


def refit_full(bundle,args,seed,epochs,label='shared-private'):
    mask=np.ones(len(bundle.y),bool);scaler=FeatureStandardizer.fit(bundle.X);seed_everything(seed);device=choose_device(args.device)
    model=_make_model(bundle.X.shape[1],len(bundle.faculty_names),args).to(device)
    opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    for epoch in range(1,max(int(epochs),1)+1):
        loader,_=_tensor_loader(bundle,scaler,mask,args,seed+epoch*1543,sampled=True);losses=train_epoch(model,loader,opt,device,args)
        if epoch==1 or epoch==epochs or epoch%args.print_every==0:
            print(f'[{label} refit] epoch={epoch:03d}/{epochs} loss={losses["total"]:.5f}')
    return model,scaler,device


def metric_rows_for_fold(fold,target,scores,args):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m]
        for model,s in scores.items():
            ss=np.asarray(s)[m];row={'fold':fold,'faculty':faculty,'model':model,'rows':int(m.sum()),'failures':int(y.sum()),'prevalence':float(y.mean()),'ap':safe_ap(y,ss),'auc':safe_auc(y,ss)}
            for b in BUDGETS:
                z=equal_budget_metrics(y,ss,b,args.threshold_beta);tag=int(b*100)
                row[f'precision_at_{tag}']=z['precision'];row[f'recall_at_{tag}']=z['recall'];row[f'f1_at_{tag}']=z['f1'];row[f'f2_at_{tag}']=z['fbeta']
            rows.append(row)
    return rows


def diagnostic_rows(fold,target,pred):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        base=target.faculty_idx==fi
        groups=[('all',base),('failures',base&(target.y==1)),('normals',base&(target.y==0)),('history0',base&(target.support_history_semesters<=0)),('history1_2',base&(target.support_history_semesters>=1)&(target.support_history_semesters<=2)),('history3plus',base&(target.support_history_semesters>=3))]
        for group,m in groups:
            if not np.any(m):continue
            rd=router_diagnostics(pred['weights'][m],pred['residuals'][m]);ad=_adapter_summary_from_pred(pred,m)
            rows.append({'fold':fold,'faculty':faculty,'group':group,'rows':int(m.sum()),'failures':int(target.y[m].sum()),'prevalence':float(target.y[m].mean()),
                         **rd,**ad,'mean_correction':float(np.mean(pred['correction'][m])),'mean_abs_correction':float(np.mean(np.abs(pred['correction'][m])))})
    return rows


def residual_correlation_rows(fold,target,residuals):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi
        if m.sum()<3:continue
        c=np.corrcoef(residuals[m],rowvar=False)
        for i in range(c.shape[0]):
            for j in range(i+1,c.shape[1]):
                rows.append({'fold':fold,'faculty':faculty,'expert_i':i+1,'expert_j':j+1,'correlation':float(c[i,j]) if np.isfinite(c[i,j]) else np.nan})
    return rows


def _load_shared_scores(shared_results_dir,fold,target,args):
    pieces=[];router_rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi
        df=load_shared_fold_frame(shared_results_dir,fold,faculty,target.y[m],target.q[m],args.shared_alignment_atol)
        pieces.append(df.residual_moe_shared.to_numpy(float))
        d=summarize_shared_router_frame(df);d.update({'fold':fold,'faculty':faculty});router_rows.append(d)
    return np.concatenate(pieces),router_rows


def run_fold(cache_dir,shared_results_dir,faculties,fold,args,out):
    print('\n'+'='*150);print(f'SUITE7 SHARED + FACULTY ADAPTER | TARGET FOLD {fold}');print('='*150)
    schema=load_shared_feature_schema(shared_results_dir,fold);shared_faculties=list(schema.get('faculties',[]))
    missing=[f for f in faculties if shared_faculties and f not in shared_faculties]
    if missing:raise ValueError(f'faculties absent from Suite5 shared fold {fold}: {missing}')
    common=list(schema['common_raw_features'])
    caches=load_fold_caches(cache_dir,faculties,fold);hist=build_historical_bundle(caches,faculties,common);target=build_target_bundle(caches,faculties,common)
    if schema.get('shared_feature_names') and list(schema['shared_feature_names'])!=list(hist.feature_names):
        raise ValueError(f'fold {fold}: Suite7 composed feature names do not exactly match Suite5 shared feature schema')
    shared_score,shared_router_rows=_load_shared_scores(shared_results_dir,fold,target,args)
    print(f'[DATA] exact Suite5 raw schema={len(common)} final inputs={hist.X.shape[1]} historical OOF rows={len(hist.y):,} failures={int(hist.y.sum()):,} ({hist.y.mean():.2%}) target rows={len(target.y):,}')
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True)
    save_json({'fold':fold,'faculties':faculties,'common_raw_features':common,'shared_feature_names':hist.feature_names,'adapter_dim':args.adapter_dim,'adapter_scale':args.adapter_scale,'adapter_initialization':'normal_down_zero_up'},fold_dir/'feature_schema.json')
    seed=args.seed+fold*100003
    best_epoch,val_rows,history=fit_with_temporal_validation(hist,args,seed,label=f'shared-private:F{fold}')
    history.to_csv(fold_dir/'shared_private_training_history.csv',index=False);val_rows.to_csv(fold_dir/'shared_private_best_validation_by_faculty.csv',index=False)
    model,scaler,device=refit_full(hist,args,seed,best_epoch,label=f'shared-private:F{fold}');scaler.save(fold_dir/'shared_private_scaler.npz')
    param_rows=adapter_parameter_rows(model,faculties)
    for r in param_rows:r['fold']=fold
    torch.save({'state_dict':model.state_dict(),'selected_epoch':best_epoch,'model_config':{'input_dim':hist.X.shape[1],'n_faculties':len(faculties),'n_experts':args.n_experts,'faculty_dim':args.faculty_dim,'encoder_hidden':args.encoder_hidden,'representation_dim':args.representation_dim,'expert_hidden':args.expert_hidden,'delta_max':args.delta_max,'dropout':args.dropout,'router_temperature':args.router_temperature,'adapter_dim':args.adapter_dim,'adapter_dropout':args.adapter_dropout,'adapter_scale':args.adapter_scale},'faculties':faculties,'common_raw_features':common,'adapter_initialization':'normal_down_zero_up'},fold_dir/'shared_private_model.pt')
    pred=predict_bundle(model,target,scaler,np.ones(len(target.y),bool),device,args)
    scores={COMMITTEE_MODEL:target.q,SHARED_MODEL:shared_score,PRIVATE_MODEL:pred['p']}
    metrics=metric_rows_for_fold(fold,target,scores,args);diag=diagnostic_rows(fold,target,pred);corr=residual_correlation_rows(fold,target,pred['residuals'])
    pd.DataFrame(metrics).to_csv(fold_dir/'fold_metrics.csv',index=False);pd.DataFrame(diag).to_csv(fold_dir/'adapter_specialization_by_faculty.csv',index=False);pd.DataFrame(corr).to_csv(fold_dir/'residual_correlations.csv',index=False);pd.DataFrame(param_rows).to_csv(fold_dir/'adapter_parameter_norms.csv',index=False)
    records=[]
    for fi,faculty in enumerate(faculties):
        m=target.faculty_idx==fi;sm={k:v[m] for k,v in scores.items()}
        frame=pd.DataFrame({'actual':target.y[m],'committee':target.q[m],SHARED_MODEL:shared_score[m],PRIVATE_MODEL:pred['p'][m],'correction_logit':pred['correction'][m],'history_semesters':target.support_history_semesters[m],
                            'adapter_norm':pred['adapter_norm'][m],'shared_representation_norm':pred['shared_norm'][m],'adapted_representation_norm':pred['adapted_norm'][m],'relative_adapter_norm':pred['relative_adapter_norm'][m],'representation_cosine':pred['representation_cosine'][m],'mean_abs_adapter_activation':pred['mean_abs_adapter_activation'][m]})
        for j in range(pred['weights'].shape[1]):frame[f'weight_{"null" if j==0 else f"expert_{j}"}']=pred['weights'][m,j]
        for j in range(pred['residuals'].shape[1]):frame[f'residual_expert_{j+1}']=pred['residuals'][m,j]
        frame.to_csv(fold_dir/f'{faculty}_scores.csv',index=False)
        records.append({'faculty':faculty,'fold':fold,'y':target.y[m],'scores':sm})
        mm={r['model']:r for r in metrics if r['faculty']==faculty}
        ad=next(r for r in diag if r['faculty']==faculty and r['group']=='all')
        print(f'[TARGET {faculty}] n={m.sum():,} prev={target.y[m].mean():.2%} | AP C/S/A={mm[COMMITTEE_MODEL]["ap"]:.4f}/{mm[SHARED_MODEL]["ap"]:.4f}/{mm[PRIVATE_MODEL]["ap"]:.4f} ΔA-S={mm[PRIVATE_MODEL]["ap"]-mm[SHARED_MODEL]["ap"]:+.4f} | AUC={mm[COMMITTEE_MODEL]["auc"]:.4f}/{mm[SHARED_MODEL]["auc"]:.4f}/{mm[PRIVATE_MODEL]["auc"]:.4f} | F2@20={mm[COMMITTEE_MODEL]["f2_at_20"]:.4f}/{mm[SHARED_MODEL]["f2_at_20"]:.4f}/{mm[PRIVATE_MODEL]["f2_at_20"]:.4f} | Arel={ad["mean_relative_adapter_norm"]:.3f} cos={ad["mean_representation_cosine"]:.4f} NULL={ad["mean_null_weight"]:.3f}')
    return records,metrics,diag,corr,param_rows,shared_router_rows,{'fold':fold,'selected_epoch':best_epoch,'validation_rows':int(len(val_rows)),'validation_score':float(history.selection_score.max()) if len(history) and history.selection_score.notna().any() else np.nan}


def pooled_ranking_rows(records,holdout_folds,args):
    rows=[]
    for faculty in sorted({r['faculty'] for r in records}):
        rr=[r for r in records if r['faculty']==faculty and r['fold'] in holdout_folds]
        if not rr:continue
        y=np.concatenate([r['y'] for r in rr]);models=set.intersection(*[set(r['scores']) for r in rr])
        for model in sorted(models):
            s=np.concatenate([r['scores'][model] for r in rr]);row={'faculty':faculty,'model':model,'rows':len(y),'failures':int(y.sum()),'prevalence':float(y.mean()),'ap':safe_ap(y,s),'auc':safe_auc(y,s)}
            for b in BUDGETS:
                z=equal_budget_metrics(y,s,b,args.threshold_beta);tag=int(b*100)
                row[f'precision_at_{tag}']=z['precision'];row[f'recall_at_{tag}']=z['recall'];row[f'f1_at_{tag}']=z['f1'];row[f'f2_at_{tag}']=z['fbeta']
            rows.append(row)
    return pd.DataFrame(rows)


def operational_summary(records,faculties,args,out):
    all_rows=[];policies=[]
    print('\n'+'='*150);print('SUITE7 OFFICIAL OPERATIONAL EVALUATION — DEVELOPMENT POLICIES FROZEN BEFORE HOLDOUT');print('='*150)
    for faculty in faculties:
        for model in [COMMITTEE_MODEL,SHARED_MODEL,PRIVATE_MODEL]:
            pol,rows=evaluate_faculty(records,faculty,model,args.dev_folds,args.holdout_folds,args.threshold_beta,args.min_precision,args.max_warning_ratio,args.min_flags,verbose=True)
            policies.append({'faculty':faculty,'model':model,'threshold':pol.threshold,'warning_ratio':pol.warning_ratio,'development_ap':pol.development_ap,'development_auc':pol.development_auc,'development_prevalence':pol.development_prevalence,'development_flag_rate':pol.flag_rate,'development_precision':pol.precision,'development_recall':pol.recall,'development_f1':pol.f1,'development_f2':pol.fbeta,'constraints_relaxed':pol.constraints_relaxed});all_rows.extend(rows)
    wdf=pd.DataFrame(all_rows);pdf=pd.DataFrame(policies);wdf.to_csv(out/'suite7_warning_metrics.csv',index=False);pdf.to_csv(out/'suite7_threshold_policies.csv',index=False)
    return wdf,pdf


def print_comparison(cmp):
    print('\n'+'='*150);print('SUITE7 HOLDOUT — COMMITTEE vs SUITE5 SHARED vs SUITE7 SHARED+FACULTY ADAPTER');print('='*150)
    cols=['faculty','committee_ap','shared_ap','private_ap','private_minus_shared_ap','committee_f2_20','shared_f2_20','private_f2_20','private_minus_shared_f2_20','private_minus_shared_fixed_f2','private_minus_shared_drift_f2','evidence_pattern']
    fmt={c:'{:+.4f}'.format for c in ['private_minus_shared_ap','private_minus_shared_f2_20','private_minus_shared_fixed_f2','private_minus_shared_drift_f2']}
    fmt.update({c:'{:.4f}'.format for c in ['committee_ap','shared_ap','private_ap','committee_f2_20','shared_f2_20','private_f2_20']})
    print(cmp[cols].to_string(index=False,formatters=fmt))
    for metric in ['private_minus_shared_ap','private_minus_shared_auc','private_minus_shared_f2_20','private_minus_shared_drift_f2']:
        v=cmp[metric].to_numpy(float);v=v[np.isfinite(v)]
        if len(v):print(f'{metric:34s} mean={v.mean():+.4f} median={np.median(v):+.4f} worst={v.min():+.4f} win_rate={(v>0).mean():.1%}')


def parse_args():
    p=argparse.ArgumentParser(description='Suite7 shared Residual-MoE + faculty-private adapter')
    p.add_argument('--cache_dir',required=True,help='Completed Suite5 canonical cache directory')
    p.add_argument('--shared_results_dir',required=True,help='Completed Suite5 shared Residual-MoE results (read-only reference)')
    p.add_argument('--models_dir',required=True)
    p.add_argument('--faculties',default='all');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--reference_metrics',default=None)
    p.add_argument('--seed',type=int,default=42);p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto')
    p.add_argument('--n_experts',type=int,default=4);p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--encoder_hidden',type=int,default=256);p.add_argument('--representation_dim',type=int,default=128);p.add_argument('--expert_hidden',type=int,default=64);p.add_argument('--delta_max',type=float,default=2.0);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--router_temperature',type=float,default=1.0)
    p.add_argument('--adapter_dim',type=int,default=16);p.add_argument('--adapter_dropout',type=float,default=0.0);p.add_argument('--adapter_scale',type=float,default=1.0)
    p.add_argument('--epochs',type=int,default=60);p.add_argument('--min_epochs',type=int,default=8);p.add_argument('--patience',type=int,default=8);p.add_argument('--fallback_epochs',type=int,default=20);p.add_argument('--val_blocks',type=int,default=2);p.add_argument('--min_train_rows',type=int,default=800);p.add_argument('--min_val_rows',type=int,default=200)
    p.add_argument('--batch_size',type=int,default=1024);p.add_argument('--predict_batch_size',type=int,default=4096);p.add_argument('--samples_per_epoch',type=int,default=0);p.add_argument('--lr',type=float,default=1e-3);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--grad_clip',type=float,default=5.0);p.add_argument('--print_every',type=int,default=5)
    p.add_argument('--router_lambda',type=float,default=.5);p.add_argument('--specialization_lambda',type=float,default=.01);p.add_argument('--magnitude_lambda',type=float,default=.01);p.add_argument('--utility_temperature',type=float,default=.25);p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--macro_f2_weight',type=float,default=1.0);p.add_argument('--min_selection_improvement',type=float,default=1e-5)
    p.add_argument('--threshold_beta',type=float,default=2.0);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--shared_alignment_atol',type=float,default=1e-6)
    a=p.parse_args();a.dev_folds=[int(x) for x in a.dev_folds.split(',') if x.strip()];a.holdout_folds=[int(x) for x in a.holdout_folds.split(',') if x.strip()];return a


def main():
    args=parse_args();out=Path(args.models_dir);out.mkdir(parents=True,exist_ok=True)
    first_fold=min(args.dev_folds+args.holdout_folds);schema=load_shared_feature_schema(args.shared_results_dir,first_fold);available=list(schema.get('faculties',[]))
    if not available:
        suffix=f'_fold_{first_fold}.joblib';available=sorted([p.name[:-len(suffix)] for p in Path(args.cache_dir).glob(f'*{suffix}')])
    faculties=parse_faculties(args.faculties,available)
    print('='*150);print('SUITE7 — CROSS-FACULTY SHARED RESIDUAL MoE + FACULTY-PRIVATE BOTTLENECK ADAPTER');print('Suite5 cache and shared predictions are reused read-only; only Suite7 is trained.');print('='*150)
    print(f'faculties={len(faculties)} folds={args.dev_folds+args.holdout_folds} experts={args.n_experts} adapter={args.representation_dim}->{args.adapter_dim}->{args.representation_dim} init=normal_down+zero_up device={args.device}')
    save_json({'faculties':faculties,'args':vars(args),'adapter_initialization':'normal_down_zero_up'},out/'suite7_run_config.json')
    records=[];metrics=[];diag=[];corr=[];param=[];shared_router=[];training=[]
    for fold in args.dev_folds+args.holdout_folds:
        rr,mm,dd,cc,pp,sr,tt=run_fold(args.cache_dir,args.shared_results_dir,faculties,fold,args,out)
        records.extend(rr);metrics.extend(mm);diag.extend(dd);corr.extend(cc);param.extend(pp);shared_router.extend(sr);training.append(tt)
    metric_df=pd.DataFrame(metrics);diag_df=pd.DataFrame(diag);corr_df=pd.DataFrame(corr);param_df=pd.DataFrame(param);shared_router_df=pd.DataFrame(shared_router);training_df=pd.DataFrame(training)
    metric_df.to_csv(out/'suite7_fold_metrics.csv',index=False);diag_df.to_csv(out/'suite7_adapter_diagnostics.csv',index=False);corr_df.to_csv(out/'suite7_residual_correlations.csv',index=False);param_df.to_csv(out/'suite7_adapter_parameter_norms.csv',index=False);shared_router_df.to_csv(out/'suite7_suite5_router_reference.csv',index=False);training_df.to_csv(out/'suite7_training_summary.csv',index=False)

    # Holdout behavior comparison: did the adapter change routing/correction in a meaningful faculty-specific way?
    behavior=[]
    if len(diag_df):
        da=diag_df[(diag_df.fold.isin(args.holdout_folds))&(diag_df.group=='all')]
        for faculty in faculties:
            a=da[da.faculty==faculty];s=shared_router_df[(shared_router_df.fold.isin(args.holdout_folds))&(shared_router_df.faculty==faculty)]
            if a.empty:continue
            row={'faculty':faculty,
                 'private_null_weight':float(a.mean_null_weight.mean()),
                 'private_router_entropy':float(a.normalized_entropy.mean()),
                 'private_max_weight':float(a.mean_max_weight.mean()),
                 'private_abs_correction':float(a.mean_abs_correction.mean()),
                 'adapter_relative_norm':float(a.mean_relative_adapter_norm.mean()),
                 'adapter_representation_cosine':float(a.mean_representation_cosine.mean()),
                 'adapter_norm':float(a.mean_adapter_norm.mean())}
            if not s.empty:
                row.update({'shared_null_weight':float(s.mean_null_weight.mean()),
                            'shared_router_entropy':float(s.normalized_entropy.mean()),
                            'shared_max_weight':float(s.mean_max_weight.mean()),
                            'shared_abs_correction':float(s.mean_abs_correction.mean()),
                            'delta_null_private_minus_shared':float(a.mean_null_weight.mean()-s.mean_null_weight.mean()),
                            'delta_entropy_private_minus_shared':float(a.normalized_entropy.mean()-s.normalized_entropy.mean()),
                            'delta_abs_correction_private_minus_shared':float(a.mean_abs_correction.mean()-s.mean_abs_correction.mean())})
            if len(corr_df):
                cr=corr_df[(corr_df.fold.isin(args.holdout_folds))&(corr_df.faculty==faculty)]
                row['private_mean_residual_correlation']=float(cr.correlation.mean()) if not cr.empty else np.nan
            behavior.append(row)
    behavior_df=pd.DataFrame(behavior);behavior_df.to_csv(out/'suite7_holdout_adapter_router_summary.csv',index=False)
    if len(behavior_df):
        print('\n'+'='*150);print('SUITE7 HOLDOUT ADAPTER / ROUTER BEHAVIOR SUMMARY');print('='*150)
        show=[c for c in ['faculty','adapter_relative_norm','adapter_representation_cosine','private_null_weight','shared_null_weight','private_router_entropy','shared_router_entropy','private_abs_correction','shared_abs_correction','private_mean_residual_correlation'] if c in behavior_df.columns]
        print(behavior_df[show].to_string(index=False,float_format=lambda x:f'{x:.4f}'))

    pooled=pooled_ranking_rows(records,args.holdout_folds,args);pooled.to_csv(out/'suite7_holdout_pooled.csv',index=False)
    wdf,_=operational_summary(records,faculties,args,out)
    cmp=build_shared_private_comparison(pooled,wdf);cmp.to_csv(out/'suite7_shared_private_comparison.csv',index=False);print_comparison(cmp)
    if args.reference_metrics:
        official=merge_external_references(cmp,args.reference_metrics);official.to_csv(out/'suite7_official_comparison.csv',index=False)
        print('\n'+'='*150);print('SUITE7 OFFICIAL COMPARISON + PROVIDED V2 / COUNCIL REFERENCES');print('='*150);print(official.to_string(index=False))
    print(f'\nCompleted. Results: {out}')


if __name__=='__main__':main()
