# -*- coding: utf-8 -*-
"""Suite8 — true specialist discovery over the Suite7 shared-private backbone.

Controlled Experiment C:
  * Committee cache is frozen/reused.
  * Suite5 shared and Suite7 shared-private target scores are read-only references.
  * Suite7 shared encoder + faculty adapters + router/loss structure remain intact.
  * The ONLY architectural change is expert-specific learned latent ownership/views.

The experiment asks whether distinct information views are enough to break the
~0.99 residual-expert correlation observed in Suite5/6/7 and make routing useful.
"""
from __future__ import annotations

import argparse
from pathlib import Path

import numpy as np
import pandas as pd
import torch
from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler

from suite2_common import safe_ap, safe_auc, equal_budget_metrics
from suite7_common import (
    FeatureStandardizer,
    load_fold_caches,
    build_historical_bundle,
    build_target_bundle,
    faculty_class_sampling_weights,
    latest_block_split,
    per_faculty_validation_rows,
    model_selection_score,
    parse_faculties,
    save_json,
)
from suite7_eval import evaluate_faculty
from suite7_reporting import (
    load_shared_feature_schema,
    load_shared_fold_frame,
    merge_external_references,
    summarize_shared_router_frame,
)
from suite8_reporting import load_suite7_fold_frame, build_true_specialist_comparison
from suite8_true_specialist_moe import (
    TrueSpecialistSharedPrivateResidualMoE,
    suite8_residual_moe_loss,
    router_diagnostics,
    adapter_parameter_rows,
    ownership_diagnostics,
    expert_view_cosine_diagnostics,
    counterfactual_utility_diagnostics,
    ownership_rows,
    top_ownership_summary,
    seed_everything,
    choose_device,
)

BUDGETS=(0.10,0.15,0.20,0.25)
COMMITTEE_MODEL='committee'
SHARED_MODEL='residual_moe_shared'
ADAPTER_MODEL='residual_moe_shared_private'
SPECIALIST_MODEL='residual_moe_true_specialist'


def _tensor_loader(bundle,scaler,mask,args,seed,sampled=True):
    mask=np.asarray(mask,bool)
    X=scaler.transform(bundle.X[mask]);y=bundle.y[mask].astype(np.float32);q=bundle.q[mask].astype(np.float32);f=bundle.faculty_idx[mask].astype(np.int64)
    ds=TensorDataset(torch.from_numpy(X),torch.from_numpy(f),torch.from_numpy(q),torch.from_numpy(y))
    if not sampled:
        return DataLoader(ds,batch_size=args.predict_batch_size,shuffle=False,num_workers=0)
    w,diag=faculty_class_sampling_weights(bundle.y[mask],bundle.faculty_idx[mask],bundle.faculty_names,args.sampling_beta)
    gen=torch.Generator();gen.manual_seed(int(seed));n=len(ds) if args.samples_per_epoch<=0 else int(args.samples_per_epoch)
    sampler=WeightedRandomSampler(torch.as_tensor(w,dtype=torch.double),num_samples=n,replacement=True,generator=gen)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0),diag


def _make_model(input_dim,n_faculties,args):
    return TrueSpecialistSharedPrivateResidualMoE(
        input_dim=input_dim,n_faculties=n_faculties,n_experts=args.n_experts,
        faculty_dim=args.faculty_dim,encoder_hidden=args.encoder_hidden,
        representation_dim=args.representation_dim,expert_hidden=args.expert_hidden,
        delta_max=args.delta_max,dropout=args.dropout,router_temperature=args.router_temperature,
        adapter_dim=args.adapter_dim,adapter_dropout=args.adapter_dropout,adapter_scale=args.adapter_scale,
        view_temperature=args.view_temperature,view_strength=args.view_strength,view_init_std=args.view_init_std,
    )


def train_epoch(model,loader,optimizer,device,args):
    model.train();keys=['total','prediction','router','specialization_mi','magnitude','view_entropy','view_balance'];acc={k:0. for k in keys};n=0
    for X,f,q,y in loader:
        X=X.to(device);f=f.to(device);q=q.to(device);y=y.to(device)
        optimizer.zero_grad(set_to_none=True);out=model(X,f,q)
        ls=suite8_residual_moe_loss(
            out,y,q,router_lambda=args.router_lambda,specialization_lambda=args.specialization_lambda,
            magnitude_lambda=args.magnitude_lambda,utility_temperature=args.utility_temperature,
            view_entropy_lambda=args.view_entropy_lambda,view_balance_lambda=args.view_balance_lambda,
        )
        ls['total'].backward();torch.nn.utils.clip_grad_norm_(model.parameters(),args.grad_clip);optimizer.step()
        b=len(y);n+=b
        for k in keys:acc[k]+=float(ls[k].detach().cpu())*b
    return {k:v/max(n,1) for k,v in acc.items()}


@torch.no_grad()
def predict_bundle(model,bundle,scaler,mask,device,args):
    model.eval();loader=_tensor_loader(bundle,scaler,mask,args,seed=0,sampled=False)
    ps=[];ws=[];rs=[];cs=[];cfs=[];an=[];hn=[];hfn=[];rel=[];cos=[];aabs=[];view_cos_sum=0.;view_cos_sq_sum=0.;view_cos_n=0
    for X,f,q,_ in loader:
        out=model(X.to(device),f.to(device),q.to(device))
        p=out['final_prob'].cpu().numpy();w=out['weights'].cpu().numpy();r=out['residuals'].cpu().numpy();c=out['correction'].cpu().numpy();cf=out['counterfactual_prob'].cpu().numpy()
        h=out['shared_representation'].cpu().numpy();a=out['adapter_output'].cpu().numpy();hf=out['adapted_representation'].cpu().numpy();views=out['expert_views'].cpu().numpy()
        ps.append(p);ws.append(w);rs.append(r);cs.append(c);cfs.append(cf)
        aa=np.linalg.norm(a,axis=1);hh=np.linalg.norm(h,axis=1);ff=np.linalg.norm(hf,axis=1)
        an.append(aa);hn.append(hh);hfn.append(ff);rel.append(aa/np.maximum(hh,1e-12));cos.append(np.clip(np.sum(h*hf,axis=1)/np.maximum(hh*ff,1e-12),-1,1));aabs.append(np.mean(np.abs(a),axis=1))
        vd=expert_view_cosine_diagnostics(views)
        # Each batch contributes K choose 2 similarities per row; reconstruct count.
        pairs=max(model.n_experts*(model.n_experts-1)//2,1);cnt=len(p)*pairs
        if np.isfinite(vd['mean_pairwise_view_cosine']):
            view_cos_sum+=vd['mean_pairwise_view_cosine']*cnt
            view_cos_sq_sum+=(vd['std_pairwise_view_cosine']**2+vd['mean_pairwise_view_cosine']**2)*cnt
            view_cos_n+=cnt
    if not ps:
        return {'p':np.array([]),'weights':np.empty((0,args.n_experts+1)),'residuals':np.empty((0,args.n_experts)),'correction':np.array([]),'counterfactual_prob':np.empty((0,args.n_experts))}
    own=model.view_ownership().detach().cpu().numpy();od=ownership_diagnostics(own)
    vm=view_cos_sum/max(view_cos_n,1);vs=max(view_cos_sq_sum/max(view_cos_n,1)-vm*vm,0.)**.5 if view_cos_n else np.nan
    return {
        'p':np.concatenate(ps),'weights':np.concatenate(ws),'residuals':np.concatenate(rs),'correction':np.concatenate(cs),'counterfactual_prob':np.concatenate(cfs),
        'adapter_norm':np.concatenate(an),'shared_norm':np.concatenate(hn),'adapted_norm':np.concatenate(hfn),'relative_adapter_norm':np.concatenate(rel),'representation_cosine':np.concatenate(cos),'mean_abs_adapter_activation':np.concatenate(aabs),
        'view_ownership':own,'view_scales':model.view_scales().detach().cpu().numpy(),'mean_pairwise_view_cosine':float(vm),'std_pairwise_view_cosine':float(vs),**od,
    }


def _adapter_summary(pred,mask=None):
    if mask is None:mask=np.ones(len(pred['p']),bool)
    mask=np.asarray(mask,bool)
    if not np.any(mask):
        return {'mean_adapter_norm':np.nan,'mean_shared_norm':np.nan,'mean_adapted_norm':np.nan,'mean_relative_adapter_norm':np.nan,'mean_representation_cosine':np.nan,'mean_abs_adapter_activation':np.nan}
    return {
        'mean_adapter_norm':float(np.mean(pred['adapter_norm'][mask])),'mean_shared_norm':float(np.mean(pred['shared_norm'][mask])),'mean_adapted_norm':float(np.mean(pred['adapted_norm'][mask])),
        'mean_relative_adapter_norm':float(np.mean(pred['relative_adapter_norm'][mask])),'mean_representation_cosine':float(np.mean(pred['representation_cosine'][mask])),'mean_abs_adapter_activation':float(np.mean(pred['mean_abs_adapter_activation'][mask])),
    }


def _mean_residual_corr(residuals):
    r=np.asarray(residuals,float)
    if r.ndim!=2 or len(r)<3 or r.shape[1]<2:return np.nan
    c=np.corrcoef(r,rowvar=False);v=c[np.triu_indices(c.shape[0],1)];v=v[np.isfinite(v)]
    return float(v.mean()) if len(v) else np.nan


def fit_with_temporal_validation(bundle,args,seed,label='true-specialist'):
    tr,va=latest_block_split(bundle.faculty_idx,bundle.blocks,args.val_blocks)
    if tr.sum()<args.min_train_rows:raise RuntimeError(f'{label}: only {tr.sum()} chronological training rows')
    if va.sum()<args.min_val_rows:
        print(f'[{label}] validation rows={va.sum():,} < {args.min_val_rows}; using fixed {args.fallback_epochs} epochs')
        return int(args.fallback_epochs),pd.DataFrame(),pd.DataFrame([{'epoch':0,'selection_score':np.nan,'note':'no_validation'}])
    scaler=FeatureStandardizer.fit(bundle.X[tr]);seed_everything(seed);device=choose_device(args.device);model=_make_model(bundle.X.shape[1],len(bundle.faculty_names),args).to(device);opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    best_score=-np.inf;best_epoch=1;pat=0;history=[];best_rows=pd.DataFrame()
    print(f'[{label}] chronological train={tr.sum():,} val={va.sum():,} faculties={len(bundle.faculty_names)} input={bundle.X.shape[1]} adapter={args.representation_dim}->{args.adapter_dim}->{args.representation_dim} views=K{args.n_experts} strength={args.view_strength:.2f} device={device}')
    for epoch in range(1,args.epochs+1):
        loader,_=_tensor_loader(bundle,scaler,tr,args,seed+epoch*997,sampled=True);losses=train_epoch(model,loader,opt,device,args);pred=predict_bundle(model,bundle,scaler,va,device,args)
        rows=per_faculty_validation_rows(bundle.y[va],bundle.q[va],pred['p'],bundle.faculty_idx[va],bundle.faculty_names,args.threshold_beta,.20);score,macro=model_selection_score(rows,args.macro_f2_weight)
        rd=router_diagnostics(pred['weights'],pred['residuals']);ad=_adapter_summary(pred);rc=_mean_residual_corr(pred['residuals'])
        hist={'epoch':epoch,'selection_score':score,**losses,**macro,'null_weight':rd['mean_null_weight'],'router_entropy':rd['normalized_entropy'],'mean_max_weight':rd['mean_max_weight'],'mean_residual_correlation':rc,
              'mean_adapter_norm':ad['mean_adapter_norm'],'mean_relative_adapter_norm':ad['mean_relative_adapter_norm'],'mean_representation_cosine':ad['mean_representation_cosine'],'mean_abs_correction':float(np.mean(np.abs(pred['correction']))),
              'ownership_entropy':pred['normalized_ownership_entropy'],'ownership_load_imbalance':pred['ownership_load_imbalance'],'ownership_cosine':pred['mean_pairwise_ownership_cosine'],'view_cosine':pred['mean_pairwise_view_cosine']}
        history.append(hist)
        if epoch==1 or epoch%args.print_every==0:
            print(f'[{label}] epoch={epoch:03d} loss={losses["total"]:.5f} sel={score:+.5f} median ΔAP={macro["median_delta_ap"]:+.4f} median ΔF2@20={macro["median_delta_f2_20"]:+.4f} AP-win={macro["win_rate_ap"]:.1%} NULL={rd["mean_null_weight"]:.3f} H={rd["normalized_entropy"]:.3f} Rcorr={rc:.3f} OwnH={pred["normalized_ownership_entropy"]:.3f} OwnCos={pred["mean_pairwise_ownership_cosine"]:.3f} ViewCos={pred["mean_pairwise_view_cosine"]:.3f} Arel={ad["mean_relative_adapter_norm"]:.3f}')
        if np.isfinite(score) and score>best_score+args.min_selection_improvement:best_score=score;best_epoch=epoch;best_rows=rows.copy();pat=0
        else:pat+=1
        if epoch>=args.min_epochs and pat>=args.patience:break
    print(f'[{label}] selected epoch={best_epoch} macro selection score={best_score:+.5f} | {top_ownership_summary(model,args.top_owned_dims)}')
    return best_epoch,best_rows,pd.DataFrame(history)


def refit_full(bundle,args,seed,epochs,label='true-specialist'):
    mask=np.ones(len(bundle.y),bool);scaler=FeatureStandardizer.fit(bundle.X);seed_everything(seed);device=choose_device(args.device);model=_make_model(bundle.X.shape[1],len(bundle.faculty_names),args).to(device);opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)
    for epoch in range(1,max(int(epochs),1)+1):
        loader,_=_tensor_loader(bundle,scaler,mask,args,seed+epoch*1543,sampled=True);losses=train_epoch(model,loader,opt,device,args)
        if epoch==1 or epoch==epochs or epoch%args.print_every==0:
            od=ownership_diagnostics(model.view_ownership().detach().cpu().numpy())
            print(f'[{label} refit] epoch={epoch:03d}/{epochs} loss={losses["total"]:.5f} OwnH={od["normalized_ownership_entropy"]:.3f} OwnCos={od["mean_pairwise_ownership_cosine"]:.3f}')
    return model,scaler,device


def metric_rows_for_fold(fold,target,scores,args):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m]
        for model,s in scores.items():
            ss=np.asarray(s)[m];row={'fold':fold,'faculty':faculty,'model':model,'rows':int(m.sum()),'failures':int(y.sum()),'prevalence':float(y.mean()),'ap':safe_ap(y,ss),'auc':safe_auc(y,ss)}
            for b in BUDGETS:
                z=equal_budget_metrics(y,ss,b,args.threshold_beta);tag=int(b*100);row[f'precision_at_{tag}']=z['precision'];row[f'recall_at_{tag}']=z['recall'];row[f'f1_at_{tag}']=z['f1'];row[f'f2_at_{tag}']=z['fbeta']
            rows.append(row)
    return rows


def specialist_diagnostic_rows(fold,target,pred):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        base=target.faculty_idx==fi;groups=[('all',base),('failures',base&(target.y==1)),('normals',base&(target.y==0)),('history0',base&(target.support_history_semesters<=0)),('history1_2',base&(target.support_history_semesters>=1)&(target.support_history_semesters<=2)),('history3plus',base&(target.support_history_semesters>=3))]
        for group,m in groups:
            if not np.any(m):continue
            rd=router_diagnostics(pred['weights'][m],pred['residuals'][m]);ad=_adapter_summary(pred,m);ut=counterfactual_utility_diagnostics(target.y[m],target.q[m],pred['counterfactual_prob'][m])
            rows.append({'fold':fold,'faculty':faculty,'group':group,'rows':int(m.sum()),'failures':int(target.y[m].sum()),'prevalence':float(target.y[m].mean()),**rd,**ad,
                         'mean_correction':float(np.mean(pred['correction'][m])),'mean_abs_correction':float(np.mean(np.abs(pred['correction'][m]))),'mean_residual_correlation':_mean_residual_corr(pred['residuals'][m]),
                         'normalized_ownership_entropy':pred['normalized_ownership_entropy'],'ownership_load_imbalance':pred['ownership_load_imbalance'],'mean_pairwise_ownership_cosine':pred['mean_pairwise_ownership_cosine'],'mean_pairwise_view_cosine':pred['mean_pairwise_view_cosine'],**ut})
    return rows


def residual_correlation_rows(fold,target,residuals):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi
        if m.sum()<3:continue
        c=np.corrcoef(residuals[m],rowvar=False)
        for i in range(c.shape[0]):
            for j in range(i+1,c.shape[1]):rows.append({'fold':fold,'faculty':faculty,'expert_i':i+1,'expert_j':j+1,'correlation':float(c[i,j]) if np.isfinite(c[i,j]) else np.nan})
    return rows


def utility_rows(fold,target,pred):
    rows=[]
    for fi,faculty in enumerate(target.faculty_names):
        base=target.faculty_idx==fi
        for group,m in [('all',base),('failures',base&(target.y==1)),('normals',base&(target.y==0))]:
            if np.any(m):rows.append({'fold':fold,'faculty':faculty,'group':group,'rows':int(m.sum()),**counterfactual_utility_diagnostics(target.y[m],target.q[m],pred['counterfactual_prob'][m])})
    return rows


def _load_reference_scores(shared_results_dir,suite7_results_dir,fold,target,args):
    shared_parts=[];adapter_parts=[];suite7_router=[]
    for fi,faculty in enumerate(target.faculty_names):
        m=target.faculty_idx==fi
        s=load_shared_fold_frame(shared_results_dir,fold,faculty,target.y[m],target.q[m],args.reference_alignment_atol);a=load_suite7_fold_frame(suite7_results_dir,fold,faculty,target.y[m],target.q[m],args.reference_alignment_atol)
        shared_parts.append(s[SHARED_MODEL].to_numpy(float));adapter_parts.append(a[ADAPTER_MODEL].to_numpy(float));d=summarize_shared_router_frame(a);d.update({'fold':fold,'faculty':faculty});suite7_router.append(d)
    return np.concatenate(shared_parts),np.concatenate(adapter_parts),suite7_router


def run_fold(cache_dir,shared_results_dir,suite7_results_dir,faculties,fold,args,out):
    print('\n'+'='*160);print(f'SUITE8 TRUE SPECIALIST DISCOVERY | TARGET FOLD {fold}');print('='*160)
    schema=load_shared_feature_schema(shared_results_dir,fold);common=list(schema['common_raw_features']);shared_faculties=list(schema.get('faculties',[]));missing=[f for f in faculties if shared_faculties and f not in shared_faculties]
    if missing:raise ValueError(f'faculties absent from Suite5 shared fold {fold}: {missing}')
    caches=load_fold_caches(cache_dir,faculties,fold);hist=build_historical_bundle(caches,faculties,common);target=build_target_bundle(caches,faculties,common)
    if schema.get('shared_feature_names') and list(schema['shared_feature_names'])!=list(hist.feature_names):raise ValueError(f'fold {fold}: composed features do not match Suite5 schema')
    shared_score,adapter_score,suite7_router=_load_reference_scores(shared_results_dir,suite7_results_dir,fold,target,args)
    print(f'[DATA] raw_common={len(common)} inputs={hist.X.shape[1]} OOF={len(hist.y):,} failures={int(hist.y.sum()):,} ({hist.y.mean():.2%}) target={len(target.y):,}')
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True)
    save_json({'fold':fold,'faculties':faculties,'common_raw_features':common,'shared_feature_names':hist.feature_names,'adapter_dim':args.adapter_dim,'view_temperature':args.view_temperature,'view_strength':args.view_strength,'view_init_std':args.view_init_std,'view_entropy_lambda':args.view_entropy_lambda,'view_balance_lambda':args.view_balance_lambda},fold_dir/'feature_schema.json')
    seed=args.seed+fold*100003;best_epoch,val_rows,history=fit_with_temporal_validation(hist,args,seed,label=f'true-specialist:F{fold}');history.to_csv(fold_dir/'true_specialist_training_history.csv',index=False);val_rows.to_csv(fold_dir/'true_specialist_best_validation_by_faculty.csv',index=False)
    model,scaler,device=refit_full(hist,args,seed,best_epoch,label=f'true-specialist:F{fold}');scaler.save(fold_dir/'true_specialist_scaler.npz')
    own_rows=ownership_rows(model,fold);pd.DataFrame(own_rows).to_csv(fold_dir/'expert_ownership.csv',index=False);param_rows=adapter_parameter_rows(model,faculties)
    for r in param_rows:r['fold']=fold
    torch.save({'state_dict':model.state_dict(),'selected_epoch':best_epoch,'model_config':{'input_dim':hist.X.shape[1],'n_faculties':len(faculties),'n_experts':args.n_experts,'faculty_dim':args.faculty_dim,'encoder_hidden':args.encoder_hidden,'representation_dim':args.representation_dim,'expert_hidden':args.expert_hidden,'delta_max':args.delta_max,'dropout':args.dropout,'router_temperature':args.router_temperature,'adapter_dim':args.adapter_dim,'adapter_dropout':args.adapter_dropout,'adapter_scale':args.adapter_scale,'view_temperature':args.view_temperature,'view_strength':args.view_strength,'view_init_std':args.view_init_std},'faculties':faculties,'common_raw_features':common},fold_dir/'true_specialist_model.pt')
    pred=predict_bundle(model,target,scaler,np.ones(len(target.y),bool),device,args);scores={COMMITTEE_MODEL:target.q,SHARED_MODEL:shared_score,ADAPTER_MODEL:adapter_score,SPECIALIST_MODEL:pred['p']};metrics=metric_rows_for_fold(fold,target,scores,args);diag=specialist_diagnostic_rows(fold,target,pred);corr=residual_correlation_rows(fold,target,pred['residuals']);utils=utility_rows(fold,target,pred)
    pd.DataFrame(metrics).to_csv(fold_dir/'fold_metrics.csv',index=False);pd.DataFrame(diag).to_csv(fold_dir/'specialist_diagnostics.csv',index=False);pd.DataFrame(corr).to_csv(fold_dir/'residual_correlations.csv',index=False);pd.DataFrame(utils).to_csv(fold_dir/'counterfactual_utility.csv',index=False);pd.DataFrame(param_rows).to_csv(fold_dir/'adapter_parameter_norms.csv',index=False)
    records=[]
    for fi,faculty in enumerate(faculties):
        m=target.faculty_idx==fi;frame=pd.DataFrame({'actual':target.y[m],'committee':target.q[m],SHARED_MODEL:shared_score[m],ADAPTER_MODEL:adapter_score[m],SPECIALIST_MODEL:pred['p'][m],'correction_logit':pred['correction'][m],'history_semesters':target.support_history_semesters[m],'relative_adapter_norm':pred['relative_adapter_norm'][m],'representation_cosine':pred['representation_cosine'][m]})
        for j in range(pred['weights'].shape[1]):frame[f'weight_{"null" if j==0 else f"expert_{j}"}']=pred['weights'][m,j]
        for j in range(pred['residuals'].shape[1]):frame[f'residual_expert_{j+1}']=pred['residuals'][m,j]
        frame.to_csv(fold_dir/f'{faculty}_scores.csv',index=False);records.append({'faculty':faculty,'fold':fold,'y':target.y[m],'scores':{k:v[m] for k,v in scores.items()}})
        mm={r['model']:r for r in metrics if r['faculty']==faculty};dd=next(r for r in diag if r['faculty']==faculty and r['group']=='all')
        print(f'[TARGET {faculty}] AP C/S/A/T={mm[COMMITTEE_MODEL]["ap"]:.4f}/{mm[SHARED_MODEL]["ap"]:.4f}/{mm[ADAPTER_MODEL]["ap"]:.4f}/{mm[SPECIALIST_MODEL]["ap"]:.4f} ΔT-A={mm[SPECIALIST_MODEL]["ap"]-mm[ADAPTER_MODEL]["ap"]:+.4f} | AUC={mm[COMMITTEE_MODEL]["auc"]:.4f}/{mm[SHARED_MODEL]["auc"]:.4f}/{mm[ADAPTER_MODEL]["auc"]:.4f}/{mm[SPECIALIST_MODEL]["auc"]:.4f} | F2@20={mm[COMMITTEE_MODEL]["f2_at_20"]:.4f}/{mm[SHARED_MODEL]["f2_at_20"]:.4f}/{mm[ADAPTER_MODEL]["f2_at_20"]:.4f}/{mm[SPECIALIST_MODEL]["f2_at_20"]:.4f} | Rcorr={dd["mean_residual_correlation"]:.3f} NULL={dd["mean_null_weight"]:.3f} H={dd["normalized_entropy"]:.3f}')
    od=ownership_diagnostics(pred['view_ownership']);print(f'[OWNERSHIP F{fold}] entropy={od["normalized_ownership_entropy"]:.4f} load_imbalance={od["ownership_load_imbalance"]:.6f} cosine={od["mean_pairwise_ownership_cosine"]:.4f} view_cosine={pred["mean_pairwise_view_cosine"]:.4f} | {top_ownership_summary(model,args.top_owned_dims)}')
    training={'fold':fold,'selected_epoch':best_epoch,'validation_rows':int(len(val_rows)),'validation_score':float(history.selection_score.max()) if len(history) and history.selection_score.notna().any() else np.nan,**od,'mean_pairwise_view_cosine':pred['mean_pairwise_view_cosine'],'mean_residual_correlation':_mean_residual_corr(pred['residuals'])}
    return records,metrics,diag,corr,utils,own_rows,param_rows,suite7_router,training


def pooled_ranking_rows(records,holdout_folds,args):
    rows=[]
    for faculty in sorted({r['faculty'] for r in records}):
        rr=[r for r in records if r['faculty']==faculty and r['fold'] in holdout_folds]
        if not rr:continue
        y=np.concatenate([r['y'] for r in rr]);models=set.intersection(*[set(r['scores']) for r in rr])
        for model in sorted(models):
            s=np.concatenate([r['scores'][model] for r in rr]);row={'faculty':faculty,'model':model,'rows':len(y),'failures':int(y.sum()),'prevalence':float(y.mean()),'ap':safe_ap(y,s),'auc':safe_auc(y,s)}
            for b in BUDGETS:
                z=equal_budget_metrics(y,s,b,args.threshold_beta);tag=int(b*100);row[f'precision_at_{tag}']=z['precision'];row[f'recall_at_{tag}']=z['recall'];row[f'f1_at_{tag}']=z['f1'];row[f'f2_at_{tag}']=z['fbeta']
            rows.append(row)
    return pd.DataFrame(rows)


def operational_summary(records,faculties,args,out):
    all_rows=[];policies=[];print('\n'+'='*160);print('SUITE8 OFFICIAL OPERATIONAL EVALUATION — EACH DEVELOPMENT POLICY FROZEN BEFORE HOLDOUT');print('='*160)
    for faculty in faculties:
        for model in [COMMITTEE_MODEL,SHARED_MODEL,ADAPTER_MODEL,SPECIALIST_MODEL]:
            pol,rows=evaluate_faculty(records,faculty,model,args.dev_folds,args.holdout_folds,args.threshold_beta,args.min_precision,args.max_warning_ratio,args.min_flags,verbose=True)
            policies.append({'faculty':faculty,'model':model,'threshold':pol.threshold,'warning_ratio':pol.warning_ratio,'development_ap':pol.development_ap,'development_auc':pol.development_auc,'development_prevalence':pol.development_prevalence,'development_flag_rate':pol.flag_rate,'development_precision':pol.precision,'development_recall':pol.recall,'development_f1':pol.f1,'development_f2':pol.fbeta,'constraints_relaxed':pol.constraints_relaxed});all_rows.extend(rows)
    wdf=pd.DataFrame(all_rows);pdf=pd.DataFrame(policies);wdf.to_csv(out/'suite8_warning_metrics.csv',index=False);pdf.to_csv(out/'suite8_threshold_policies.csv',index=False);return wdf,pdf


def print_comparison(cmp):
    print('\n'+'='*160);print('SUITE8 HOLDOUT — COMMITTEE vs SUITE5 SHARED vs SUITE7 ADAPTER vs SUITE8 TRUE SPECIALIST');print('='*160)
    cols=['faculty','committee_ap','shared_ap','adapter_ap','specialist_ap','specialist_minus_adapter_ap','committee_f2_20','shared_f2_20','adapter_f2_20','specialist_f2_20','specialist_minus_adapter_f2_20','specialist_minus_adapter_fixed_f2','specialist_minus_adapter_drift_f2','evidence_pattern']
    print(cmp[cols].to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    for metric in ['specialist_minus_adapter_ap','specialist_minus_adapter_auc','specialist_minus_adapter_f2_20','specialist_minus_adapter_fixed_f2','specialist_minus_adapter_drift_f2']:
        v=cmp[metric].to_numpy(float);v=v[np.isfinite(v)]
        if len(v):print(f'{metric:38s} mean={v.mean():+.4f} median={np.median(v):+.4f} worst={v.min():+.4f} win_rate={(v>0).mean():.1%}')


def collapse_summary(cmp,diag_df,suite7_router_df,holdout_folds):
    rows=[];d=diag_df[(diag_df.fold.isin(holdout_folds))&(diag_df.group=='all')]
    for faculty in sorted(cmp.faculty.unique()):
        a=d[d.faculty==faculty];s=suite7_router_df[(suite7_router_df.fold.isin(holdout_folds))&(suite7_router_df.faculty==faculty)];c=cmp[cmp.faculty==faculty].iloc[0]
        if a.empty:continue
        row={'faculty':faculty,'specialist_minus_adapter_ap':c.specialist_minus_adapter_ap,'specialist_minus_adapter_f2_20':c.specialist_minus_adapter_f2_20,'specialist_minus_adapter_drift_f2':c.specialist_minus_adapter_drift_f2,
             'suite8_residual_correlation':float(a.mean_residual_correlation.mean()),'suite8_router_entropy':float(a.normalized_entropy.mean()),'suite8_null_weight':float(a.mean_null_weight.mean()),'ownership_entropy':float(a.normalized_ownership_entropy.mean()),'ownership_cosine':float(a.mean_pairwise_ownership_cosine.mean()),'expert_view_cosine':float(a.mean_pairwise_view_cosine.mean())}
        if not s.empty:
            row.update({'suite7_residual_correlation':float(s.mean_pairwise_residual_correlation.mean()),'suite7_router_entropy':float(s.normalized_entropy.mean()),'suite7_null_weight':float(s.mean_null_weight.mean()),'delta_residual_correlation':float(a.mean_residual_correlation.mean()-s.mean_pairwise_residual_correlation.mean()),'delta_router_entropy':float(a.normalized_entropy.mean()-s.normalized_entropy.mean())})
        rows.append(row)
    return pd.DataFrame(rows)


def parse_args():
    p=argparse.ArgumentParser(description='Suite8 true specialist discovery: Suite7 shared-private backbone + learned expert-specific latent views')
    p.add_argument('--cache_dir',required=True);p.add_argument('--shared_results_dir',required=True);p.add_argument('--suite7_results_dir',required=True);p.add_argument('--models_dir',required=True);p.add_argument('--faculties',default='all');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--reference_metrics',default=None)
    p.add_argument('--seed',type=int,default=42);p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto');p.add_argument('--n_experts',type=int,default=4);p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--encoder_hidden',type=int,default=256);p.add_argument('--representation_dim',type=int,default=128);p.add_argument('--expert_hidden',type=int,default=64);p.add_argument('--delta_max',type=float,default=2.0);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--router_temperature',type=float,default=1.0)
    p.add_argument('--adapter_dim',type=int,default=16);p.add_argument('--adapter_dropout',type=float,default=0.0);p.add_argument('--adapter_scale',type=float,default=1.0)
    p.add_argument('--view_temperature',type=float,default=1.0);p.add_argument('--view_strength',type=float,default=.5);p.add_argument('--view_init_std',type=float,default=.01);p.add_argument('--view_entropy_lambda',type=float,default=.02);p.add_argument('--view_balance_lambda',type=float,default=.02);p.add_argument('--top_owned_dims',type=int,default=8)
    p.add_argument('--epochs',type=int,default=60);p.add_argument('--min_epochs',type=int,default=8);p.add_argument('--patience',type=int,default=8);p.add_argument('--fallback_epochs',type=int,default=20);p.add_argument('--val_blocks',type=int,default=2);p.add_argument('--min_train_rows',type=int,default=800);p.add_argument('--min_val_rows',type=int,default=200)
    p.add_argument('--batch_size',type=int,default=1024);p.add_argument('--predict_batch_size',type=int,default=4096);p.add_argument('--samples_per_epoch',type=int,default=0);p.add_argument('--lr',type=float,default=1e-3);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--grad_clip',type=float,default=5.0);p.add_argument('--print_every',type=int,default=5)
    p.add_argument('--router_lambda',type=float,default=.5);p.add_argument('--specialization_lambda',type=float,default=.01);p.add_argument('--magnitude_lambda',type=float,default=.01);p.add_argument('--utility_temperature',type=float,default=.25);p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--macro_f2_weight',type=float,default=1.0);p.add_argument('--min_selection_improvement',type=float,default=1e-5)
    p.add_argument('--threshold_beta',type=float,default=2.0);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--reference_alignment_atol',type=float,default=1e-6)
    a=p.parse_args();a.dev_folds=[int(x) for x in a.dev_folds.split(',') if x.strip()];a.holdout_folds=[int(x) for x in a.holdout_folds.split(',') if x.strip()];return a


def main():
    args=parse_args();out=Path(args.models_dir);out.mkdir(parents=True,exist_ok=True);first=min(args.dev_folds+args.holdout_folds);schema=load_shared_feature_schema(args.shared_results_dir,first);available=list(schema.get('faculties',[]))
    if not available:
        suffix=f'_fold_{first}.joblib';available=sorted([p.name[:-len(suffix)] for p in Path(args.cache_dir).glob(f'*{suffix}')])
    faculties=parse_faculties(args.faculties,available)
    print('='*160);print('SUITE8 — TRUE SPECIALIST DISCOVERY: SHARED-PRIVATE BACKBONE + LEARNED EXPERT LATENT OWNERSHIP');print('Suite5 cache, Suite5 shared scores, and Suite7 adapter scores are reused read-only. Only Suite8 trains.');print('='*160)
    print(f'faculties={len(faculties)} folds={args.dev_folds+args.holdout_folds} experts={args.n_experts} view_strength={args.view_strength} view_T={args.view_temperature} view_entropy_lambda={args.view_entropy_lambda} view_balance_lambda={args.view_balance_lambda}')
    save_json({'faculties':faculties,'args':vars(args),'controlled_baseline':'Suite7','specialization':'softmax latent ownership across experts'},out/'suite8_run_config.json')
    records=[];metrics=[];diag=[];corr=[];utils=[];own=[];params=[];suite7_router=[];training=[]
    for fold in args.dev_folds+args.holdout_folds:
        rr,mm,dd,cc,uu,oo,pp,sr,tt=run_fold(args.cache_dir,args.shared_results_dir,args.suite7_results_dir,faculties,fold,args,out);records.extend(rr);metrics.extend(mm);diag.extend(dd);corr.extend(cc);utils.extend(uu);own.extend(oo);params.extend(pp);suite7_router.extend(sr);training.append(tt)
    metric_df=pd.DataFrame(metrics);diag_df=pd.DataFrame(diag);corr_df=pd.DataFrame(corr);util_df=pd.DataFrame(utils);own_df=pd.DataFrame(own);param_df=pd.DataFrame(params);suite7_router_df=pd.DataFrame(suite7_router);training_df=pd.DataFrame(training)
    metric_df.to_csv(out/'suite8_fold_metrics.csv',index=False);diag_df.to_csv(out/'suite8_specialist_diagnostics.csv',index=False);corr_df.to_csv(out/'suite8_residual_correlations.csv',index=False);util_df.to_csv(out/'suite8_counterfactual_utility.csv',index=False);own_df.to_csv(out/'suite8_expert_ownership.csv',index=False);param_df.to_csv(out/'suite8_adapter_parameter_norms.csv',index=False);training_df.to_csv(out/'suite8_training_summary.csv',index=False)
    pooled=pooled_ranking_rows(records,args.holdout_folds,args);pooled.to_csv(out/'suite8_holdout_pooled.csv',index=False);wdf,_=operational_summary(records,faculties,args,out);cmp=build_true_specialist_comparison(pooled,wdf);cmp.to_csv(out/'suite8_true_specialist_comparison.csv',index=False);print_comparison(cmp)
    collapse=collapse_summary(cmp,diag_df,suite7_router_df,args.holdout_folds);collapse.to_csv(out/'suite8_collapse_summary.csv',index=False)
    print('\n'+'='*160);print('SUITE8 EXPERT-COLLAPSE DIAGNOSTIC — HOLDOUT');print('Goal: ownership/view diversity should rise and residual correlation/router entropy should fall WITHOUT sacrificing AP/F2.');print('='*160)
    if len(collapse):print(collapse.to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    if args.reference_metrics:
        official=merge_external_references(cmp,args.reference_metrics);official.to_csv(out/'suite8_official_comparison.csv',index=False);print('\nOFFICIAL V2/COUNCIL-AUGMENTED COMPARISON');print(official.to_string(index=False))
    print(f'\nCompleted. Results: {out}')


if __name__=='__main__':main()
