# -*- coding: utf-8 -*-
"""Suite12 — Collaborative Evidence MoE with a distilled shared expert.

This experiment removes the residual-on-committee constraint. The frozen RF/ET/HGB
committee is used only as a teacher/reference. All deployed Suite12 modules are neural,
receive heterogeneous chronology-safe evidence, communicate through gated latent evidence
exchange, and are fine-tuned end-to-end for direct failure prediction.
"""
from __future__ import annotations
import argparse,copy,json,random
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
import torch

from suite12_pipeline import discover_faculties,parse_faculties,load_fold_bundle,context_feature_names
from suite12_data import Suite12Standardizers
from suite12_models import CollaborativeEvidenceMoE
from suite12_training import train_phase,train_fixed_phase,predict,safe_ap,safe_auc
from suite12_reporting import *
from suite12_structured_cache import build_raw_index,RawEvidenceIndex,ensure_fold_structured,structured_cache_compatible,COMP_TYPES
from suite7_common import latest_block_split


def parse_ints(s):return [int(x) for x in str(s).split(',') if x.strip()]

def seed_all(seed):
    random.seed(seed);np.random.seed(seed);torch.manual_seed(seed)
    if torch.cuda.is_available():torch.cuda.manual_seed_all(seed)

def device_of(s):return torch.device('cuda' if s=='auto' and torch.cuda.is_available() else ('cpu' if s=='auto' else s))

def split_rel_sel(bundle,val_mask):
    val=np.asarray(val_mask,bool);rel=np.zeros(len(val),bool);sel=np.zeros(len(val),bool)
    for fi in np.unique(bundle.faculty_idx[val]):
        m=val&(bundle.faculty_idx==fi)&np.isfinite(bundle.blocks);bs=sorted(pd.unique(bundle.blocks[m]))
        if len(bs)>=2:sel[m&(bundle.blocks==bs[-1])]=1;rel[m&(bundle.blocks!=bs[-1])]=1
        elif len(bs)==1:rel[m]=1;sel[m]=1
    return rel,sel

def model_from_bundle(b,args):
    return CollaborativeEvidenceMoE(flat_dim=b.flat.shape[1],e0_context_dim=b.e0_context.shape[1],fusion_context_dim=b.fusion_context.shape[1],e1_context_dim=b.e1_context.shape[1],e2_context_dim=b.e2_context.shape[1],e3_context_dim=b.e3_context.shape[1],e4_context_dim=b.e4_context.shape[1],student_dim=b.student_seq.shape[-1],course_dim=b.course_target.shape[-1],relation_dim=b.relation_events.shape[-1],component_numeric_dim=b.component_events.shape[-1],n_faculties=len(b.faculty_names),n_component_types=len(COMP_TYPES),d_model=args.d_model,dropout=args.dropout,e0_dropout=args.e0_dropout,expert_dropout=args.expert_dropout)

def phase_lrs(args):return [args.e0_lr,args.private_lr,args.fusion_lr,args.joint_lr]
def phase_epochs(args):return [args.e0_epochs,args.private_epochs,args.fusion_epochs,args.joint_epochs]
def phase_distill(args):return [args.distill_warmup,0.0,0.0,args.distill_joint]

def select_epochs(hist,args,seed,device):
    tr,va=latest_block_split(hist.faculty_idx,hist.blocks,args.val_blocks);rel,sel=split_rel_sel(hist,va)
    if rel.sum()==0:rel=va.copy()
    std=Suite12Standardizers.fit(hist,tr);h=std.transform(hist);seed_all(seed);model=model_from_bundle(h,args).to(device)
    selected=[];all_hist=[]
    for phase,(ep,lr,dl) in enumerate(zip(phase_epochs(args),phase_lrs(args),phase_distill(args))):
        be,hh=train_phase(model,h,tr,rel,phase,ep,lr,args.weight_decay,args.batch_size,args.sampling_beta,seed+phase*701,device,args.aux_lambda,dl,args.patience,args.samples_per_epoch);selected.append(be);all_hist.extend(hh)
    pr=predict(model,h,sel,args.predict_batch_size,device);committee=h.teacher[sel].mean(1);e0=pr['private'][:,0]
    rows=[]
    for fi,name in enumerate(h.faculty_names):
        m=(h.faculty_idx[sel]==fi)
        if not np.any(m):continue
        yy=h.y[sel][m];rows.append({'faculty':name,'rows':int(m.sum()),'failures':int(yy.sum()),'committee_ap':safe_ap(yy,committee[m]),'e0_ap':safe_ap(yy,e0[m]),'suite12_ap':safe_ap(yy,pr['score'][m]),'delta_vs_committee':safe_ap(yy,pr['score'][m])-safe_ap(yy,committee[m]),'delta_vs_e0':safe_ap(yy,pr['score'][m])-safe_ap(yy,e0[m])})
    rdf=pd.DataFrame(rows);print(f'[SELECTION] phase_epochs={selected} medianΔAP vs committee={rdf.delta_vs_committee.median():+.4f} vs E0={rdf.delta_vs_e0.median():+.4f} AP-wins vs committee={(rdf.delta_vs_committee>0).sum()}/{len(rdf)}')
    return selected,pd.DataFrame(all_hist),rdf

def refit(hist,target,args,selected,seed,device):
    std=Suite12Standardizers.fit(hist,np.ones(len(hist.y),bool));h=std.transform(hist);t=std.transform(target);seed_all(seed);model=model_from_bundle(h,args).to(device);allm=np.ones(len(h.y),bool)
    for phase,(ep,lr,dl) in enumerate(zip(selected,phase_lrs(args),phase_distill(args))):train_fixed_phase(model,h,allm,phase,ep,lr,args.weight_decay,args.batch_size,args.sampling_beta,seed+phase*1709,device,args.aux_lambda,dl,args.samples_per_epoch)
    return model,std,h,t

def availability_report(bundle):
    return {'E1_student_history':float(bundle.student_mask.any(1).mean()),'E2_course_target':1.0,'E2_course_neighbors':float(bundle.course_neighbor_mask.any(1).mean()),'E3_relation_history':float(bundle.relation_mask.any(1).mean()),'E4_component_history':float(bundle.component_mask.any(1).mean())}

def ensure_structured(args,faculties,folds):
    sd=Path(args.structured_cache_dir);missing=[sd/f'{f}_fold_{fold}.joblib' for fold in folds for f in faculties if not structured_cache_compatible(sd/f'{f}_fold_{fold}.joblib')]
    if not missing:return
    if not args.database_dir and (not args.data1_zip or not args.data2_zip):raise FileNotFoundError(f'{len(missing)} structured caches missing. Provide --database_dir (preferred) or both legacy ZIP paths; first={missing[0]}')
    raw=sd/'raw_index';build_raw_index(args.cache_dir,args.data1_zip,args.data2_zip,faculties,folds,raw,args.force_raw,args.raw_chunksize,database_dir=args.database_dir,strict_alignment=not args.skip_alignment_audit);idx=RawEvidenceIndex(raw)
    for fold in folds:
      for fac in faculties:
        p=sd/f'{fac}_fold_{fold}.joblib'
        if p.exists() and not args.force_structured:continue
        d=joblib.load(Path(args.cache_dir)/f'{fac}_fold_{fold}.joblib');x=ensure_fold_structured(d,idx,p,args.max_semesters,args.max_course_neighbors,args.max_relation_events,args.max_component_events,args.force_structured)
        print(f'[STRUCTURED F{fold} {fac}] OOF={len(x["train_row_index"]):,} target={len(x["target_row_index"]):,} E1={x["train"]["student_mask"].any(1).mean():.1%} E3={x["train"]["relation_mask"].any(1).mean():.1%} E4={x["train"]["component_mask"].any(1).mean():.1%}')

def run_fold(args,faculties,fold,out,device):
    print('\n'+'='*180);print(f'SUITE12 COLLABORATIVE EVIDENCE MoE | TARGET FOLD {fold}');print('='*180)
    hist,target,common=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold)
    print(f'[DATA] raw_common={len(common)} flat={hist.flat.shape[1]} contexts(e0/fusion/e1/e2/e3/e4)={hist.e0_context.shape[1]}/{hist.fusion_context.shape[1]}/{hist.e1_context.shape[1]}/{hist.e2_context.shape[1]}/{hist.e3_context.shape[1]}/{hist.e4_context.shape[1]} OOF={len(hist.y):,} fail={hist.y.sum():,} ({hist.y.mean():.2%}) target={len(target.y):,}')
    print(f'[EVIDENCE OOF] {availability_report(hist)}');print(f'[EVIDENCE TARGET] {availability_report(target)}')
    selected,history,selrows=select_epochs(hist,args,args.seed+fold*100003,device);fd=out/f'fold_{fold}';fd.mkdir(parents=True,exist_ok=True);history.to_csv(fd/'training_history.csv',index=False);selrows.to_csv(fd/'historical_selection.csv',index=False)
    model,std,h,t=refit(hist,target,args,selected,args.seed+fold*200003,device);pr=predict(model,t,np.ones(len(t.y),bool),args.predict_batch_size,device)
    abl={k:predict(model,t,np.ones(len(t.y),bool),args.predict_batch_size,device,ablate=k)['score'] for k in range(5)}
    joblib.dump(std,fd/'suite12_standardizers.joblib',compress=3)
    torch.save({'state_dict':model.state_dict(),'selected_phase_epochs':selected,
                'model_config':{'d_model':args.d_model,'dropout':args.dropout,'e0_dropout':args.e0_dropout,'expert_dropout':args.expert_dropout,
                                'flat_dim':h.flat.shape[1],'e0_context_dim':h.e0_context.shape[1],'fusion_context_dim':h.fusion_context.shape[1],
                                'e1_context_dim':h.e1_context.shape[1],'e2_context_dim':h.e2_context.shape[1],'e3_context_dim':h.e3_context.shape[1],'e4_context_dim':h.e4_context.shape[1],
                                'student_dim':h.student_seq.shape[-1],'course_dim':h.course_target.shape[-1],'relation_dim':h.relation_events.shape[-1],'component_numeric_dim':h.component_events.shape[-1]},
                'faculty_names':list(h.faculty_names),'common_raw_features':common,'context_features':context_feature_names(),'component_types':COMP_TYPES},fd/'suite12_model.pt')
    records=[];metrics=[];diag=[];rep=[]
    for fi,fac in enumerate(faculties):
        m=t.faculty_idx==fi;y=t.y[m];q=t.teacher[m].mean(1);scores=reference_for_faculty(args,fold,fac,y,q);scores[E0]=pr['private'][m,0];scores[MODEL]=pr['score'][m]
        for k in range(1,5):scores[f'suite12_private_e{k}']=pr['private'][m,k]
        sf=pd.DataFrame({'actual':y,'committee':q,E0:scores[E0],MODEL:scores[MODEL]})
        for k in range(1,5):sf[f'suite12_private_e{k}']=scores[f'suite12_private_e{k}']
        for k,v in scores.items():
            if k not in sf:sf[k]=v
        for k in range(5):sf[f'gate_e{k}']=pr['gate'][m,k];sf[f'ablate_e{k}']=abl[k][m]
        sf.to_csv(fd/f'{fac}_scores.csv',index=False)
        records.append({'fold':fold,'faculty':fac,'y':y,'scores':scores})
        for name,s in scores.items():metrics.append(metric_row(fold,fac,name,y,s,args.threshold_beta))
        base=scores[COMMITTEE];trans=top20_transitions(y,base,scores[MODEL]);row={'fold':fold,'faculty':fac,'rows':int(m.sum()),'failures':int(y.sum()),'ap_committee':safe_ap(y,base),'ap_e0':safe_ap(y,scores[E0]),'ap_suite12':safe_ap(y,scores[MODEL]),'auc_suite12':safe_auc(y,scores[MODEL]),**trans}
        private_aps=[]
        for k in range(1,5):
            ap_k=safe_ap(y,scores[f'suite12_private_e{k}']);private_aps.append(ap_k);row[f'private_e{k}_ap']=ap_k
        row['best_private_ap']=float(np.nanmax(private_aps)) if np.isfinite(private_aps).any() else np.nan
        row['collaboration_delta_vs_e0_ap']=row['ap_suite12']-row['ap_e0'];row['collaboration_delta_vs_best_private_ap']=row['ap_suite12']-row['best_private_ap'] if np.isfinite(row['best_private_ap']) else np.nan
        gm=pr['gate'][m];row['gate_entropy_norm']=float(np.mean(-np.sum(gm*np.log(np.clip(gm,1e-12,1.0)),axis=1)/np.log(5.0)));row['gate_max_mean']=float(np.mean(gm.max(1)))
        teacher_true=t.teacher[m];teacher_hat=pr['teacher_pred'][m];row['teacher_head_mae']=float(np.mean(np.abs(teacher_hat-teacher_true)));row['teacher_mean_mae']=float(np.mean(np.abs(teacher_hat.mean(1)-teacher_true.mean(1))))
        for k,nm in enumerate(['rf','et','hgb']):row[f'teacher_{nm}_mae']=float(np.mean(np.abs(teacher_hat[:,k]-teacher_true[:,k])))
        av=pr['availability'][m]
        for k in range(5):row[f'expert_e{k}_availability']=float(av[:,k].mean())
        for k in range(5):
            row[f'gate_e{k}_mean']=float(gm[:,k].mean());row[f'ablate_e{k}_delta_ap']=safe_ap(y,abl[k][m])-safe_ap(y,scores[MODEL])
            z=metric_row(fold,fac,'ablation',y,abl[k][m],args.threshold_beta);basez=metric_row(fold,fac,'suite12',y,scores[MODEL],args.threshold_beta);row[f'ablate_e{k}_delta_f2_at_20']=z['f2_at_20']-basez['f2_at_20']
            fm=m&(t.y==1);nm=m&(t.y==0);row[f'gate_e{k}_failure_mean']=float(pr['gate'][fm,k].mean()) if np.any(fm) else np.nan;row[f'gate_e{k}_normal_mean']=float(pr['gate'][nm,k].mean()) if np.any(nm) else np.nan
        for k in range(5):
            a=pr['tokens'][m,k];b=pr['collab_tokens'][m,k];den=np.linalg.norm(a,axis=1)*np.linalg.norm(b,axis=1)+1e-12;row[f'e{k}_private_to_collab_cosine']=float(np.mean(np.sum(a*b,axis=1)/den))
        diag.append(row)
        for rr in representation_cosine(pr['tokens'][m]):rep.append({'fold':fold,'faculty':fac,'stage':'private',**rr})
        for rr in representation_cosine(pr['collab_tokens'][m]):rep.append({'fold':fold,'faculty':fac,'stage':'collaborative',**rr})
        pap=' '.join(f'E{k}:{row[f"private_e{k}_ap"]:.3f}' for k in range(1,5));ab=' '.join(f'E{k}:{row[f"ablate_e{k}_delta_ap"]:+.3f}' for k in range(5))
        print(f'[TARGET {fac}] AP C/E0/12={row["ap_committee"]:.4f}/{row["ap_e0"]:.4f}/{row["ap_suite12"]:.4f} | privateAP {pap} | gateH={row["gate_entropy_norm"]:.3f} gate='+' '.join(f'E{k}:{row[f"gate_e{k}_mean"]:.2f}' for k in range(5))+f' | ablateΔAP {ab} | teacherMAE={row["teacher_head_mae"]:.4f} | top20 rescue/lost={trans["rescued_failures"]}/{trans["lost_failures"]}')
    pd.DataFrame(diag).to_csv(fd/'suite12_diagnostics.csv',index=False);pd.DataFrame(rep).to_csv(fd/'representation_cosine.csv',index=False)
    def mean_attention(mask):
        aa=pr['attention'][mask]
        if len(aa)==0:return np.full((5,5),np.nan)
        return aa.mean(axis=(0,1)) if aa.ndim==4 else aa.mean(0)
    att=mean_attention(np.ones(len(t.y),bool));att_fail=mean_attention(t.y==1);att_norm=mean_attention(t.y==0)
    labels=[f'E{i}' for i in range(5)]
    attdf=pd.DataFrame(att,index=labels,columns=labels);attdf.to_csv(fd/'mean_expert_attention.csv')
    pd.DataFrame(att_fail,index=labels,columns=labels).to_csv(fd/'mean_expert_attention_failures.csv');pd.DataFrame(att_norm,index=labels,columns=labels).to_csv(fd/'mean_expert_attention_normals.csv')
    print('[EXPERT ATTENTION] target mean rows=query, cols=source');print(attdf.to_string(float_format=lambda x:f'{x:.3f}'))
    print('[EXPERT ATTENTION Δ failure-normal]');print(pd.DataFrame(att_fail-att_norm,index=labels,columns=labels).to_string(float_format=lambda x:f'{x:+.3f}'))
    summary={'fold':fold,'selected_phase_epochs':selected,'target_evidence_availability':availability_report(t),'mean_gate':[float(x) for x in pr['gate'].mean(0)],'mean_attention':att.tolist(),'mean_attention_failures':att_fail.tolist(),'mean_attention_normals':att_norm.tolist(),'mean_teacher_head_mae':float(np.mean(np.abs(pr['teacher_pred']-t.teacher))),'context_features':context_feature_names()};(fd/'suite12_fold_summary.json').write_text(json.dumps(summary,indent=2))
    return records,metrics,diag,rep

def parse_args():
    p=argparse.ArgumentParser(description='Suite12 Collaborative Evidence MoE')
    p.add_argument('--cache_dir',required=True);p.add_argument('--structured_cache_dir',required=True);p.add_argument('--models_dir',required=True);p.add_argument('--database_dir',default=None,help='Preferred: extracted raw database directory containing DDTBHK.csv, DDTBHP.csv, DDiemTPTG.csv, MKQDK.csv, KToHoc.csv, curriculum tables, etc.');p.add_argument('--data1_zip',default=None,help='Legacy fallback only');p.add_argument('--data2_zip',default=None,help='Legacy fallback only');p.add_argument('--faculties',default='all');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--primary_faculty',default='khoa_cong_nghe_thong_tin')
    p.add_argument('--shared_results_dir',default=None);p.add_argument('--suite7_results_dir',default=None);p.add_argument('--suite8_results_dir',default=None);p.add_argument('--suite9_results_dir',default=None);p.add_argument('--suite10_results_dir',default=None);p.add_argument('--suite11_results_dir',default=None);p.add_argument('--v2_results_dir',default=None);p.add_argument('--council_results_dir',default=None)
    p.add_argument('--seed',type=int,default=42);p.add_argument('--device',choices=['auto','cpu','cuda'],default='auto');p.add_argument('--d_model',type=int,default=64);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--e0_dropout',type=float,default=.15);p.add_argument('--expert_dropout',type=float,default=.05)
    p.add_argument('--e0_epochs',type=int,default=8);p.add_argument('--private_epochs',type=int,default=8);p.add_argument('--fusion_epochs',type=int,default=8);p.add_argument('--joint_epochs',type=int,default=8);p.add_argument('--e0_lr',type=float,default=1e-3);p.add_argument('--private_lr',type=float,default=8e-4);p.add_argument('--fusion_lr',type=float,default=8e-4);p.add_argument('--joint_lr',type=float,default=3e-4);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--aux_lambda',type=float,default=.25);p.add_argument('--distill_warmup',type=float,default=1.0);p.add_argument('--distill_joint',type=float,default=.10);p.add_argument('--patience',type=int,default=3);p.add_argument('--val_blocks',type=int,default=3);p.add_argument('--batch_size',type=int,default=512);p.add_argument('--predict_batch_size',type=int,default=2048);p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--samples_per_epoch',type=int,default=250000,help='0=all historical rows; positive values bound sampled rows per epoch for speed')
    p.add_argument('--max_semesters',type=int,default=6);p.add_argument('--max_course_neighbors',type=int,default=8);p.add_argument('--max_relation_events',type=int,default=16);p.add_argument('--max_component_events',type=int,default=24);p.add_argument('--raw_chunksize',type=int,default=250000);p.add_argument('--force_raw',action='store_true');p.add_argument('--force_structured',action='store_true');p.add_argument('--skip_alignment_audit',action='store_true',help='Not recommended: skip strict raw DB vs canonical expanding-fold identity/target audit')
    p.add_argument('--threshold_beta',type=float,default=2.0);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--verbose_operational',action='store_true')
    a=p.parse_args();a.dev_folds=parse_ints(a.dev_folds);a.holdout_folds=parse_ints(a.holdout_folds);return a

def main():
    args=parse_args();folds=args.dev_folds+args.holdout_folds;available=discover_faculties(args.cache_dir,folds[0]);faculties=parse_faculties(args.faculties,available);out=Path(args.models_dir);out.mkdir(parents=True,exist_ok=True);device=device_of(args.device)
    print('='*180);print('SUITE12 — COLLABORATIVE EVIDENCE MoE WITH DISTILLED SHARED EXPERT');print('direct end-to-end prediction | E0 global distilled | E1 trajectory | E2 curriculum | E3 compatibility | E4 mastery');print('='*180);print(f'faculties={len(faculties)} folds={folds} device={device} d_model={args.d_model} phases={phase_epochs(args)}')
    ensure_structured(args,faculties,folds)
    records=[];metrics=[];diag=[];rep=[]
    for fold in folds:
        rr,mm,dd,rp=run_fold(args,faculties,fold,out,device);records.extend(rr);metrics.extend(mm);diag.extend(dd);rep.extend(rp)
    pd.DataFrame(metrics).to_csv(out/'suite12_fold_metrics.csv',index=False);pd.DataFrame(diag).to_csv(out/'suite12_diagnostics.csv',index=False);pd.DataFrame(rep).to_csv(out/'suite12_representation_cosine.csv',index=False)
    meanf=mean_fold_summary(metrics,args.holdout_folds);meanf.to_csv(out/'suite12_holdout_mean_fold.csv',index=False);warning=operational(records,faculties,args);warning.to_csv(out/'suite12_warning_metrics.csv',index=False);macro=macro_summary(meanf,warning);macro.to_csv(out/'suite12_macro_model_comparison.csv',index=False)
    cols=['model','faculties','mean_fold_ap','mean_fold_auc','mean_fold_precision_at_20','mean_fold_recall_at_20','mean_fold_f1_at_20','mean_fold_f2_at_20','fixed_precision','fixed_recall','fixed_f1','fixed_fbeta','drift_precision','drift_recall','drift_f1','drift_fbeta','drift_tp_total','drift_fn_total','drift_fp_total','drift_tn_total']
    print('\nSUITE12 PRIMARY HOLDOUT COMPARISON — SAME EVALUATOR');print(macro[[c for c in cols if c in macro]].to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    pf=args.primary_faculty;mf=meanf[meanf.faculty==pf];po=warning[(warning.faculty==pf)&(warning.fold.astype(str)=='POOLED')]
    if len(mf):
        rows=[]
        for model in sorted(mf.model.unique()):
            r=mf[mf.model==model].iloc[0];z={'model':model,'AP':r.mean_fold_ap,'AUC':r.mean_fold_auc,'P@20':r.mean_fold_precision_at_20,'R@20':r.mean_fold_recall_at_20,'F1@20':r.mean_fold_f1_at_20,'F2@20':r.mean_fold_f2_at_20}
            for pol in ['fixed','drift']:
                d=po[(po.model==model)&(po.policy==pol)]
                if len(d):
                    x=d.iloc[0]
                    for c in ['precision','recall','f1','fbeta','tp','fn','fp','tn']:z[f'{pol}_{c}']=x[c]
            rows.append(z)
        print(f'\nPRIMARY FACULTY COMPARISON — {pf}');print(pd.DataFrame(rows).to_string(index=False,float_format=lambda x:f'{x:.4f}'))
    print_win_rates(meanf,warning,MODEL)
    (out/'suite12_run_config.json').write_text(json.dumps({'args':vars(args),'faculties':faculties,'architecture':'direct collaborative evidence MoE; frozen tree committee is teacher/reference only','context_features':context_feature_names()},indent=2,default=str))
    print(f'\nCompleted. Results: {out}')
if __name__=='__main__':main()
