# -*- coding: utf-8 -*-
from __future__ import annotations
import argparse,gc,json,hashlib
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
import torch
from sklearn.metrics import average_precision_score

from suite16_pipeline import discover_faculties,parse_faculties,load_fold_bundle
from suite16_structured_cache import build_raw_index,RawEvidenceIndex,structured_cache_compatible,ensure_fold_structured
from suite16_specialists import LatentFactorStore,RelativeMasteryStore,build_student_curriculum_compatibility,build_student_dynamics
from suite16_training import select_hgb_stage,select_grande_stage,stage_crossfit_hgb,stage_crossfit_grande,fit_hgb_refit_predict,fit_predict_grande_fixed
from suite16_temporal import stage_admission
from suite16_hgb import logit,sigmoid,apply_stage_delta,model_feature_usage as hgb_feature_usage
from suite16_grande import model_feature_usage as grande_feature_usage
from suite16_reporting import *

STAGES=(('S1_COMPATIBILITY','Student–Curriculum Compatibility'),('S2_DYNAMICS','Student Dynamics'),('S3_MASTERY','Relative Mastery Phenotype'))
SUITE15_TREE_REFERENCE={'S1_COMPATIBILITY':.0092,'S2_DYNAMICS':.0028,'S3_MASTERY':.0001}

def seed_all(seed):
    np.random.seed(int(seed));torch.manual_seed(int(seed))
    if torch.cuda.is_available():torch.cuda.manual_seed_all(int(seed))

def parse_ints(s):return [int(x.strip()) for x in str(s).split(',') if x.strip()]
def parse_floats(s):return tuple(float(x.strip()) for x in str(s).split(',') if x.strip())
def parse_iters(s):return tuple(int(x.strip()) for x in str(s).split(',') if x.strip())

def safe_ap(y,s):
    y=np.asarray(y,int);return float(average_precision_score(y,np.asarray(s,float))) if np.unique(y).size>1 else np.nan

def macro_ap(y,s,faculty_idx,faculty_names):
    vals=[]
    for fi,_ in enumerate(faculty_names):
        m=np.asarray(faculty_idx)==fi
        if m.sum()>1 and np.unique(np.asarray(y)[m]).size>1:vals.append(safe_ap(np.asarray(y)[m],np.asarray(s)[m]))
    return float(np.mean(vals)) if vals else np.nan

def faculty_ap_rows(y,base,cand,faculty_idx,faculty_names):
    rows=[]
    for fi,name in enumerate(faculty_names):
        m=np.asarray(faculty_idx)==fi
        if m.sum()<2 or np.unique(np.asarray(y)[m]).size<2:continue
        a=safe_ap(np.asarray(y)[m],np.asarray(base)[m]);b=safe_ap(np.asarray(y)[m],np.asarray(cand)[m]);rows.append({'faculty':name,'base_ap':a,'candidate_ap':b,'delta_ap':b-a,'rows':int(m.sum()),'failures':int(np.sum(np.asarray(y)[m]))})
    return rows

def ensure_structured_and_raw(args,faculties,folds):
    sd=Path(args.structured_cache_dir);sd.mkdir(parents=True,exist_ok=True);raw=sd/'raw_index'
    needed=['student_semester.joblib','course_history.joblib','component_history.joblib','course_static.joblib','course_edges.joblib']
    missing_raw=any(not (raw/x).exists() for x in needed)
    missing_struct=[sd/f'{f}_fold_{fold}.joblib' for fold in folds for f in faculties if not structured_cache_compatible(sd/f'{f}_fold_{fold}.joblib')]
    if missing_raw or missing_struct:
        if not args.database_dir and (not args.data1_zip or not args.data2_zip):raise FileNotFoundError('Suite16 needs the Suite12 raw_index or access to raw database/ZIPs.')
        print(f'[CACHE] raw_missing={missing_raw} structured_missing={len(missing_struct)} -> preparing chronology-safe evidence index')
        build_raw_index(args.cache_dir,args.data1_zip,args.data2_zip,faculties,folds,raw,args.force_raw,args.raw_chunksize,database_dir=args.database_dir,strict_alignment=not args.skip_alignment_audit);idx=RawEvidenceIndex(raw)
        for fold in folds:
            for fac in faculties:
                p=sd/f'{fac}_fold_{fold}.joblib'
                if structured_cache_compatible(p) and not args.force_structured:continue
                d=joblib.load(Path(args.cache_dir)/f'{fac}_fold_{fold}.joblib');ensure_fold_structured(d,idx,p,args.max_semesters,args.max_course_neighbors,args.max_relation_events,args.max_component_events,args.force_structured)
    return raw

def feature_signature(args,fold,hist,target):
    return {'version':4,'view_dtype':'float32','fold':int(fold),'hist':len(hist.y),'target':len(target.y),'latent_dim':args.latent_dim,'mastery_events':args.mastery_max_events,'peer_count':args.mastery_min_peer_count}

def build_or_load_features(hist,target,raw_index,raw_index_dir,args,fold,out):
    fd=out/f'fold_{fold}';fd.mkdir(parents=True,exist_ok=True);p=fd/'redesigned_views.joblib';sig=feature_signature(args,fold,hist,target)
    if p.exists() and not args.force_views:
        z=joblib.load(p)
        if z.get('signature')==sig:
            print(f'[VIEWS] reuse {p} dims='+','.join(f'{k}:{z["hist"][k].shape[1]}' for k,_ in STAGES));return z
    print('[VIEWS] building redesigned chronology-safe specialists')
    latent=LatentFactorStore.from_raw_index(raw_index_dir,out/'derived_support'/'latent_factors',args.latent_dim,args.seed)
    lh=latent.features_for_rows(hist.student_id,hist.course_id,hist.target_semester);lt_=latent.features_for_rows(target.student_id,target.course_id,target.target_semester)
    s1h,n1=build_student_curriculum_compatibility(hist,lh,True);s1t=build_student_curriculum_compatibility(target,lt_,False)
    s2h,n2=build_student_dynamics(hist.student_seq,hist.student_mask,True);s2t=build_student_dynamics(target.student_seq,target.student_mask,False)
    mastery=RelativeMasteryStore.from_raw_index(raw_index_dir,out/'derived_support'/'mastery_peer_stats',args.mastery_min_peer_count,args.mastery_max_events,raw_index.relation_type)
    s3h=mastery.features_for_rows(hist.student_id,hist.course_id,hist.target_semester);s3t=mastery.features_for_rows(target.student_id,target.course_id,target.target_semester);n3=mastery.feature_names
    def _finite_view(x,label):
        a=np.asarray(x,np.float32)
        bad=~np.isfinite(a)
        if bad.any():
            print(f'[VIEW SANITIZE WARN] {label}: replacing {int(bad.sum()):,} non-finite values with 0 before caching')
            a=a.copy();a[bad]=0.
        return a
    z={'signature':sig,'hist':{'S1_COMPATIBILITY':_finite_view(s1h,'hist/S1'),'S2_DYNAMICS':_finite_view(s2h,'hist/S2'),'S3_MASTERY':_finite_view(s3h,'hist/S3')},'target':{'S1_COMPATIBILITY':_finite_view(s1t,'target/S1'),'S2_DYNAMICS':_finite_view(s2t,'target/S2'),'S3_MASTERY':_finite_view(s3t,'target/S3')},'names':{'S1_COMPATIBILITY':n1,'S2_DYNAMICS':n2,'S3_MASTERY':n3}}
    joblib.dump(z,p,compress=3);print('[VIEWS] ready dtype=float32 dims='+','.join(f'{k}:{z["hist"][k].shape[1]}' for k,_ in STAGES))
    for k,_ in STAGES:
        ah=z['hist'][k];at=z['target'][k]
        print(f'[VIEW RANGE {k}] hist max|x|={float(np.max(np.abs(ah))) if ah.size else 0.:.3g} target max|x|={float(np.max(np.abs(at))) if at.size else 0.:.3g} finite={np.isfinite(ah).all() and np.isfinite(at).all()}')
    try:
        pc=n1.index('prereq_coverage');pm=n1.index('prereq_mean_grade');dd=n2.index('delta1_DTBHK');ac=n3.index('all_count');az=n3.index('all_mean_z');lt=n3.index('all_low_tail_frac')
        print(f'[VIEW INSIGHT S1] latent seen student/course hist={lh[:,-2].mean():.1%}/{lh[:,-1].mean():.1%} target={lt_[:,-2].mean():.1%}/{lt_[:,-1].mean():.1%} | prereq evidence rows hist/target={(s1h[:,pc]>0).mean():.1%}/{(s1t[:,pc]>0).mean():.1%} mean prereq grade hist={np.mean(s1h[s1h[:,pc]>0,pm]) if np.any(s1h[:,pc]>0) else np.nan:.3f}')
        print(f'[VIEW INSIGHT S2] |Δ latest GPA| hist/target={np.mean(np.abs(s2h[:,dd])):.3f}/{np.mean(np.abs(s2t[:,dd])):.3f} deterioration hist/target={(s2h[:,dd]<0).mean():.1%}/{(s2t[:,dd]<0).mean():.1%}')
        print(f'[VIEW INSIGHT S3] mastery coverage hist/target={(s3h[:,ac]>0).mean():.1%}/{(s3t[:,ac]>0).mean():.1%} | mean relative-z hist/target={np.mean(s3h[s3h[:,ac]>0,az]) if np.any(s3h[:,ac]>0) else np.nan:+.3f}/{np.mean(s3t[s3t[:,ac]>0,az]) if np.any(s3t[:,ac]>0) else np.nan:+.3f} low-tail hist/target={np.mean(s3h[s3h[:,ac]>0,lt]) if np.any(s3h[:,ac]>0) else np.nan:.3f}/{np.mean(s3t[s3t[:,ac]>0,lt]) if np.any(s3t[:,ac]>0) else np.nan:.3f}')
    except Exception as e:print(f'[VIEW INSIGHT WARN] {e}')
    return z

def select_kwargs(args,family,stage_seed):
    common=dict(eta_grid=args.eta_grid,n_windows=args.meta_windows,min_prior_blocks=args.meta_min_prior_blocks,min_faculties=args.window_min_faculties,min_failures=args.window_min_failures,min_rows=args.window_min_rows,sampling_beta=args.sampling_beta,seed=stage_seed)
    if family=='HGB':return dict(common,iter_candidates=args.hgb_iter_candidates,max_leaf_nodes=args.hgb_max_leaf_nodes,learning_rate=args.hgb_lr,l2=args.hgb_l2,correction_bound=args.correction_bound,max_rows=args.hgb_max_rows)
    return dict(common,max_epochs=args.grande_epochs,n_estimators=args.grande_estimators,depth=args.grande_depth,selected_variables=args.grande_selected_variables,split_temperature=args.grande_split_temperature,feature_temperature=args.grande_feature_temperature,correction_bound=args.correction_bound,train_eta=args.grande_train_eta,lr=args.grande_lr,batch_size=args.grande_batch_size,samples_per_epoch=args.grande_samples_per_epoch,device=args.device_resolved)

def crossfit_kwargs(args,family,stage_seed):
    common=dict(min_prior_blocks=args.crossfit_min_prior_blocks,min_faculties=args.window_min_faculties,min_failures=args.window_min_failures,min_rows=args.window_min_rows,sampling_beta=args.sampling_beta,seed=stage_seed)
    if family=='HGB':return dict(common,max_leaf_nodes=args.hgb_max_leaf_nodes,learning_rate=args.hgb_lr,l2=args.hgb_l2,correction_bound=args.correction_bound,max_rows=args.hgb_max_rows)
    return dict(common,n_estimators=args.grande_estimators,depth=args.grande_depth,selected_variables=args.grande_selected_variables,split_temperature=args.grande_split_temperature,feature_temperature=args.grande_feature_temperature,correction_bound=args.correction_bound,train_eta=args.grande_train_eta,lr=args.grande_lr,batch_size=args.grande_batch_size,samples_per_epoch=args.grande_samples_per_epoch)

def admission(args,sel):
    if sel.eta==0 or sel.complexity==0:return False,'identity_selected'
    return stage_admission(sel.stats,args.admit_min_mean_ap,args.admit_min_median_ap,args.admit_min_win_rate,args.admit_min_recent_ap,args.admit_max_f1_drop,args.admit_max_f2_drop)

def correction_diag(fold,family,stage,y,base,cand,delta,eta,covered=None):
    d=np.asarray(delta,float);y=np.asarray(y,int);base=np.asarray(base,float);cand=np.asarray(cand,float);res=y-base;m=np.ones(len(y),bool) if covered is None else np.asarray(covered,bool)
    corr=float(np.corrcoef(d[m],res[m])[0,1]) if m.sum()>2 and np.std(d[m])>0 and np.std(res[m])>0 else np.nan
    return {'fold':fold,'family':family,'stage':stage,'eta':eta,'rows':int(m.sum()),'mean_abs_delta':float(np.mean(np.abs(d[m]))) if m.any() else 0.,'p95_abs_delta':float(np.quantile(np.abs(d[m]),.95)) if m.any() else 0.,'mean_delta_failure':float(np.mean(d[m&(y==1)])) if np.any(m&(y==1)) else np.nan,'mean_delta_normal':float(np.mean(d[m&(y==0)])) if np.any(m&(y==0)) else np.nan,'corr_delta_residual':corr,'ap_before':safe_ap(y[m],base[m]) if m.any() else np.nan,'ap_after':safe_ap(y[m],cand[m]) if m.any() else np.nan}

def run_hgb_single_audits(hist,target,views,args,fold,out):
    base_h=hist.teacher.mean(1);base_t=target.teacher.mean(1);rows=[];facrows=[]
    print('\n[REDESIGNED SINGLE-STAGE HGB AUDIT] — each view conditioned directly on Committee')
    for si,(stage,label) in enumerate(STAGES):
        Xh=np.asarray(views['hist'][stage]);Xt=np.asarray(views['target'][stage]);sel=select_hgb_stage(Xh,hist.y,base_h,hist.faculty_idx,hist.faculty_names,hist.blocks,**select_kwargs(args,'HGB',args.seed+fold*10000+si*211))
        if sel.eta and sel.complexity:
            d,model=fit_hgb_refit_predict(Xh,Xt,hist.y,base_h,hist.faculty_idx,hist.faculty_names,np.arange(len(hist.y)),sel,**crossfit_kwargs(args,'HGB',args.seed+fold*11000+si*223));score=sigmoid(apply_stage_delta(logit(base_t),d,sel.eta));top=hgb_feature_usage(model,views['names'][stage],12)
        else:d=np.zeros(len(target.y),np.float32);score=base_t.copy();top=[]
        fr=faculty_ap_rows(target.y,base_t,score,target.faculty_idx,target.faculty_names);gain=np.mean([r['delta_ap'] for r in fr]) if fr else np.nan;wins=sum(r['delta_ap']>0 for r in fr);accepted,reason=admission(args,sel)
        print(f'  {stage:17s} {label:34s} eta={sel.eta:.2f} complexity={sel.complexity:3d} meta mean/med/recent={sel.stats.get("mean_delta_ap",np.nan):+.4f}/{sel.stats.get("median_delta_ap",np.nan):+.4f}/{sel.stats.get("recent_delta_ap",np.nan):+.4f} wins={sel.stats.get("win_rate",0):.1%} admission={accepted}:{reason} | TARGET macroΔAP={gain:+.4f} wins={wins}/{len(fr)} | Suite15 old tree≈{SUITE15_TREE_REFERENCE[stage]:+.4f}')
        if top:print('    top HGB splits: '+', '.join(f'{n}:{c}' for n,c in top))
        rows.append({'fold':fold,'stage':stage,'label':label,'eta':sel.eta,'complexity':sel.complexity,'meta_score':sel.stats.get('score',np.nan),'meta_mean_delta_ap':sel.stats.get('mean_delta_ap',np.nan),'meta_median_delta_ap':sel.stats.get('median_delta_ap',np.nan),'meta_recent_delta_ap':sel.stats.get('recent_delta_ap',np.nan),'meta_win_rate':sel.stats.get('win_rate',np.nan),'admitted':accepted,'admission_reason':reason,'target_macro_delta_ap':gain,'target_wins':wins,'target_faculties':len(fr),'suite15_old_tree_delta_ap':SUITE15_TREE_REFERENCE[stage]});facrows += [dict(fold=fold,stage=stage,**r) for r in fr]
    return rows,facrows

def run_chain(family,hist,target,views,args,fold,out):
    base_h=hist.teacher.mean(1).astype(np.float32);base_t=target.teacher.mean(1).astype(np.float32);current_h=base_h.copy();current_t=base_t.copy();active=np.ones(len(hist.y),bool);stage_outputs={};records=[];corr=[];usage=[]
    print(f'\n[{family} SEQUENTIAL CONDITIONAL BOOSTING] Committee -> S1 -> S2 -> S3')
    for si,(stage,label) in enumerate(STAGES):
        Xh=np.asarray(views['hist'][stage]);Xt=np.asarray(views['target'][stage]);seed=args.seed+fold*100000+(1 if family=='HGB' else 2)*10000+si*307
        if family=='HGB':sel=select_hgb_stage(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,active_mask=active,**select_kwargs(args,'HGB',seed))
        else:sel=select_grande_stage(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,active_mask=active,**select_kwargs(args,'GRANDE',seed))
        ok,reason=admission(args,sel);prev_h=current_h.copy();prev_t=current_t.copy();fit_active=active.copy();covered=np.zeros(len(hist.y),bool);target_delta=np.zeros(len(target.y),np.float32);top=[]
        if ok:
            if family=='HGB':
                new_h,covered,cw,_=stage_crossfit_hgb(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,active,sel,**crossfit_kwargs(args,'HGB',seed+1000))
                if not covered.any():ok=False;reason='no_crossfit_coverage'
                else:
                    target_delta,model=fit_hgb_refit_predict(Xh,Xt,hist.y,current_h,hist.faculty_idx,hist.faculty_names,np.where(fit_active)[0],sel,**crossfit_kwargs(args,'HGB',seed+2000));top=hgb_feature_usage(model,views['names'][stage],15)
            else:
                new_h,covered,cw,_=stage_crossfit_grande(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,active,sel,args.device_resolved,**crossfit_kwargs(args,'GRANDE',seed+1000))
                if not covered.any():ok=False;reason='no_crossfit_coverage'
                else:
                    target_delta,model,sc=fit_predict_grande_fixed(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,np.where(fit_active)[0],Xt,sel,args.device_resolved,**crossfit_kwargs(args,'GRANDE',seed+2000));top=grande_feature_usage(model,views['names'][stage],15)
            if ok:
                current_h=new_h;current_t=sigmoid(apply_stage_delta(logit(current_t),target_delta,sel.eta));active=covered
        if not ok:
            current_h=prev_h;current_t=prev_t;covered=fit_active
        stage_outputs[stage]=current_t.copy();fr=faculty_ap_rows(target.y,prev_t,current_t,target.faculty_idx,target.faculty_names);gain=np.mean([r['delta_ap'] for r in fr]) if fr else 0.;wins=sum(r['delta_ap']>0 for r in fr);comm_gain=macro_ap(target.y,current_t,target.faculty_idx,target.faculty_names)-macro_ap(target.y,base_t,target.faculty_idx,target.faculty_names)
        print(f'  [{stage}] {label} | selected eta={sel.eta:.2f} complexity={sel.complexity} metaScore={sel.stats.get("score",np.nan):+.5f} mean/med/q25/recent={sel.stats.get("mean_delta_ap",np.nan):+.4f}/{sel.stats.get("median_delta_ap",np.nan):+.4f}/{sel.stats.get("q25_delta_ap",np.nan):+.4f}/{sel.stats.get("recent_delta_ap",np.nan):+.4f} winRate={sel.stats.get("win_rate",0):.1%} -> {"ACCEPT" if ok else "SKIP"} ({reason}) | crossfitCoverage={active.mean():.1%} | TARGET ΔAPprev={gain:+.4f} wins={wins}/{len(fr)} cumulativeΔCommittee={comm_gain:+.4f}')
        for j,r in enumerate(sel.window_rows):
            wb=sel.meta_windows[j].val_blocks if j<len(sel.meta_windows) else []
            print(f'      metaW{j+1} blocks={list(map(int,wb))} ΔAPmed={r.get("delta_ap",np.nan):+.4f} ΔAPmean={r.get("delta_ap_mean",np.nan):+.4f} APwins={r.get("wins",0)}/{r.get("faculties",0)} ΔF1={r.get("delta_f1",np.nan):+.4f} ΔF2={r.get("delta_f2",np.nan):+.4f}')
        if top:print('      top tree features: '+', '.join(f'{n}:{c}' for n,c in top))
        records.append({'fold':fold,'family':family,'stage':stage,'label':label,'selected_eta':sel.eta,'selected_complexity':sel.complexity,'accepted':ok,'reason':reason,'meta_score':sel.stats.get('score',np.nan),'meta_mean_delta_ap':sel.stats.get('mean_delta_ap',np.nan),'meta_median_delta_ap':sel.stats.get('median_delta_ap',np.nan),'meta_q25_delta_ap':sel.stats.get('q25_delta_ap',np.nan),'meta_recent_delta_ap':sel.stats.get('recent_delta_ap',np.nan),'meta_win_rate':sel.stats.get('win_rate',np.nan),'crossfit_coverage':float(active.mean()),'target_delta_ap_vs_previous':gain,'target_ap_wins':wins,'target_faculties':len(fr),'target_cumulative_delta_ap_vs_committee':comm_gain})
        corr.append(correction_diag(fold,family,stage,target.y,prev_t,current_t,target_delta,sel.eta));usage += [{'fold':fold,'family':family,'stage':stage,'rank':i+1,'feature':n,'count':c} for i,(n,c) in enumerate(top)]
        gc.collect()
    return current_t,stage_outputs,records,corr,usage

def run_fold(args,faculties,fold,out,raw_index):
    print('\n'+'='*170);print(f'SUITE16 — TARGET FOLD {fold}');print('='*170)
    hist,target,common=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold);committee_h=hist.teacher.mean(1);committee_t=target.teacher.mean(1)
    print(f'[DATA] faculties={len(faculties)} flat={len(common)} OOF={len(hist.y):,} fail={np.sum(hist.y):,} ({np.mean(hist.y):.2%}) target={len(target.y):,} | Committee target macroAP={macro_ap(target.y,committee_t,target.faculty_idx,target.faculty_names):.4f}')
    views=build_or_load_features(hist,target,raw_index,args.raw_index_dir,args,fold,out)
    audit,audit_fac=run_hgb_single_audits(hist,target,views,args,fold,out)
    hgb_final,hgb_stages,hgb_rec,hgb_corr,hgb_usage=run_chain('HGB',hist,target,views,args,fold,out)
    grande_final,grande_stages,grande_rec,grande_corr,grande_usage=run_chain('GRANDE',hist,target,views,args,fold,out)
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True);records=[];metrics=[]
    stage_cols={COMMITTEE:committee_t,HGB:hgb_final,GRANDE:grande_final}
    for st,_ in STAGES:stage_cols[f'hgb_after_{st.lower()}']=hgb_stages[st];stage_cols[f'grande_after_{st.lower()}']=grande_stages[st]
    cmp=[]
    for fi,fac in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;c=safe_ap(target.y[m],committee_t[m]);h=safe_ap(target.y[m],hgb_final[m]);g=safe_ap(target.y[m],grande_final[m]);cmp.append({'faculty':fac,'committee_ap':c,'hgb_ap':h,'hgb_delta':h-c,'grande_ap':g,'grande_delta':g-c,'rows':int(m.sum()),'failures':int(target.y[m].sum())})
    print_table('[TARGET FACULTY RANKING] Committee vs Suite16 final families',pd.DataFrame(cmp).sort_values('grande_delta',ascending=False))
    for fi,fac in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m];q=committee_t[m];scores=reference_for_faculty(args,fold,fac,y,q)
        for k,v in stage_cols.items():scores[k]=np.asarray(v)[m]
        records.append({'faculty':fac,'fold':fold,'y':y,'scores':scores})
        for model,s in scores.items():metrics.append(metric_row(fold,fac,model,y,s,args.threshold_beta))
        sf=pd.DataFrame({'actual':y,'student_id':target.student_id[m],'course_id':target.course_id[m],'target_semester':target.target_semester[m]})
        for model,s in scores.items():sf[model]=s
        sf.to_csv(fold_dir/f'{fac}_scores.csv',index=False)
    pd.DataFrame(audit).to_csv(fold_dir/'redesigned_view_audit.csv',index=False);pd.DataFrame(audit_fac).to_csv(fold_dir/'redesigned_view_audit_faculty.csv',index=False);pd.DataFrame(hgb_rec+grande_rec).to_csv(fold_dir/'stage_decisions.csv',index=False);pd.DataFrame(hgb_corr+grande_corr).to_csv(fold_dir/'correction_diagnostics.csv',index=False);pd.DataFrame(hgb_usage+grande_usage).to_csv(fold_dir/'feature_usage.csv',index=False)
    return records,metrics,audit,audit_fac,hgb_rec+grande_rec,hgb_corr+grande_corr,hgb_usage+grande_usage

def postprocess(args,out,faculties,records,metrics,audit,audit_fac,stage_rec,corr,usage):
    mdf=pd.DataFrame(metrics);mdf.to_csv(out/'fold_metrics.csv',index=False);meanf=mean_fold_summary(mdf,args.holdout_folds);meanf.to_csv(out/'mean_fold_holdout.csv',index=False)
    warn=operational(records,faculties,args);warn.to_csv(out/'operational_metrics.csv',index=False);macro=macro_summary(meanf,warn);macro.to_csv(out/'primary_holdout_comparison.csv',index=False);print_table('SUITE16 PRIMARY HOLDOUT COMPARISON — SAME EVALUATOR',macro)
    for cand in [HGB,GRANDE]:print_win_rates(meanf,warn,cand)
    adf=pd.DataFrame(audit);adf.to_csv(out/'redesigned_view_audit.csv',index=False);af=pd.DataFrame(audit_fac);af.to_csv(out/'redesigned_view_audit_faculty.csv',index=False)
    if len(adf):
        z=adf[adf.fold.isin(args.holdout_folds)].groupby('stage').agg(target_macro_delta_ap=('target_macro_delta_ap','mean'),target_wins=('target_wins','sum'),cells=('target_faculties','sum'),admission_rate=('admitted','mean'),meta_mean_delta_ap=('meta_mean_delta_ap','mean'),suite15_old_tree_delta_ap=('suite15_old_tree_delta_ap','mean')).reset_index();print_table('REDESIGNED SPECIALIST AUDIT — HOLDOUT MEAN',z)
    sr=pd.DataFrame(stage_rec);sr.to_csv(out/'stage_decisions.csv',index=False)
    if len(sr):
        hs=sr[sr.fold.isin(args.holdout_folds)].groupby(['family','stage']).agg(accept_rate=('accepted','mean'),target_delta_ap_vs_previous=('target_delta_ap_vs_previous','mean'),cumulative_delta_ap_vs_committee=('target_cumulative_delta_ap_vs_committee','mean'),meta_mean_delta_ap=('meta_mean_delta_ap','mean'),meta_recent_delta_ap=('meta_recent_delta_ap','mean'),crossfit_coverage=('crossfit_coverage','mean')).reset_index();print_table('SEQUENTIAL STAGE CONTRIBUTIONS — HOLDOUT',hs)
    cdf=pd.DataFrame(corr);cdf.to_csv(out/'correction_diagnostics.csv',index=False);pd.DataFrame(usage).to_csv(out/'feature_usage.csv',index=False)
    if len(cdf):
        cz=cdf[cdf.fold.isin(args.holdout_folds)].groupby(['family','stage']).agg(mean_abs_delta=('mean_abs_delta','mean'),p95_abs_delta=('p95_abs_delta','mean'),mean_delta_failure=('mean_delta_failure','mean'),mean_delta_normal=('mean_delta_normal','mean'),corr_delta_residual=('corr_delta_residual','mean')).reset_index();print_table('CONDITIONAL CORRECTION DIAGNOSTICS — HOLDOUT',cz)
    # Faculty head-to-head for our two final families.
    for cand in [HGB,GRANDE]:
        a=meanf[meanf.model==cand][['faculty','mean_fold_ap','mean_fold_auc','mean_fold_f1_at_20','mean_fold_f2_at_20']].copy();b=meanf[meanf.model==COMMITTEE][['faculty','mean_fold_ap','mean_fold_f2_at_20']].rename(columns={'mean_fold_ap':'committee_ap','mean_fold_f2_at_20':'committee_f2_at20'});z=a.merge(b,on='faculty');z['delta_ap_vs_committee']=z.mean_fold_ap-z.committee_ap;z['delta_f2_at20_vs_committee']=z.mean_fold_f2_at_20-z.committee_f2_at20;z.to_csv(out/f'{cand}_faculty_comparison.csv',index=False);print_table(f'{cand} — FACULTY HOLDOUT COMPARISON vs COMMITTEE',z.sort_values('delta_ap_vs_committee',ascending=False))

def build_parser():
    root='/root/disk_3tb/earlywarning/30072026';p=argparse.ArgumentParser(description='Suite16 Sequential Conditional Gradient-Tree Boosting')
    p.add_argument('--cache_dir',default=f'{root}/results/cross_faculty_residual_moe/cache');p.add_argument('--structured_cache_dir',default=f'{root}/results/suite12_structured_cache');p.add_argument('--out_dir',default=f'{root}/results/suite16_conditional_tree_boosting');p.add_argument('--database_dir',default='/root/disk_3tb/earlywarning/database');p.add_argument('--data1_zip',default='');p.add_argument('--data2_zip',default='');p.add_argument('--faculties',default='all');p.add_argument('--folds',default='1,2,3,4,5,6,7');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--seed',type=int,default=20260830);p.add_argument('--device',default='auto')
    p.add_argument('--latent_dim',type=int,default=12);p.add_argument('--mastery_min_peer_count',type=int,default=5);p.add_argument('--mastery_max_events',type=int,default=48);p.add_argument('--force_views',action='store_true')
    p.add_argument('--eta_grid',default='0,.1,.2,.3,.5');p.add_argument('--meta_windows',type=int,default=3);p.add_argument('--meta_min_prior_blocks',type=int,default=4);p.add_argument('--crossfit_min_prior_blocks',type=int,default=3);p.add_argument('--window_min_faculties',type=int,default=8);p.add_argument('--window_min_failures',type=int,default=120);p.add_argument('--window_min_rows',type=int,default=5000)
    p.add_argument('--admit_min_mean_ap',type=float,default=.002);p.add_argument('--admit_min_median_ap',type=float,default=0.);p.add_argument('--admit_min_win_rate',type=float,default=.55);p.add_argument('--admit_min_recent_ap',type=float,default=-.002);p.add_argument('--admit_max_f1_drop',type=float,default=-.005);p.add_argument('--admit_max_f2_drop',type=float,default=-.005)
    p.add_argument('--sampling_beta',type=float,default=.9999);p.add_argument('--correction_bound',type=float,default=3.)
    p.add_argument('--hgb_iter_candidates',default='20,40,60,80');p.add_argument('--hgb_max_leaf_nodes',type=int,default=31);p.add_argument('--hgb_lr',type=float,default=.05);p.add_argument('--hgb_l2',type=float,default=1.);p.add_argument('--hgb_max_rows',type=int,default=180000)
    p.add_argument('--grande_estimators',type=int,default=96);p.add_argument('--grande_depth',type=int,default=4);p.add_argument('--grande_epochs',type=int,default=5);p.add_argument('--grande_selected_variables',type=float,default=.8);p.add_argument('--grande_split_temperature',type=float,default=.5);p.add_argument('--grande_feature_temperature',type=float,default=.5);p.add_argument('--grande_train_eta',type=float,default=.2);p.add_argument('--grande_lr',type=float,default=.003);p.add_argument('--grande_batch_size',type=int,default=1024);p.add_argument('--grande_samples_per_epoch',type=int,default=60000)
    p.add_argument('--threshold_beta',type=float,default=2.);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--primary_faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--verbose_operational',action='store_true')
    p.add_argument('--shared_results_dir',default=f'{root}/results/cross_faculty_residual_moe');p.add_argument('--suite7_results_dir',default=f'{root}/results/suite7_shared_private_adapter');p.add_argument('--suite8_results_dir',default=f'{root}/results/suite8_true_specialist');p.add_argument('--suite9_results_dir',default=f'{root}/results/suite9_competitive_responsibility');p.add_argument('--suite10_results_dir',default=f'{root}/results/suite10_semantic_specialists');p.add_argument('--suite11_results_dir',default=f'{root}/results/suite11_opportunity_latent_domains');p.add_argument('--suite12_results_dir',default=f'{root}/results/suite12_collaborative_evidence_moe');p.add_argument('--suite13_results_dir',default=f'{root}/results/suite13_committee_preserving_collaborative_moe');p.add_argument('--suite14_results_dir',default=f'{root}/results/suite14_authority_safe_collaborative_moe');p.add_argument('--v2_results_dir',default=f'{root}/results/final_train_v2');p.add_argument('--council_results_dir',default=f'{root}/results/suite4/2_competence_council')
    p.add_argument('--force_raw',action='store_true');p.add_argument('--force_structured',action='store_true');p.add_argument('--skip_alignment_audit',action='store_true');p.add_argument('--raw_chunksize',type=int,default=250000);p.add_argument('--max_semesters',type=int,default=6);p.add_argument('--max_course_neighbors',type=int,default=8);p.add_argument('--max_relation_events',type=int,default=16);p.add_argument('--max_component_events',type=int,default=24)
    return p

def main(argv=None):
    args=build_parser().parse_args(argv);args.folds=parse_ints(args.folds);args.dev_folds=parse_ints(args.dev_folds);args.holdout_folds=parse_ints(args.holdout_folds);args.eta_grid=parse_floats(args.eta_grid);args.hgb_iter_candidates=parse_iters(args.hgb_iter_candidates);args.device_resolved='cuda' if args.device in ('auto','cuda') and torch.cuda.is_available() else 'cpu';args.raw_index_dir=str(Path(args.structured_cache_dir)/'raw_index')
    out=Path(args.out_dir);out.mkdir(parents=True,exist_ok=True);available=discover_faculties(args.cache_dir,args.folds[0]);faculties=parse_faculties(args.faculties,available);raw_dir=ensure_structured_and_raw(args,faculties,args.folds);args.raw_index_dir=str(raw_dir);raw_index=RawEvidenceIndex(raw_dir);seed_all(args.seed)
    print('='*170);print('SUITE16 — SEQUENTIAL CONDITIONAL GRADIENT-TREE BOOSTING');print('Immutable Committee | redesigned S1/S2/S3 | HGB/Newton reference + GRANDE-style hard differentiable trees | multi-window temporal admission | no gate/MoE/E0');print('='*170);print(f'faculties={len(faculties)} folds={args.folds} device={args.device_resolved} eta_grid={args.eta_grid} meta_windows={args.meta_windows} window_min(fac/fail/rows)={args.window_min_faculties}/{args.window_min_failures}/{args.window_min_rows}')
    records=[];metrics=[];aud=[];audf=[];st=[];corr=[];usage=[]
    for fold in args.folds:
        a,b,c,d,e,f,g=run_fold(args,faculties,fold,out,raw_index);records+=a;metrics+=b;aud+=c;audf+=d;st+=e;corr+=f;usage+=g;gc.collect()
    postprocess(args,out,faculties,records,metrics,aud,audf,st,corr,usage);print(f'\n[DONE] Suite16 results: {out}')

if __name__=='__main__':main()
