# -*- coding: utf-8 -*-
from __future__ import annotations
import argparse,gc,hashlib,json
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
from sklearn.metrics import average_precision_score

from suite17_pipeline import discover_faculties,parse_faculties,load_fold_bundle
from suite17_structured_cache import build_raw_index,RawEvidenceIndex,structured_cache_compatible,ensure_fold_structured
from suite17_specialists import LatentFactorStore,RelativeMasteryStore,build_relational_variant_family
from suite17_local import build_faculty_local_evidence,LOCAL_FEATURE_NAMES
from suite17_training import select_hgb_stage,stage_crossfit_hgb_dense,fit_hgb_refit_predict,ForwardCandidate,choose_forward_candidate
from suite17_temporal import stage_admission
from suite17_hgb import logit,sigmoid,apply_stage_delta,model_feature_usage
from suite17_alignment import align_by_faculty
from suite17_reporting import *

R_VARIANTS=('R_RELATION','R_RELATION_CURRICULUM','R_RELATION_LATENT','R_FULL')
SPECIALISTS=('R','M','L')
LABELS={'R':'Relational Compatibility','M':'Relative Mastery','L':'Faculty-Local Temporal Tabular Evidence'}

def seed_all(seed):np.random.seed(int(seed))
def parse_ints(s):return [int(x.strip()) for x in str(s).split(',') if x.strip()]
def parse_floats(s):return tuple(float(x.strip()) for x in str(s).split(',') if x.strip())
def parse_iters(s):return tuple(int(x.strip()) for x in str(s).split(',') if x.strip())

def safe_ap(y,s):
    y=np.asarray(y,int);return float(average_precision_score(y,np.asarray(s,float))) if np.unique(y).size>1 else np.nan

def macro_ap(y,s,faculty_idx,faculty_names):
    vals=[]
    for fi,_ in enumerate(faculty_names):
        m=np.asarray(faculty_idx)==fi
        if m.sum()>1 and np.unique(np.asarray(y)[m]).size>1:vals.append(safe_ap(np.asarray(y)[m],np.asarray(s)[m]))
    return float(np.mean(vals)) if vals else np.nan

def faculty_ap_rows(y,base,cand,faculty_idx,faculty_names):
    rows=[]
    for fi,name in enumerate(faculty_names):
        m=np.asarray(faculty_idx)==fi
        if m.sum()<2 or np.unique(np.asarray(y)[m]).size<2:continue
        a=safe_ap(np.asarray(y)[m],np.asarray(base)[m]);b=safe_ap(np.asarray(y)[m],np.asarray(cand)[m]);rows.append({'faculty':name,'base_ap':a,'candidate_ap':b,'delta_ap':b-a,'rows':int(m.sum()),'failures':int(np.sum(np.asarray(y)[m]))})
    return rows

def ensure_structured_and_raw(args,faculties,folds):
    sd=Path(args.structured_cache_dir);sd.mkdir(parents=True,exist_ok=True);raw=sd/'raw_index';needed=['student_semester.joblib','course_history.joblib','component_history.joblib','course_static.joblib','course_edges.joblib'];missing_raw=any(not (raw/x).exists() for x in needed);missing_struct=[sd/f'{f}_fold_{fold}.joblib' for fold in folds for f in faculties if not structured_cache_compatible(sd/f'{f}_fold_{fold}.joblib')]
    if missing_raw or missing_struct:
        if not args.database_dir and (not args.data1_zip or not args.data2_zip):raise FileNotFoundError('Suite17 needs the Suite12 raw_index or access to raw database/ZIPs.')
        print(f'[CACHE] raw_missing={missing_raw} structured_missing={len(missing_struct)} -> preparing chronology-safe evidence index');build_raw_index(args.cache_dir,args.data1_zip,args.data2_zip,faculties,folds,raw,args.force_raw,args.raw_chunksize,database_dir=args.database_dir,strict_alignment=not args.skip_alignment_audit);idx=RawEvidenceIndex(raw)
        for fold in folds:
            for fac in faculties:
                p=sd/f'{fac}_fold_{fold}.joblib'
                if structured_cache_compatible(p) and not args.force_structured:continue
                d=joblib.load(Path(args.cache_dir)/f'{fac}_fold_{fold}.joblib');ensure_fold_structured(d,idx,p,args.max_semesters,args.max_course_neighbors,args.max_relation_events,args.max_component_events,args.force_structured)
    return raw

def _finite(x,label):
    a=np.asarray(x,np.float32);bad=~np.isfinite(a)
    if bad.any():print(f'[VIEW SANITIZE WARN] {label}: replacing {int(bad.sum()):,} non-finite values with 0');a=a.copy();a[bad]=0.
    return a

def view_signature(args,fold,hist,target):return {'version':1,'fold':int(fold),'hist':len(hist.y),'target':len(target.y),'latent_dim':args.latent_dim,'mastery_events':args.mastery_max_events,'peer_count':args.mastery_min_peer_count}

def build_or_load_structured_views(hist,target,raw_index,args,fold,out):
    fd=out/f'fold_{fold}';fd.mkdir(parents=True,exist_ok=True);p=fd/'suite17_structured_views.joblib';sig=view_signature(args,fold,hist,target)
    if p.exists() and not args.force_views:
        z=joblib.load(p)
        if z.get('signature')==sig:
            print(f'[VIEWS] reuse {p} R='+','.join(f'{k}:{z["hist"][k].shape[1]}' for k in R_VARIANTS)+f' M:{z["hist"]["M"].shape[1]}');return z
    print('[VIEWS] building R ablations + preserved relative mastery')
    support=Path(args.support_cache_dir);support.mkdir(parents=True,exist_ok=True)
    latent=LatentFactorStore.from_raw_index(args.raw_index_dir,support/'latent_factors',args.latent_dim,args.seed);lh=latent.features_for_rows(hist.student_id,hist.course_id,hist.target_semester);lt=latent.features_for_rows(target.student_id,target.course_id,target.target_semester)
    rh=build_relational_variant_family(hist,lh);rt=build_relational_variant_family(target,lt)
    mastery=RelativeMasteryStore.from_raw_index(args.raw_index_dir,support/'mastery_peer_stats',args.mastery_min_peer_count,args.mastery_max_events,raw_index.relation_type);mh=mastery.features_for_rows(hist.student_id,hist.course_id,hist.target_semester);mt=mastery.features_for_rows(target.student_id,target.course_id,target.target_semester)
    z={'signature':sig,'hist':{},'target':{},'names':{}}
    for k in R_VARIANTS:z['hist'][k]=_finite(rh[k][0],f'hist/{k}');z['target'][k]=_finite(rt[k][0],f'target/{k}');z['names'][k]=rh[k][1]
    z['hist']['M']=_finite(mh,'hist/M');z['target']['M']=_finite(mt,'target/M');z['names']['M']=mastery.feature_names;joblib.dump(z,p,compress=3)
    print('[VIEWS] ready '+', '.join(f'{k}:{z["hist"][k].shape[1]}' for k in (*R_VARIANTS,'M')))
    for k in (*R_VARIANTS,'M'):
        print(f'[VIEW RANGE {k}] hist max|x|={np.max(np.abs(z["hist"][k])):.3g} target max|x|={np.max(np.abs(z["target"][k])):.3g} finite={np.isfinite(z["hist"][k]).all() and np.isfinite(z["target"][k]).all()}')
    return z

def local_signature(args,fold,hist,target,source_dim):
    return {'version':1,'fold':int(fold),'hist':len(hist.y),'target':len(target.y),'source_dim':int(source_dim),'recent_blocks':args.local_recent_blocks,'window_blocks':args.local_crossfit_window_blocks,'min_rows':args.local_min_train_rows,'min_failures':args.local_min_failures,'iter':args.local_hgb_iter}

def build_or_load_local(hist,target,args,fold,out):
    fd=out/f'fold_{fold}';p=fd/'faculty_local_evidence.joblib';xh=np.concatenate([hist.flat,hist.e0_context],1).astype(np.float32);xt=np.concatenate([target.flat,target.e0_context],1).astype(np.float32);sig=local_signature(args,fold,hist,target,xh.shape[1])
    if p.exists() and not args.force_local:
        z=joblib.load(p)
        if z.get('signature')==sig:
            print(f'[LOCAL] reuse {p} dim={z["hist"].shape[1]} global/recent availability hist={z["hist_available"][:,0].mean():.1%}/{z["hist_available"][:,1].mean():.1%} target={z["target_available"][:,0].mean():.1%}/{z["target_available"][:,1].mean():.1%}');return z
    print('[LOCAL] building chronology-safe faculty-local all-history/recent alternative predictions')
    r=build_faculty_local_evidence(xh,hist.teacher,hist.y,hist.teacher.mean(1),hist.faculty_idx,hist.blocks,xt,target.teacher,target.teacher.mean(1),target.faculty_idx,recent_blocks=args.local_recent_blocks,min_prior_blocks=args.local_min_prior_blocks,crossfit_window_blocks=args.local_crossfit_window_blocks,min_train_rows=args.local_min_train_rows,min_failures=args.local_min_failures,max_iter=args.local_hgb_iter,max_leaf_nodes=args.local_hgb_max_leaf_nodes,learning_rate=args.local_hgb_lr,l2=args.local_hgb_l2,sampling_beta=args.sampling_beta,seed=args.seed+fold*701)
    z={'signature':sig,'hist':r.hist_features,'target':r.target_features,'names':list(LOCAL_FEATURE_NAMES),'hist_available':r.hist_available,'target_available':r.target_available,'hist_global':r.hist_global,'hist_recent':r.hist_recent,'target_global':r.target_global,'target_recent':r.target_recent,'diagnostics':r.diagnostics};joblib.dump(z,p,compress=3)
    print(f'[LOCAL] ready dim={z["hist"].shape[1]} global/recent availability hist={z["hist_available"][:,0].mean():.1%}/{z["hist_available"][:,1].mean():.1%} target={z["target_available"][:,0].mean():.1%}/{z["target_available"][:,1].mean():.1%}')
    return z

def select_kwargs(args,seed):return dict(eta_grid=args.eta_grid,iter_candidates=args.hgb_iter_candidates,n_windows=args.meta_windows,min_prior_blocks=args.meta_min_prior_blocks,min_faculties=args.window_min_faculties,min_failures=args.window_min_failures,min_rows=args.window_min_rows,max_leaf_nodes=args.hgb_max_leaf_nodes,learning_rate=args.hgb_lr,l2=args.hgb_l2,correction_bound=args.correction_bound,max_rows=args.hgb_max_rows,sampling_beta=args.sampling_beta,seed=seed)
def crossfit_kwargs(args,seed):return dict(min_prior_blocks=args.crossfit_min_prior_blocks,min_faculties=args.window_min_faculties,min_failures=args.window_min_failures,min_rows=args.window_min_rows,max_leaf_nodes=args.hgb_max_leaf_nodes,learning_rate=args.hgb_lr,l2=args.hgb_l2,correction_bound=args.correction_bound,max_rows=args.hgb_max_rows,sampling_beta=args.sampling_beta,seed=seed)

def admission(args,sel,third=False):
    if sel.eta==0 or sel.complexity==0:return False,'identity_selected'
    if third:
        return stage_admission(sel.stats,args.third_admit_min_mean_ap,args.third_admit_min_median_ap,args.third_admit_min_win_rate,args.third_admit_min_recent_mean_ap,args.third_admit_min_recent_median_ap,args.third_admit_max_recent_window_mean_drop,args.admit_max_f1_drop,args.admit_max_f2_drop)
    return stage_admission(sel.stats,args.admit_min_mean_ap,args.admit_min_median_ap,args.admit_min_win_rate,args.admit_min_recent_mean_ap,args.admit_min_recent_median_ap,args.admit_max_recent_window_mean_drop,args.admit_max_f1_drop,args.admit_max_f2_drop)

def augment_view(x,names,committee,current,availability):
    x=np.asarray(x,np.float32);controls=[np.asarray(committee,np.float32)[:,None],np.asarray(current,np.float32)[:,None],(np.asarray(current,np.float32)-np.asarray(committee,np.float32))[:,None]];nn=list(names)+['committee_p','current_p','current_minus_committee']
    for k in sorted(availability):controls.append(np.asarray(availability[k],np.float32)[:,None]);nn.append(f'previous_{k}_crossfit_available')
    return np.concatenate([x]+controls,1).astype(np.float32),nn

def correction_diag(fold,stage_index,stage,y,base,cand,delta,eta,bound,covered):
    d=np.asarray(delta,float);eff=float(eta)*d;y=np.asarray(y,int);base=np.asarray(base,float);res=y-base;m=np.asarray(covered,bool);corr=float(np.corrcoef(eff[m],res[m])[0,1]) if m.sum()>2 and np.std(eff[m])>0 and np.std(res[m])>0 else np.nan
    return {'fold':fold,'stage_index':stage_index,'stage':stage,'eta':eta,'covered_rows':int(m.sum()),'coverage':float(m.mean()),'mean_abs_delta':float(np.mean(np.abs(d[m]))) if m.any() else 0.,'mean_abs_effective_delta':float(np.mean(np.abs(eff[m]))) if m.any() else 0.,'p95_abs_effective_delta':float(np.quantile(np.abs(eff[m]),.95)) if m.any() else 0.,'saturation_rate':float(np.mean(np.abs(d[m])>=.95*float(bound))) if m.any() else 0.,'mean_effective_delta_failure':float(np.mean(eff[m&(y==1)])) if np.any(m&(y==1)) else np.nan,'mean_effective_delta_normal':float(np.mean(eff[m&(y==0)])) if np.any(m&(y==0)) else np.nan,'corr_effective_delta_residual':corr,'ap_before':safe_ap(y[m],base[m]) if m.any() else np.nan,'ap_after':safe_ap(y[m],cand[m]) if m.any() else np.nan}

def fit_target_counterfactual(Xh,Xt,hist,current_h,target_current,sel,args,seed):
    if sel.eta==0 or sel.complexity==0:return target_current.copy(),np.zeros(len(target_current),np.float32),None
    d,m=fit_hgb_refit_predict(Xh,Xt,hist.y,current_h,hist.faculty_idx,hist.faculty_names,np.arange(len(hist.y)),sel,**crossfit_kwargs(args,seed));s=sigmoid(apply_stage_delta(logit(target_current),d,sel.eta)).astype(np.float32);return s,d,m

def direct_audit_one(name,label,Xh,Xt,names,hist,target,args,fold,seed):
    ch=hist.teacher.mean(1);ct=target.teacher.mean(1);Ah,an=augment_view(Xh,names,ch,ch,{});At,_=augment_view(Xt,names,ct,ct,{})
    sel=select_hgb_stage(Ah,hist.y,ch,hist.faculty_idx,hist.faculty_names,hist.blocks,**select_kwargs(args,seed));ok,reason=admission(args,sel);score,delta,model=fit_target_counterfactual(Ah,At,hist,ch,ct,sel,args,seed+900);fr=faculty_ap_rows(target.y,ct,score,target.faculty_idx,target.faculty_names);gain=np.mean([r['delta_ap'] for r in fr]) if fr else np.nan;wins=int(sum(r['delta_ap']>0 for r in fr));top=model_feature_usage(model,an,12) if model else []
    row={'fold':fold,'specialist':name,'label':label,'eta':sel.eta,'complexity':sel.complexity,'admitted':ok,'admission_reason':reason,'meta_score':sel.stats.get('score',np.nan),'meta_mean_delta_ap':sel.stats.get('mean_delta_ap',np.nan),'meta_median_delta_ap':sel.stats.get('median_delta_ap',np.nan),'meta_recent_mean_delta_ap':sel.stats.get('recent_mean_delta_ap',np.nan),'meta_recent_median_delta_ap':sel.stats.get('recent_median_delta_ap',np.nan),'meta_worst_recent_mean_delta_ap':sel.stats.get('worst_recent_window_mean_delta_ap',np.nan),'meta_win_rate':sel.stats.get('win_rate',np.nan),'target_macro_delta_ap':gain,'target_wins':wins,'target_faculties':len(fr)}
    print(f'  {name:24s} eta={sel.eta:.2f} iter={sel.complexity:3d} score={row["meta_score"]:+.5f} mean/med={row["meta_mean_delta_ap"]:+.4f}/{row["meta_median_delta_ap"]:+.4f} recent mean/med={row["meta_recent_mean_delta_ap"]:+.4f}/{row["meta_recent_median_delta_ap"]:+.4f} worstRecentMean={row["meta_worst_recent_mean_delta_ap"]:+.4f} winRate={row["meta_win_rate"]:.1%} -> {"ADMIT" if ok else "REJECT"}({reason}) | TARGET ΔAP={gain:+.4f} wins={wins}/{len(fr)}')
    if top:print('      top splits: '+', '.join(f'{n}:{c}' for n,c in top))
    return row,[dict(fold=fold,specialist=name,**r) for r in fr],sel,score,top

def run_standalone_audits(hist,target,views,local,args,fold):
    print('\n[STANDALONE SPECIALIST AUDIT] Committee-conditioned; target results diagnostic only')
    rows=[];facrows=[];sels={};scores={};tops={}
    for i,v in enumerate(R_VARIANTS):
        row,fr,sel,score,top=direct_audit_one(v,v,views['hist'][v],views['target'][v],views['names'][v],hist,target,args,fold,args.seed+fold*10000+i*137);rows.append(row);facrows+=fr;sels[v]=sel;scores[v]=score;tops[v]=top
    admitted=[r for r in rows if r['specialist'] in R_VARIANTS and r['admitted']];pool=admitted if admitted else [r for r in rows if r['specialist'] in R_VARIANTS];best=max(pool,key=lambda r:(r['meta_score'],r['meta_mean_delta_ap'],r['meta_median_delta_ap']));rsel=best['specialist'];print(f'[R ABLATION SELECT] {rsel} chosen from history only; target ΔAP diagnostic={best["target_macro_delta_ap"]:+.4f}')
    for i,(name,label,Xh,Xt,names) in enumerate([('M',LABELS['M'],views['hist']['M'],views['target']['M'],views['names']['M']),('L',LABELS['L'],local['hist'],local['target'],local['names'])]):
        row,fr,sel,score,top=direct_audit_one(name,label,Xh,Xt,names,hist,target,args,fold,args.seed+fold*12000+700+i*149);rows.append(row);facrows+=fr;sels[name]=sel;scores[name]=score;tops[name]=top
    # Raw local alternative-predictor insight, before conditional boosting.
    ct=target.teacher.mean(1)
    for key,pred in [('LOCAL_GLOBAL_RAW',local['target_global']),('LOCAL_RECENT_RAW',local['target_recent'])]:
        fr=faculty_ap_rows(target.y,ct,pred,target.faculty_idx,target.faculty_names);print(f'[LOCAL RAW {key}] target macroΔAP={np.mean([r["delta_ap"] for r in fr]):+.4f} wins={sum(r["delta_ap"]>0 for r in fr)}/{len(fr)}')
    return rsel,rows,facrows,sels,scores,tops

def candidate_base_view(name,rsel,views,local):
    if name=='R':return views['hist'][rsel],views['target'][rsel],views['names'][rsel]
    if name=='M':return views['hist']['M'],views['target']['M'],views['names']['M']
    return local['hist'],local['target'],local['names']

def run_forward_selection(hist,target,views,local,rsel,args,fold):
    ch=hist.teacher.mean(1).astype(np.float32);ct=target.teacher.mean(1).astype(np.float32);current_h=ch.copy();current_t=ct.copy();remaining=list(SPECIALISTS);avail_h={};avail_t={};stage_outputs={};candidate_rows=[];stage_rows=[];corr_rows=[];usage=[];accepted=[]
    print('\n[ROBUST FORWARD COMPLEMENTARY BOOSTING] candidate pool={R,M,L}; dense identity-filled OOF ledger')
    stage_index=1
    while remaining and stage_index<=3:
        third=stage_index>int(args.max_deploy_stages);fc=[];counter={};prepared={}
        print(f'\n  [FORWARD STAGE {stage_index}] current accepted={accepted or ["Committee"]} remaining={remaining}'+(' [DIAGNOSTIC/STRICT]' if third else ''))
        for ci,name in enumerate(remaining):
            bh,bt,bn=candidate_base_view(name,rsel,views,local);Xh,names=augment_view(bh,bn,ch,current_h,avail_h);Xt,_=augment_view(bt,bn,ct,current_t,avail_t);seed=args.seed+fold*100000+stage_index*10000+ci*311
            sel=select_hgb_stage(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,**select_kwargs(args,seed));ok,reason=admission(args,sel,third=third);score_t,d_t,model=fit_target_counterfactual(Xh,Xt,hist,current_h,current_t,sel,args,seed+5000);fr=faculty_ap_rows(target.y,current_t,score_t,target.faculty_idx,target.faculty_names);tg=np.mean([r['delta_ap'] for r in fr]) if fr else np.nan;tw=int(sum(r['delta_ap']>0 for r in fr));prepared[name]=(Xh,Xt,names,sel,score_t,d_t,model,seed);counter[name]=score_t
            row={'fold':fold,'stage_index':stage_index,'candidate':name,'label':LABELS[name],'selected_eta':sel.eta,'selected_complexity':sel.complexity,'admitted':ok,'reason':reason,'meta_score':sel.stats.get('score',np.nan),'meta_mean_delta_ap':sel.stats.get('mean_delta_ap',np.nan),'meta_median_delta_ap':sel.stats.get('median_delta_ap',np.nan),'meta_q25_delta_ap':sel.stats.get('q25_delta_ap',np.nan),'meta_recent_mean_delta_ap':sel.stats.get('recent_mean_delta_ap',np.nan),'meta_recent_median_delta_ap':sel.stats.get('recent_median_delta_ap',np.nan),'meta_worst_recent_mean_delta_ap':sel.stats.get('worst_recent_window_mean_delta_ap',np.nan),'meta_win_rate':sel.stats.get('win_rate',np.nan),'target_counterfactual_delta_ap':tg,'target_counterfactual_wins':tw,'target_faculties':len(fr),'diagnostic_third_stage':third};candidate_rows.append(row);fc.append(ForwardCandidate(name,ok,sel.stats,sel,reason))
            print(f'    candidate {name} | eta={sel.eta:.2f} iter={sel.complexity:3d} score={row["meta_score"]:+.5f} mean/med={row["meta_mean_delta_ap"]:+.4f}/{row["meta_median_delta_ap"]:+.4f} recent mean/med={row["meta_recent_mean_delta_ap"]:+.4f}/{row["meta_recent_median_delta_ap"]:+.4f} worstRecentMean={row["meta_worst_recent_mean_delta_ap"]:+.4f} winRate={row["meta_win_rate"]:.1%} -> {"ADMIT" if ok else "REJECT"}({reason}) | TARGET counterfactual ΔAP={tg:+.4f} wins={tw}/{len(fr)}')
        chosen=choose_forward_candidate(fc)
        if chosen is None:
            print('    [FORWARD STOP] no remaining candidate passes robust admission');break
        if third and not args.deploy_third_stage:
            print(f'    [FORWARD DIAGNOSTIC] best third candidate={chosen.name} but max_deploy_stages={args.max_deploy_stages}; not deployed');stage_rows.append({'fold':fold,'stage_index':stage_index,'selected_specialist':chosen.name,'deployed':False,'reason':'third_stage_diagnostic_only','meta_score':chosen.stats.get('score',np.nan),'target_counterfactual_delta_ap':next(r['target_counterfactual_delta_ap'] for r in candidate_rows[::-1] if r['stage_index']==stage_index and r['candidate']==chosen.name)});break
        name=chosen.name;Xh,Xt,names,sel,score_t,d_t,model,seed=prepared[name];prev_h=current_h.copy();prev_t=current_t.copy();new_h,covered,cw,cwrows=stage_crossfit_hgb_dense(Xh,hist.y,current_h,hist.faculty_idx,hist.faculty_names,hist.blocks,sel,**crossfit_kwargs(args,seed+7000));current_h=new_h.astype(np.float32);current_t=score_t.astype(np.float32);avail_h[name]=covered.astype(np.float32);avail_t[name]=np.ones(len(target.y),np.float32);accepted.append(name);remaining.remove(name);stage_outputs[f'stage{stage_index}_{name}']=current_t.copy();fr=faculty_ap_rows(target.y,prev_t,current_t,target.faculty_idx,target.faculty_names);gain=np.mean([r['delta_ap'] for r in fr]) if fr else np.nan;commgain=macro_ap(target.y,current_t,target.faculty_idx,target.faculty_names)-macro_ap(target.y,ct,target.faculty_idx,target.faculty_names);top=model_feature_usage(model,names,18) if model else []
        print(f'    [FORWARD SELECT] {name} -> DEPLOY | dense ledger=100.0%, true crossfit coverage={covered.mean():.1%}, identity fallback={(~covered).mean():.1%} | TARGET ΔAPprev={gain:+.4f} cumulativeΔCommittee={commgain:+.4f}')
        if top:print('      top splits: '+', '.join(f'{n}:{c}' for n,c in top))
        stage_rows.append({'fold':fold,'stage_index':stage_index,'selected_specialist':name,'deployed':True,'reason':'accepted_best_remaining','selected_eta':sel.eta,'selected_complexity':sel.complexity,'meta_score':sel.stats.get('score',np.nan),'meta_mean_delta_ap':sel.stats.get('mean_delta_ap',np.nan),'meta_recent_mean_delta_ap':sel.stats.get('recent_mean_delta_ap',np.nan),'crossfit_coverage':float(covered.mean()),'identity_fallback_rate':float((~covered).mean()),'target_delta_ap_vs_previous':gain,'target_cumulative_delta_ap_vs_committee':commgain})
        corr_rows.append(correction_diag(fold,stage_index,name,target.y,prev_t,current_t,d_t,sel.eta,args.correction_bound,np.ones(len(target.y),bool)));usage += [{'fold':fold,'stage_index':stage_index,'specialist':name,'rank':i+1,'feature':n,'count':c} for i,(n,c) in enumerate(top)];stage_index+=1;gc.collect()
    return current_t,stage_outputs,candidate_rows,stage_rows,corr_rows,usage,accepted

def run_fold(args,faculties,fold,out,raw_index):
    print('\n'+'='*180);print(f'SUITE17 — TARGET FOLD {fold}');print('='*180);hist,target,common=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold);ch=hist.teacher.mean(1);ct=target.teacher.mean(1);print(f'[DATA] faculties={len(faculties)} flat={len(common)} OOF={len(hist.y):,} fail={np.sum(hist.y):,} ({np.mean(hist.y):.2%}) target={len(target.y):,} | Committee target macroAP={macro_ap(target.y,ct,target.faculty_idx,target.faculty_names):.4f}')
    views=build_or_load_structured_views(hist,target,raw_index,args,fold,out);local=build_or_load_local(hist,target,args,fold,out)
    local_raw=[]
    for key,pred in [('LOCAL_GLOBAL_RAW',local['target_global']),('LOCAL_RECENT_RAW',local['target_recent'])]:
        for r in faculty_ap_rows(target.y,ct,pred,target.faculty_idx,target.faculty_names):local_raw.append({'fold':fold,'predictor':key,**r})
    print_table('[LOCAL RAW PREDICTOR FACULTY AUDIT]',pd.DataFrame(local_raw).sort_values(['predictor','delta_ap'],ascending=[True,False]))
    rsel,audit,audit_fac,_,_,_=run_standalone_audits(hist,target,views,local,args,fold);raw_final,stage_outputs,candidates,stages,corr,usage,accepted=run_forward_selection(hist,target,views,local,rsel,args,fold);aligned,align_rows=align_by_faculty(raw_final,ct,target.faculty_idx,target.faculty_names);ap_raw=macro_ap(target.y,raw_final,target.faculty_idx,target.faculty_names);ap_aligned=macro_ap(target.y,aligned,target.faculty_idx,target.faculty_names);print(f'\n[ALIGNMENT] raw/aligned macroAP={ap_raw:.6f}/{ap_aligned:.6f} (must match ranking) accepted_order={accepted}');print_table('[ALIGNMENT PARAMETERS]',pd.DataFrame(align_rows),max_rows=20)
    fd=out/f'fold_{fold}';records=[];metrics=[];cmp=[]
    stage_cols={COMMITTEE:ct,RAW:raw_final,FINAL:aligned};stage_cols.update({f'suite17_{k}':v for k,v in stage_outputs.items()})
    for fi,fac in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;y=target.y[m];q=ct[m];scores=reference_for_faculty(args,fold,fac,y,q)
        for k,v in stage_cols.items():scores[k]=np.asarray(v)[m]
        records.append({'faculty':fac,'fold':fold,'y':y,'scores':scores});[metrics.append(metric_row(fold,fac,model,y,s,args.threshold_beta)) for model,s in scores.items()]
        row={'faculty':fac,'committee_ap':safe_ap(y,q),'suite17_ap':safe_ap(y,aligned[m]),'delta_ap':safe_ap(y,aligned[m])-safe_ap(y,q),'rows':int(m.sum()),'failures':int(y.sum())}
        if COUNCIL in scores:row['council_ap']=safe_ap(y,scores[COUNCIL]);row['delta_vs_council']=row['suite17_ap']-row['council_ap']
        if S16 in scores:row['suite16_ap']=safe_ap(y,scores[S16]);row['delta_vs_suite16']=row['suite17_ap']-row['suite16_ap']
        cmp.append(row)
        sf=pd.DataFrame({'actual':y,'student_id':target.student_id[m],'course_id':target.course_id[m],'target_semester':target.target_semester[m]});[sf.__setitem__(model,s) for model,s in scores.items()];sf.to_csv(fd/f'{fac}_scores.csv',index=False)
    print_table('[TARGET FACULTY COMPARISON] Suite17 vs Committee / Council / Suite16',pd.DataFrame(cmp).sort_values('delta_ap',ascending=False))
    pd.DataFrame(audit).to_csv(fd/'standalone_specialist_audit.csv',index=False);pd.DataFrame(audit_fac).to_csv(fd/'standalone_specialist_audit_faculty.csv',index=False);pd.DataFrame(candidates).to_csv(fd/'forward_candidate_audit.csv',index=False);pd.DataFrame(stages).to_csv(fd/'forward_stage_decisions.csv',index=False);pd.DataFrame(corr).to_csv(fd/'correction_diagnostics.csv',index=False);pd.DataFrame(usage).to_csv(fd/'feature_usage.csv',index=False);pd.DataFrame(align_rows).to_csv(fd/'alignment.csv',index=False);pd.DataFrame(local['diagnostics']).to_csv(fd/'local_evidence_diagnostics.csv',index=False);pd.DataFrame(local_raw).to_csv(fd/'local_raw_predictor_audit.csv',index=False);(fd/'selected_relational_variant.txt').write_text(rsel+'\n');(fd/'accepted_order.txt').write_text(' -> '.join(accepted)+'\n')
    return records,metrics,audit,audit_fac,candidates,stages,corr,usage,align_rows,[{'fold':fold,'selected_relational_variant':rsel,'accepted_order':'->'.join(accepted)}]

def postprocess(args,out,faculties,records,metrics,audit,audit_fac,candidates,stages,corr,usage,alignment,fold_summary):
    mdf=pd.DataFrame(metrics);mdf.to_csv(out/'fold_metrics.csv',index=False);meanf=mean_fold_summary(mdf,args.holdout_folds);meanf.to_csv(out/'mean_fold_holdout.csv',index=False);warn=operational(records,faculties,args);warn.to_csv(out/'operational_metrics.csv',index=False);macro=macro_summary(meanf,warn);macro.to_csv(out/'primary_holdout_comparison.csv',index=False);print_table('SUITE17 PRIMARY HOLDOUT COMPARISON — SAME EVALUATOR',macro);print_win_rates(meanf,FINAL)
    adf=pd.DataFrame(audit);adf.to_csv(out/'standalone_specialist_audit.csv',index=False);pd.DataFrame(audit_fac).to_csv(out/'standalone_specialist_audit_faculty.csv',index=False)
    if len(adf):
        h=adf[adf.fold.isin(args.holdout_folds)].groupby('specialist').agg(target_macro_delta_ap=('target_macro_delta_ap','mean'),target_wins=('target_wins','sum'),cells=('target_faculties','sum'),admission_rate=('admitted','mean'),meta_mean_delta_ap=('meta_mean_delta_ap','mean'),meta_recent_mean_delta_ap=('meta_recent_mean_delta_ap','mean')).reset_index();print_table('STANDALONE SPECIALIST / R-ABLATION AUDIT — HOLDOUT',h)
    cdf=pd.DataFrame(candidates);cdf.to_csv(out/'forward_candidate_audit.csv',index=False)
    if len(cdf):
        z=cdf[cdf.fold.isin(args.holdout_folds)].groupby(['stage_index','candidate']).agg(admission_rate=('admitted','mean'),meta_score=('meta_score','mean'),meta_mean_delta_ap=('meta_mean_delta_ap','mean'),recent_mean_delta_ap=('meta_recent_mean_delta_ap','mean'),target_counterfactual_delta_ap=('target_counterfactual_delta_ap','mean'),target_counterfactual_wins=('target_counterfactual_wins','sum')).reset_index();print_table('FORWARD CANDIDATE COUNTERFACTUAL AUDIT — HOLDOUT',z)
    sdf=pd.DataFrame(stages);sdf.to_csv(out/'forward_stage_decisions.csv',index=False)
    if len(sdf):print_table('FORWARD SELECTED STAGES — ALL FOLDS',sdf)
    fs=pd.DataFrame(fold_summary);fs.to_csv(out/'fold_selection_summary.csv',index=False);print_table('SELECTED R VARIANT + DEPLOYED ORDER',fs)
    cc=pd.DataFrame(corr);cc.to_csv(out/'correction_diagnostics.csv',index=False)
    if len(cc):print_table('CONDITIONAL CORRECTION DIAGNOSTICS — HOLDOUT',cc[cc.fold.isin(args.holdout_folds)].groupby('stage').agg(mean_abs_effective_delta=('mean_abs_effective_delta','mean'),p95_abs_effective_delta=('p95_abs_effective_delta','mean'),saturation_rate=('saturation_rate','mean'),corr_effective_delta_residual=('corr_effective_delta_residual','mean')).reset_index())
    pd.DataFrame(usage).to_csv(out/'feature_usage.csv',index=False);pd.DataFrame(alignment).to_csv(out/'alignment.csv',index=False)
    overlap=council_overlap_table(meanf,FINAL);overlap.to_csv(out/'suite17_vs_council_faculty.csv',index=False)
    if len(overlap):print_table('SUITE17 vs COUNCIL — ALL OVERLAPPING FACULTIES',overlap.sort_values('delta_ap',ascending=False));summ=council_overlap_summary(overlap);summ.to_csv(out/'suite17_vs_council_summary.csv',index=False);print_table('SUITE17 vs COUNCIL — OVERLAP SUMMARY',summ)
    # Faculty comparison against Committee, Suite16, Suite14, Council when available.
    base=meanf[meanf.model==FINAL][['faculty','mean_fold_ap','mean_fold_auc','mean_fold_f2_at_20']].rename(columns={'mean_fold_ap':'suite17_ap','mean_fold_auc':'suite17_auc','mean_fold_f2_at_20':'suite17_f2'})
    for model,prefix in [(COMMITTEE,'committee'),(S16,'suite16'),(S14,'suite14'),(COUNCIL,'council')]:
        q=meanf[meanf.model==model][['faculty','mean_fold_ap','mean_fold_auc','mean_fold_f2_at_20']].rename(columns={'mean_fold_ap':f'{prefix}_ap','mean_fold_auc':f'{prefix}_auc','mean_fold_f2_at_20':f'{prefix}_f2'});base=base.merge(q,on='faculty',how='left')
        base[f'delta_ap_vs_{prefix}']=base.suite17_ap-base[f'{prefix}_ap']
    base.to_csv(out/'faculty_head_to_head.csv',index=False);print_table('SUITE17 FACULTY HEAD-TO-HEAD',base.sort_values('delta_ap_vs_committee',ascending=False))

def build_parser():
    root='/root/disk_3tb/earlywarning/30072026';p=argparse.ArgumentParser(description='Suite17 Robust Forward Complementary Specialist Boosting')
    p.add_argument('--cache_dir',default=f'{root}/results/cross_faculty_residual_moe/cache');p.add_argument('--structured_cache_dir',default=f'{root}/results/suite12_structured_cache');p.add_argument('--out_dir',default=f'{root}/results/suite17_robust_forward_specialist_boosting');p.add_argument('--support_cache_dir',default=f'{root}/results/suite16_conditional_tree_boosting/derived_support');p.add_argument('--database_dir',default='/root/disk_3tb/earlywarning/database');p.add_argument('--data1_zip',default='');p.add_argument('--data2_zip',default='');p.add_argument('--faculties',default='all');p.add_argument('--folds',default='1,2,3,4,5,6,7');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--seed',type=int,default=20260831)
    p.add_argument('--latent_dim',type=int,default=12);p.add_argument('--mastery_min_peer_count',type=int,default=5);p.add_argument('--mastery_max_events',type=int,default=48);p.add_argument('--force_views',action='store_true');p.add_argument('--force_local',action='store_true')
    p.add_argument('--local_recent_blocks',type=int,default=4);p.add_argument('--local_min_prior_blocks',type=int,default=4);p.add_argument('--local_crossfit_window_blocks',type=int,default=2);p.add_argument('--local_min_train_rows',type=int,default=300);p.add_argument('--local_min_failures',type=int,default=10);p.add_argument('--local_hgb_iter',type=int,default=60);p.add_argument('--local_hgb_max_leaf_nodes',type=int,default=31);p.add_argument('--local_hgb_lr',type=float,default=.05);p.add_argument('--local_hgb_l2',type=float,default=1.)
    p.add_argument('--eta_grid',default='0,.05,.10,.20,.30');p.add_argument('--correction_bound',type=float,default=2.5);p.add_argument('--hgb_iter_candidates',default='20,40,60,80');p.add_argument('--hgb_max_leaf_nodes',type=int,default=31);p.add_argument('--hgb_lr',type=float,default=.05);p.add_argument('--hgb_l2',type=float,default=1.);p.add_argument('--hgb_max_rows',type=int,default=180000);p.add_argument('--sampling_beta',type=float,default=.9999)
    p.add_argument('--meta_windows',type=int,default=3);p.add_argument('--meta_min_prior_blocks',type=int,default=4);p.add_argument('--crossfit_min_prior_blocks',type=int,default=3);p.add_argument('--window_min_faculties',type=int,default=8);p.add_argument('--window_min_failures',type=int,default=120);p.add_argument('--window_min_rows',type=int,default=5000)
    p.add_argument('--admit_min_mean_ap',type=float,default=.002);p.add_argument('--admit_min_median_ap',type=float,default=0.);p.add_argument('--admit_min_win_rate',type=float,default=.60);p.add_argument('--admit_min_recent_mean_ap',type=float,default=0.);p.add_argument('--admit_min_recent_median_ap',type=float,default=0.);p.add_argument('--admit_max_recent_window_mean_drop',type=float,default=-.003);p.add_argument('--admit_max_f1_drop',type=float,default=-.005);p.add_argument('--admit_max_f2_drop',type=float,default=-.005)
    p.add_argument('--max_deploy_stages',type=int,default=2);p.add_argument('--deploy_third_stage',action='store_true');p.add_argument('--third_admit_min_mean_ap',type=float,default=.004);p.add_argument('--third_admit_min_median_ap',type=float,default=.002);p.add_argument('--third_admit_min_win_rate',type=float,default=.65);p.add_argument('--third_admit_min_recent_mean_ap',type=float,default=.001);p.add_argument('--third_admit_min_recent_median_ap',type=float,default=.001);p.add_argument('--third_admit_max_recent_window_mean_drop',type=float,default=0.)
    p.add_argument('--threshold_beta',type=float,default=2.);p.add_argument('--min_precision',type=float,default=.15);p.add_argument('--max_warning_ratio',type=float,default=2.5);p.add_argument('--min_flags',type=int,default=5);p.add_argument('--primary_faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--verbose_operational',action='store_true')
    p.add_argument('--shared_results_dir',default=f'{root}/results/cross_faculty_residual_moe');p.add_argument('--suite7_results_dir',default=f'{root}/results/suite7_shared_private_adapter');p.add_argument('--suite8_results_dir',default=f'{root}/results/suite8_true_specialist');p.add_argument('--suite9_results_dir',default=f'{root}/results/suite9_competitive_responsibility');p.add_argument('--suite10_results_dir',default=f'{root}/results/suite10_semantic_specialists');p.add_argument('--suite11_results_dir',default=f'{root}/results/suite11_opportunity_latent_domains');p.add_argument('--suite12_results_dir',default=f'{root}/results/suite12_collaborative_evidence_moe');p.add_argument('--suite13_results_dir',default=f'{root}/results/suite13_committee_preserving_collaborative_moe');p.add_argument('--suite14_results_dir',default=f'{root}/results/suite14_authority_safe_collaborative_moe');p.add_argument('--suite16_results_dir',default=f'{root}/results/suite16_conditional_tree_boosting');p.add_argument('--v2_results_dir',default=f'{root}/results/final_train_v2');p.add_argument('--council_results_dir',default=f'{root}/results/suite4/2_competence_council')
    p.add_argument('--force_raw',action='store_true');p.add_argument('--force_structured',action='store_true');p.add_argument('--skip_alignment_audit',action='store_true');p.add_argument('--raw_chunksize',type=int,default=250000);p.add_argument('--max_semesters',type=int,default=6);p.add_argument('--max_course_neighbors',type=int,default=8);p.add_argument('--max_relation_events',type=int,default=16);p.add_argument('--max_component_events',type=int,default=24)
    return p

def main(argv=None):
    args=build_parser().parse_args(argv);args.folds=parse_ints(args.folds);args.dev_folds=parse_ints(args.dev_folds);args.holdout_folds=parse_ints(args.holdout_folds);args.eta_grid=parse_floats(args.eta_grid);args.hgb_iter_candidates=parse_iters(args.hgb_iter_candidates);args.raw_index_dir=str(Path(args.structured_cache_dir)/'raw_index');out=Path(args.out_dir);out.mkdir(parents=True,exist_ok=True);available=discover_faculties(args.cache_dir,args.folds[0]);faculties=parse_faculties(args.faculties,available);raw_dir=ensure_structured_and_raw(args,faculties,args.folds);args.raw_index_dir=str(raw_dir);raw_index=RawEvidenceIndex(raw_dir);seed_all(args.seed)
    print('='*180);print('SUITE17 — ROBUST FORWARD COMPLEMENTARY SPECIALIST BOOSTING');print('Immutable Committee | R ablation + M mastery + L faculty-local temporal evidence | adaptive forward selection | dense OOF ledger | no S2 Dynamics | no homemade GRANDE');print('='*180);print(f'faculties={len(faculties)} folds={args.folds} eta_grid={args.eta_grid} deploy_stages={args.max_deploy_stages} correction_bound={args.correction_bound} meta_windows={args.meta_windows} strict recent mean/median guards enabled')
    records=[];metrics=[];audit=[];audit_fac=[];candidates=[];stages=[];corr=[];usage=[];align=[];fs=[]
    for fold in args.folds:
        a,b,c,d,e,f,g,h,i,j=run_fold(args,faculties,fold,out,raw_index);records+=a;metrics+=b;audit+=c;audit_fac+=d;candidates+=e;stages+=f;corr+=g;usage+=h;align+=i;fs+=j;gc.collect()
    postprocess(args,out,faculties,records,metrics,audit,audit_fac,candidates,stages,corr,usage,align,fs);print(f'\n[DONE] Suite17 results: {out}')

if __name__=='__main__':main()
