# -*- coding: utf-8 -*-
from __future__ import annotations

import argparse, copy, gc, time
from pathlib import Path
import joblib
import numpy as np
import pandas as pd
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader, WeightedRandomSampler

from suite26_common import *
from suite26_data import *
from suite26_tree import TreeNativeEncoder
from suite26_model import TreeNativeAcademicMemory
from suite26_reporting import *

VARIANTS=[('memory','suite26_tree_memory'),('tair','suite26_tair')]


def move_batch(batch,device):
    return {k:(v.to(device,non_blocking=True) if torch.is_tensor(v) else v) for k,v in batch.items()}


def pairwise_rank_loss(logits,y,faculty,max_pairs=2048):
    losses=[]
    for f in torch.unique(faculty):
        m=faculty==f; pos=torch.where(m&(y>0.5))[0]; neg=torch.where(m&(y<=0.5))[0]
        if not len(pos) or not len(neg): continue
        n=min(max_pairs,max(len(pos),len(neg))*4)
        ip=pos[torch.randint(len(pos),(n,),device=logits.device)]; ine=neg[torch.randint(len(neg),(n,),device=logits.device)]
        losses.append(F.softplus(-(logits[ip]-logits[ine])).mean())
    return torch.stack(losses).mean() if losses else logits.sum()*0


def train_epoch(model,loader,opt,device,args):
    model.train(); tot=bce_sum=rank_sum=joint_sum=ability_sum=delta_sum=0.; n=0
    for batch in loader:
        b=move_batch(batch,device); opt.zero_grad(set_to_none=True); z,aux=model(b,return_aux=True); y=b['y']
        bce=F.binary_cross_entropy_with_logits(z,y); rank=pairwise_rank_loss(z,y,b['faculty_idx'])
        joint=F.binary_cross_entropy_with_logits(aux['joint_logit'],y); ability=z.sum()*0
        if model.variant=='tair': ability=F.binary_cross_entropy_with_logits(aux['ability_logit'],y)
        dreg=aux['delta'].square().mean(); loss=bce+args.rank_weight*rank+args.joint_weight*joint+args.ability_weight*ability+args.delta_reg*dreg
        loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),args.grad_clip); opt.step(); bs=len(z); n+=bs
        tot+=float(loss.detach())*bs;bce_sum+=float(bce.detach())*bs;rank_sum+=float(rank.detach())*bs;joint_sum+=float(joint.detach())*bs;ability_sum+=float(ability.detach())*bs;delta_sum+=float(dreg.detach())*bs
    den=max(n,1);return tuple(x/den for x in [tot,bce_sum,rank_sum,joint_sum,ability_sum,delta_sum])


@torch.no_grad()
def predict_model(model,dataset,device,batch_size=512,ablate=None,return_aux=False):
    model.eval();loader=DataLoader(dataset,batch_size=batch_size,shuffle=False,num_workers=0,pin_memory=True); pp=[]; jj=[]; idx=[]; auxs=[]
    for batch in loader:
        b=move_batch(batch,device);z,a=model(b,return_aux=True,ablate=ablate);pp.append(torch.sigmoid(z).cpu().numpy());jj.append(torch.sigmoid(a['joint_logit']).cpu().numpy());idx.append(batch['bundle_index'].numpy())
        if return_aux: auxs.append({k:v.detach().cpu() for k,v in a.items() if torch.is_tensor(v)})
    p=np.concatenate(pp) if pp else np.zeros(0);j=np.concatenate(jj) if jj else np.zeros(0);idx=np.concatenate(idx) if idx else np.zeros(0,int)
    if not return_aux:return p,j,idx
    merged={k:torch.cat([a[k] for a in auxs],0).numpy() for k in auxs[0]} if auxs else {};return p,j,idx,merged


def validation_metrics(bundle,mask,score):
    aps=[];f2=[];br=[]
    for fi in range(len(bundle.faculty_names)):
        m=np.asarray(mask)&(bundle.faculty_idx==fi)
        if not m.any():continue
        aps.append(safe_ap(bundle.y[m],score[m]));md=metric_dict(bundle.y[m],score[m]);f2.append(md['fbeta_at_20']);br.append(md['brier'])
    return float(np.nanmean(aps)),float(np.nanmean(f2)),float(np.nanmean(br))


def make_model(variant,n_features,n_faculties,tree_meta,args):
    return TreeNativeAcademicMemory(n_features,len(COURSE_FEATURES),n_faculties,tree_meta.leaf_stats,tree_meta.n_trees,variant,
                                    args.state_dim,args.course_dim,args.memory_dim,args.ability_dim,args.leaf_dim,args.faculty_dim,args.dropout,args.delta_cap)


def make_loader(query,bank,qtree,btree,store,indices,args,weighted=True,grade_override=None,y_override=None):
    ds=Suite26Dataset(query,bank,qtree,btree,store,indices,grade_override,y_override)
    if not weighted:return DataLoader(ds,batch_size=args.batch_size,shuffle=False,num_workers=0,pin_memory=True)
    idx=np.asarray(indices,np.int64);weights=faculty_sampling_weights(query.faculty_idx[idx],len(query.faculty_names));ns=min(args.samples_per_epoch,len(idx)) if args.samples_per_epoch>0 else len(idx);sampler=WeightedRandomSampler(torch.as_tensor(weights,dtype=torch.double),num_samples=max(ns,1),replacement=True)
    return DataLoader(ds,batch_size=args.batch_size,sampler=sampler,num_workers=0,pin_memory=True)


def selection_train(variant,query,bank,qtree,btree,store,tr_mask,va_mask,tree_meta,args,device,seed):
    tr_idx=np.where(tr_mask)[0];va_idx=np.where(va_mask)[0];set_seed(seed);model=make_model(variant,query.current.shape[1],len(query.faculty_names),tree_meta,args).to(device);opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay);loader=make_loader(query,bank,qtree,btree,store,tr_idx,args,True);vds=Suite26Dataset(query,bank,qtree,btree,store,va_idx);best=None;bad=0;hist=[]
    for ep in range(1,args.max_epochs+1):
        loss,bce,rank,joint,ability,dreg=train_epoch(model,loader,opt,device,args);pv,jv,ii=predict_model(model,vds,device,args.eval_batch_size);full=np.full(len(query.y),np.nan);jfull=np.full(len(query.y),np.nan);full[ii]=pv;jfull[ii]=jv;ap,f2,br=validation_metrics(query,va_mask,full);key=(ap,f2,-br);row={'epoch':ep,'loss':loss,'bce':bce,'rank':rank,'joint_bce':joint,'ability_bce':ability,'delta_reg':dreg,'macro_ap':ap,'macro_f2_at20':f2,'brier':br};hist.append(row)
        if best is None or key>best['key']:best={'key':key,'epoch':ep,'state':copy.deepcopy({k:v.detach().cpu() for k,v in model.state_dict().items()}),'val_score':full.copy(),'val_joint':jfull.copy()};bad=0
        elif ep>=args.min_epochs:bad+=1
        print(f'[SELECT {variant.upper()}] epoch={ep:02d}/{args.max_epochs} loss={loss:.5f} BCE={bce:.5f} rank={rank:.5f} joint={joint:.5f} ability={ability:.5f} macroAP={ap:.4f} F2@20={f2:.4f} Brier={br:.4f} best={best["epoch"]:02d}')
        if ep>=args.min_epochs and bad>=args.patience:break
    model.load_state_dict(best['state']);best['history']=hist;return best


def refit_predict(variant,query,bank,qtree,btree,store,target,ttree,target_store,tree_meta,epochs,args,device,seed,return_aux=False):
    idx=np.arange(len(query));set_seed(seed);model=make_model(variant,query.current.shape[1],len(query.faculty_names),tree_meta,args).to(device);opt=torch.optim.AdamW(model.parameters(),lr=args.lr,weight_decay=args.weight_decay);loader=make_loader(query,bank,qtree,btree,store,idx,args,True)
    for ep in range(1,int(epochs)+1):
        vals=train_epoch(model,loader,opt,device,args)
        if ep==1 or ep==epochs:print(f'[REFIT {variant.upper()}] epoch={ep:02d}/{epochs} loss={vals[0]:.5f} BCE={vals[1]:.5f} rank={vals[2]:.5f} joint={vals[3]:.5f} ability={vals[4]:.5f}')
    ds=Suite26Dataset(target,bank,ttree,btree,target_store);res=predict_model(model,ds,device,args.eval_batch_size,return_aux=return_aux);return model,res


def policy_masks(bundle,val_mask):
    cal=np.zeros(len(bundle.y),bool);pol=np.zeros(len(bundle.y),bool)
    for fi in range(len(bundle.faculty_names)):
        m=np.asarray(val_mask)&(bundle.faculty_idx==fi);blocks=sorted(pd.unique(bundle.blocks[m]));
        if not blocks:continue
        if len(blocks)==1:cal[m]=True;pol[m]=True
        else:pol[m&(bundle.blocks==blocks[-1])]=True;cal[m&np.isin(bundle.blocks,blocks[:-1])]=True
    return cal,pol


def select_blend(bundle,pol,primary,alpha_grid):
    best=None;rows=[]
    for a in alpha_grid:
        p=np.full(len(bundle.y),np.nan);p[pol]=blend_logits(bundle.committee[pol],primary[pol],a);ap,f2,br=validation_metrics(bundle,pol,p);row={'alpha':float(a),'policy_macro_ap':ap,'policy_macro_f2':f2,'policy_brier':br};rows.append(row);key=(ap,f2,-float(a))
        if best is None or key>best[0]:best=(key,row)
    return best[1],pd.DataFrame(rows)


def history_support_frame(store,query,bank):
    rows=[]
    for fi,fac in enumerate(query.faculty_names):
        m=query.faculty_idx==fi
        if not m.any():continue
        valid=store.idx[m]>=0; safe=np.where(valid,store.idx[m],0); y=np.asarray(bank.y,int)[safe]; g=np.asarray(bank.grade10,float)[safe];
        rows.append({'faculty':fac,'rows':int(m.sum()),'prevalence':float(query.y[m].mean()),'history_mean':float(store.support[m].mean()),'history_zero_rate':float(np.mean(store.support[m]==0)),'history_failure_mean':float(store.failure_support[m].mean()),'same_course_history_mean':float((store.same_course[m]*valid).sum(1).mean()),'course_seen_mean':float(store.course_seen_count[m].mean()),'new_course_rate':float(np.mean(store.course_seen_count[m]==0)),'history_grade_present_rate':float(np.sum(np.isfinite(g)&valid)/max(valid.sum(),1))})
    return pd.DataFrame(rows)


def shuffled_history_store(store,query,bank,seed):
    """Permute complete student histories only within faculty+target-semester groups.
    This preserves chronology because donor and recipient share the same query semester.
    Same-course flags are recomputed relative to the recipient target course.
    """
    rng=np.random.default_rng(seed); idx=store.idx.copy();rec=store.recency.copy();same=np.zeros_like(store.same_course);sup=store.support.copy();fs=store.failure_support.copy();gs=store.grade_support.copy();cs=store.course_seen_count.copy()
    keys=np.asarray([f'{int(f)}|{float(s):.3f}' for f,s in zip(query.faculty_idx,query.semester_ordinal)],object)
    for k in np.unique(keys):
        loc=np.where(keys==k)[0]
        if len(loc)<=1:continue
        donor=loc[rng.permutation(len(loc))];idx[loc]=store.idx[donor];rec[loc]=store.recency[donor];sup[loc]=store.support[donor];fs[loc]=store.failure_support[donor];gs[loc]=store.grade_support[donor]
    valid=idx>=0
    for i in range(len(query)):
        m=valid[i]
        if m.any():same[i,m]=(bank.course_id[idx[i,m]]==str(query.course_id[i])).astype(np.float32)
    # Donor semester equals recipient semester, so chronology remains strict.
    if valid.any():
        qi=np.repeat(np.arange(len(query))[:,None],idx.shape[1],axis=1)[valid];bi=idx[valid]
        if np.any(bank.semester_ordinal[bi]>=query.semester_ordinal[qi]):raise RuntimeError('shuffle chronology violation')
    return HistoryStore(idx,rec,same,sup,fs,gs,cs,store.max_history)


def trim_history_store(store,k):
    k=int(k);idx=store.idx.copy();rec=store.recency.copy();same=store.same_course.copy();H=idx.shape[1]
    if k>=H:return store
    # Store rows are chronological with newest events at the right edge among valid positions.
    for i in range(len(idx)):
        valid=np.where(idx[i]>=0)[0]
        if len(valid)>k:
            drop=valid[:-k];idx[i,drop]=-1;rec[i,drop]=0;same[i,drop]=0
    sup=(idx>=0).sum(1).astype(np.int16);return HistoryStore(idx,rec,same,sup,store.failure_support.copy(),store.grade_support.copy(),store.course_seen_count.copy(),store.max_history)


def ablation_rows_for_fold(fold,model,target,bank,ttree,btree,store,base_score,calibration,device,args):
    rows=[];ds=Suite26Dataset(target,bank,ttree,btree,store)
    for mode in ['no_memory','no_labels','no_grades','no_recency','no_tree_leaf','no_ability','uniform_attention','tree_only']:
        p,_,ii=predict_model(model,ds,device,args.eval_batch_size,ablate=[mode]);score=apply_platt(p,*calibration);full=np.full(len(target.y),np.nan);full[ii]=score
        for fi,fac in enumerate(target.faculty_names):
            m=target.faculty_idx==fi;rows.append({'fold':fold,'faculty':fac,'ablation':mode,'delta_ap':safe_ap(target.y[m],full[m])-safe_ap(target.y[m],base_score[m]),'mean_abs_probability_change':float(np.mean(np.abs(full[m]-base_score[m])))})
    return rows


def shuffle_rows_for_fold(fold,model,target,bank,ttree,btree,store,base_score,calibration,device,args):
    sh=shuffled_history_store(store,target,bank,args.seed+fold*991);ds=Suite26Dataset(target,bank,ttree,btree,sh);p,_,ii=predict_model(model,ds,device,args.eval_batch_size);full=np.full(len(target.y),np.nan);full[ii]=apply_platt(p,*calibration);rows=[]
    for fi,fac in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;rows.append({'fold':fold,'faculty':fac,'delta_ap':safe_ap(target.y[m],full[m])-safe_ap(target.y[m],base_score[m]),'mean_abs_probability_change':float(np.mean(np.abs(full[m]-base_score[m])))})
    return rows


def mechanism_rows(fold,target,bank,store,aux):
    att=np.asarray(aux['attention'],float);mask=store.idx>=0;safe=np.where(mask,store.idx,0);hy=bank.y[safe];hg=np.nan_to_num(bank.grade10[safe],nan=5.)/10.;rows=[]
    for fi,fac in enumerate(target.faculty_names):
        m=target.faculty_idx==fi;yy=target.y[m];aa=att[m];den=aa.sum(1);wfail=(aa*hy[m]).sum(1);wgrade=(aa*hg[m]).sum(1);same=(aa*store.same_course[m]).sum(1);wrec=(aa*store.recency[m]).sum(1);top=aa.max(1) if len(aa) else np.zeros(0);ability=np.asarray(aux['ability_logit'])[m];delta=np.asarray(aux['delta'])[m];theta=np.linalg.norm(np.asarray(aux['theta'])[m],axis=1);req=np.linalg.norm(np.asarray(aux['requirement'])[m],axis=1)
        rows.append({'fold':fold,'faculty':fac,'rows':int(m.sum()),'history_mean':float(store.support[m].mean()),'weighted_history_fail_rate':float(wfail.mean()),'weighted_fail_rate_for_failures':float(wfail[yy==1].mean()) if np.any(yy==1) else np.nan,'weighted_fail_rate_for_normals':float(wfail[yy==0].mean()) if np.any(yy==0) else np.nan,'weighted_grade':float(wgrade.mean()),'same_course_attention_mass':float(same.mean()),'weighted_recency':float(wrec.mean()),'attention_top1':float(top.mean()),'ability_logit_mean':float(ability.mean()),'ability_logit_failures':float(ability[yy==1].mean()) if np.any(yy==1) else np.nan,'ability_logit_normals':float(ability[yy==0].mean()) if np.any(yy==0) else np.nan,'delta_mean':float(delta.mean()),'delta_failures':float(delta[yy==1].mean()) if np.any(yy==1) else np.nan,'delta_normals':float(delta[yy==0].mean()) if np.any(yy==0) else np.nan,'theta_norm':float(theta.mean()),'requirement_norm':float(req.mean())})
    return rows


def attention_group_rows(fold,target,bank,store,aux):
    att=np.asarray(aux['attention'],float);valid=store.idx>=0;safe=np.where(valid,store.idx,0);hy=bank.y[safe];hg=np.nan_to_num(bank.grade10[safe],nan=5.);rows=[]
    groups={'history_fail':(hy==1)&valid,'history_pass':(hy==0)&valid,'same_course':(store.same_course>0)&valid,'other_course':(store.same_course<=0)&valid,'low_grade':(hg<5)&valid,'mid_grade':(hg>=5)&(hg<7)&valid,'high_grade':(hg>=7)&valid}
    for name,g in groups.items():
        mass=(att*g).sum();tok=g.sum();rows.append({'fold':fold,'group':name,'tokens':int(tok),'attention_mass':float(mass),'mean_attention':float(mass/max(tok,1))})
    return rows


def context_sensitivity_rows(fold,model,target,bank,ttree,btree,store,calib,device,args):
    rows=[]
    for k in [4,8,16,32]:
        st=trim_history_store(store,k);ds=Suite26Dataset(target,bank,ttree,btree,st);p,_,ii=predict_model(model,ds,device,args.eval_batch_size);full=np.full(len(target.y),np.nan);full[ii]=apply_platt(p,*calib)
        for fi,fac in enumerate(target.faculty_names):
            m=target.faculty_idx==fi;rows.append({'fold':fold,'faculty':fac,'history_k':k,'ap':safe_ap(target.y[m],full[m]),'auc':safe_auc(target.y[m],full[m])})
    return rows


def run(args):
    t0=time.time();out=Path(args.out_dir);out.mkdir(parents=True,exist_ok=True);device=torch.device(args.device if args.device!='auto' else ('cuda' if torch.cuda.is_available() else 'cpu'));faculties=parse_faculties(args.faculties,discover_faculties(args.cache_dir,1));folds=[int(x) for x in args.folds.split(',')];dev=[int(x) for x in args.dev_folds.split(',')];hold=[int(x) for x in args.holdout_folds.split(',')];common_names=discover_common_feature_names(args.cache_dir,faculties,1);feat_names=current_feature_names(common_names)
    print('='*190);print('SUITE26 — TREE-NATIVE ENCODER + TARGET-CONDITIONED STUDENT MEMORY + LATENT ABILITY–REQUIREMENT INTERACTION (TAIR)');print('single internal ExtraTrees current-state learner | compact leaf embeddings | same-student episodic memory with observed grades/outcomes | target-conditioned attention | multidimensional student ability vs course requirement | Committee only as reference/blend ablation');print('='*190)
    print(f'[ENV] device={device} torch={torch.__version__} faculties={len(faculties)} folds={folds} dev={dev} holdout={hold} common_raw={len(common_names)} current_features={len(feat_names)}')
    print('[SCIENTIFIC CONTRACT] A=suite26_tree (single ExtraTrees current-state baseline, NOT Committee); B=suite26_tree_memory adds target-conditioned same-student episodic memory; C=suite26_tair PRIMARY adds latent ability–requirement interaction. Historical events must be strictly earlier than the query semester; same-semester outcomes are forbidden. Committee is not an input to A/B/C and is used only as a reference plus secondary blend ablation. F6/F7 never select tree/model epochs, calibration, or blend.')
    all_metrics=[];records=[];reference_meta=[];support_rows=[];abl_rows=[];shuffle_rows=[];diag_rows=[];att_rows=[];strata_rows=[];novelty_all=[];support_gain=[];comp_rows=[];krows=[];blend_grid_all=[];importance_all=[]
    ref_dirs={'suite14':args.suite14_dir,'suite16':args.suite16_dir,'suite17':args.suite17_dir,'suite18':args.suite18_dir,'suite20':args.suite20_dir,'suite21':args.suite21_dir,'suite22':args.suite22_dir,'suite22_primary':args.suite22_dir,'suite22_relational':args.suite22_dir,'suite23':args.suite23_dir,'suite23_graph':args.suite23_dir,'suite24':args.suite24_dir,'suite24_posthoc':args.suite24_dir,'suite25':args.suite25_dir,'suite25_blend':args.suite25_dir,'council':args.council_dir,'v2':args.v2_dir}
    for fold in folds:
        fd=out/f'fold_{fold}';fd.mkdir(exist_ok=True);print('\n'+'='*190+f'\nSUITE26 TARGET FOLD {fold}\n'+'='*190)
        hist_raw,target_raw,_=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold,common_names);tr,va=latest_block_split(hist_raw.faculty_idx,hist_raw.blocks,args.val_blocks);cal,pol=policy_masks(hist_raw,va);print(f'[DATA] history={len(hist_raw):,} fail={hist_raw.y.sum():,} ({hist_raw.y.mean():.2%}) train={tr.sum():,} validation={va.sum():,} target={len(target_raw):,} fail={target_raw.y.sum():,} ({target_raw.y.mean():.2%})')
        # ---------------- selection tree ----------------
        print('[TREE] fitting selection tree on older training blocks only...');tsel=TreeNativeEncoder(args.seed+fold*100,args.tree_estimators,args.tree_max_leaf_nodes,args.tree_min_leaf,args.tree_max_features,args.tree_jobs).fit(hist_raw.current[tr],hist_raw.y[tr]);print(f'[TREE] selection summary={tsel.summary()}')
        th_sel=tsel.transform(hist_raw.current,args.tree_chunk,True,'TREE-HISTORY-SEL'); tree_val_raw=th_sel.score.copy(); tree_cal=fit_platt(hist_raw.y[cal],tree_val_raw[cal]);tree_val=apply_platt(tree_val_raw,*tree_cal);
        # Downstream training receives OOB current-state scores for rows used to fit the forest.
        if tsel.training_oob_score_ is not None:
            oo=np.asarray(tsel.training_oob_score_,np.float32);good=np.isfinite(oo);loc=np.where(tr)[0];th_sel.score[loc[good]]=oo[good]
        print(f'[TREE] calibration a={tree_cal[0]:.4f} b={tree_cal[1]:.4f} validation macroAP={validation_metrics(hist_raw,va,tree_val)[0]:.4f} training_OOB_AP={tsel.oob_ap_}')
        prep_sel=Standardizer.fit(hist_raw,tr);hist_sel=prep_sel.transform(hist_raw);bank_sel=hist_sel
        print('[HISTORY STORE] selection phase (memory candidates restricted to training partition)...');store_sel=build_history_store(bank_sel,hist_sel,tr,args.max_history,args.keep_recent,args.history_progress_every);print_table(f'FOLD {fold} SELECTION STUDENT-HISTORY SUPPORT',history_support_frame(store_sel,hist_sel,bank_sel))
        bests={};calibs={};val_scores={};val_joint={}
        for vi,(variant,name) in enumerate(VARIANTS):
            print('\n'+'-'*150+f'\n[VARIANT {name}]');best=selection_train(variant,hist_sel,bank_sel,th_sel,th_sel,store_sel,tr,va,tsel.meta,args,device,args.seed+fold*1000+vi*100);bests[name]=best;aa,bb=fit_platt(hist_raw.y[cal],best['val_score'][cal]);calibs[name]=(aa,bb);val_scores[name]=apply_platt(best['val_score'],aa,bb);ja,jb=fit_platt(hist_raw.y[cal],best['val_joint'][cal]);val_joint[name]=apply_platt(best['val_joint'],ja,jb);calibs[name+'_joint']=(ja,jb);pd.DataFrame(best['history']).to_csv(fd/f'{name}_selection_history.csv',index=False);save_json({'selected_epoch':best['epoch'],'platt_a':aa,'platt_b':bb,'joint_platt_a':ja,'joint_platt_b':jb},fd/f'{name}_selection.json')
        blend_sel,blend_grid=select_blend(hist_raw,pol,val_scores['suite26_tair'],tuple(float(x) for x in args.alpha_grid.split(',')));alpha=float(blend_sel['alpha']);blend_grid_all.extend(blend_grid.assign(fold=fold).to_dict('records'));print(f'[COMMITTEE BLEND ABLATION] selected alpha={alpha:.2f} policyAP={blend_sel["policy_macro_ap"]:.4f} policyF2={blend_sel["policy_macro_f2"]:.4f}')
        del store_sel,hist_sel,bank_sel,th_sel,tsel;gc.collect()
        # ---------------- production refit ----------------
        print('[TREE] fitting production tree on all permissible history...');tprod=TreeNativeEncoder(args.seed+fold*100+50000,args.tree_estimators,args.tree_max_leaf_nodes,args.tree_min_leaf,args.tree_max_features,args.tree_jobs).fit(hist_raw.current,hist_raw.y);print(f'[TREE] production summary={tprod.summary()}')
        th=tprod.transform(hist_raw.current,args.tree_chunk,True,'TREE-HISTORY-PROD');tt=tprod.transform(target_raw.current,args.tree_chunk,True,'TREE-TARGET-PROD');
        if tprod.training_oob_score_ is not None:
            oo=np.asarray(tprod.training_oob_score_,np.float32);good=np.isfinite(oo);th.score[good]=oo[good]
        prep=Standardizer.fit(hist_raw,np.ones(len(hist_raw),bool));hist=prep.transform(hist_raw);target=prep.transform(target_raw);bank=hist
        print('[HISTORY STORE] production history...');store_hist=build_history_store(bank,hist,None,args.max_history,args.keep_recent,args.history_progress_every);print('[HISTORY STORE] production target...');store_target=build_history_store(bank,target,None,args.max_history,args.keep_recent,args.history_progress_every);sf=history_support_frame(store_target,target,bank);sf['fold']=fold;support_rows.extend(sf.to_dict('records'));sf.to_csv(fd/'history_support_target.csv',index=False);print_table(f'FOLD {fold} TARGET STUDENT-HISTORY SUPPORT',sf)
        tree_target=apply_platt(tt.score,*tree_cal);target_scores={'suite26_tree':tree_target};models={};aux_primary=None;joint_raw=None
        for vi,(variant,name) in enumerate(VARIANTS):
            model,res=refit_predict(variant,hist,bank,th,th,store_hist,target,tt,store_target,tprod.meta,bests[name]['epoch'],args,device,args.seed+fold*1000+vi*100,return_aux=(variant=='tair'))
            if variant=='tair':praw,jraw,ii,aux=res;aux_primary=aux;joint_raw=jraw;models[name]=model
            else:praw,jraw,ii=res;models[name]=model
            full=np.full(len(target),np.nan);full[ii]=apply_platt(praw,*calibs[name]);target_scores[name]=full
            if variant=='tair':
                jf=np.full(len(target),np.nan);jf[ii]=apply_platt(jraw,*calibs[name+'_joint']);target_scores['suite26_joint_head']=jf
            print(f'[VARIANT {name}] selected_epochs={bests[name]["epoch"]} target macroAP={np.nanmean([safe_ap(target.y[target.faculty_idx==j],full[target.faculty_idx==j]) for j in range(len(faculties))]):.4f}')
            if args.save_models:torch.save({'state_dict':model.state_dict(),'variant':variant,'feature_names':feat_names,'selected_epoch':bests[name]['epoch'],'args':vars(args),'tree_summary':tprod.summary()},fd/f'{name}_model.pt')
        # Tree feature importance is stable/interpretable and cheap to retain.
        imp=tprod.feature_importance_rows(feat_names,args.print_tree_features);[r.update({'fold':fold}) for r in imp];importance_all.extend(imp);pd.DataFrame(imp).to_csv(fd/'tree_feature_importance.csv',index=False);print_table(f'FOLD {fold} TREE CURRENT-STATE FEATURE IMPORTANCE',imp)
        primary=target_scores['suite26_tair'];committee_blend=blend_logits(target.committee,primary,alpha);scores={'committee':target.committee,**target_scores,'suite26_committee_blend':committee_blend}
        refs,meta=load_references(ref_dirs,fold,target);reference_meta.extend(meta);scores.update(refs)
        # Mechanism diagnostics on C.
        ar=ablation_rows_for_fold(fold,models['suite26_tair'],target,bank,tt,th,store_target,primary,calibs['suite26_tair'],device,args);abl_rows.extend(ar);pd.DataFrame(ar).to_csv(fd/'memory_ablations.csv',index=False)
        sr=shuffle_rows_for_fold(fold,models['suite26_tair'],target,bank,tt,th,store_target,primary,calibs['suite26_tair'],device,args);shuffle_rows.extend(sr);pd.DataFrame(sr).to_csv(fd/'shuffled_student_history.csv',index=False)
        dr=mechanism_rows(fold,target,bank,store_target,aux_primary);diag_rows.extend(dr);pd.DataFrame(dr).to_csv(fd/'memory_mechanism_diagnostics.csv',index=False)
        at=attention_group_rows(fold,target,bank,store_target,aux_primary);att_rows.extend(at);pd.DataFrame(at).to_csv(fd/'attention_groups.csv',index=False)
        kr=context_sensitivity_rows(fold,models['suite26_tair'],target,bank,tt,th,store_target,calibs['suite26_tair'],device,args);krows.extend(kr);pd.DataFrame(kr).to_csv(fd/'history_k_sensitivity.csv',index=False)
        diag_scores={'suite26_tair':primary,'suite26_tree_memory':target_scores['suite26_tree_memory'],'suite26_tree':tree_target,'committee':target.committee}
        st=history_strata_rows(fold,target,store_target,diag_scores);strata_rows.extend(st);pd.DataFrame(st).to_csv(fd/'history_support_strata.csv',index=False);nv=novelty_rows(fold,target,store_target,diag_scores);novelty_all.extend(nv);pd.DataFrame(nv).to_csv(fd/'novelty_regimes.csv',index=False);sg=support_gain_rows(fold,target,store_target,tree_target,primary);support_gain.extend(sg);pd.DataFrame(sg).to_csv(fd/'support_gain.csv',index=False);cr=complementarity_rows(fold,target,target.committee,primary,committee_blend);comp_rows.extend(cr);pd.DataFrame(cr).to_csv(fd/'committee_complementarity.csv',index=False)
        # Metrics/reference outputs.
        rows=faculty_fold_metric_rows(fold,target,{k:v for k,v in scores.items() if np.isfinite(v).all()})
        for rn,rs in refs.items():
            if np.isfinite(rs).all():continue
            for fi,fac in enumerate(faculties):
                m=target.faculty_idx==fi;z=rs[m]
                if np.isfinite(z).all():rows.append({'fold':fold,'faculty':fac,'model':rn,'rows':int(m.sum()),'failures':int(target.y[m].sum()),'prevalence':float(target.y[m].mean()),**metric_dict(target.y[m],z)})
        all_metrics.extend(rows);print_table(f'SUITE26 TARGET FOLD {fold} — NEW + AVAILABLE REFERENCE RESULTS',pd.DataFrame(rows).sort_values(['faculty','model']))
        for fi,fac in enumerate(faculties):
            m=target.faculty_idx==fi;fs={k:np.asarray(v)[m] for k,v in scores.items() if np.isfinite(np.asarray(v)[m]).all()};records.append({'fold':fold,'faculty':fac,'y':target.y[m],'student_id':target.student_id[m],'scores':fs});df=pd.DataFrame({'actual':target.y[m],'student_id':target.student_id[m],'course_id':target.course_id[m]});
            for k,v in fs.items():df[k]=v
            df.to_csv(fd/f'{fac}_scores.csv',index=False)
        save_json({'tree_selection_calibration':tree_cal,'tree_production_summary':tprod.summary(),'committee_blend_alpha':alpha,'selected_epochs':{k:v['epoch'] for k,v in bests.items()}},fd/'fold_summary.json')
        del store_hist,store_target,models,bank,hist,target,tprod,th,tt;gc.collect();
        if torch.cuda.is_available():torch.cuda.empty_cache()
    # ---------------- final report ----------------
    metrics=pd.DataFrame(all_metrics);metrics.to_csv(out/'all_fold_metrics.csv',index=False);pd.DataFrame(reference_meta).to_csv(out/'reference_loading.csv',index=False);pd.DataFrame(support_rows).to_csv(out/'history_support_all.csv',index=False);pd.DataFrame(abl_rows).to_csv(out/'memory_ablations_all.csv',index=False);pd.DataFrame(shuffle_rows).to_csv(out/'shuffled_student_history_all.csv',index=False);pd.DataFrame(diag_rows).to_csv(out/'memory_diagnostics_all.csv',index=False);pd.DataFrame(att_rows).to_csv(out/'attention_groups_all.csv',index=False);pd.DataFrame(strata_rows).to_csv(out/'history_support_strata_all.csv',index=False);pd.DataFrame(novelty_all).to_csv(out/'novelty_regimes_all.csv',index=False);pd.DataFrame(support_gain).to_csv(out/'support_gain_all.csv',index=False);pd.DataFrame(comp_rows).to_csv(out/'committee_complementarity_all.csv',index=False);pd.DataFrame(krows).to_csv(out/'history_k_sensitivity_all.csv',index=False);pd.DataFrame(blend_grid_all).to_csv(out/'committee_blend_grid_all.csv',index=False);pd.DataFrame(importance_all).to_csv(out/'tree_feature_importance_all.csv',index=False)
    print_table('SUITE26 ALL-FOLD MACRO SUMMARY — F1..F7',macro_summary(metrics,folds));print_table('SUITE26 HOLDOUT MACRO SUMMARY — F6/F7',macro_summary(metrics,hold));primary='suite26_tair';comparators=['suite26_tree','suite26_tree_memory','suite26_joint_head','committee','suite17','suite18','suite21','suite22','suite23','suite24','suite25','suite25_blend','suite22_relational','suite23_graph','suite24_posthoc','council'];ps=[]
    for b in comparators:
        z=paired_summary(metrics,primary,b,hold)
        if z.get('cells',0):ps.append(z)
    print_table('SUITE26 PRIMARY PAIRED HOLDOUT COMPARISON',ps);pd.DataFrame(ps).to_csv(out/'paired_holdout_summary.csv',index=False)
    hh=[]
    for fac in faculties:
        d=metrics[(metrics.faculty==fac)&metrics.fold.isin(hold)];a=d[d.model==primary].ap.mean();row={'faculty':fac,'suite26_ap':a}
        for b in comparators+['suite26_committee_blend']:
            z=d[d.model==b]
            if len(z):row[f'{b}_ap']=z.ap.mean();row[f'delta_vs_{b}']=a-z.ap.mean()
        hh.append(row)
    hh=pd.DataFrame(hh);hh.to_csv(out/'holdout_faculty_head_to_head.csv',index=False);print_table('SUITE26 HOLDOUT FACULTY HEAD-TO-HEAD',hh.sort_values('delta_vs_committee',ascending=False))
    op=operational_metrics(records,dev,hold);op.to_csv(out/'operational_metrics.csv',index=False);piv=op[op.fold.astype(str)=='POOLED'].groupby(['model','policy'],as_index=False).agg(precision=('precision','mean'),recall=('recall','mean'),f2=('fbeta','mean'),flag_rate=('flag_rate','mean'));print_table('SUITE26 OPERATIONAL HOLDOUT SUMMARY',piv.sort_values(['policy','f2'],ascending=[True,False]))
    for model in [primary,'suite26_tree_memory','suite26_tree','suite26_joint_head','suite26_committee_blend','committee','suite22','suite23','suite24','suite25','council','v2']:
        for pol in ['fixed','drift']:print_confusion('POOLED F6/F7 CONFUSION MATRIX',op,'khoa_cong_nghe_thong_tin',model,pol)
    br=[]
    for fac in faculties:
        rr=[r for r in records if r['faculty']==fac and int(r['fold']) in hold]
        if not rr or not all(primary in r['scores'] for r in rr):continue
        y=np.concatenate([r['y'] for r in rr]);cand=np.concatenate([r['scores'][primary] for r in rr]);cl=np.concatenate([r['student_id'] for r in rr])
        for ref in ['suite26_tree','committee','suite18','suite21','suite22','suite23','suite24','suite25','council']:
            if not all(ref in r['scores'] for r in rr):continue
            s=np.concatenate([r['scores'][ref] for r in rr]);d,lo,hi=cluster_bootstrap_ap_delta(y,cand,s,cl,args.bootstrap,args.seed+len(br)*31);br.append({'faculty':fac,'reference':ref,'delta_ap':d,'ci_low':lo,'ci_high':hi,'positive_ci':lo>0,'negative_ci':hi<0})
    br=pd.DataFrame(br);br.to_csv(out/'paired_cluster_bootstrap_holdout.csv',index=False);print_table('SUITE26 STUDENT-CLUSTER BOOTSTRAP — HOLDOUT',br)
    cu=component_uplift(metrics,hold);cu.to_csv(out/'component_uplift_holdout.csv',index=False);print_table('SUITE26 COMPONENT UPLIFT — HOLDOUT',cu)
    if abl_rows:
        a=pd.DataFrame(abl_rows);print_table('SUITE26 MEMORY / ABILITY ABLATION SUMMARY — HOLDOUT',a[a.fold.isin(hold)].groupby('ablation',as_index=False).agg(cells=('delta_ap','size'),mean_delta_ap=('delta_ap','mean'),median_delta_ap=('delta_ap','median'),loss_cells=('delta_ap',lambda x:int((x<0).sum())),mean_abs_prob_change=('mean_abs_probability_change','mean')))
    if shuffle_rows:
        s=pd.DataFrame(shuffle_rows);print_table('SUITE26 SHUFFLED-STUDENT-HISTORY INTERVENTION — HOLDOUT',s[s.fold.isin(hold)].groupby(lambda _:0).agg(cells=('delta_ap','size'),mean_delta_ap=('delta_ap','mean'),median_delta_ap=('delta_ap','median'),loss_cells=('delta_ap',lambda x:int((x<0).sum())),mean_abs_prob_change=('mean_abs_probability_change','mean')).reset_index(drop=True))
    if diag_rows:print_table('SUITE26 TREE-MEMORY / ABILITY MECHANISM DIAGNOSTICS — HOLDOUT',pd.DataFrame(diag_rows)[pd.DataFrame(diag_rows).fold.isin(hold)])
    if att_rows:print_table('SUITE26 HISTORY ATTENTION GROUPS — HOLDOUT',pd.DataFrame(att_rows)[pd.DataFrame(att_rows).fold.isin(hold)].groupby('group',as_index=False).agg(tokens=('tokens','sum'),mean_attention=('mean_attention','mean'),attention_mass=('attention_mass','sum')))
    if comp_rows:print_table('SUITE26 COMMITTEE COMPLEMENTARITY — HOLDOUT',pd.DataFrame(comp_rows)[pd.DataFrame(comp_rows).fold.isin(hold)])
    if novelty_all:print_table('SUITE26 COLD/NEW MEMORY REGIMES — HOLDOUT',pd.DataFrame(novelty_all)[pd.DataFrame(novelty_all).fold.isin(hold)])
    if support_gain:
        sgc=support_gain_correlations(support_gain,hold);sgc.to_csv(out/'support_gain_correlations.csv',index=False);print_table('SUITE26 HISTORY-SUPPORT vs GAIN CORRELATIONS — HOLDOUT',sgc)
    if krows:print_table('SUITE26 HISTORY-K SENSITIVITY — HOLDOUT',pd.DataFrame(krows)[pd.DataFrame(krows).fold.isin(hold)].groupby('history_k',as_index=False).agg(macro_ap=('ap','mean'),macro_auc=('auc','mean')))
    audit=premise_audit(metrics,abl_rows,shuffle_rows,hold);audit.to_csv(out/'premise_audit.csv',index=False);print_table('SUITE26 PREDECLARED PREMISE / SUCCESS AUDIT',audit)
    elapsed=(time.time()-t0)/3600.;save_json({'elapsed_hours':elapsed,'folds':folds,'faculties':faculties,'primary':'suite26_tair','tree_baseline':'suite26_tree','memory_variant':'suite26_tree_memory','committee_blend':'suite26_committee_blend','current_features':len(feat_names)},out/'run_summary.json');print(f'\n[DONE] Suite26 complete in {elapsed:.2f} hours | output={out}')


def parser():
    root='/root/disk_3tb/earlywarning/30072026';p=argparse.ArgumentParser();p.add_argument('--cache_dir',default=f'{root}/results/cross_faculty_residual_moe/cache');p.add_argument('--structured_cache_dir',default=f'{root}/results/suite12_structured_cache');p.add_argument('--out_dir',default=f'{root}/results/suite26_tree_native_student_memory');p.add_argument('--faculties',default='all');p.add_argument('--folds',default='1,2,3,4,5,6,7');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--seed',type=int,default=20260930);p.add_argument('--device',default='auto');
    p.add_argument('--tree_estimators',type=int,default=96);p.add_argument('--tree_max_leaf_nodes',type=int,default=768);p.add_argument('--tree_min_leaf',type=int,default=4);p.add_argument('--tree_max_features',type=float,default=.70);p.add_argument('--tree_jobs',type=int,default=-1);p.add_argument('--tree_chunk',type=int,default=50000);p.add_argument('--print_tree_features',type=int,default=40)
    p.add_argument('--state_dim',type=int,default=64);p.add_argument('--course_dim',type=int,default=32);p.add_argument('--memory_dim',type=int,default=64);p.add_argument('--ability_dim',type=int,default=16);p.add_argument('--leaf_dim',type=int,default=16);p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--delta_cap',type=float,default=2.0)
    p.add_argument('--lr',type=float,default=8e-4);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--rank_weight',type=float,default=.12);p.add_argument('--joint_weight',type=float,default=.20);p.add_argument('--ability_weight',type=float,default=.05);p.add_argument('--delta_reg',type=float,default=.001);p.add_argument('--grad_clip',type=float,default=5.);p.add_argument('--batch_size',type=int,default=192);p.add_argument('--eval_batch_size',type=int,default=384);p.add_argument('--max_epochs',type=int,default=12);p.add_argument('--min_epochs',type=int,default=3);p.add_argument('--patience',type=int,default=3);p.add_argument('--samples_per_epoch',type=int,default=180000);p.add_argument('--val_blocks',type=int,default=3)
    p.add_argument('--max_history',type=int,default=32);p.add_argument('--keep_recent',type=int,default=24);p.add_argument('--history_progress_every',type=int,default=50000);p.add_argument('--alpha_grid',default='0,.10,.25,.50,.75');p.add_argument('--bootstrap',type=int,default=500);p.add_argument('--save_models',action='store_true')
    p.add_argument('--suite14_dir',default=f'{root}/results/suite14_authority_safe_collaborative_moe');p.add_argument('--suite16_dir',default=f'{root}/results/suite16_conditional_tree_boosting');p.add_argument('--suite17_dir',default=f'{root}/results/suite17_robust_forward_specialist_boosting');p.add_argument('--suite18_dir',default=f'{root}/results/suite18_fmm_forward_specialist_boosting');p.add_argument('--suite20_dir',default=f'{root}/results/suite20_semester_target_transformer');p.add_argument('--suite21_dir',default=f'{root}/results/suite21_guarded_temporal_residual');p.add_argument('--suite22_dir',default=f'{root}/results/suite22_sftr_tabm');p.add_argument('--suite23_dir',default=f'{root}/results/suite23_cs_hagt');p.add_argument('--suite24_dir',default=f'{root}/results/suite24_target_conditioned_structural_residual');p.add_argument('--suite25_dir',default=f'{root}/results/suite25_structure_aware_chronological_retrieval');p.add_argument('--council_dir',default=f'{root}/results/suite4/2_competence_council');p.add_argument('--v2_dir',default=f'{root}/results/final_train_v2');return p

if __name__=='__main__':run(parser().parse_args())
