#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Suite15 — Conditional Information Lattice Audit.

No MoE, no gate, no authority controller. Every structured-view probe is an
identity-preserving correction around a frozen chronology-safe anchor.
"""
from __future__ import annotations
import argparse,gc,json,math,os,random
from pathlib import Path
from itertools import combinations
import numpy as np
import pandas as pd
import joblib
import torch
from sklearn.ensemble import HistGradientBoostingClassifier

from suite15_pipeline import discover_faculties,parse_faculties,load_fold_bundle
from suite15_structured_cache import structured_cache_compatible,build_raw_index,RawEvidenceIndex,ensure_fold_structured
from suite15_views import ViewMatrixBuilder,all_view_subsets,subset_name,VIEWS
from suite15_training import generate_crossfit_e0,fit_target_e0,train_residual_probe,refit_residual_probe,predict_residual_probe,_sampling_weights
from suite15_information import safe_ap,safe_auc,exact_shapley,pairwise_synergy,absorption_index,cluster_bootstrap_delta_ap,classify_view_verdict
from suite15_reporting import faculty_threshold_maps,target_metric_rows,macro_summary,print_table,write_json
from suite2_common import equal_budget_metrics


def seed_all(seed):
    random.seed(seed);np.random.seed(seed);torch.manual_seed(seed)
    if torch.cuda.is_available():torch.cuda.manual_seed_all(seed)

def parse_ints(s):return [int(x.strip()) for x in str(s).split(',') if x.strip()]
def parse_hidden(s):return tuple(int(x.strip()) for x in str(s).split(',') if x.strip())
def logit_np(p):
    p=np.clip(np.asarray(p,float),1e-5,1-1e-5);return np.log(p)-np.log1p(-p)
def bce_rows(y,p):
    p=np.clip(np.asarray(p,float),1e-6,1-1e-6);y=np.asarray(y,float);return -(y*np.log(p)+(1-y)*np.log(1-p))

def ensure_structured(args,faculties,folds):
    sd=Path(args.structured_cache_dir);missing=[sd/f'{f}_fold_{fold}.joblib' for fold in folds for f in faculties if not structured_cache_compatible(sd/f'{f}_fold_{fold}.joblib')]
    if not missing:return
    if not args.database_dir and (not args.data1_zip or not args.data2_zip):raise FileNotFoundError(f'{len(missing)} structured caches missing. Provide --database_dir or both raw DB ZIPs; first={missing[0]}')
    print(f'[STRUCTURED] rebuilding {len(missing)} incompatible/missing schema-v4 caches')
    raw=sd/'raw_index';build_raw_index(args.cache_dir,args.data1_zip,args.data2_zip,faculties,folds,raw,args.force_raw,args.raw_chunksize,database_dir=args.database_dir,strict_alignment=not args.skip_alignment_audit);idx=RawEvidenceIndex(raw)
    for fold in folds:
        for fac in faculties:
            p=sd/f'{fac}_fold_{fold}.joblib'
            if structured_cache_compatible(p) and not args.force_structured:continue
            d=joblib.load(Path(args.cache_dir)/f'{fac}_fold_{fold}.joblib');ensure_fold_structured(d,idx,p,args.max_semesters,args.max_course_neighbors,args.max_relation_events,args.max_component_events,args.force_structured);print(f'[STRUCTURED F{fold} {fac}] ready')

def anchor_cache_path(out,fold):return out/'anchor_cache'/f'fold_{int(fold)}.joblib'

def get_e0_anchors(hist,target,args,fold,out,device):
    p=anchor_cache_path(out,fold);p.parent.mkdir(parents=True,exist_ok=True)
    if p.exists() and not args.force_e0_anchor:
        x=joblib.load(p)
        if len(x.get('oof_score',[]))==len(hist.y) and len(x.get('target_score',[]))==len(target.y):
            print(f'[E0 ANCHOR] reuse {p} coverage={np.mean(np.isfinite(x["oof_score"])):.1%} target_epoch={x.get("target_epoch",0)}');return x
    print('[E0 ANCHOR] generating strict walk-forward cross-fitted E0 scores')
    cf=generate_crossfit_e0(hist,args,args.seed+fold*100003,device,args.min_e0_prior_blocks,verbose=True);tg=fit_target_e0(hist,target,args,args.seed+fold*200003,device)
    x={'oof_score':cf['score'],'oof_delta':cf['delta'],'covered':cf['covered'],'crossfit_history':cf['history'],'target_score':tg['score'],'target_delta':tg['delta'],'target_epoch':tg['epoch'],'hist_rows':len(hist.y),'target_rows':len(target.y),'row_index':np.asarray(hist.row_index),'faculty_idx':np.asarray(hist.faculty_idx)};joblib.dump(x,p,compress=3)
    print(f'[E0 ANCHOR] coverage={cf["covered"].mean():.1%} target_epoch={tg["epoch"]} target AP C/E0={safe_ap(target.y,target.teacher.mean(1)):.4f}/{safe_ap(target.y,tg["score"]):.4f}')
    return x

def latest_probe_split(bundle):
    u=np.array(sorted(np.unique(np.asarray(bundle.blocks,float))),float)
    if len(u)<2:raise RuntimeError(f'Need >=2 cross-fitted temporal blocks for probe selection; got {u.tolist()}')
    val=u[-1];return np.asarray(bundle.blocks)<val,np.asarray(bundle.blocks)==val,float(val)

def combine_features(controls,view):
    if view.shape[1]==0:return np.asarray(controls,np.float16)
    return np.concatenate([controls,view],1).astype(np.float16,copy=False)

def build_X(builder,bundle,subset,full_context=False):
    c=builder.controls(bundle);v=builder.transform(bundle,subset,include_full_context=full_context);return combine_features(c,v)

def probe_signature(args,anchor,config,full_context):
    return {'anchor':anchor,'config':config,'full_context':bool(full_context),'max_epochs':args.probe_max_epochs,'hidden':args.probe_hidden,'bound':args.probe_residual_bound,'lr':args.probe_lr,'seed':args.seed}

def run_probe(anchor_name,subset,audit,target,anchor_hist,anchor_target,tr_mask,val_mask,args,fold,fold_dir,device,full_context=False):
    cfg=subset_name(subset)+('_FULLCTX' if full_context else '');cache=fold_dir/'probe_cache'/f'{anchor_name}_{cfg.replace("+","_")}.joblib';cache.parent.mkdir(parents=True,exist_ok=True);sig=probe_signature(args,anchor_name,cfg,full_context)
    if cache.exists() and not args.force_probes:
        x=joblib.load(cache)
        if x.get('signature')==sig and len(x.get('target_score',[]))==len(target.y):return x
    bs=ViewMatrixBuilder.fit(audit,tr_mask);Xsel=build_X(bs,audit,subset,full_context);model,best=train_residual_probe(Xsel,audit.y,anchor_hist,audit.faculty_idx,audit.faculty_names,tr_mask,val_mask,max_epochs=args.probe_max_epochs,patience=args.probe_patience,batch_size=args.batch_size,lr=args.probe_lr,weight_decay=args.weight_decay,beta=args.sampling_beta,samples_per_epoch=args.probe_samples_per_epoch,residual_bound=args.probe_residual_bound,hidden=parse_hidden(args.probe_hidden),dropout=args.dropout,seed=args.seed+fold*1009+sum(ord(c) for c in cfg+anchor_name),device=device)
    val_idx=np.where(val_mask)[0];val_score=predict_residual_probe(model,Xsel[val_idx],logit_np(anchor_hist[val_idx]),args.predict_batch_size,device);f1map,f2map=faculty_threshold_maps(audit.y[val_idx],val_score,audit.faculty_idx[val_idx],audit.faculty_names)
    del Xsel,model,bs;gc.collect()
    br=ViewMatrixBuilder.fit(audit,np.ones(len(audit.y),bool));Xr=build_X(br,audit,subset,full_context);Xt=build_X(br,target,subset,full_context);refit=refit_residual_probe(Xr,audit.y,anchor_hist,audit.faculty_idx,best['epoch'],hidden=parse_hidden(args.probe_hidden),residual_bound=args.probe_residual_bound,dropout=args.dropout,lr=args.probe_lr,weight_decay=args.weight_decay,batch_size=args.batch_size,beta=args.sampling_beta,seed=args.seed+fold*2017+sum(ord(c) for c in cfg+anchor_name),samples_per_epoch=args.probe_samples_per_epoch,device=device);target_score=predict_residual_probe(refit,Xt,logit_np(anchor_target),args.predict_batch_size,device)
    out={'signature':sig,'anchor':anchor_name,'config':cfg,'subset':tuple(subset),'selected_epoch':int(best['epoch']),'val_stats':best['stats'],'val_score':val_score.astype(np.float32),'val_index':val_idx.astype(np.int32),'f1_thresholds':f1map,'f2_thresholds':f2map,'target_score':target_score.astype(np.float32),'feature_dim':int(Xr.shape[1])};joblib.dump(out,cache,compress=3);del Xr,Xt,refit;br=None;gc.collect();return out

def add_anchor_metric_rows(rows,fold,anchor_name,target,score,val_y,val_score,val_fac):
    f1,f2=faculty_threshold_maps(val_y,val_score,val_fac,target.faculty_names);rows.extend(target_metric_rows(fold,anchor_name,'ANCHOR',target.y,score,score,score,target.faculty_idx,target.faculty_names,target.student_id,f1,f2))

def correction_diagnostic(fold,anchor,config,y,anchor_score,score):
    a=logit_np(anchor_score);c=logit_np(score)-a;conf=np.abs(np.asarray(anchor_score)-.5);corr=float(np.corrcoef(np.abs(c),conf)[0,1]) if np.std(c)>0 and np.std(conf)>0 else np.nan;loss=bce_rows(y,anchor_score);cut=np.quantile(loss,.8);hard=loss>=cut;improve=float(np.mean(bce_rows(y,anchor_score)[hard]-bce_rows(y,score)[hard])) if hard.any() else np.nan
    return {'fold':fold,'anchor':anchor,'config':config,'mean_abs_logit_correction':float(np.mean(np.abs(c))),'p95_abs_logit_correction':float(np.quantile(np.abs(c),.95)),'corr_abs_correction_anchor_confidence':corr,'hardest20_bce_improvement':improve}

def fit_tree_target(base_train,view_train,y_train,fac_train,base_target,view_target,args,seed):
    n=len(y_train);idx=np.arange(n)
    if args.tree_max_rows>0 and n>args.tree_max_rows:
        w=_sampling_weights(y_train,fac_train,args.sampling_beta);rng=np.random.default_rng(seed);idx=rng.choice(idx,size=args.tree_max_rows,replace=False,p=w/w.sum())
    sw=_sampling_weights(np.asarray(y_train)[idx],np.asarray(fac_train)[idx],args.sampling_beta)
    def fit(X):return HistGradientBoostingClassifier(max_iter=args.tree_max_iter,max_leaf_nodes=args.tree_max_leaf_nodes,learning_rate=args.tree_lr,l2_regularization=args.tree_l2,random_state=seed).fit(np.asarray(X)[idx],np.asarray(y_train)[idx],sample_weight=sw)
    c0=fit(base_train);c1=fit(np.concatenate([base_train,view_train],1));return c0.predict_proba(base_target)[:,1],c1.predict_proba(np.concatenate([base_target,view_target],1))[:,1]

def run_tree_probes(audit,target,e0_hist,e0_target,args,fold,fold_dir):
    if not args.tree_probe:return []
    builder=ViewMatrixBuilder.fit(audit,np.ones(len(audit.y),bool));controls_h=builder.controls(audit).astype(np.float32);controls_t=builder.controls(target).astype(np.float32);base_h=np.concatenate([logit_np(e0_hist)[:,None],controls_h],1);base_t=np.concatenate([logit_np(e0_target)[:,None],controls_t],1);rows=[]
    for subset in [('V1',),('V2',),('V3',),('V4',),tuple(VIEWS)]:
        cfg=subset_name(subset);vh=builder.transform(audit,subset).astype(np.float16);vt=builder.transform(target,subset).astype(np.float16);p0,p1=fit_tree_target(base_h,vh,audit.y,audit.faculty_idx,base_t,vt,args,args.seed+fold*3701+len(cfg));
        for fi,name in enumerate(target.faculty_names):
            m=target.faculty_idx==fi
            if np.any(m) and np.unique(target.y[m]).size>1:rows.append({'fold':fold,'faculty':name,'config':cfg,'control_ap':safe_ap(target.y[m],p0[m]),'view_ap':safe_ap(target.y[m],p1[m]),'delta_ap':safe_ap(target.y[m],p1[m])-safe_ap(target.y[m],p0[m])})
        print(f'[TREE {cfg}] macroΔAP={np.mean([r["delta_ap"] for r in rows if r["config"]==cfg]):+.4f}')
        del vh,vt,p0,p1;gc.collect()
    return rows

def run_fold(args,faculties,fold,out,device,secondary):
    print('\n'+'='*170);print(f'SUITE15 CONDITIONAL INFORMATION LATTICE AUDIT | TARGET FOLD {fold}');print('='*170)
    hist,target,common=load_fold_bundle(args.cache_dir,args.structured_cache_dir,faculties,fold);print(f'[DATA] faculties={len(faculties)} flat={hist.flat.shape[1]} OOF={len(hist.y):,} fail={hist.y.sum():,} ({hist.y.mean():.2%}) target={len(target.y):,} raw_common={len(common)}')
    anchors=get_e0_anchors(hist,target,args,fold,out,device);covered=np.asarray(anchors['covered'],bool);audit=hist.subset(covered);e0_hist=np.asarray(anchors['oof_score'])[covered];e0_target=np.asarray(anchors['target_score']);committee_hist=audit.teacher.mean(1);committee_target=target.teacher.mean(1);print(f'[AUDIT COVERAGE] {covered.sum():,}/{len(covered):,} ({covered.mean():.1%}) blocks={sorted(np.unique(audit.blocks).tolist())}');del hist;gc.collect()
    tr,val,val_block=latest_probe_split(audit);print(f'[PROBE SPLIT] train={tr.sum():,} val={val.sum():,} latest_validation_block={val_block:g}')
    fold_dir=out/f'fold_{fold}';fold_dir.mkdir(parents=True,exist_ok=True);pd.DataFrame(anchors['crossfit_history']).to_csv(fold_dir/'e0_crossfit_history.csv',index=False)
    metric_rows=[];corr_rows=[];score_frame=pd.DataFrame({'faculty':[audit.faculty_names[i] for i in target.faculty_idx],'student_id':target.student_id.astype(str),'course_id':target.course_id.astype(str),'y':target.y,'committee':committee_target,'e0':e0_target})
    # Anchor thresholds are determined on the same latest historical block.
    add_anchor_metric_rows(metric_rows,fold,'E0',target,e0_target,audit.y[val],e0_hist[val],audit.faculty_idx[val]);add_anchor_metric_rows(metric_rows,fold,'COMMITTEE',target,committee_target,audit.y[val],committee_hist[val],audit.faculty_idx[val])
    results={};control_scores={}
    for subset in all_view_subsets():
        r=run_probe('E0',subset,audit,target,e0_hist,e0_target,tr,val,args,fold,fold_dir,device,False);results[('E0',tuple(subset))]=r;cfg=r['config'];score_frame['e0_'+cfg]=r['target_score'];
        if not subset:control_scores['E0']=r['target_score']
        metric_rows.extend(target_metric_rows(fold,'E0',cfg,target.y,r['target_score'],e0_target,control_scores.get('E0',r['target_score']),target.faculty_idx,target.faculty_names,target.student_id,r['f1_thresholds'],r['f2_thresholds']));corr_rows.append(correction_diagnostic(fold,'E0',cfg,target.y,e0_target,r['target_score']))
        st=r['val_stats'];print(f'[E0 {cfg:11s}] ep={r["selected_epoch"]} dim={r["feature_dim"]:4d} val medΔAP={st["median_delta_ap"]:+.4f} meanΔ={st["mean_delta_ap"]:+.4f} wins={st["wins"]}/{st["faculties"]} | target macroAP={np.mean([x["ap"] for x in metric_rows if x["fold"]==fold and x["anchor"]=="E0" and x["config"]==cfg]):.4f}')
    if secondary:
        csubs=[(),('V1',),('V2',),('V3',),('V4',),tuple(VIEWS)]
        for subset in csubs:
            r=run_probe('COMMITTEE',subset,audit,target,committee_hist,committee_target,tr,val,args,fold,fold_dir,device,False);results[('COMMITTEE',tuple(subset))]=r;cfg=r['config'];score_frame['committee_'+cfg]=r['target_score'];
            if not subset:control_scores['COMMITTEE']=r['target_score']
            metric_rows.extend(target_metric_rows(fold,'COMMITTEE',cfg,target.y,r['target_score'],committee_target,control_scores.get('COMMITTEE',r['target_score']),target.faculty_idx,target.faculty_names,target.student_id,r['f1_thresholds'],r['f2_thresholds']))
        for v in VIEWS:
            r=run_probe('E0',(v,),audit,target,e0_hist,e0_target,tr,val,args,fold,fold_dir,device,True);cfg=r['config'];score_frame['e0_'+cfg]=r['target_score'];metric_rows.extend(target_metric_rows(fold,'E0',cfg,target.y,r['target_score'],e0_target,control_scores['E0'],target.faculty_idx,target.faculty_names,target.student_id,r['f1_thresholds'],r['f2_thresholds']))
    tree_rows=run_tree_probes(audit,target,e0_hist,e0_target,args,fold,fold_dir) if secondary else []
    fmdf=pd.DataFrame(metric_rows)
    anchors=fmdf[(fmdf.anchor=='E0')&(fmdf.config=='ANCHOR')][['faculty','ap']].rename(columns={'ap':'E0_AP'}).merge(fmdf[(fmdf.anchor=='COMMITTEE')&(fmdf.config=='ANCHOR')][['faculty','ap']].rename(columns={'ap':'Committee_AP'}),on='faculty',how='outer')
    print_table('[TARGET ANCHORS] Committee vs E0 by faculty',anchors,['faculty','Committee_AP','E0_AP'])
    focus=['V1','V2','V3','V4','V1+V2+V3+V4'];z=fmdf[(fmdf.anchor=='E0')&(fmdf.config.isin(focus))].pivot(index='faculty',columns='config',values='delta_ap_vs_control').reset_index()
    print_table('[TARGET CONDITIONAL ΔAP vs CONTROL] core views',z)
    fmdf.to_csv(fold_dir/'faculty_metrics.csv',index=False);pd.DataFrame(corr_rows).to_csv(fold_dir/'correction_diagnostics.csv',index=False);score_frame.to_csv(fold_dir/'target_scores.csv.gz',index=False,compression='gzip');pd.DataFrame(tree_rows).to_csv(fold_dir/'tree_probe.csv',index=False)
    return metric_rows,corr_rows,tree_rows

def postprocess(args,out,all_metrics,all_corr,all_tree):
    metrics=pd.DataFrame(all_metrics);metrics.to_csv(out/'faculty_fold_metrics.csv',index=False);pd.DataFrame(all_corr).to_csv(out/'correction_diagnostics.csv',index=False);pd.DataFrame(all_tree).to_csv(out/'tree_probe.csv',index=False)
    core_metrics=metrics[(metrics.anchor!='E0') | (~metrics.config.str.endswith('_FULLCTX'))].copy();summary=macro_summary(core_metrics,args.holdout_folds,'E0');summary.to_csv(out/'information_lattice.csv',index=False);print_table('SUITE15 PRIMARY HOLDOUT INFORMATION LATTICE — E0 ANCHOR',summary,['config','cells','mean_ap','mean_delta_ap_vs_anchor','mean_delta_ap_vs_control','wins_vs_control','mean_auc','mean_f1','mean_f2','mean_f2_at20'])
    # Cell-level Shapley and synergy for the E0 core lattice.
    shap=[];syn=[]
    for (fold,fac),g in metrics[(metrics.anchor=='E0')&(~metrics.config.str.endswith('_FULLCTX'))&(~metrics.config.eq('ANCHOR'))].groupby(['fold','faculty']):
        perf={}
        for s in all_view_subsets():
            cfg=subset_name(s);z=g[g.config==cfg]
            if len(z):perf[tuple(s)]=float(z.iloc[0].ap)
        if len(perf)==16:
            perf_f1={};perf_f2={}
            for ss in all_view_subsets():
                cfg=subset_name(ss);zz=g[g.config==cfg]
                if len(zz):perf_f1[tuple(ss)]=float(zz.iloc[0].f1);perf_f2[tuple(ss)]=float(zz.iloc[0].f2)
            ph=exact_shapley(perf,VIEWS);ph1=exact_shapley(perf_f1,VIEWS);ph2=exact_shapley(perf_f2,VIEWS)
            for v,x in ph.items():shap.append({'fold':fold,'faculty':fac,'view':v,'shapley_ap':x,'shapley_f1':ph1[v],'shapley_f2':ph2[v]})
            sy=pairwise_synergy(perf,VIEWS)
            for (a,b),x in sy.items():syn.append({'fold':fold,'faculty':fac,'view_a':a,'view_b':b,'synergy_ap':x})
    shapdf=pd.DataFrame(shap);syndf=pd.DataFrame(syn);shapdf.to_csv(out/'view_shapley.csv',index=False);syndf.to_csv(out/'pairwise_synergy.csv',index=False)
    if len(shapdf):print_table('HOLDOUT SHAPLEY AP ATTRIBUTION',shapdf[shapdf.fold.isin(args.holdout_folds)].groupby('view').agg(shapley_ap_mean=('shapley_ap','mean'),shapley_ap_median=('shapley_ap','median'),shapley_f1_mean=('shapley_f1','mean'),shapley_f2_mean=('shapley_f2','mean'),count=('view','size')).reset_index())
    if len(syndf):print_table('HOLDOUT PAIRWISE PREDICTIVE SYNERGY',syndf[syndf.fold.isin(args.holdout_folds)].groupby(['view_a','view_b']).synergy_ap.agg(['mean','median','count']).reset_index())
    loo=[]
    core=metrics[(metrics.anchor=='E0')&(~metrics.config.str.endswith('_FULLCTX'))]
    for (fold,fac),g in core.groupby(['fold','faculty']):
        allr=g[g.config=='V1+V2+V3+V4']
        if len(allr)==0:continue
        ap_all=float(allr.iloc[0].ap)
        for v in VIEWS:
            keep=tuple(x for x in VIEWS if x!=v);zz=g[g.config==subset_name(keep)]
            if len(zz):loo.append({'fold':fold,'faculty':fac,'view_removed':v,'all_view_ap':ap_all,'without_view_ap':float(zz.iloc[0].ap),'loss_when_removed':ap_all-float(zz.iloc[0].ap)})
    loodf=pd.DataFrame(loo);loodf.to_csv(out/'leave_one_view_out.csv',index=False)
    if len(loodf):print_table('HOLDOUT LEAVE-ONE-VIEW-OUT LOSS',loodf[loodf.fold.isin(args.holdout_folds)].groupby('view_removed').loss_when_removed.agg(['mean','median','count']).reset_index())
    wins=core[(core.fold.isin(args.holdout_folds))&(core.config.isin(['V1','V2','V3','V4','V1+V2+V3+V4']))].pivot_table(index=['fold','faculty'],columns='config',values='delta_ap_vs_control').reset_index();wins.to_csv(out/'view_win_matrix.csv',index=False)
    # Committee -> E0 absorption using matched CONTROL baselines.
    absorption=[]
    for fold in args.secondary_folds:
        for fac in metrics.faculty.unique():
            e=metrics[(metrics.fold==fold)&(metrics.faculty==fac)&(metrics.anchor=='E0')];c=metrics[(metrics.fold==fold)&(metrics.faculty==fac)&(metrics.anchor=='COMMITTEE')]
            if len(e)==0 or len(c)==0:continue
            ec=e[e.config=='CONTROL'];cc=c[c.config=='CONTROL']
            if len(ec)==0 or len(cc)==0:continue
            for cfg in ['V1','V2','V3','V4','V1+V2+V3+V4']:
                er=e[e.config==cfg];cr=c[c.config==cfg]
                if len(er) and len(cr):
                    eg=float(er.iloc[0].ap-ec.iloc[0].ap);cg=float(cr.iloc[0].ap-cc.iloc[0].ap);absorption.append({'fold':fold,'faculty':fac,'view':cfg,'committee_gain':cg,'e0_gain':eg,'absorption_index':absorption_index(cg,eg)})
    absdf=pd.DataFrame(absorption);absdf.to_csv(out/'absorption.csv',index=False)
    if len(absdf):print_table('COMMITTEE → E0 EMPIRICAL ABSORPTION',absdf.groupby('view')[['committee_gain','e0_gain','absorption_index']].mean().reset_index())
    tree_agree=pd.DataFrame()
    if len(all_tree):
        td=pd.DataFrame(all_tree);md=metrics[(metrics.anchor=='E0')&(metrics.config.isin(td.config.unique()))][['fold','faculty','config','delta_ap_vs_control']].rename(columns={'delta_ap_vs_control':'mlp_delta_ap'});tree_agree=td.merge(md,on=['fold','faculty','config'],how='left');tree_agree['same_sign']=np.sign(tree_agree.delta_ap)==np.sign(tree_agree.mlp_delta_ap);tree_agree.to_csv(out/'tree_mlp_agreement.csv',index=False)
        agg=tree_agree.groupby('config').agg(tree_delta_ap=('delta_ap','mean'),mlp_delta_ap=('mlp_delta_ap','mean'),same_sign_rate=('same_sign','mean'),cells=('faculty','size')).reset_index();print_table('TREE ↔ MLP CONDITIONAL-VALUE AGREEMENT',agg)
    else:pd.DataFrame(columns=['fold','faculty','config','delta_ap','mlp_delta_ap','same_sign']).to_csv(out/'tree_mlp_agreement.csv',index=False)
    # Bootstrap holdout cell deltas vs CONTROL for core E0 lattice.
    boots=[];boot_samples={v:[] for v in VIEWS}
    if args.bootstrap_reps>0:
        for fold in args.holdout_folds:
            p=out/f'fold_{fold}'/'target_scores.csv.gz'
            if not p.exists():continue
            sf=pd.read_csv(p)
            for fac,g in sf.groupby('faculty'):
                base=g['e0_CONTROL'].to_numpy(float)
                for subset in all_view_subsets():
                    if not subset:continue
                    cfg=subset_name(subset);col='e0_'+cfg
                    if col not in g:continue
                    b=cluster_bootstrap_delta_ap(g.y.to_numpy(int),base,g[col].to_numpy(float),g.student_id.astype(str).to_numpy(),args.bootstrap_reps,args.seed+fold*97+sum(ord(c) for c in fac+cfg));boots.append({'fold':fold,'faculty':fac,'config':cfg,'mean':b['mean'],'ci_low':b['ci_low'],'ci_high':b['ci_high']})
                    if len(subset)==1:boot_samples[subset[0]].append(np.asarray(b['samples'],float))
        pd.DataFrame(boots).to_csv(out/'bootstrap_ci.csv',index=False)
    # View verdicts from core single-view gains relative to CONTROL.
    verdict={};d=metrics[(metrics.anchor=='E0')&(metrics.fold.isin(args.holdout_folds))]
    for v in VIEWS:
        g=d[d.config==v];delta=g.delta_ap_vs_control.to_numpy(float);ci_low=np.nan
        if boot_samples[v]:
            arr=np.vstack([x for x in boot_samples[v] if len(x)==args.bootstrap_reps]);macro=np.nanmean(arr,axis=0);ci_low=float(np.nanquantile(macro,.025))
        verdict[v]={'mean_delta_ap':float(np.nanmean(delta)) if len(delta) else np.nan,'median_delta_ap':float(np.nanmedian(delta)) if len(delta) else np.nan,'wins':int(np.sum(delta>0)),'cells':int(len(delta)),'macro_bootstrap_ci_low':ci_low,'verdict':classify_view_verdict(float(np.nanmean(delta)) if len(delta) else 0.,int(np.sum(delta>0)),len(delta),ci_low)}
    write_json(out/'information_verdict.json',verdict)
    print('\n[SUITE15 INFORMATION VERDICT]')
    for v in VIEWS:
        x=verdict[v];print(f'  {v}: {x["verdict"]:22s} meanΔAP={x["mean_delta_ap"]:+.4f} medΔAP={x["median_delta_ap"]:+.4f} wins={x["wins"]}/{x["cells"]} macroCI_low={x["macro_bootstrap_ci_low"]:+.4f}')
    # Fold6 -> fold7 conditional-gain drift.
    drift=[]
    for v in VIEWS:
        g=d[d.config==v].groupby('fold').delta_ap_vs_control.mean()
        if 6 in g.index and 7 in g.index:drift.append({'view':v,'fold6_delta_ap':g.loc[6],'fold7_delta_ap':g.loc[7],'fold7_minus_fold6':g.loc[7]-g.loc[6]})
    pd.DataFrame(drift).to_csv(out/'holdout_gain_drift.csv',index=False)
    if drift:print_table('FOLD6 → FOLD7 CONDITIONAL GAIN DRIFT',pd.DataFrame(drift))
    # Automatic best subset and strongest pair for planning Suite16.
    if len(summary):
        eligible=summary[~summary.config.isin(['ANCHOR','CONTROL'])];best=eligible.iloc[0] if len(eligible) else summary.iloc[0]
        pair_best=None
        if len(syndf):
            z=syndf[syndf.fold.isin(args.holdout_folds)].groupby(['view_a','view_b']).synergy_ap.mean().sort_values(ascending=False);pair_best=(z.index[0],float(z.iloc[0])) if len(z) else None
        hint={'best_subset':str(best['config']),'best_subset_mean_ap':float(best['mean_ap']),'best_subset_delta_vs_control':float(best['mean_delta_ap_vs_control']),'strongest_pair':pair_best,'view_verdicts':verdict};write_json(out/'next_experiment_hints.json',hint);print(f'\n[NEXT-EXPERIMENT HINT] best subset={hint["best_subset"]} ΔAPvsCONTROL={hint["best_subset_delta_vs_control"]:+.4f} strongest_pair={pair_best}')

def build_parser():
    root='/root/disk_3tb/earlywarning/30072026';p=argparse.ArgumentParser(description='Suite15 Conditional Information Lattice Audit')
    p.add_argument('--cache_dir',default=f'{root}/results/cross_faculty_residual_moe/cache');p.add_argument('--structured_cache_dir',default=f'{root}/results/suite12_structured_cache');p.add_argument('--out_dir',default=f'{root}/results/suite15_conditional_information_audit');p.add_argument('--database_dir',default='/root/disk_3tb/earlywarning/database');p.add_argument('--data1_zip',default='');p.add_argument('--data2_zip',default='');p.add_argument('--faculties',default='all');p.add_argument('--folds',default='1,2,3,4,5,6,7');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--secondary_folds',default='6,7');p.add_argument('--device',default='cuda');p.add_argument('--seed',type=int,default=20260830)
    p.add_argument('--d_model',type=int,default=64);p.add_argument('--faculty_dim',type=int,default=8);p.add_argument('--dropout',type=float,default=.10);p.add_argument('--e0_residual_bound',type=float,default=2.5);p.add_argument('--e0_lr',type=float,default=1e-3);p.add_argument('--e0_max_epochs',type=int,default=4);p.add_argument('--e0_patience',type=int,default=2);p.add_argument('--e0_samples_per_epoch',type=int,default=120000);p.add_argument('--e0_preserve_lambda',type=float,default=.5);p.add_argument('--e0_recon_lambda',type=float,default=.1);p.add_argument('--min_e0_prior_blocks',type=int,default=2)
    p.add_argument('--probe_hidden',default='256,128');p.add_argument('--probe_residual_bound',type=float,default=2.0);p.add_argument('--probe_lr',type=float,default=1e-3);p.add_argument('--probe_max_epochs',type=int,default=5);p.add_argument('--probe_patience',type=int,default=2);p.add_argument('--probe_samples_per_epoch',type=int,default=120000);p.add_argument('--batch_size',type=int,default=4096);p.add_argument('--predict_batch_size',type=int,default=8192);p.add_argument('--weight_decay',type=float,default=1e-4);p.add_argument('--sampling_beta',type=float,default=.9999)
    p.add_argument('--bootstrap_reps',type=int,default=500);p.add_argument('--tree_probe',action=argparse.BooleanOptionalAction,default=True);p.add_argument('--tree_max_rows',type=int,default=150000);p.add_argument('--tree_max_iter',type=int,default=80);p.add_argument('--tree_max_leaf_nodes',type=int,default=31);p.add_argument('--tree_lr',type=float,default=.05);p.add_argument('--tree_l2',type=float,default=1.)
    p.add_argument('--force_e0_anchor',action='store_true');p.add_argument('--force_probes',action='store_true');p.add_argument('--force_raw',action='store_true');p.add_argument('--force_structured',action='store_true');p.add_argument('--skip_alignment_audit',action='store_true');p.add_argument('--raw_chunksize',type=int,default=250000);p.add_argument('--max_semesters',type=int,default=6);p.add_argument('--max_course_neighbors',type=int,default=8);p.add_argument('--max_relation_events',type=int,default=16);p.add_argument('--max_component_events',type=int,default=24)
    return p

def main(argv=None):
    args=build_parser().parse_args(argv);args.folds=parse_ints(args.folds);args.holdout_folds=parse_ints(args.holdout_folds);args.secondary_folds=parse_ints(args.secondary_folds);out=Path(args.out_dir);out.mkdir(parents=True,exist_ok=True);available=discover_faculties(args.cache_dir,args.folds[0]);faculties=parse_faculties(args.faculties,available);ensure_structured(args,faculties,args.folds);device=args.device if args.device!='cuda' or torch.cuda.is_available() else 'cpu';seed_all(args.seed)
    print('='*170);print('SUITE15 — CONDITIONAL INFORMATION LATTICE AUDIT');print('Frozen chronology-safe E0 | raw structured views | 16-state lattice | no gate/MoE/authority | Committee absorption + tree confirmation');print('='*170);print(f'faculties={len(faculties)} folds={args.folds} holdout={args.holdout_folds} secondary={args.secondary_folds} device={device} probe_epochs={args.probe_max_epochs} bootstrap={args.bootstrap_reps}')
    metrics=[];corr=[];trees=[]
    for fold in args.folds:
        a,b,c=run_fold(args,faculties,fold,out,device,fold in args.secondary_folds);metrics.extend(a);corr.extend(b);trees.extend(c);gc.collect()
    postprocess(args,out,metrics,corr,trees);print(f'\n[DONE] Suite15 results: {out}')

if __name__=='__main__':main()
