# -*- coding: utf-8 -*-
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from suite23_common import *

REFERENCE_ALIASES={
 'suite14':['suite14_authority_safe_final','suite14','final'],
 'suite16':['suite16_hgb_final','suite16','final'],
 'suite17':['suite17_hgb_final','suite17_final','final'],
 'suite18':['suite18_hgb_final','suite18_final','final'],
 'suite20':['suite20_transformer','suite20_semester_target_transformer','transformer','final'],
 'suite21':['suite21_guarded_temporal_residual','suite21_guarded','final'],
 'suite22':['suite22_safe_blend','safe_blend','final'],
 'suite22_primary':['suite22_sftr_tabm','sftr_tabm','primary'],
 'suite22_relational':['suite22_relational_ft_tabm','relational_ft_tabm','relational'],
 'council':['suite4_competence_council','competence_council','council','final'],
 'v2':['failure_probability_lt5','failure_probability','stacked_probability','final'],
}

def _candidates(root,fold,faculty):
    r=Path(root);return [r/f'fold_{fold}'/f'{faculty}_scores.csv',r/f'{faculty}_fold_{fold}_scores.csv',r/f'fold_{fold}'/f'{faculty}_target_scores.csv',r/f'{faculty}_fold_{fold}_target_scores.csv',r/f'{faculty}_fold_{fold}_test_results.csv']

def load_reference_score(root,fold,faculty,y,aliases):
    if not root:return None,None,None
    p=next((x for x in _candidates(root,fold,faculty) if x.exists()),None)
    if p is None:return None,None,None
    df=pd.read_csv(p);actual=next((c for c in ['actual','actual_failure_lt5','y','label','target'] if c in df.columns),None)
    if actual:
        yy=pd.to_numeric(df[actual],errors='coerce').to_numpy()
        if len(yy)!=len(y) or not np.array_equal(yy.astype(int),np.asarray(y,int)):raise ValueError(f'reference alignment mismatch {p}')
    col=next((c for c in aliases if c in df.columns),None)
    if col is None:return None,p,None
    s=pd.to_numeric(df[col],errors='coerce').to_numpy(float)
    if len(s)!=len(y) or not np.isfinite(s).all():raise ValueError(f'invalid reference {p}:{col}')
    return s,p,col

def load_references(reference_dirs,fold,bundle):
    scores={};meta=[]
    for name,root in reference_dirs.items():
        full=np.full(len(bundle.y),np.nan,float);ok=True;loaded=0
        for fi,fac in enumerate(bundle.faculty_names):
            m=bundle.faculty_idx==fi
            try:s,p,col=load_reference_score(root,fold,fac,bundle.y[m],REFERENCE_ALIASES.get(name,[name,'final']))
            except Exception as e:meta.append({'reference':name,'fold':fold,'faculty':fac,'status':'error','detail':str(e)});ok=False;continue
            if s is None:meta.append({'reference':name,'fold':fold,'faculty':fac,'status':'missing','path':str(p) if p else ''});ok=False;continue
            full[m]=s;loaded+=1;meta.append({'reference':name,'fold':fold,'faculty':fac,'status':'loaded','path':str(p),'column':col})
        if loaded:
            scores[name]=full
    return scores,meta

def append_reference_metric_rows(rows,fold,bundle,ref_scores):
    for fi,fac in enumerate(bundle.faculty_names):
        m=bundle.faculty_idx==fi
        for name,s in ref_scores.items():
            z=np.asarray(s)[m];good=np.isfinite(z)
            if good.all():rows.extend(faculty_fold_metric_rows(fold,bundle.subset(m),{name:z}) if False else [{'fold':int(fold),'faculty':fac,'model':name,'rows':int(m.sum()),'failures':int(bundle.y[m].sum()),'prevalence':float(bundle.y[m].mean()),**metric_dict(bundle.y[m],z)}])

def component_uplift(metrics,models,folds=(6,7)):
    d=pd.DataFrame(metrics);d=d[d.fold.isin(folds)];out=[]
    for a,b,label in zip(models[:-1],models[1:],['heterogeneous_structure_gain','full_time_readout_gain']):
        aa=d[d.model==a][['fold','faculty','ap','fbeta_at_20']].rename(columns={'ap':'a','fbeta_at_20':'af2'});bb=d[d.model==b][['fold','faculty','ap','fbeta_at_20']].rename(columns={'ap':'b','fbeta_at_20':'bf2'});z=aa.merge(bb,on=['fold','faculty']);z['component']=label;z['delta_ap']=z.b-z.a;z['delta_f2_20']=z.bf2-z.af2;out.append(z[['fold','faculty','component','delta_ap','delta_f2_20']])
    return pd.concat(out,ignore_index=True) if out else pd.DataFrame()

def support_strata_rows(fold,bundle,store,scores):
    sup=np.asarray(store.data['support']);rows=[]
    defs=[('history_courses',sup[:,0],[(-.5,.5,'0'),(.5,2.5,'1-2'),(2.5,5.5,'3-5'),(5.5,99,'6+')]),('cohort_students',sup[:,1],[(-.5,.5,'0'),(.5,4.5,'1-4'),(4.5,8.5,'5-8'),(8.5,99,'9+')]),('structural_courses',sup[:,2],[(-.5,.5,'0'),(.5,2.5,'1-2'),(2.5,99,'3+')])]
    for fi,fac in enumerate(bundle.faculty_names):
        fm=bundle.faculty_idx==fi
        for dim,val,bins in defs:
            for lo,hi,label in bins:
                m=fm&(val>lo)&(val<=hi)
                if m.sum()<20:continue
                for model,s in scores.items():rows.append({'fold':fold,'faculty':fac,'support_dimension':dim,'support_bin':label,'model':model,'rows':int(m.sum()),'failures':int(bundle.y[m].sum()),'ap':safe_ap(bundle.y[m],np.asarray(s)[m]),'auc':safe_auc(bundle.y[m],np.asarray(s)[m])})
    return rows

def complementarity_rows(fold,bundle,committee,graph,fusion,rate=.20):
    rows=[]
    for fi,fac in enumerate(bundle.faculty_names):
        m=bundle.faculty_idx==fi;y=bundle.y[m];q=committee[m];g=graph[m];f=fusion[m];pq=warn_by_rate(q,rate).astype(bool);pg=warn_by_rate(g,rate).astype(bool);pf=warn_by_rate(f,rate).astype(bool)
        fn=(y==1)&(~pq);fa=(y==0)&pq
        rows.append({'fold':fold,'faculty':fac,'rows':int(m.sum()),'committee_false_negatives_at20':int(fn.sum()),'graph_rescued_fn_at20':int(np.sum(fn&pg)),'fusion_rescued_fn_at20':int(np.sum(fn&pf)),'committee_false_alarms_at20':int(fa.sum()),'graph_corrected_fa_at20':int(np.sum(fa&(~pg))),'fusion_corrected_fa_at20':int(np.sum(fa&(~pf))),'both_correct':int(np.sum((pq==(y==1))&(pg==(y==1)))),'committee_only_correct':int(np.sum((pq==(y==1))&(pg!=(y==1)))),'graph_only_correct':int(np.sum((pq!=(y==1))&(pg==(y==1)))),'both_wrong':int(np.sum((pq!=(y==1))&(pg!=(y==1))))})
    return rows

def bootstrap_ap_delta(y,a,b,n_boot=500,seed=42):
    y=np.asarray(y,int);a=np.asarray(a,float);b=np.asarray(b,float);rng=np.random.default_rng(seed);vals=[]
    if np.unique(y).size<2:return np.nan,np.nan,np.nan
    for _ in range(int(n_boot)):
        idx=rng.integers(0,len(y),len(y));yy=y[idx]
        if np.unique(yy).size<2:continue
        vals.append(safe_ap(yy,a[idx])-safe_ap(yy,b[idx]))
    if not vals:return np.nan,np.nan,np.nan
    return float(safe_ap(y,a)-safe_ap(y,b)),float(np.quantile(vals,.025)),float(np.quantile(vals,.975))

def bootstrap_rows(bundle,candidate,refs,n_boot=500,seed=42):
    out=[]
    for fi,fac in enumerate(bundle.faculty_names):
        m=bundle.faculty_idx==fi
        for name,s in refs.items():
            z=np.asarray(s)[m]
            if not np.isfinite(z).all():continue
            d,lo,hi=bootstrap_ap_delta(bundle.y[m],candidate[m],z,n_boot,seed+fi*37+len(name));out.append({'faculty':fac,'reference':name,'delta_ap':d,'ci_low':lo,'ci_high':hi,'significant_positive':bool(lo>0),'significant_negative':bool(hi<0)})
    return pd.DataFrame(out)

def print_confusion(title,op,faculty,model,policy='fixed'):
    z=op[(op.faculty==faculty)&(op.model==model)&(op.fold.astype(str)=='POOLED')&(op.policy==policy)]
    if not len(z):return
    r=z.iloc[0];print(f"\n{title} | {faculty} | {model} | {policy}\nActual\\Pred      Normal      Failure\nNormal        {int(r.tn):10d} {int(r.fp):10d}\nFailure       {int(r.fn):10d} {int(r.tp):10d}\nP={r.precision:.4f} R={r.recall:.4f} F2={r.fbeta:.4f} flag={r.flag_rate:.4f}")

def novelty_regime_rows(fold,bundle,store,scores):
    """Cold/new-state diagnostics from chronology-safe graph support.

    new_student := no historical student-course edge before query.
    new_course  := no historical cohort member for target course before query.
    These are operational graph-coldness definitions, not demographic labels.
    """
    sup=np.asarray(store.data['support']);new_s=sup[:,0]==0;new_c=sup[:,1]==0
    labels=np.where(new_s&new_c,'new_student+new_course',np.where(new_s,'new_student+seen_course',np.where(new_c,'returning_student+new_course','returning_student+seen_course')))
    rows=[]
    for fi,fac in enumerate(bundle.faculty_names):
        fm=bundle.faculty_idx==fi
        for regime in ['returning_student+seen_course','returning_student+new_course','new_student+seen_course','new_student+new_course']:
            m=fm&(labels==regime)
            if not m.any():continue
            for model,s in scores.items():
                rows.append({'fold':int(fold),'faculty':fac,'regime':regime,'model':model,'rows':int(m.sum()),'failures':int(bundle.y[m].sum()),'prevalence':float(bundle.y[m].mean()),'ap':safe_ap(bundle.y[m],np.asarray(s)[m]),'auc':safe_auc(bundle.y[m],np.asarray(s)[m])})
    return rows


def support_gain_rows(fold,bundle,store,committee,graph,fusion):
    sup=np.asarray(store.data['support']);rows=[]
    for fi,fac in enumerate(bundle.faculty_names):
        m=bundle.faculty_idx==fi
        if not m.any():continue
        rows.append({'fold':int(fold),'faculty':fac,'rows':int(m.sum()),'failures':int(bundle.y[m].sum()),
                     'history_courses_mean':float(sup[m,0].mean()),'cohort_students_mean':float(sup[m,1].mean()),'structural_courses_mean':float(sup[m,2].mean()),'groups_mean':float(sup[m,3].mean()),
                     'graph_delta_ap_vs_committee':safe_ap(bundle.y[m],graph[m])-safe_ap(bundle.y[m],committee[m]),
                     'fusion_delta_ap_vs_committee':safe_ap(bundle.y[m],fusion[m])-safe_ap(bundle.y[m],committee[m])})
    return rows


def _pearson(x,y):
    x=np.asarray(x,float);y=np.asarray(y,float);m=np.isfinite(x)&np.isfinite(y)
    if m.sum()<3 or np.nanstd(x[m])<1e-12 or np.nanstd(y[m])<1e-12:return np.nan
    return float(np.corrcoef(x[m],y[m])[0,1])


def support_gain_correlations(rows,folds=(6,7)):
    d=pd.DataFrame(rows);d=d[d.fold.isin(list(folds))]
    out=[]
    for target in ['graph_delta_ap_vs_committee','fusion_delta_ap_vs_committee']:
        for support in ['history_courses_mean','cohort_students_mean','structural_courses_mean','groups_mean']:
            out.append({'target':target,'support':support,'cells':int(len(d)),'pearson_r':_pearson(d[support],d[target])})
    return pd.DataFrame(out)


def premise_audit(metrics,perturb_rows,ablation_rows,support_gain,primary='suite23_faculty_safe_fusion',holdout=(6,7),it_faculty='khoa_cong_nghe_thong_tin'):
    """Compact falsifiable audit of Suite23's predeclared premises."""
    d=pd.DataFrame(metrics);h=d[d.fold.isin(list(holdout))];rows=[]
    def delta(a,b):
        z=paired_summary(h,a,b)
        return z.get('mean_delta_ap',np.nan),z.get('wins',0),z.get('cells',0),z.get('mean_delta_f2_at20',np.nan)
    dab,wab,cab,fab=delta('suite23_heterogeneous_graph','suite23_bipartite_graph')
    dbc,wbc,cbc,fbc=delta('suite23_cs_hagt','suite23_heterogeneous_graph')
    dfus,wfus,cfus,ffus=delta(primary,'committee')
    dlocal,wlocal,clocal,flocal=delta('suite23_cs_hagt','suite22_relational') if 'suite22_relational' in set(h.model) else (np.nan,0,0,np.nan)
    def verdict(v,w,c,strong=.003):
        if not np.isfinite(v):return 'UNAVAILABLE'
        if v>=strong and c and w/c>=.60:return 'SUPPORTED'
        if v>0:return 'WEAK_SUPPORT'
        return 'NOT_SUPPORTED'
    rows += [
        {'premise':'heterogeneous_structure_B_minus_A','mean_delta_ap':dab,'wins':wab,'cells':cab,'verdict':verdict(dab,wab,cab)},
        {'premise':'time_plus_dual_readout_C_minus_B','mean_delta_ap':dbc,'wins':wbc,'cells':cbc,'verdict':verdict(dbc,wbc,cbc)},
        {'premise':'graph_complements_committee_primary_fusion','mean_delta_ap':dfus,'wins':wfus,'cells':cfus,'verdict':verdict(dfus,wfus,cfus,strong=.005)},
        {'premise':'global_graph_beats_suite22_local_relational','mean_delta_ap':dlocal,'wins':wlocal,'cells':clocal,'verdict':verdict(dlocal,wlocal,clocal)},
    ]
    p=pd.DataFrame(perturb_rows)
    if len(p):
        p=p[p.fold.isin(list(holdout))]
        for mode in ['zero_history','shuffle_history','zero_cohort','shuffle_cohort']:
            z=p[p.perturbation==mode];v=float(z.delta_ap.mean()) if len(z) else np.nan;loss=int((z.delta_ap<0).sum()) if len(z) else 0;c=len(z)
            # A successful identity/content perturbation should hurt: negative delta.
            ver='SUPPORTED' if np.isfinite(v) and v<=-.003 and c and loss/c>=.75 else ('WEAK_SUPPORT' if np.isfinite(v) and v<0 else 'NOT_SUPPORTED')
            rows.append({'premise':f'relational_identity_{mode}_should_hurt','mean_delta_ap':v,'wins':loss,'cells':c,'verdict':ver})
    a=pd.DataFrame(ablation_rows)
    if len(a):
        a=a[a.fold.isin(list(holdout))]
        for mode in ['no_time','no_student_history_readout','no_course_cohort_readout','no_structural','no_groups','no_faculty_edges']:
            z=a[a.ablation==mode];v=float(z.delta_ap.mean()) if len(z) else np.nan;loss=int((z.delta_ap<0).sum()) if len(z) else 0;c=len(z)
            ver='SUPPORTED' if np.isfinite(v) and v<=-.003 and c and loss/c>=.60 else ('WEAK_SUPPORT' if np.isfinite(v) and v<0 else 'NOT_SUPPORTED')
            rows.append({'premise':f'trained_dependency_{mode}','mean_delta_ap':v,'wins':loss,'cells':c,'verdict':ver})
    # project success against Council overlap
    if 'council' in set(h.model):
        aa=h[h.model==primary][['fold','faculty','ap','fbeta_at_20']].rename(columns={'ap':'a','fbeta_at_20':'af2'});bb=h[h.model=='council'][['fold','faculty','ap','fbeta_at_20']].rename(columns={'ap':'b','fbeta_at_20':'bf2'});z=aa.merge(bb,on=['fold','faculty'])
        if len(z):
            fac=z.groupby('faculty').agg(a=('a','mean'),b=('b','mean'),af2=('af2','mean'),bf2=('bf2','mean'));fac['delta']=fac.a-fac.b
            macro=float(fac.a.mean()-fac.b.mean());meaningful=int((fac.delta<-.003).sum());strict=int((fac.delta>.003).sum());it=float(fac.loc[it_faculty,'delta']) if it_faculty in fac.index else np.nan
            fd=z.assign(delta=z.a-z.b).groupby('fold').delta.mean().to_dict();f2=float((z.af2-z.bf2).mean())
            success=bool(macro>=.010 and meaningful==0 and np.isfinite(it) and it>=0 and all(fd.get(f,np.nan)>0 for f in holdout) and f2>=-.005)
            near=bool(macro>0 and meaningful<=1 and (not np.isfinite(it) or it>=-.005))
            rows.append({'premise':'PROJECT_COUNCIL_BREAKTHROUGH','mean_delta_ap':macro,'wins':strict,'cells':len(fac),'verdict':'SUCCESS' if success else ('NEAR_SUCCESS' if near else 'NOT_SUCCESSFUL'),'meaningful_faculty_losses':meaningful,'it_delta_ap':it,'mean_delta_f2':f2,**{f'fold_{int(k)}_delta_ap':float(v) for k,v in fd.items()}})
    return pd.DataFrame(rows)
