# -*- coding: utf-8 -*-
"""Suite2 Experiment C: explicit history-depth regime modeling.

Regimes are based on strictly prior semesters, not merely whether an ID appeared
somewhere in the training file:
  history0   : no prior semester
  history1_2 : one or two prior semesters
  history3plus: three or more prior semesters

History-rich rows use a relational+committee blend.  History-limited/no-history
rows use dedicated HGB meta models with committee fallback.
"""
from __future__ import annotations
import sys
from pathlib import Path as _BootstrapPath
_PARENT = str(_BootstrapPath(__file__).resolve().parents[1])
if _PARENT not in sys.path:
    sys.path.insert(0, _PARENT)
import argparse
from pathlib import Path
import numpy as np
import pandas as pd

import train_v3 as core
from exp_b_relational_stacker import fit_relational_predict
from suite2_common import (
    load_binary_fold,crossfit_committee,fit_production_committee,
    chronological_support_features,history_regime_labels,v2_meta_features,support_matrix,
    select_hgb_with_authority,fit_hgb_classifier,predict_positive,effective_weights,
    blend_probability,safe_ap,safe_auc,score_metrics,equal_budget_metrics,save_csv,parse_folds,
    permutation_rows,print_metric_line,
)

HISTORY_PREFIXES=('prev_','hist_','recent3_')

def no_history_feature_indices(names):
    return [i for i,n in enumerate(names) if not str(n).startswith(HISTORY_PREFIXES)]


def build_features(X,p,support,names,raw_idx=None):
    raw_idx=list(range(X.shape[1])) if raw_idx is None else list(raw_idx);meta,mn=v2_meta_features(p);sup,sn=support_matrix(support)
    return np.column_stack([core.clean_X(X)[:,raw_idx],meta,sup]),[names[i] for i in raw_idx]+mn+sn


def _select_alpha(y,q,r,grid):
    rows=[]
    for a in grid:
        s=blend_probability(q,r,a);rows.append({'alpha':float(a),'ap':safe_ap(y,s),'auc':safe_auc(y,s),'best_f2':score_metrics(y,s)['best_f2']})
    return max(rows,key=lambda x:(np.nan_to_num(x['ap'],nan=-1),np.nan_to_num(x['auc'],nan=-1),np.nan_to_num(x['best_f2'],nan=-1),-x['alpha'])),rows


def run_fold(a,fold,outdir):
    d=load_binary_fold(a.splits_dir,a.faculty,fold)
    if d is None:return None
    Xtr,Xte,ytr,yte,tr,te,names=d
    print('\n'+'='*118);print(f'FOLD {fold} | history-depth regime experiment | historical={len(ytr):,} test={len(yte):,}')
    cf=crossfit_committee(Xtr,ytr,tr,seed=a.seed+fold*31,n_estimators=a.n_estimators,hist_max_iter=a.hist_max_iter,crossfit_folds=a.crossfit_folds,verbose=False);idx=np.where(cf.covered_mask)[0]
    Xo=Xtr[idx];yo=ytr[idx];po=cf.probabilities[idx];bo=cf.block_ids[idx];qo=core.committee_statistics(po)['committee_mean']
    sup_hist_all=chronological_support_features(tr);sup_o=sup_hist_all.iloc[idx].reset_index(drop=True);reg_o=history_regime_labels(sup_o.history_semesters)
    mt,mv,vinfo=core.choose_multiblock_validation(yo,bo,cf.mode,a.validation_min_rows,a.validation_min_failures,a.validation_min_blocks,a.seed+fold)
    _,pte,qte=fit_production_committee(Xtr,ytr,Xte,seed=a.seed+fold*17,n_estimators=a.n_estimators,hist_max_iter=a.hist_max_iter);sup_te=chronological_support_features(tr,te);reg_te=history_regime_labels(sup_te.history_semesters)
    final=qte.copy();selection_rows=[];importance_rows=[]
    print('[Regime support]')
    for g in ['history0','history1_2','history3plus']:
        print(f"  {g:12s} historical OOF={int(np.sum(reg_o==g)):,} val={int(np.sum(reg_o[mv]==g)):,} target={int(np.sum(reg_te==g)):,}")

    # Rich-history relational expert. Train on all meta-training history so its
    # embeddings/side encoder can exploit the full context, but select alpha only
    # on history-rich validation rows.
    rich_val=mv[reg_o[mv]=='history3plus'];rel_te=np.full(len(yte),np.nan)
    if len(rich_val)>=a.min_regime_val_rows and np.unique(yo[rich_val]).size==2:
        train_orig=idx[mt];val_orig=idx[rich_val]
        rel_val,info_val=fit_relational_predict(Xtr[train_orig],ytr[train_orig],tr.iloc[train_orig].reset_index(drop=True),Xtr[val_orig],tr.iloc[val_orig].reset_index(drop=True),names,a,a.seed+fold*5000)
        best_alpha,arows=_select_alpha(yo[rich_val],qo[rich_val],rel_val,a.rel_alpha_grid)
        rel_full,info_full=fit_relational_predict(Xtr,ytr,tr,Xte,te,names,a,a.seed+fold*6000);rel_te=rel_full
        m=reg_te=='history3plus';final[m]=blend_probability(qte[m],rel_full[m],best_alpha['alpha'])
        selection_rows += [{'fold':fold,'regime':'history3plus','model':'relational_blend',**r} for r in arows]
        print(f"[history3plus] relational alpha={best_alpha['alpha']:.2f} val AP/AUC={best_alpha['ap']:.4f}/{best_alpha['auc']:.4f} target cold student/course={info_full.get('new_student_rate',np.nan):.1%}/{info_full.get('new_course_rate',np.nan):.1%}")
    else:
        print('[history3plus] relational fallback to committee (insufficient rich validation support)')

    # History0 and history1_2 HGB meta models.
    nohist_idx=no_history_feature_indices(names)
    for g in ['history0','history1_2']:
        trm=mt[reg_o[mt]==g];vam=mv[reg_o[mv]==g];allm=np.where(reg_o==g)[0];tm=np.where(reg_te==g)[0]
        if len(trm)<a.min_regime_train_rows or yo[trm].sum()<a.min_regime_train_failures or len(vam)<a.min_regime_val_rows or np.unique(yo[vam]).size<2:
            print(f'[{g}] fallback to committee: train={len(trm):,}/{int(yo[trm].sum())} failures val={len(vam):,}')
            selection_rows.append({'fold':fold,'regime':g,'model':'committee_fallback','authority':0.0,'train_rows':len(trm),'val_rows':len(vam)})
            continue
        ridx=nohist_idx if g=='history0' else None
        Z,nm=build_features(Xo,po,sup_o,names,ridx);Zt,_=build_features(Xte,pte,sup_te,names,ridx)
        sel=select_hgb_with_authority(Z[trm],yo[trm],Z[vam],yo[vam],qo[vam],seed=a.seed+fold*7000+(0 if g=='history0' else 1),max_iter=a.specialist_max_iter,authority_grid=a.authority_grid,beta=a.effective_beta)
        w,_=effective_weights(yo[allm],a.effective_beta);model=fit_hgb_classifier(Z[allm],yo[allm],seed=a.seed+fold*8000+(0 if g=='history0' else 1),sample_weight=w,**sel['params'])
        if len(tm):
            ps=predict_positive(model,Zt[tm]);final[tm]=blend_probability(qte[tm],ps,sel['authority'])
        print(f"[{g}] HGB authority={sel['authority']:.2f} val AP/AUC={sel['validation']['ap']:.4f}/{sel['validation']['auc']:.4f} raw features={len(ridx) if ridx is not None else len(names)}")
        selection_rows += [{'fold':fold,'regime':g,'model':'hgb_meta',**r} for r in sel['candidate_rows']]
        importance_rows += [{'fold':fold,'regime':g,**r} for r in permutation_rows(sel['model'],Z[vam],yo[vam],nm,seed=a.seed+fold,top_k=20)]

    print('[Temporal test]');print_metric_line('committee',yte,qte);print_metric_line('C history-regime model',yte,final,qte)
    for g in ['history0','history1_2','history3plus']:
        m=reg_te==g
        if m.any(): print(f"  {g:12s} rows={m.sum():5d} prev={yte[m].mean():.2%} AP {safe_ap(yte[m],qte[m]):.4f}->{safe_ap(yte[m],final[m]):.4f} Δ={safe_ap(yte[m],final[m])-safe_ap(yte[m],qte[m]):+.4f}")
    tag=f'{a.faculty}_fold_{fold}';pd.DataFrame({'actual':yte,'committee':qte,'history_regime_model':final,'history_regime':reg_te,'relational':rel_te,**{c:sup_te[c].to_numpy() for c in sup_te.columns}}).to_csv(outdir/f'{tag}_scores.csv',index=False);save_csv(selection_rows,outdir/f'{tag}_regime_selection.csv');save_csv(importance_rows,outdir/f'{tag}_feature_importance.csv')
    rr=[]
    for g in ['history0','history1_2','history3plus']:
        m=reg_te==g
        if not m.any():continue
        for nm,sc in [('committee',qte),('history_regime_model',final)]:rr.append({'fold':fold,'regime':g,'model':nm,'rows':int(m.sum()),'failures':int(yte[m].sum()),'prevalence':float(yte[m].mean()),'ap':safe_ap(yte[m],sc[m]),'auc':safe_auc(yte[m],sc[m])})
    save_csv(rr,outdir/f'{tag}_regime_metrics.csv')
    return {'fold':fold,'y':yte,'committee':qte,'history_regime_model':final}


def summary(split,recs):
    y=np.concatenate([r['y'] for r in recs]);rows=[]
    for nm in ['committee','history_regime_model']:
        s=np.concatenate([r[nm] for r in recs]);rows.append({'split':split,'model':nm,**score_metrics(y,s),**{f'budget20_{k}':v for k,v in equal_budget_metrics(y,s,.20).items() if k in ['precision','recall','fbeta','flag_rate']}})
    return rows


def parse_args():
    p=argparse.ArgumentParser();p.add_argument('--splits_dir',required=True);p.add_argument('--faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--models_dir',default='results/suite2/c_history_regime');p.add_argument('--seed',type=int,default=42);p.add_argument('--n_estimators',type=int,default=500);p.add_argument('--hist_max_iter',type=int,default=250);p.add_argument('--crossfit_folds',type=int,default=5);p.add_argument('--specialist_max_iter',type=int,default=250);p.add_argument('--validation_min_rows',type=int,default=5000);p.add_argument('--validation_min_failures',type=int,default=150);p.add_argument('--validation_min_blocks',type=int,default=2);p.add_argument('--effective_beta',type=float,default=.9999);p.add_argument('--authority_grid',type=lambda s:[float(x) for x in s.split(',')],default=[0,.25,.5,.75,1]);p.add_argument('--min_regime_train_rows',type=int,default=1000);p.add_argument('--min_regime_train_failures',type=int,default=40);p.add_argument('--min_regime_val_rows',type=int,default=150);p.add_argument('--rel_alpha_grid',type=lambda s:[float(x) for x in s.split(',')],default=[0,.1,.2,.3,.4,.5,.75]);p.add_argument('--top_raw_features',type=int,default=48);p.add_argument('--student_dim',type=int,default=16);p.add_argument('--course_dim',type=int,default=16);p.add_argument('--section_dim',type=int,default=8);p.add_argument('--rel_epochs',type=int,default=40);p.add_argument('--rel_patience',type=int,default=6);p.add_argument('--rel_lr',type=float,default=2e-3);p.add_argument('--rel_batch_size',type=int,default=1024);p.add_argument('--rel_predict_batch',type=int,default=4096);p.add_argument('--rel_min_train_rows',type=int,default=1200);p.add_argument('--rel_val_min_rows',type=int,default=300);p.add_argument('--rel_val_min_failures',type=int,default=15);p.add_argument('--device',choices=['auto','cpu'],default='auto');return p.parse_args()

def main():
    a=parse_args();out=Path(a.models_dir);out.mkdir(parents=True,exist_ok=True);print('='*118);print('SUITE2 EXPERIMENT C — HISTORY-DEPTH REGIME MODELS');print('history0 -> no-history HGB | history1_2 -> limited-history HGB | history3plus -> relational+committee');print('='*118)
    dev=[r for f in parse_folds(a.dev_folds) if (r:=run_fold(a,f,out)) is not None];hold=[r for f in parse_folds(a.holdout_folds) if (r:=run_fold(a,f,out)) is not None];rows=[]
    if dev:rows+=summary('development',dev)
    if hold:rows+=summary('holdout',hold)
    save_csv(rows,out/f'{a.faculty}_summary.csv')
    for r in rows:print(f"{r['split'].upper():11s} {r['model']:22s} AP/AUC={r['ap']:.4f}/{r['auc']:.4f} F2@20%={r['budget20_fbeta']:.4f}")
if __name__=='__main__':main()
