# -*- coding: utf-8 -*-
"""Suite2 Experiment A: strong nonlinear HGB meta/specialist.

A1 trains a global HGB meta-classifier on the full prepared representation plus
cross-fitted V2 committee outputs and chronology-safe support indicators.
A2 trains the same family only on the correction-opportunity routed population.
Both retain authority=0 as an exact committee fallback.
"""
from __future__ import annotations
import sys
from pathlib import Path as _BootstrapPath
_PARENT = str(_BootstrapPath(__file__).resolve().parents[1])
if _PARENT not in sys.path:
    sys.path.insert(0, _PARENT)
import argparse
from pathlib import Path
import numpy as np
import pandas as pd

import train_v3 as core
from suite2_common import (
    load_binary_fold, crossfit_committee, fit_production_committee,
    chronological_support_features, support_matrix, v2_meta_features,
    select_hgb_with_authority, fit_hgb_classifier, predict_positive,
    effective_weights, blend_probability, safe_ap, safe_auc, score_metrics,
    equal_budget_metrics, permutation_rows, save_csv, parse_folds, print_metric_line,
)


def build_features(X, p, support, raw_names):
    meta, mn = v2_meta_features(p)
    sup, sn = support_matrix(support)
    return np.column_stack([core.clean_X(X), meta, sup]), list(raw_names) + mn + sn


def apply_routed_blend(q, specialist, routed, authority):
    q = np.asarray(q, float); specialist = np.asarray(specialist, float); routed = np.asarray(routed, bool)
    out = q.copy()
    out[routed] = blend_probability(q[routed], specialist[routed], authority)
    return out


def _refit(selected, X, y, seed, beta):
    w,_ = effective_weights(y, beta)
    return fit_hgb_classifier(X, y, seed=seed, sample_weight=w, **selected['params'])


def run_fold(a, fold, outdir):
    d = load_binary_fold(a.splits_dir, a.faculty, fold)
    if d is None: return None
    Xtr,Xte,ytr,yte,tr,te,names = d
    print('\n' + '='*118)
    print(f'FOLD {fold} | historical={len(ytr):,} test={len(yte):,} features={len(names)} | fail train/test={ytr.mean():.2%}/{yte.mean():.2%}')
    cf = crossfit_committee(Xtr,ytr,tr,seed=a.seed+fold*31,n_estimators=a.n_estimators,hist_max_iter=a.hist_max_iter,crossfit_folds=a.crossfit_folds,verbose=False)
    idx=np.where(cf.covered_mask)[0]; Xo=Xtr[idx]; yo=ytr[idx]; po=cf.probabilities[idx]; bo=cf.block_ids[idx]
    qo=core.committee_statistics(po)['committee_mean']
    support_all=chronological_support_features(tr); support_o=support_all.iloc[idx].reset_index(drop=True)
    Zo,feat_names=build_features(Xo,po,support_o,names)
    meta_tr,meta_va,vinfo=core.choose_multiblock_validation(yo,bo,cf.mode,a.validation_min_rows,a.validation_min_failures,a.validation_min_blocks,a.seed+fold)
    print(f"[Temporal selection] mode={vinfo['mode']} train={len(meta_tr):,} val={len(meta_va):,} failures={int(yo[meta_va].sum())} blocks={vinfo.get('validation_blocks')}")

    # A1 global meta-HGB
    global_sel=select_hgb_with_authority(Zo[meta_tr],yo[meta_tr],Zo[meta_va],yo[meta_va],qo[meta_va],seed=a.seed+1000*fold,max_iter=a.specialist_max_iter,authority_grid=a.authority_grid,beta=a.effective_beta)
    global_model=_refit(global_sel,Zo,yo,a.seed+2000*fold,a.effective_beta)

    # Production committee and target representation
    _,pte,qte=fit_production_committee(Xtr,ytr,Xte,seed=a.seed+fold*17,n_estimators=a.n_estimators,hist_max_iter=a.hist_max_iter)
    support_te=chronological_support_features(tr,te); Zte,_=build_features(Xte,pte,support_te,names)
    pglobal=predict_positive(global_model,Zte); sglobal=blend_probability(qte,pglobal,global_sel['authority'])

    # A2 routed HGB
    route_cands=core.generate_correction_route_candidates(yo[meta_tr],po[meta_tr],bo[meta_tr],a.route_min_rate,a.route_max_rate,a.target_route_rate,top_k=1)
    policy,_=route_cands[0]
    ro=core.apply_correction_routing_policy(po,policy,bo)['routed']
    rt=core.apply_correction_routing_policy(pte,policy,None)['routed']
    routed_sel=None; routed_model=None; proute_all=qte.copy(); srouted=qte.copy()
    rtr=meta_tr[ro[meta_tr]]; rva=meta_va[ro[meta_va]]
    if len(rtr)>=a.min_specialist_rows and yo[rtr].sum()>=a.min_specialist_failures and len(rva)>=50 and np.unique(yo[rva]).size==2:
        routed_sel=select_hgb_with_authority(Zo[rtr],yo[rtr],Zo[rva],yo[rva],qo[rva],seed=a.seed+3000*fold,max_iter=a.specialist_max_iter,authority_grid=a.authority_grid,beta=a.effective_beta)
        rall=np.where(ro)[0]
        routed_model=_refit(routed_sel,Zo[rall],yo[rall],a.seed+4000*fold,a.effective_beta)
        if rt.any(): proute_all[rt]=predict_positive(routed_model,Zte[rt])
        srouted=apply_routed_blend(qte,proute_all,rt,routed_sel['authority'])
    else:
        print(f'[Routed HGB] fallback: train rows/failures={len(rtr)}/{int(yo[rtr].sum())} val={len(rva)}')

    print('[Validation selection]')
    print(f"  A1 all-row HGB: authority={global_sel['authority']:.2f} AP/AUC={global_sel['validation']['ap']:.4f}/{global_sel['validation']['auc']:.4f}")
    if routed_sel:
        print(f"  A2 routed HGB : route={ro.mean():.1%} failure enrichment={(yo[ro].mean()/max(yo.mean(),1e-12)):.2f}x authority={routed_sel['authority']:.2f} AP/AUC={routed_sel['validation']['ap']:.4f}/{routed_sel['validation']['auc']:.4f}")
    else: print('  A2 routed HGB : committee fallback')
    print('[Temporal test]')
    print_metric_line('committee',yte,qte)
    print_metric_line('A1 all-row meta-HGB',yte,sglobal,qte)
    print_metric_line('A2 routed meta-HGB',yte,srouted,qte)
    print(f"  target route={rt.mean():.1%} target routed failure prevalence={yte[rt].mean() if rt.any() else 0:.2%}")

    tag=f'{a.faculty}_fold_{fold}'
    pd.DataFrame({'actual':yte,'committee':qte,'allrow_meta_hgb':sglobal,'routed_meta_hgb':srouted,'allrow_specialist':pglobal,'routed_specialist':proute_all,'routed':rt.astype(int),**{c:support_te[c].to_numpy() for c in support_te.columns}}).to_csv(outdir/f'{tag}_scores.csv',index=False)
    rows=[]
    for nm,sc in [('committee',qte),('allrow_meta_hgb',sglobal),('routed_meta_hgb',srouted)]:
        rows.append({'fold':fold,'model':nm,**score_metrics(yte,sc),**{f'budget20_{k}':v for k,v in equal_budget_metrics(yte,sc,.20).items() if k in ['precision','recall','fbeta','flag_rate']}})
    save_csv(rows,outdir/f'{tag}_metrics.csv')
    cand=[]
    for r in global_sel['candidate_rows']: cand.append({'fold':fold,'variant':'allrow',**r})
    if routed_sel:
        for r in routed_sel['candidate_rows']: cand.append({'fold':fold,'variant':'routed',**r})
    save_csv(cand,outdir/f'{tag}_candidate_grid.csv')
    route_info=core.correction_opportunity_metrics(yo,qo,ro); route_info.update({'fold':fold,'train_route_rate':float(ro.mean()),'test_route_rate':float(rt.mean()),'train_failure_enrichment':float(yo[ro].mean()/max(yo.mean(),1e-12)),'test_failure_enrichment':float(yte[rt].mean()/max(yte.mean(),1e-12)) if rt.any() else np.nan})
    save_csv([route_info],outdir/f'{tag}_routing_diagnostics.csv')
    # Validation permutation importance for interpretability.
    imp=permutation_rows(global_sel['model'],Zo[meta_va],yo[meta_va],feat_names,seed=a.seed+fold,top_k=25)
    save_csv([{'fold':fold,'variant':'allrow',**r} for r in imp],outdir/f'{tag}_feature_importance.csv')
    return {'fold':fold,'y':yte,'committee':qte,'allrow_meta_hgb':sglobal,'routed_meta_hgb':srouted}


def _summary(split,recs):
    y=np.concatenate([r['y'] for r in recs]); rows=[]
    for nm in ['committee','allrow_meta_hgb','routed_meta_hgb']:
        s=np.concatenate([r[nm] for r in recs]); rows.append({'split':split,'model':nm,**score_metrics(y,s),**{f'budget20_{k}':v for k,v in equal_budget_metrics(y,s,.20).items() if k in ['precision','recall','fbeta','flag_rate']}})
    return rows


def parse_args():
    p=argparse.ArgumentParser();p.add_argument('--splits_dir',required=True);p.add_argument('--faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--models_dir',default='results/suite2/a_tree_specialist');p.add_argument('--seed',type=int,default=42);p.add_argument('--n_estimators',type=int,default=500);p.add_argument('--hist_max_iter',type=int,default=250);p.add_argument('--crossfit_folds',type=int,default=5);p.add_argument('--specialist_max_iter',type=int,default=250);p.add_argument('--validation_min_rows',type=int,default=5000);p.add_argument('--validation_min_failures',type=int,default=150);p.add_argument('--validation_min_blocks',type=int,default=2);p.add_argument('--effective_beta',type=float,default=.9999);p.add_argument('--route_min_rate',type=float,default=.25);p.add_argument('--route_max_rate',type=float,default=.65);p.add_argument('--target_route_rate',type=float,default=.45);p.add_argument('--min_specialist_rows',type=int,default=1000);p.add_argument('--min_specialist_failures',type=int,default=50);p.add_argument('--authority_grid',type=lambda s:[float(x) for x in s.split(',')],default=[0,.25,.5,.75,1]);return p.parse_args()


def main():
    a=parse_args();out=Path(a.models_dir);out.mkdir(parents=True,exist_ok=True)
    print('='*118);print('SUITE2 EXPERIMENT A — STRONG TREE META / ROUTED SPECIALIST');print('A1: all 275 prepared features + V2 committee meta + support -> HGB');print('A2: correction-opportunity routed subset -> HGB, committee outside route');print('='*118)
    dev=[r for f in parse_folds(a.dev_folds) if (r:=run_fold(a,f,out)) is not None]
    hold=[r for f in parse_folds(a.holdout_folds) if (r:=run_fold(a,f,out)) is not None]
    rows=[]
    if dev: rows+=_summary('development',dev)
    if hold: rows+=_summary('holdout',hold)
    save_csv(rows,out/f'{a.faculty}_summary.csv')
    for r in rows: print(f"{r['split'].upper():11s} {r['model']:20s} AP/AUC={r['ap']:.4f}/{r['auc']:.4f} F2@20%={r['budget20_fbeta']:.4f}")

if __name__=='__main__':main()
