# -*- coding: utf-8 -*-
"""Shared utilities for Suite4 specialist-role experiments.

All deployable experiment paths consume a cached canonical RF/ET/HGB committee.
Historical specialist targets/features are chronology-safe; target labels are used
only for evaluation after predictions have been produced.
"""
from __future__ import annotations
from pathlib import Path
import json, math
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
from sklearn.metrics import average_precision_score, roc_auc_score

import train_v3 as core
from suite2_common import (
    v2_meta_features, support_matrix, chronological_support_features,
    fit_hgb_classifier, predict_positive, effective_weights,
    safe_ap, safe_auc, equal_budget_metrics, score_metrics,
)

EPS=1e-7


def clip_prob(p): return np.clip(np.asarray(p,float),EPS,1-EPS)
def logit(p):
    p=clip_prob(p); return np.log(p/(1-p))
def sigmoid(z): return 1/(1+np.exp(-np.clip(np.asarray(z,float),-35,35)))


def numeric_blocks(x):
    return np.asarray(pd.to_numeric(pd.Series(np.asarray(x)), errors='coerce'), float)


def rank01(x):
    s=pd.Series(np.asarray(x,float))
    if len(s)<=1:return np.zeros(len(s),float)
    return ((s.rank(method='average').to_numpy(float)-1)/max(len(s)-1,1)).astype(float)


def blockwise_rank01(x,blocks):
    x=np.asarray(x,float);b=np.asarray(blocks,object);out=np.zeros(len(x),float)
    for v in pd.unique(b):
        m=b==v; out[m]=rank01(x[m])
    return out

def map_score_to_reference_distribution(score,reference):
    """Monotone unlabeled mapping of a score onto the reference score scale."""
    score=np.asarray(score,float);reference=np.asarray(reference,float)
    if len(score)==0:return score.copy()
    if np.all(score==score[0]):return np.full(len(score),float(np.median(reference)))
    r=rank01(score);return np.quantile(reference,r,method='linear')


def rescue_labels(y,q,warning_rate=.20):
    """Failures not contained in the committee's top warning_rate rows."""
    y=np.asarray(y,int);q=np.asarray(q,float)
    warn=core.warn_by_rate(q,float(warning_rate)).astype(bool)
    return ((y==1)&(~warn)).astype(np.int8)


def build_rescue_pairs(y,q,blocks,warning_rate=.20,pairs_per_positive=4,max_pairs=50000,seed=42):
    """Pair missed failures with same-block normal rows having similar q.

    Returns arrays of positive and negative row indices.  Matching in q removes
    the easy committee-risk signal and forces the ranker to learn residual cues.
    """
    y=np.asarray(y,int);q=np.asarray(q,float);blocks=np.asarray(blocks,object)
    rng=np.random.default_rng(seed);pos_idx=[];neg_idx=[]
    for b in pd.unique(blocks):
        m=np.where(blocks==b)[0]
        if len(m)<5:continue
        lab=rescue_labels(y[m],q[m],warning_rate)
        pos=m[lab==1]
        # Hard normals are below the existing warning line; prefer nearest q.
        warned=core.warn_by_rate(q[m],float(warning_rate)).astype(bool)
        neg=m[(y[m]==0)&(~warned)]
        if len(pos)==0 or len(neg)==0:continue
        for i in pos:
            d=np.abs(q[neg]-q[i]);order=np.argsort(d)
            pool=neg[order[:max(16,int(pairs_per_positive)*4)]]
            k=min(int(pairs_per_positive),len(pool))
            chosen=rng.choice(pool,k,replace=False) if len(pool)>k else pool
            pos_idx.extend([int(i)]*len(chosen));neg_idx.extend([int(j) for j in chosen])
    if len(pos_idx)>max_pairs:
        keep=rng.choice(len(pos_idx),int(max_pairs),replace=False)
        pos_idx=np.asarray(pos_idx,int)[keep];neg_idx=np.asarray(neg_idx,int)[keep]
    else:
        pos_idx=np.asarray(pos_idx,int);neg_idx=np.asarray(neg_idx,int)
    return pos_idx,neg_idx


def monotonic_cumulative(p):
    p=np.clip(np.asarray(p,float),0,1)
    if p.ndim!=2:raise ValueError('cumulative probabilities must be 2-D')
    return np.maximum.accumulate(p,axis=1)


def ordinal_severity_features(cumulative):
    """Return 10 dense severity features and six disjoint grade-bin masses."""
    p=monotonic_cumulative(cumulative)
    if p.shape[1]!=5:raise ValueError('expected P(G<3),P(G<4),P(G<5),P(G<6),P(G<7)')
    bins=np.column_stack([p[:,0],p[:,1]-p[:,0],p[:,2]-p[:,1],p[:,3]-p[:,2],p[:,4]-p[:,3],1-p[:,4]])
    bins=np.clip(bins,0,1);bins=bins/np.clip(bins.sum(axis=1,keepdims=True),EPS,None)
    ent=-(bins*np.log(np.clip(bins,EPS,1))).sum(axis=1)
    expected=(bins*np.arange(6)[None,:]).sum(axis=1)
    z=np.column_stack([p,p[:,2]-p[:,1],p[:,3]-p[:,2],p[:,3]-p[:,1],ent,expected])
    names=['p_grade_lt3','p_grade_lt4','p_grade_lt5','p_grade_lt6','p_grade_lt7',
           'mass_grade_4_to5','mass_grade_5_to6','mass_grade_4_to6','severity_entropy','expected_grade_bin']
    return np.nan_to_num(z),names,bins


def competence_weights(predicted_regrets,temperature=1.0):
    """Softmax weights for committee baseline regret 0 + specialist regrets."""
    r=np.asarray(predicted_regrets,float)
    if r.ndim==1:r=r[:,None]
    z=np.column_stack([np.zeros(len(r)),r])/max(float(temperature),1e-6)
    z=z-z.max(axis=1,keepdims=True);e=np.exp(np.clip(z,-50,50));return e/e.sum(axis=1,keepdims=True)


def oracle_logit_correction(oracle,committee,cap=3.0):
    return np.clip(logit(oracle)-logit(committee),-abs(float(cap)),abs(float(cap)))


def retrieval_evidence(distances,indices,ref_y,ref_grade,ref_q,ref_course,query_course):
    d=np.asarray(distances,float);ind=np.asarray(indices,int)
    y=np.asarray(ref_y,int);g=np.asarray(ref_grade,float);q=np.asarray(ref_q,float)
    rc=np.asarray(ref_course,object);qc=np.asarray(query_course,object)
    if d.shape!=ind.shape:raise ValueError('distances/indices shape mismatch')
    w=1/np.clip(d,1e-3,None);w=w/np.clip(w.sum(axis=1,keepdims=True),EPS,None)
    yn=y[ind];gn=g[ind];qn=q[ind]
    same=(rc[ind]==qc[:,None])
    same_count=same.sum(axis=1)
    same_fail=np.divide((same*yn).sum(axis=1),np.maximum(same_count,1))
    weighted_fail=(w*yn).sum(axis=1)
    weighted_grade=(w*gn).sum(axis=1)
    weighted_q=(w*qn).sum(axis=1)
    residual=(w*(yn-qn)).sum(axis=1)
    under=(w*((yn==1)&(qn<.5))).sum(axis=1)
    z=np.column_stack([
        d[:,0],d.mean(axis=1),d.std(axis=1),d[:,-1],
        yn.mean(axis=1),weighted_fail,weighted_grade,weighted_q,residual,under,
        same_count,same_count/ind.shape[1],same_fail,
    ])
    names=['nn_min_distance','nn_mean_distance','nn_std_distance','nn_max_distance',
           'nn_failure_rate','nn_weighted_failure_rate','nn_weighted_grade','nn_weighted_committee',
           'nn_weighted_residual','nn_weighted_under_risk','nn_same_course_count','nn_same_course_fraction','nn_same_course_failure_rate']
    return np.nan_to_num(z),names


def full_meta_features(X,expert_p,support,raw_names=None):
    meta,mn=v2_meta_features(expert_p);sup,sn=support_matrix(support);X=core.clean_X(X)
    rn=list(raw_names) if raw_names is not None else [f'raw_{i}' for i in range(X.shape[1])]
    return np.column_stack([X,meta,sup]),rn+mn+sn


def compact_meta_features(expert_p,support,extra=None,extra_names=None):
    meta,mn=v2_meta_features(expert_p);sup,sn=support_matrix(support)
    parts=[meta,sup];names=mn+sn
    if extra is not None:
        parts.append(np.asarray(extra,float));names+=list(extra_names or [f'extra_{i}' for i in range(np.asarray(extra).shape[1])])
    return np.column_stack(parts),names


def fit_hgb_regressor(X,y,seed=42,max_iter=250,max_leaf_nodes=31,min_samples_leaf=20,learning_rate=.05,l2_regularization=1.0,sample_weight=None):
    m=HistGradientBoostingRegressor(learning_rate=learning_rate,max_iter=max_iter,max_leaf_nodes=max_leaf_nodes,
        min_samples_leaf=min_samples_leaf,l2_regularization=l2_regularization,early_stopping=True,random_state=int(seed))
    m.fit(core.clean_X(X),np.asarray(y,float),sample_weight=sample_weight);return m


def binary_logloss_each(y,p):
    y=np.asarray(y,int);p=clip_prob(p);return -(y*np.log(p)+(1-y)*np.log(1-p))


def history_regimes(support):
    d=np.asarray(support['history_semesters'],int)
    return np.where(d<=0,'history0',np.where(d<=2,'history1_2','history3plus'))


def metric_row(fold,model,y,p,q=None,budget=.20):
    y=np.asarray(y,int);p=np.asarray(p,float);m=equal_budget_metrics(y,p,budget,2.0)
    row={'fold':fold,'model':model,'rows':len(y),'failures':int(y.sum()),'prevalence':float(y.mean()),
         'ap':safe_ap(y,p),'auc':safe_auc(y,p),'budget':budget,'budget_precision':m['precision'],'budget_recall':m['recall'],
         'budget_f1':m['f1'],'budget_f2':m['fbeta'],'budget_flag_rate':m['flag_rate'],
         'tn':m['tn'],'fp':m['fp'],'fn':m['fn'],'tp':m['tp']}
    if q is not None:
        row['delta_ap']=row['ap']-safe_ap(y,q);row['delta_auc']=row['auc']-safe_auc(y,q)
    return row


def print_metrics(label,y,p,q=None,budget=.20):
    r=metric_row('-',label,y,p,q,budget);s=f"{label:28s} AP/AUC={r['ap']:.4f}/{r['auc']:.4f} F2@{budget:.0%}={r['budget_f2']:.4f} P/R={r['budget_precision']:.4f}/{r['budget_recall']:.4f}"
    if q is not None:s+=f" ΔAP={r['delta_ap']:+.4f}"
    print(s);return r


def save_csv(rows,path):
    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True);pd.DataFrame(rows).to_csv(path,index=False)


def fold_cache_path(cache_dir,faculty,fold): return Path(cache_dir)/f'{faculty}_fold_{int(fold)}.joblib'
def load_cache(cache_dir,faculty,fold): return joblib.load(fold_cache_path(cache_dir,faculty,fold))


def parse_folds(s): return core.parse_folds(s)


def eligible_numeric_blocks(blocks,y,min_rows=200,min_failures=5):
    b=numeric_blocks(blocks);y=np.asarray(y,int);out=[]
    for v in sorted(pd.unique(b[np.isfinite(b)])):
        m=b==v
        if m.sum()>=min_rows and y[m].sum()>=min_failures:out.append(float(v))
    return out


def summarize_records(records,out_path,faculty):
    rows=[]
    for split,recs in records.items():
        if not recs:continue
        models=sorted(set(k for r in recs for k in r['scores'].keys()))
        for model in models:
            ys=[];ps=[]
            fold_ap=[];fold_auc=[];fold_f2=[]
            for r in recs:
                if model not in r['scores']:continue
                y=r['y'];p=r['scores'][model];ys.append(y);ps.append(p)
                mm=metric_row(r['fold'],model,y,p,r['scores'].get('committee'))
                fold_ap.append(mm['ap']);fold_auc.append(mm['auc']);fold_f2.append(mm['budget_f2'])
            y=np.concatenate(ys);p=np.concatenate(ps);m=metric_row('POOLED',model,y,p)
            rows.append({'split':split,'model':model,**m,'mean_fold_ap':float(np.nanmean(fold_ap)),'mean_fold_auc':float(np.nanmean(fold_auc)),'mean_fold_f2':float(np.nanmean(fold_f2))})
    save_csv(rows,Path(out_path)/f'{faculty}_summary.csv');return rows
