# -*- coding: utf-8 -*-
"""Shared utilities for Suite5 failure-aware competence gating."""
from __future__ import annotations
import math
import numpy as np
import pandas as pd

import train_v3 as core
from suite3_common import quantile_align_to_reference
from suite4_common import binary_logloss_each, clip_prob

EPS=1e-12


def _rank_pct(x):
    """Average-tie percentile ranks in (0,1), ascending with score."""
    a=np.asarray(x,float).reshape(-1)
    if len(a)==0:return a.copy()
    r=pd.Series(a).rank(method='average').to_numpy(float)
    return (r-.5)/len(a)


def boundary_context(expert_scores,budget=.20,boundary_sigma=.10):
    """Unlabeled rank/boundary features for a batch of expert scores.

    Column 0 must be the stable committee. All features are derived from scores
    only; labels are never required.
    """
    E=np.asarray(expert_scores,float)
    if E.ndim!=2 or E.shape[1]<2:raise ValueError('expert_scores must be [n, >=2]')
    n,k=E.shape;q=E[:,0];cut=1-float(budget)
    ranks=np.column_stack([_rank_pct(E[:,j]) for j in range(k)])
    qwarn=core.warn_by_rate(q,float(budget)).astype(int)
    distance=np.abs(ranks[:,0]-cut)
    proximity=np.exp(-distance/max(float(boundary_sigma),1e-6))
    # Score-space margin is useful but rank-space distance is more stable across semesters.
    qcut=float(np.quantile(q,cut)) if n else np.nan
    score_margin=q-qcut
    gaps=ranks[:,1:]-ranks[:,[0]]
    abs_gaps=np.abs(gaps)
    expert_warn=np.column_stack([core.warn_by_rate(E[:,j],float(budget)).astype(int) for j in range(k)])
    vote_count=expert_warn.sum(axis=1)
    Z=np.column_stack([
        ranks[:,0],distance,proximity,score_margin,qwarn,
        ranks[:,1:],gaps,abs_gaps,vote_count,
    ])
    names=['committee_rank_pct','distance_to_budget_boundary','boundary_proximity','committee_score_margin','committee_warning_flag']
    names += [f'expert_{j}_rank_pct' for j in range(1,k)]
    names += [f'expert_{j}_rank_gap' for j in range(1,k)]
    names += [f'expert_{j}_abs_rank_gap' for j in range(1,k)]
    names += ['expert_warning_vote_count']
    ctx={'ranks':ranks,'committee_warn':qwarn.astype(bool),'expert_warn':expert_warn.astype(bool),
         'boundary_distance':distance,'boundary_proximity':proximity,'score_margin':score_margin,
         'cutoff_rank':cut,'cutoff_score':qcut}
    return np.nan_to_num(Z,nan=0.0,posinf=0.0,neginf=0.0),names,ctx


def failure_aware_utility(y,expert_scores,budget=.20,positive_regret_scale=2.0,
                          rescue_bonus=1.5,lost_tp_penalty=2.0,
                          fp_remove_bonus=.35,fp_add_penalty=.50,
                          boundary_scale=.50,regret_cap=3.0):
    """Realized per-specialist utility relative to committee.

    Produces one dense target per non-committee expert. It combines weighted
    log-loss regret with fixed-budget event utility. Target labels are used only
    on historical OOF/forward blocks when fitting competence models.
    """
    y=np.asarray(y,int).reshape(-1);E=np.asarray(expert_scores,float)
    if len(y)!=len(E):raise ValueError('length mismatch')
    if E.shape[1]<2:raise ValueError('need committee + specialist')
    q=clip_prob(E[:,0]);_,_,ctx=boundary_context(E,budget)
    qwarn=ctx['committee_warn'];ew=ctx['expert_warn'];prox=ctx['boundary_proximity']
    lq=binary_logloss_each(y,q)
    U=[];resc=[];lost=[];fpr=[];fpa=[]
    for j in range(1,E.shape[1]):
        pj=clip_prob(E[:,j]);dense=lq-binary_logloss_each(y,pj)
        dense=dense*np.where(y==1,float(positive_regret_scale),1.0)
        rescue=(y==1)&(~qwarn)&ew[:,j]
        lost_tp=(y==1)&qwarn&(~ew[:,j])
        fp_removed=(y==0)&qwarn&(~ew[:,j])
        fp_added=(y==0)&(~qwarn)&ew[:,j]
        # Decision changes near the warning boundary receive moderately larger value.
        bmult=1+float(boundary_scale)*prox
        u=dense + bmult*(float(rescue_bonus)*rescue.astype(float)
                          -float(lost_tp_penalty)*lost_tp.astype(float)
                          +float(fp_remove_bonus)*fp_removed.astype(float)
                          -float(fp_add_penalty)*fp_added.astype(float))
        U.append(np.clip(u,-abs(float(regret_cap)),abs(float(regret_cap))))
        resc.append(int(rescue.sum()));lost.append(int(lost_tp.sum()));fpr.append(int(fp_removed.sum()));fpa.append(int(fp_added.sum()))
    return np.column_stack(U),{'rescues_by_expert':resc,'lost_tp_by_expert':lost,'fp_removed_by_expert':fpr,'fp_added_by_expert':fpa,
                               'committee_missed_failures':int(((y==1)&(~qwarn)).sum()),'committee_detected_failures':int(((y==1)&qwarn).sum())}


def competence_sample_weights(y,expert_scores,budget=.20,positive_weight=3.0,rescue_weight=4.0,
                              boundary_weight=1.5,boundary_sigma=.10):
    """Sample weights that stop normal rows from dominating competence fitting."""
    y=np.asarray(y,int).reshape(-1);E=np.asarray(expert_scores,float)
    _,_,ctx=boundary_context(E,budget,boundary_sigma)
    qwarn=ctx['committee_warn'];missed=(y==1)&(~qwarn)
    w=np.ones(len(y),float)
    w += (float(positive_weight)-1.0)*(y==1)
    w += float(rescue_weight)*missed
    w += float(boundary_weight)*ctx['boundary_proximity']
    # Normalize for stable HGB regularization while preserving ratios.
    w=w/max(float(np.mean(w)),EPS)
    return w,{'missed_failures':int(missed.sum()),'failures':int((y==1).sum()),'mean_weight':float(np.mean(w)),
              'failure_mean_weight':float(np.mean(w[y==1])) if np.any(y==1) else np.nan,
              'normal_mean_weight':float(np.mean(w[y==0])) if np.any(y==0) else np.nan}


def competence_softmax(predicted_utility,temperature=1.0):
    r=np.asarray(predicted_utility,float)
    if r.ndim==1:r=r[:,None]
    z=np.column_stack([np.zeros(len(r)),r])/max(float(temperature),1e-6)
    z=z-z.max(axis=1,keepdims=True)
    e=np.exp(np.clip(z,-50,50))
    return e/np.clip(e.sum(axis=1,keepdims=True),EPS,None)


def mix_council(expert_scores,predicted_utility,alpha=1.0,temperature=1.0):
    E=np.asarray(expert_scores,float);a=float(alpha)
    W=competence_softmax(predicted_utility,temperature)
    if E.shape!=W.shape:raise ValueError(f'expert/weight shape mismatch {E.shape} vs {W.shape}')
    if a<=0:return E[:,0].copy(),W
    mix=(W*E).sum(axis=1)
    return clip_prob((1-a)*E[:,0]+a*mix),W


def align_expert_matrix(expert_scores):
    E=np.asarray(expert_scores,float);A=E.copy()
    for j in range(1,E.shape[1]):A[:,j]=quantile_align_to_reference(E[:,j],E[:,0])
    return A


def weight_diagnostics(weights):
    W=np.asarray(weights,float)
    ent=-(W*np.log(np.clip(W,EPS,1))).sum(axis=1)
    eff=np.exp(ent)
    ordered=np.sort(W,axis=1)
    return {'mean_entropy':float(np.mean(ent)),'normalized_entropy':float(np.mean(ent)/math.log(W.shape[1])),
            'mean_effective_experts':float(np.mean(eff)),'mean_max_weight':float(np.mean(W.max(axis=1))),
            'mean_top1_margin':float(np.mean(ordered[:,-1]-ordered[:,-2]))}


def decision_change_diagnostics(y,committee,score,budget=.20):
    y=np.asarray(y,int);q=np.asarray(committee,float);p=np.asarray(score,float)
    qw=core.warn_by_rate(q,float(budget)).astype(bool);pw=core.warn_by_rate(p,float(budget)).astype(bool)
    return {
        'rescued_failures':int(((y==1)&(~qw)&pw).sum()),
        'lost_existing_tp':int(((y==1)&qw&(~pw)).sum()),
        'false_positives_removed':int(((y==0)&qw&(~pw)).sum()),
        'false_positives_added':int(((y==0)&(~qw)&pw).sum()),
        'net_tp_change':int(((y==1)&pw).sum()-((y==1)&qw).sum()),
        'net_fp_change':int(((y==0)&pw).sum()-((y==0)&qw).sum()),
    }


def choose_robust_candidate(rows,min_combined_gain=0.0,min_win_rate=.5,
                            min_worst_ap=-.01,min_worst_f2=-.015,f2_weight=1.0):
    """Choose a prequential policy with exact committee fallback.

    Sharper specialization is never rewarded directly; when performance is tied,
    the candidate with more effective experts (higher-entropy mixture) wins.
    """
    rows=[dict(r) for r in rows]
    committee=next((r for r in rows if r.get('gate_mode')=='committee'),None)
    if committee is None:
        committee={'gate_mode':'committee','median_delta_ap':0.,'median_delta_f2':0.,'mean_delta_ap':0.,'mean_delta_f2':0.,
                   'worst_delta_ap':0.,'worst_delta_f2':0.,'combined_win_rate':0.,'mean_effective_experts':4.0}
    eligible=[]
    for r in rows:
        if r.get('gate_mode')=='committee':continue
        combined=float(r.get('median_delta_ap',-np.inf))+float(f2_weight)*float(r.get('median_delta_f2',-np.inf))
        if (combined >= float(min_combined_gain)
            and float(r.get('combined_win_rate',0.0)) >= float(min_win_rate)
            and float(r.get('worst_delta_ap',-np.inf)) >= float(min_worst_ap)
            and float(r.get('worst_delta_f2',-np.inf)) >= float(min_worst_f2)):
            rr=dict(r);rr['combined_median_gain']=combined;eligible.append(rr)
    if not eligible:return dict(committee)
    # Performance first. If effectively tied, prefer the smoother/more diversified gate.
    best=max(eligible,key=lambda r:(
        round(float(r['combined_median_gain']),10),
        round(float(r.get('median_delta_ap',0.0)),10),
        round(float(r.get('median_delta_f2',0.0)),10),
        round(float(r.get('mean_delta_ap',0.0))+float(f2_weight)*float(r.get('mean_delta_f2',0.0)),10),
        float(r.get('mean_effective_experts',0.0)),
    ))
    return best
