# -*- coding: utf-8 -*-
"""Representation, geometry, drift, interpretability and hard-case diagnostics."""
from __future__ import annotations

import numpy as np
import pandas as pd
from sklearn.neighbors import NearestNeighbors
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.metrics import r2_score, average_precision_score

from suite19_common import safe_ap, safe_auc


def _finite_rows(z, available=None):
    z=np.asarray(z,float);m=np.isfinite(z).all(1)
    if available is not None:m &= np.asarray(available,bool)
    return m


def covariance_spectrum(z):
    z=np.asarray(z,float)
    if z.ndim!=2:return np.asarray([],float)
    if len(z)<2:return np.zeros(z.shape[1],float)
    c=np.cov(z,rowvar=False)
    if np.ndim(c)==0:return np.asarray([float(c)])
    vals=np.linalg.eigvalsh((c+c.T)/2)[::-1]
    return np.clip(vals,0,None)


def effective_rank(z_or_eigs):
    a=np.asarray(z_or_eigs,float)
    ev=a if a.ndim==1 else covariance_spectrum(a)
    s=ev.sum()
    if s<=0:return 0.0
    p=ev/s;p=p[p>0]
    return float(np.exp(-np.sum(p*np.log(p))))


def _pair_stats(z,max_pairs,seed):
    n=len(z)
    if n<2:return {'pair_distance_mean':np.nan,'pair_distance_std':np.nan,'pair_distance_q10':np.nan,'pair_distance_q50':np.nan,'pair_distance_q90':np.nan,'pair_cosine_mean':np.nan,'pair_cosine_std':np.nan}
    rng=np.random.default_rng(seed);k=min(int(max_pairs),max(1,n*(n-1)//2));i=rng.integers(0,n,size=k);j=rng.integers(0,n,size=k);bad=i==j
    while bad.any():j[bad]=rng.integers(0,n,size=bad.sum());bad=i==j
    a=z[i];b=z[j];dist=np.linalg.norm(a-b,axis=1);cos=np.sum(a*b,1)/(np.linalg.norm(a,axis=1)*np.linalg.norm(b,axis=1)+1e-12)
    return {'pair_distance_mean':float(np.mean(dist)),'pair_distance_std':float(np.std(dist)),'pair_distance_q10':float(np.quantile(dist,.1)),'pair_distance_q50':float(np.quantile(dist,.5)),'pair_distance_q90':float(np.quantile(dist,.9)),'pair_cosine_mean':float(np.mean(cos)),'pair_cosine_std':float(np.std(cos))}


def representation_diagnostics(z,y,faculty,committee,available=None,max_pairs=20000,nn_k=10,seed=42):
    z=np.asarray(z,float);y=np.asarray(y,int);faculty=np.asarray(faculty,int);q=np.asarray(committee,float);m=_finite_rows(z,available);zz=z[m];yy=y[m];ff=faculty[m];qq=q[m]
    out={'rows':int(len(z)),'available_rows':int(m.sum()),'availability_rate':float(m.mean()) if len(m) else 0.,'latent_dim':int(z.shape[1])}
    if len(zz)==0:return out
    mu=zz.mean(0);std=zz.std(0);out.update({'dead_dimension_fraction':float(np.mean(std<1e-7)),'dim_mean_abs_mean':float(np.mean(np.abs(mu))),'dim_mean_min':float(np.min(mu)),'dim_mean_max':float(np.max(mu)),'dim_std_mean':float(np.mean(std)),'dim_std_min':float(np.min(std)),'dim_std_max':float(np.max(std)),'dim_min':float(np.min(zz)),'dim_max':float(np.max(zz))})
    ev=covariance_spectrum(zz);out['effective_rank']=effective_rank(ev);out['covariance_trace']=float(ev.sum());out['top_covariance_eigenvalues']=';'.join(f'{x:.8g}' for x in ev[:10]);out['top_singular_values']=';'.join(f'{x:.8g}' for x in np.sqrt(np.clip(ev[:10],0,None)))
    if ev.sum()>0:
        cum=np.cumsum(ev)/ev.sum();out['dims_for_50pct_energy']=int(np.searchsorted(cum,.5)+1);out['dims_for_90pct_energy']=int(np.searchsorted(cum,.9)+1)
    out.update(_pair_stats(zz,max_pairs,seed))
    if np.unique(yy).size==2:
        c0=zz[yy==0].mean(0);c1=zz[yy==1].mean(0);out['class_centroid_distance']=float(np.linalg.norm(c1-c0))
        w0=np.linalg.norm(zz[yy==0]-c0,axis=1);w1=np.linalg.norm(zz[yy==1]-c1,axis=1);out['within_class_distance_mean']=float(np.mean(np.r_[w0,w1]));out['between_over_within']=out['class_centroid_distance']/max(out['within_class_distance_mean'],1e-12)
    else:out['class_centroid_distance']=np.nan;out['within_class_distance_mean']=np.nan;out['between_over_within']=np.nan
    if len(zz)>=3:
        k=min(max(2,int(nn_k)+1),len(zz));nbr=NearestNeighbors(n_neighbors=k,metric='euclidean').fit(zz);dist,idx=nbr.kneighbors(zz);idx=idx[:,1:];dist=dist[:,1:];labs=yy[idx];out['neighborhood_label_purity']=float(np.mean(labs==yy[:,None]));out['mean_knn_distance']=float(np.mean(dist));out['failure_neighbor_rate']=float(np.mean(labs[yy==1])) if np.any(yy==1) else np.nan
        if np.any(yy==1) and np.any(yy==0):
            n0=NearestNeighbors(n_neighbors=1).fit(zz[yy==0]);d10,_=n0.kneighbors(zz[yy==1]);n1=NearestNeighbors(n_neighbors=1).fit(zz[yy==1]);d01,_=n1.kneighbors(zz[yy==0]);out['failure_to_normal_nn_distance']=float(d10.mean());out['normal_to_failure_nn_distance']=float(d01.mean());out['failure_to_normal_nn_ratio']=float(d10.mean()/max(d01.mean(),1e-12))
        high=np.quantile(qq,.8);low=np.quantile(qq,.2);hard=((yy==0)&(qq>=high))|((yy==1)&(qq<=low));out['committee_hard_rows']=int(hard.sum());out['committee_hard_neighbor_purity']=float(np.mean(labs[hard]==yy[hard,None])) if hard.any() else np.nan
    cents={fi:zz[ff==fi].mean(0) for fi in np.unique(ff) if np.any(ff==fi)}
    if len(cents)>1:
        keys=np.array(sorted(cents));C=np.vstack([cents[k] for k in keys]);pred=keys[np.argmin(((zz[:,None,:]-C[None,:,:])**2).sum(2),axis=1)];out['faculty_centroid_separability_accuracy']=float(np.mean(pred==ff))
        between=[]
        for i in range(len(C)):
            for j in range(i+1,len(C)):between.append(float(np.linalg.norm(C[i]-C[j])))
        out['faculty_centroid_pair_distance_mean']=float(np.mean(between)) if between else np.nan
    else:out['faculty_centroid_separability_accuracy']=np.nan;out['faculty_centroid_pair_distance_mean']=np.nan
    return out


def temporal_drift_rows(z,blocks,available=None,faculty=None):
    z=np.asarray(z,float);b=np.asarray(blocks,float);m=_finite_rows(z,available);rows=[]
    if faculty is None:groups=[('ALL',m)]
    else:
        f=np.asarray(faculty,int);groups=[('ALL',m)]+[(str(fi),m&(f==fi)) for fi in np.unique(f[m])]
    for group,gm in groups:
        vals=np.array(sorted(v for v in np.unique(b[gm]) if np.isfinite(v)),float)
        for a,c in zip(vals[:-1],vals[1:]):
            za=z[gm&(b==a)];zb=z[gm&(b==c)]
            if not len(za) or not len(zb):continue
            ca=za.mean(0);cb=zb.mean(0);A=np.cov(za,rowvar=False) if len(za)>1 else np.zeros((z.shape[1],z.shape[1]));B=np.cov(zb,rowvar=False) if len(zb)>1 else np.zeros((z.shape[1],z.shape[1]));rows.append({'group':group,'from_block':int(a) if float(a).is_integer() else float(a),'to_block':int(c) if float(c).is_integer() else float(c),'from_rows':len(za),'to_rows':len(zb),'centroid_shift':float(np.linalg.norm(cb-ca)),'covariance_shift_fro':float(np.linalg.norm(B-A,ord='fro')),'centroid_shift_per_dim':float(np.linalg.norm(cb-ca)/np.sqrt(max(z.shape[1],1)))})
    return rows


def hard_case_masks(y,committee,warning_rate=.20):
    y=np.asarray(y,int);q=np.asarray(committee,float);k=max(1,int(round(len(y)*float(warning_rate))));ord=np.argsort(-q,kind='mergesort');warn=np.zeros(len(y),bool);warn[ord[:k]]=True
    low=np.quantile(q,.10) if len(q) else 0;high=np.quantile(q,.90) if len(q) else 1
    return {'committee_false_negative_at20':(y==1)&(~warn),'committee_false_alarm_at20':(y==0)&warn,'committee_high_q_normal':(y==0)&(q>=high),'missed_failure_very_low_q':(y==1)&(q<=low)}


def _rank_percentile(s):
    s=np.asarray(s,float);o=np.argsort(s,kind='mergesort');r=np.empty(len(s),float);r[o]=np.arange(len(s));return r/max(len(s)-1,1)


def hard_case_audit_rows(y,committee,candidates:dict[str,np.ndarray],warning_rate=.20,extra_masks:dict[str,np.ndarray]|None=None):
    y=np.asarray(y,int);q=np.asarray(committee,float);masks=hard_case_masks(y,q,warning_rate);masks.update(extra_masks or {});base_rank=_rank_percentile(q);rows=[]
    for group,m in masks.items():
        m=np.asarray(m,bool)
        if not m.any():continue
        for name,s in candidates.items():
            s=np.asarray(s,float);cand_rank=_rank_percentile(s);top=np.zeros(len(s),bool);top[np.argsort(-s,kind='mergesort')[:max(1,int(round(len(s)*warning_rate)))]]=True
            rows.append({'group':group,'model':name,'rows':int(m.sum()),'failures':int(y[m].sum()),'prevalence':float(y[m].mean()),'committee_ap':safe_ap(y[m],q[m]),'candidate_ap':safe_ap(y[m],s[m]),'committee_auc':safe_auc(y[m],q[m]),'candidate_auc':safe_auc(y[m],s[m]),'mean_score_change':float(np.mean(s[m]-q[m])),'mean_rank_movement':float(np.mean(cand_rank[m]-base_rank[m])),'failure_mean_rank_movement':float(np.mean(cand_rank[m&(y==1)]-base_rank[m&(y==1)])) if np.any(m&(y==1)) else np.nan,'candidate_recall_at20_on_group_failures':float(np.mean(top[m&(y==1)])) if np.any(m&(y==1)) else np.nan})
    return rows


def mechanism_probe_rows(z,mechanism_targets,names=None,available=None,max_rows=30000,seed=42):
    """Post-hoc interpretability probe only; never used to tune base encoders."""
    z=np.asarray(z,float);t=np.asarray(mechanism_targets,float);m=_finite_rows(z,available)&np.isfinite(t).all(1);idx=np.where(m)[0]
    if len(idx)<20:return []
    if len(idx)>max_rows:idx=np.sort(np.random.default_rng(seed).choice(idx,max_rows,replace=False))
    zz=z[idx];tt=t[idx];names=list(names) if names is not None else [f'mechanism_{i}' for i in range(tt.shape[1])];rows=[];cut=max(10,int(.8*len(idx)));tr=np.arange(cut);va=np.arange(cut,len(idx))
    if not len(va):return []
    for j,name in enumerate(names):
        if np.nanstd(tt[:,j])<1e-8:continue
        model=Ridge(alpha=1.0).fit(zz[tr],tt[tr,j]);p=model.predict(zz[va]);rows.append({'mechanism':name,'rows':len(idx),'r2_from_z':float(r2_score(tt[va,j],p))})
    return rows


def residual_sign_probe(z,y,committee,available=None,max_rows=30000,seed=42):
    z=np.asarray(z,float);y=np.asarray(y,int);q=np.asarray(committee,float);m=_finite_rows(z,available);idx=np.where(m)[0]
    if len(idx)<50:return []
    if len(idx)>max_rows:idx=np.sort(np.random.default_rng(seed).choice(idx,max_rows,replace=False))
    target=(y[idx]-q[idx]>0).astype(int);cut=max(20,int(.8*len(idx)));tr=np.arange(cut);va=np.arange(cut,len(idx))
    if len(np.unique(target[tr]))<2 or len(va)<5:return []
    lr=LogisticRegression(C=.5,max_iter=2000,class_weight='balanced',random_state=seed).fit(z[idx][tr],target[tr]);p=lr.predict_proba(z[idx][va])[:,1]
    return [{'mechanism':'committee_residual_positive_sign','rows':len(idx),'ap_from_z':float(average_precision_score(target[va],p)) if len(np.unique(target[va]))>1 else np.nan}]


def redundancy_audit_rows(z,X,feature_names=None,available=None,max_features=24,max_rows=30000,seed=42):
    z=np.asarray(z,float);X=np.asarray(X,float);m=_finite_rows(z,available)&np.isfinite(X).all(1);idx=np.where(m)[0]
    if len(idx)<30:return []
    rng=np.random.default_rng(seed)
    if len(idx)>max_rows:idx=np.sort(rng.choice(idx,max_rows,replace=False))
    zz=z[idx];xx=X[idx];var=xx.var(0);sel=np.argsort(-var)[:min(max_features,xx.shape[1])];names=list(feature_names) if feature_names is not None else [f'x{i}' for i in range(xx.shape[1])];cut=max(20,int(.8*len(idx)));tr=np.arange(cut);va=np.arange(cut,len(idx));rows=[]
    for j in sel:
        if np.std(xx[va,j])<1e-10:continue
        model=Ridge(alpha=1.).fit(zz[tr],xx[tr,j]);rows.append({'direction':'z_to_raw','target':names[j],'r2':float(r2_score(xx[va,j],model.predict(zz[va])))})
    for j in range(min(z.shape[1],24)):
        if np.std(zz[va,j])<1e-10:continue
        model=Ridge(alpha=1.).fit(xx[tr],zz[tr,j]);rows.append({'direction':'raw_to_z','target':f'z[{j}]','r2':float(r2_score(zz[va,j],model.predict(xx[va])))})
    return rows


def rfm_eigenvector_loadings(eigvals,eigvecs,feature_names,top_k=10):
    vals=np.asarray(eigvals,float);vec=np.asarray(eigvecs,float);names=list(feature_names);rows=[];total=max(vals.sum(),1e-12);cum=0.
    # eigvecs accepted as d x k or list from metadata.
    if vec.ndim!=2:return []
    for j in range(min(vec.shape[1],len(vals))):
        cum+=max(vals[j],0);order=np.argsort(-np.abs(vec[:,j]))[:min(top_k,len(names))]
        for rank,i in enumerate(order,1):rows.append({'direction':j,'eigenvalue':float(vals[j]),'cumulative_metric_energy':float(cum/total),'rank':rank,'feature':names[i],'loading':float(vec[i,j]),'abs_loading':float(abs(vec[i,j]))})
    return rows


def nca_difficulty_rows(y,committee,native_score,faculty=None,bins=(0,.2,.4,.6,.8,1.0)):
    y=np.asarray(y,int);q=np.asarray(committee,float);p=np.asarray(native_score,float);difficulty=np.where(y==1,1-q,q);rows=[]
    edges=np.asarray(bins,float)
    for i in range(len(edges)-1):
        lo,hi=edges[i],edges[i+1];m=(difficulty>=lo)&(difficulty<(hi if i<len(edges)-2 else hi+1e-12))
        if not m.any():continue
        rows.append({'difficulty_bin':f'[{lo:.1f},{hi:.1f}]','rows':int(m.sum()),'failures':int(y[m].sum()),'prevalence':float(y[m].mean()),'committee_ap':safe_ap(y[m],q[m]),'native_metric_ap':safe_ap(y[m],p[m]),'delta_ap':safe_ap(y[m],p[m])-safe_ap(y[m],q[m]) if np.unique(y[m]).size==2 else np.nan,'committee_auc':safe_auc(y[m],q[m]),'native_metric_auc':safe_auc(y[m],p[m])})
    return rows


def direction_stability_rows(direction_sets:dict[str,np.ndarray],top_k=16):
    """Cosine/subspace stability across fold-level learned direction matrices."""
    keys=sorted(direction_sets);rows=[]
    for ai,a in enumerate(keys):
        A=np.asarray(direction_sets[a],float)
        for b in keys[ai+1:]:
            B=np.asarray(direction_sets[b],float);k=min(int(top_k),A.shape[1],B.shape[1])
            if k<1:continue
            QA,_=np.linalg.qr(A[:,:k]);QB,_=np.linalg.qr(B[:,:k]);sv=np.linalg.svd(QA.T@QB,compute_uv=False);rows.append({'a':a,'b':b,'k':k,'mean_principal_cosine':float(np.mean(sv)),'min_principal_cosine':float(np.min(sv)),'max_principal_angle_deg':float(np.degrees(np.arccos(np.clip(np.min(sv),-1,1))))})
    return rows
