"""Consolidated scientific reports; no holdout-based candidate selection."""
from __future__ import annotations
from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.metrics import average_precision_score
from s33_common import log,save_csv,save_json,binary_stats,freeze_threshold

HISTORIES={'committee':.350152,'suite25_committee_blend':.369526,'suite22':.369159,'suite31_full_history_hgb':.353807,'suite32_local_common':.346168,'council_13_faculty_archive':.364204,'council_IT_only':.444927}

def make_macro(scores,folds):
    v=scores[scores.fold.isin(folds)]
    return v.groupby('arm',sort=False).agg(cells=('ap','count'),faculties=('faculty','nunique'),ap=('ap','mean'),auc=('auc','mean'),brier=('brier','mean'),ece10=('ece10','mean'),f2_at_20=('f2_at_20','mean')).reset_index().sort_values('ap',ascending=False)

def pair(scores,candidate,reference,folds,subset=None):
    v=scores[scores.fold.isin(folds)]
    if subset is not None:v=v.merge(subset[['fold','faculty']].drop_duplicates(),on=['fold','faculty'],how='inner')
    a=v[v.arm==candidate][['fold','faculty','ap']];b=v[v.arm==reference][['fold','faculty','ap']]
    z=a.merge(b,on=['fold','faculty'],suffixes=('_a','_b'))
    if z.empty:return None
    d=z.ap_a-z.ap_b
    return dict(candidate=candidate,reference=reference,cells=len(d),mean_delta_ap=float(d.mean()),median_delta_ap=float(d.median()),wins=int((d>1e-10).sum()),losses=int((d < -1e-10).sum()),ties=int((abs(d)<=1e-10).sum()),worst=float(d.min()),best=float(d.max()))

def matched_placebo(preds,folds,minimum_rows=30,minimum_positives=5):
    cells=[]
    z=preds[(preds.fold.isin(folds))&(preds.arm.isin(['S4','P1']))&(preds.placebo_matchable>0)]
    for (fold,faculty),p in z.groupby(['fold','faculty']):
        a=p[p.arm=='S4'].sort_values('row_index');b=p[p.arm=='P1'].sort_values('row_index')
        if len(a)!=len(b) or not np.array_equal(a.row_index,b.row_index):raise ValueError('Placebo target row mismatch')
        if len(a)<minimum_rows or int(a.y.sum())<minimum_positives or int((1-a.y).sum())<minimum_positives:continue
        ap_a=average_precision_score(a.y,a.p);ap_b=average_precision_score(b.y,b.p)
        cells.append(dict(fold=fold,faculty=faculty,rows=len(a),failures=int(a.y.sum()),S4_ap=ap_a,P1_ap=ap_b,delta_ap=ap_a-ap_b))
    out=pd.DataFrame(cells)
    return out

def freeze(scores,preds,output):
    dev=scores[scores.fold<=5];m=make_macro(dev,[1,2,3,4,5]);
    byfold=scores[(scores.fold<=5)].groupby(['fold','arm']).ap.mean().unstack()
    byfac=scores[(scores.fold<=5)].groupby(['faculty','arm']).ap.mean().unstack()
    d1=pair(scores,'S4','S1',range(1,6));d2=pair(scores,'S3','S2',range(1,6))
    d3=pair(scores,'S4','P1',range(1,6))
    matched=matched_placebo(preds,range(1,6))
    save_csv(Path(output)/'development'/'matched_placebo_comparison.csv',matched)
    winfolds=int(((byfold.S4-byfold.S1)>0).sum());winfac=int(((byfac.S4-byfac.S1)>0).sum())
    placebo_scope=pd.read_csv(Path(output)/'development'/'placebo_scope.csv') if (Path(output)/'development'/'placebo_scope.csv').exists() else pd.DataFrame()
    matchable=float(placebo_scope.matchable_rows.sum()/max(1,placebo_scope.rows.sum())) if len(placebo_scope) else 0
    reasons=[]
    if (not d1) or d1['mean_delta_ap']<.01:reasons.append('S4-S1 below preregistered +0.010 AP')
    if (not d2) or d2['mean_delta_ap']<=0:reasons.append('No S3-S2 positive macro improvement')
    if winfolds<3:reasons.append('Fewer than 3 of 5 development folds improve S4-S1')
    if winfac<8:reasons.append('Fewer than 8 of 14 development faculties improve S4-S1')
    if matchable<.15:reasons.append('Insufficient matched placebo scope: under 15% of rows')
    # A subset-specific placebo contrast is reported separately; overall effect here is only a prefilter.
    if matched.empty or matched.delta_ap.mean()<=0:reasons.append('S4 has no positive matched-section placebo contrast at qualified development cells')
    selected='S4' if not reasons else 'S0'
    result=dict(selected=selected,promoted=selected=='S4',frozen_from='F1-F5_ONLY',reasons=reasons,dev_macro=m.to_dict('records'),deltas={'S4_minus_S1':d1,'S3_minus_S2':d2,'S4_minus_P1':d3},positive_folds=winfolds,positive_faculties=winfac,placebo_matchable_fraction=matchable,qualified_placebo_cells=len(matched),qualified_placebo_delta=float(matched.delta_ap.mean()) if len(matched) else None)
    save_json(Path(output)/'development'/'frozen_decision.json',result)
    log('DECISION',selected=selected,positive_folds=winfolds,positive_faculties=winfac,placebo_matched=matchable,reason=reasons)
    return result

def operational(preds,out):
    rows=[];thresholds=[]
    for (fac,arm),g in preds.groupby(['faculty','arm'],sort=False):
        dev=g[g.fold<=5];late=g[g.fold>=6]
        if dev.empty:continue
        for beta in (1,2):
            cut=freeze_threshold(dev.y.to_numpy(),dev.p.to_numpy(),beta)
            thresholds.append(dict(faculty=fac,arm=arm,beta=beta,threshold=cut,fit_folds='F1-F5'))
            for fold,z in g.groupby('fold',sort=False):
                train=dev[dev.fold<fold] if fold<=5 else dev
                th=freeze_threshold(train.y.to_numpy(),train.p.to_numpy(),beta) if len(train) else np.nan
                if not np.isfinite(th):
                    log('THRESHOLD_FREEZE',faculty=fac,fold=int(fold),arm=arm,beta=beta,reason='NO_PRIOR_DEVELOPMENT_LABELS')
                    continue
                bs=binary_stats(z.y.to_numpy(),z.p.to_numpy()>=th)
                students=z.loc[z.p>=th,'student_id'].nunique()
                rows.append(dict(fold=int(fold),faculty=fac,arm=arm,beta=beta,threshold=th,fit_folds=('F1-F'+str(fold-1) if fold<=5 else 'F1-F5'),flagged_students=int(students),alerts_per_1000=1000*bs['flag_rate'],**bs))
                log('THRESHOLD_FREEZE',faculty=fac,fold=int(fold),arm=arm,beta=beta,threshold=th,fit_folds=('F1-F'+str(fold-1) if fold<=5 else 'F1-F5'),f2=bs['f2'],recall=bs['recall'],flagged_students=students)
    save_csv(Path(out)/'reports'/'operational.csv',rows);save_csv(Path(out)/'development'/'frozen_thresholds.csv',thresholds)
    return pd.DataFrame(rows)

def bootstrap(preds,candidate,base,n_draw=100,unit='student_id',seed=42):
    d=preds[(preds.fold>=6)&(preds.arm.isin([candidate,base]))]
    if d.empty:return {}
    target=[];uniques=set()
    for (fold,faculty),cell in d.groupby(['fold','faculty']):
        a=cell[cell.arm==candidate].sort_values('row_index');b=cell[cell.arm==base].sort_values('row_index')
        if len(a)!=len(b) or not np.array_equal(a.row_index,b.row_index):raise ValueError('Bootstrap row mismatch')
        keys=a[unit].astype(str).to_numpy();uniques.update(keys)
        target.append((a.y.to_numpy(int),a.p.to_numpy(float),b.p.to_numpy(float),keys))
    units=np.array(sorted(uniques),dtype=object);rng=np.random.default_rng(seed)
    values=[];invalid=0
    for _ in range(n_draw):
        ids=rng.integers(0,len(units),size=len(units));counts=np.bincount(ids,minlength=len(units))
        counter=dict(zip(units[counts>0],counts[counts>0]))
        delta=[]
        for y,pa,pb,keys in target:
            w=np.fromiter((counter.get(k,0) for k in keys),dtype=float,count=len(keys))
            if np.sum(w[y==1])<=0 or np.sum(w[y==0])<=0:invalid+=1;delta=[];break
            delta.append(average_precision_score(y,pa,sample_weight=w)-average_precision_score(y,pb,sample_weight=w))
        if delta:values.append(np.mean(delta))
    result=dict(unit=unit,draws=n_draw,valid=len(values),invalid=invalid,ci95=([float(x) for x in np.quantile(values,[.025,.975])] if values else None),mean=float(np.mean(values)) if values else np.nan,note='Descriptive observed-period resampling; not a future-transport guarantee')
    return result

def final(scores,preds,root,decision,bootstrap_draws=0,include_archived=True):
    root=Path(root)
    for name,folds in [('development',list(range(1,6))),('retrospective',[6,7])]:
        v=make_macro(scores,folds)
        log('MACRO_RESULTS',stage=name,table='\n'+v.to_string(index=False,float_format=lambda x:f'{x:.6f}'))
        save_csv(root/'reports'/f'{name}_macro.csv',v)
        fold_wide=scores[scores.fold.isin(folds)].groupby(['fold','arm']).ap.mean().unstack()
        fac_wide=scores[scores.fold.isin(folds)].groupby(['faculty','arm']).ap.mean().unstack()
        log('FOLD_RESULTS',stage=name,table='\n'+fold_wide.to_string(float_format=lambda x:f'{x:.6f}'))
        log('FACULTY_RESULTS_MATRIX',stage=name,table='\n'+fac_wide.to_string(float_format=lambda x:f'{x:.6f}'))
        save_csv(root/'reports'/f'{name}_fold_matrix.csv',fold_wide.reset_index());save_csv(root/'reports'/f'{name}_faculty_matrix.csv',fac_wide.reset_index())
        for a,b in [('S4','S0'),('S4','S1'),('S3','S2'),('S4','P1'),('S2','S1')]:
            p=pair(scores,a,b,folds)
            if p:log('CONTEXT_INCREMENT',stage=name,**p)
    for name,folds in [('development',list(range(1,6))),('retrospective',[6,7])]:
        matched=matched_placebo(preds,folds)
        save_csv(root/'reports'/f'{name}_matched_placebo.csv',matched)
        log('PLACEBO_MATCH',stage=name,qualified_cells=len(matched),mean_delta_ap=(matched.delta_ap.mean() if len(matched) else np.nan),note='MATCHED_MULTISECTION_ONLY')
    op=operational(preds,root)
    for ref,ap in (HISTORIES.items() if include_archived else []):
        if ref=='council_13_faculty_archive' or ref=='council_IT_only':
            log('RETROSPECTIVE_HISTORICAL_COMPARISON',historical=ref,reference_ap=ap,note='DIFFERENT_SUPPORT_NO_MACRO_SUBTRACTION')
        else:
            p=make_macro(scores,[6,7]);cand=float(p.loc[p.arm==decision['selected'],'ap'].iloc[0]);log('RETROSPECTIVE_HISTORICAL_COMPARISON',historical=ref,reference_ap=ap,selected=decision['selected'],candidate_ap=cand,descriptive_delta=cand-ap,note='ARCHIVED_NO_ROW_PAIRED_CI')
    if bootstrap_draws:
        results=[]
        for unit in ('student_id','offering_key'):
            z=bootstrap(preds,decision['selected'],'S0',bootstrap_draws,unit,seed=42)
            results.append(z);log('UNCERTAINTY',**z)
        save_json(root/'reports'/'uncertainty.json',results)
    finalmacro=make_macro(scores,[6,7]);save_json(root/'reports'/'final_verdict.json',dict(frozen_decision=decision,retrospective=finalmacro.to_dict('records'),archived_reference=HISTORIES if include_archived else {},holdout_status='REPEATEDLY_INSPECTED_RETROSPECTIVE_NOT_PRISTINE' if include_archived else 'SYNTHETIC_SMOKE_ONLY'))
    log('INSIGHT',message='Faculty-/fold-macro gains are not causal peer effects; pay special attention to matched placebo, target support, rosters and archived comparator support.')
