ó
    Y}³jv[  ã                  ó¨  • S SK Jr  S SKJr  S SKrS SKrS SKJ	r	J
r
JrJr  Sr0 SS_SS	_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_S S!_S"S#_S$S%_S&S'_S(S)S*S+S,.ErS- rS. rS/ rSKS0 jrSLS1 jrSMS2 jrSKS3 jrS4 rS5 rS6 rS7 rS8 rS9 rS: rS; rS< rSNS= jrS> r S? r!SKS@ jr"SKSOSA jjr#SPSB jr$SQSC jr%SQSD jr&SE r'SRSF jr(SG r)SH r*SI r+SSSJ jr,g)Té    )Úannotations)ÚPathN)Úaverage_precision_scoreÚbrier_score_lossÚprecision_recall_curveÚroc_auc_score)gš™™™™™¹?g333333Ã?çš™™™™™É?ç      Ð?Úsuite21_guarded)Úsuite21_results_dirÚ!suite21_guarded_temporal_residualÚsuite21_temporal)r   Úsuite21_temporal_residualÚsuite21_flat)r   Úsuite21_flat_residualÚshared)Úshared_results_dirÚresidual_moe_sharedÚsuite7)Úsuite7_results_dirÚresidual_moe_shared_privateÚsuite8)Úsuite8_results_dirÚresidual_moe_true_specialistÚsuite9_candidate)Úsuite9_results_dirÚ"residual_moe_competitive_candidateÚsuite9_final)r   Úresidual_moe_competitive_finalÚsuite10_candidate)Úsuite10_results_dirÚsemantic_specialist_candidateÚsuite10_final)r!   Úsemantic_specialist_finalÚsuite11_candidate)Úsuite11_results_dirÚopportunity_latent_candidateÚsuite11_final)r&   Úopportunity_latent_finalÚsuite12)Úsuite12_results_dirÚ"suite12_collaborative_evidence_moeÚsuite13)Úsuite13_results_dirÚ.suite13_committee_preserving_collaborative_moeÚsuite14)Úsuite14_results_dirÚsuite14_authority_safe_finalÚsuite16)Úsuite16_results_dirÚsuite16_hgb_finalÚsuite17)Úsuite17_results_dirÚsuite17_hgb_final)Úsuite18_results_dirÚsuite18_hgb_final)Úsuite20_results_dirÚsuite20_transformer)r;   Úsuite20_gru)r;   Úsuite20_flat)Úsuite18r<   r=   r>   c                óÈ   • [         R                  " U [        5      n [         R                  " U 5      R                  S:”  a  [        [        X5      5      $ [         R                  $ ©Né   )ÚnpÚasarrayÚintÚuniqueÚsizeÚfloatr   Únan©ÚyÚps     ÚK/root/disk_3tb/earlywarning/30072026/suite22_sftr_tabm/suite22_reporting.pyÚsafe_aprN   )   sA   € Ü
�
Š
�1”cÓ€AÜ35·9²9¸Q³<×3DÑ3DÀqÓ3HŒ5Ô(¨Ó.Ó/ÐTÌbÏfÉfÐTó    c                óÈ   • [         R                  " U [        5      n [         R                  " U 5      R                  S:”  a  [        [        X5      5      $ [         R                  $ rA   )rC   rD   rE   rF   rG   rH   r   rI   rJ   s     rM   Úsafe_aucrQ   .   s@   € Ü
�
Š
�1”cÓ€AÜ)+¯ª°1«×):Ñ):¸QÓ)>Œ5”˜qÓ$Ó%ÐJÄBÇFÁFÐJrO   c                óJ  • [         R                  " U [        5      n [        U 5      n[	        S[        U[        [        [        U5      U-  5      5      5      5      n[         R                  " U[         R                  5      nU(       a  SU[         R                  " U * SS9S U '   U$ )Nr   rB   Ú	mergesort©Úkind)rC   rD   rH   ÚlenÚmaxÚminrE   ÚroundÚzerosÚint8Úargsort)ÚscoreÚrateÚnÚkÚpreds        rM   Úwarn_by_raterb   3   sz   € Ü�JŠJ�uœeÓ$€EÜˆE‹
€AÜˆAŒs�1”cœ%¤ d£¨a¡Ó0Ó1Ó2Ó3€AÜ�8Š8�A”r—w‘wÓ€DÞØ9:ˆŒR�ZŠZ˜˜ [Ñ1°"°1Ð5Ñ6Ø€KrO   c                ó¢  • [         R                  " U [        5      n [         R                  " U[        5      n[        U S:H  US:H  -  R                  5       5      n[        U S:H  US:H  -  R                  5       5      n[        U S:H  US:H  -  R                  5       5      n[        U S:H  US:H  -  R                  5       5      nU[	        X4-   S5      -  nU[	        X5-   S5      -  nSU-  U-  [	        Xx-   S5      -  n	X"-  n
SU
-   U-  U-  [	        X§-  U-   S5      -  nX4XVXxX›X6-   [	        [        U 5      S5      -  [        UR                  5       5      S.
$ )NrB   r   é   çê-�™—q=)
ÚtpÚfpÚfnÚtnÚ	precisionÚrecallÚf1ÚfbetaÚaccuracyÚ	flag_rate)rC   rD   rE   ÚsumrW   rV   rH   Úmean)rK   ra   Úbetarf   rg   rh   ri   rL   Úrrl   Úb2rm   s               rM   Úclassification_metricsru   =   sA  € Ü
�
Š
�1”cÓ€AÜ�:Š:�dœCÓ €DÜ	ˆq�A‰v˜$ !™)Ñ$×)Ñ)Ó+Ó	,€BÜ	ˆq�A‰v˜$ !™)Ñ$×)Ñ)Ó+Ó	,€BÜ	ˆq�A‰v˜$ !™)Ñ$×)Ñ)Ó+Ó	,€BÜ	ˆq�A‰v˜$ !™)Ñ$×)Ñ)Ó+Ó	,€BØ
ŒS�‘˜!‹_Ñ€AØ
ŒS�‘˜!‹_Ñ€AØ	
ˆQ‰�‰”S˜™ Ó&Ñ	&€BØ	‰€BØ�‰V�q‰L˜1Ñœs 2¡6¨A¡:¨uÓ5Ñ5€Eà "Ø¨2Ø‘W¤¤C¨£F¨A£Ñ.¼UÀ4Ç9Á9Ã;Ó=Oñð rO   c                ó.   • [        U [        X5      U5      $ ©N)ru   rb   )rK   r]   r^   rr   s       rM   Úbudget_metricsrx   P   s   € Ü! !¤\°%Ó%>ÀÓEÐErO   c           
     óª  • [         R                  " U [        5      n [         R                  " [         R                  " U[        5      SS5      n[         R
                  " SS[        U5      S-   5      n[        [        U 5      S5      nSn[        [        U5      S-
  5       H§  nU[        U5      S-
  :X  a  XU   :¬  XUS-      :*  -  nOXU   :¬  XUS-      :  -  nUR                  5       (       d  MQ  XWR                  5       U-  [        [	        X   R                  5       5      [	        X   R                  5       5      -
  5      -  -  nM©     [	        U5      $ )Nr   rB   ç        rd   )rC   rD   rE   ÚcliprH   ÚlinspacerW   rV   ÚrangeÚanyrp   Úabsrq   )rK   r]   ÚbinsÚedgesÚtotalÚeceÚiÚms           rM   Úexpected_calibration_errorr†   T   s  € Ü
�
Š
�1”cÓ€AÜ�GŠG”B—J’J˜u¤eÓ,¨a°Ó3€EÜ�KŠK˜˜1œc $›i¨!™mÓ,€EÜ”�A“˜‹N€EØ
€CÜ”3�u“: ‘>Ö"ˆØ”�E“
˜Q‘ÓØ ™(Ñ" u°a¸!±e±Ñ'<Ñ=‰Aà ™(Ñ" u°Q¸±U©|Ñ';Ñ<ˆAØ�5‰5�7‹7Ø—E‘E“G˜e‘O¤s¬5°±·±³Ó+AÄEÈ!É$Ï)É)Ë+ÓDVÑ+VÓ'WÑWÑWŠCñ #ô �‹:ÐrO   c                óÂ  • [         R                  " U[        5      n[         R                  " U[        5      n[        U 5      X[	        U5      [        UR                  5       5      [        UR                  5       5      [        X45      [        X45      [        [        U[         R                  " USS5      5      5      [        X4S5      [        [         R                  " U5      5      [        [         R                  " U5      [         R                  " U5      -
  5      S.n[         H:  n[        X4Xu5      n[        [        SU-  5      5      n	S H  n
XŠ   Xj SU	 3'   M     M<     U$ )Nr   rB   é
   )ÚfoldÚfacultyÚmodelÚrowsÚfailuresÚ
prevalenceÚapÚaucÚbrierÚece10Ú
mean_scoreÚcalibration_gapéd   )rj   rk   rl   rm   ro   Ú_at_)rC   rD   rE   rH   rV   rp   rq   rN   rQ   r   r{   r†   ÚBUDGETSrx   rY   )r‰   rŠ   r‹   rK   r]   rr   ÚrowÚbÚzÚtagr`   s              rM   Ú
metric_rowrœ   d   s  € Ü
�
Š
�1”cÓ€AÜ�JŠJ�uœeÓ$€Eä�D“	 gÜ�A“¤C¨¯©«£LÄÀaÇfÁfÃhÃÜ�aÓ¬°Ó(:ÜÔ'¨¬2¯7ª7°5¸!¸QÓ+?Ó@ÓAÜ+¨A°bÓ9ÜœBŸGšG E›NÓ+Ü ¤§¢¨£´"·'²'¸!³*Ñ!<Ó=ñ€C÷ ˆÜ˜1 QÓ-ˆÜ”%˜˜a™“.Ó!ˆÛDˆAØ#$¡4ˆC�#�T˜#˜�Ó ó Eñ ð
 €JrO   c                ó  • U (       d  g [        U 5      SU 3-  U S3-  nUR                  5       (       d  g [        R                  " U5      nX6;  a  g SU;   az  UR                  R                  [        5      n[        U5      [        U5      :w  d5  [        R                  " U[        R                  " U[        5      5      (       d  [        SU 35      e[        U5      [        U5      :w  a  [        SU 35      eXc   R                  [        5      $ )NÚfold_ú_scores.csvÚactualzReference alignment mismatch: zReference row mismatch: )r   ÚexistsÚpdÚread_csvr    Úto_numpyrE   rV   rC   Úarray_equalrD   Ú
ValueErrorrH   )Úrootr‰   rŠ   ÚcolumnrK   rL   Údfr    s           rM   Ú_optional_scoresrª   x   sÛ   € ÞØÜˆT‹
�u˜T˜F�^Ñ#¨¨	°Ð&=Ñ=€AØ�8‰8�:‰:ØÜ	�Š�Q‹€BØÓØØ�2ƒ~Ø—‘×#Ñ#¤CÓ(ˆÜˆv‹;œ#˜a›&Ó ¬¯ª°v¼r¿zºzÈ!ÌSÓ?Q×(RÑ(RÜÐ=¸a¸SÐAÓBÐBÜ
ˆ2ƒw”#�a“&ÓÜÐ3°A°3Ð7Ó8Ð8Ø‰:×ÑœuÓ%Ð%rO   c                ó
  • U (       d  g [        U 5      U SU S3-  nUR                  5       (       d  g [        R                  " U5      nSS1nUR	                  UR
                  5      (       d  g [        U5      [        U5      :w  dR  [        R                  " UR                  R                  [        5      [        R                  " U[        5      5      (       d  [        SU 35      eUR                  R                  [        5      $ )NÚ_fold_z_test_results.csvÚactual_failure_lt5Úfailure_probability_lt5zV2 alignment mismatch: )r   r¡   r¢   r£   ÚissubsetÚcolumnsrV   rC   r¥   r­   r¤   rE   rD   r¦   r®   rH   )r§   r‰   rŠ   rK   rL   r©   Úreqs          rM   Ú	_v2_scorer²   Š   sË   € ÞØÜˆT‹
˜�y  t fÐ,=Ð>Ñ>€AØ�8‰8�:‰:ØÜ	�Š�Q‹€BØÐ!:Ð
;€CØ�<‰<˜Ÿ
™
×#Ñ#ØÜ
ˆ2ƒw”#�a“&Ó¤§¢¨r×/DÑ/D×/MÑ/MÌcÓ/RÔTV×T^ÒT^Ð_`ÔbeÓTf× gÑ gÜÐ2°1°#Ð6Ó7Ð7Ø×%Ñ%×.Ñ.¬uÓ5Ð5rO   c                ó  • U (       d  g [        U 5      U SU S3-  [        U 5      SU 3-  U S3-  /n[        S U 5       S 5      nUc  g [        R                  " U5      nSU;   a  SO	SU;   a  SOS nUb  SU;  a  g [	        U5      [	        U5      :w  dR  [
        R                  " UR                  R                  [        5      [
        R                  " U[        5      5      (       d  [        SU 35      eXg   R                  [        5      $ )	Nr¬   rŸ   rž   c              3  óR   #   • U  H  oR                  5       (       d  M  Uv •  M     g 7frw   )r¡   )Ú.0Úxs     rM   Ú	<genexpr>Ú!_council_score.<locals>.<genexpr>    s   é € Ð2š�A§x¡x§z�a‰ašùs   ‚'ž	'ÚfinalÚcompetence_councilr    zCouncil alignment mismatch: )r   Únextr¢   r£   rV   rC   r¥   r    r¤   rE   rD   r¦   rH   )r§   r‰   rŠ   rK   Ú
candidatesrL   r©   Ú	score_cols           rM   Ú_council_scorer¾   ™   sû   € ÞØäˆT‹
˜�y  t f¨KÐ8Ñ8ÜˆT‹
�u˜T˜F�^Ñ#¨¨	°Ð&=Ñ=ð€Jô 	Ñ2™Ó2°DÓ9€AØ�yØÜ	�Š�Q‹€BØ" b›=‘ÐG[Ð_aÓGaÑ/CÐgk€IØÑ˜H¨BÓ.ØÜ
ˆ2ƒw”#�a“&Ó¤§¢¨r¯y©y×/AÑ/AÄ#Ó/FÌÏ
Ê
ÐSTÔVYÓHZ× [Ñ [ÜÐ7¸°sÐ;Ó<Ð<Ø‰=×!Ñ!¤%Ó(Ð(rO   c                ó<  • S[         R                  " U[        5      0n[        R	                  5        H(  u  nu  px[        [        XS 5      XXƒ5      n	U	c  M$  X•U'   M*     [        [        U SS 5      XU5      n	U	b  X•S'   [        [        U SS 5      XU5      n	U	b  X•S'   U$ )NÚ	committeeÚv2_results_dirÚv2Úcouncil_results_dirÚcouncil)	rC   rD   rH   ÚREFERENCE_COLUMNSÚitemsrª   Úgetattrr²   r¾   )
Úargsr‰   rŠ   rK   rÀ   ÚscoresÚlabelÚarg_namer¨   rš   s
             rM   Úreference_scoresrÌ   ¬   s¤   € Øœ2Ÿ:š: i´Ó7Ð8€FÜ%6×%<Ñ%<Ö%>Ñ!ˆÑ!�ÜœW T°TÓ:¸DÈ6ÓUˆØ‹=Ø�5‹Mñ &?ô 	”'˜$Ð 0°$Ó7¸ÈÓJ€AØ�}Øˆt‰Ü”w˜tÐ%:¸DÓAÀ4ÐRSÓT€AØ�}ØˆyÑØ€MrO   c                ó¾  • [         R                  " U 5      nX"R                  R                  [	        U5      5         n/ nUR                  SS/5       Hs  u  u  pEnXE[        U5      [        UR                  R                  5       5      S.nS H$  n[        Xh   R                  5       5      USU 3'   M&     UR                  U5        Mu     [         R                  " U5      $ )NrŠ   r‹   )rŠ   r‹   ÚfoldsrŒ   )
r�   r�   r‘   r’   r“   r”   Úprecision_at_20Úrecall_at_20Úf1_at_20Úfbeta_at_20Ú
mean_fold_)r¢   Ú	DataFramer‰   ÚisinÚlistÚgroupbyrV   rE   rŒ   rp   rH   rq   Úappend)	ÚmetricsrÎ   ÚdrŒ   rŠ   r‹   Úgr˜   Úcs	            rM   Úmean_fold_summaryrÝ   »   s»   € Ü
�Š�WÓ€AØ	�&‰&�+‰+”d˜5“kÓ
"Ñ#€AØ€DØ Ÿy™y¨)°WÐ)=Ö>ÑÑˆ˜!à¼¸Q»Ü˜Ÿ™Ÿ
™
›Ó%ñ
ˆó
ˆAô %*¨!©$¯)©)«+Ó$6ˆC�*˜Q˜CÐ Ó!ñ	
ð
 	�‰�CÖñ  ?ô �<Š<˜ÓÐrO   c                ó(  • / nU R                  S5       HX  u  p#X#R                  R                  5       S.nS H   n[        X5   R	                  5       5      XE'   M"     UR                  U5        MZ     [        R                  " U5      R                  SSS9$ )Nr‹   ©r‹   Ú	faculties)
Úmean_fold_apÚmean_fold_aucÚmean_fold_brierÚmean_fold_ece10Úmean_fold_mean_scoreÚmean_fold_calibration_gapÚmean_fold_precision_at_20Úmean_fold_recall_at_20Úmean_fold_f1_at_20Úmean_fold_fbeta_at_20rá   F©Ú	ascending)	r×   rŠ   ÚnuniquerH   rq   rØ   r¢   rÔ   Úsort_values)ÚmeanfrŒ   r‹   rÛ   r˜   rÜ   s         rM   Úmacro_summaryrð   Í   s‚   € Ø€DØ—M‘M 'Ö*‰ˆØ¯I©I×,=Ñ,=Ó,?Ñ@ˆó
ˆAô ˜1™4Ÿ9™9›;Ó'ˆC‹Fñ	
ð
 	�‰�CÖñ +ô �<Š<˜Ó×)Ñ)¨.ÀEÐ)ÐJÐJrO   c           
     ó¢  • X R                   U:H     n/ n[        [        U R                   5      U1-
  5       GH   nX R                   U:H     nUR                  USSS9n[	        U5      (       d  M8  UR
                  UR                  -
  nUR                  UR                  -
  nUR                  0 SU_SU_S[	        U5      _S[        UR
                  R                  5       5      _S[        UR                  R                  5       5      _S	[        UR                  5       5      _S
[        UR                  5       5      _S[        UR                  S5      5      _S[        UR                  5       5      _S[        US:„  R!                  5       5      _S[        US:„  R                  5       5      _S[        UR                  5       5      _S[        UR                  5       5      _S[        UR                  5       5      _S[        US:„  R!                  5       5      _S[        US:„  R                  5       5      _5        GM     ["        R$                  " U5      $ )NrŠ   )Ú_candÚ_ref)ÚonÚsuffixesÚ	candidateÚ	referencerà   Úcandidate_macro_apÚreference_macro_apÚmean_delta_apÚmedian_delta_apÚq25_delta_apr
   Úworst_delta_apÚap_winsr   Úap_win_rateÚmean_delta_f2Úmedian_delta_f2Úworst_delta_f2Úf2_winsÚf2_win_rate)r‹   ÚsortedÚsetÚmergerV   Úmean_fold_ap_candÚmean_fold_ap_refÚmean_fold_fbeta_at_20_candÚmean_fold_fbeta_at_20_refrØ   rH   rq   ÚmedianÚquantilerX   rE   rp   r¢   rÔ   )	rï   rö   ÚarŒ   Úrefr™   rš   ÚdaÚdf2s	            rM   Úhead_to_headr  Ú   s*  € Ø�k‰k˜YÑ&Ñ'€AØ€DÜ”c˜%Ÿ+™+Ó&¨)¨Ñ4×5ˆØ—+‘+ Ñ$Ñ%ˆØ�G‰G�A˜)Ð.?ˆGÐ@ˆÜ�1�v‰vÙØ× Ñ  1×#5Ñ#5Ñ5ˆØ×*Ñ*¨Q×-HÑ-HÑHˆØ�‰ð 

Ø˜ð

Ø$/°ð

Ø6AÄ3ÀqÃ6ð

à ¤%¨×(;Ñ(;×(@Ñ(@Ó(BÓ"Cð

ð !¤%¨×(:Ñ(:×(?Ñ(?Ó(AÓ"Bð

ð œU 2§7¡7£9Ó-ð	

ð 0AÄ%ÈÏ	É	ËÓBTð	

ð
 œE "§+¡+¨cÓ"2Ó3ð

ð
 6FÄuÈRÏVÉVËXÃð

ð ”s˜B ™FŸ<™<›>Ó*ð

ð -:¼5À"ÀqÁ&ÇÁÃÓ;Qð

ð œU 3§8¡8£:Ó.ð

ð 1BÄ5ÈÏÉËÓCVð

ð œe C§G¡G£IÓ.ð

ð 1:¼3ÀÀaÁ¿}¹}»Ó;Oð

ð œ5 #¨¡'§¡Ó!1Ó2ð

÷ 
	ñ 6ô$ �<Š<˜ÓÐrO   c                ó¸  • [         R                  " U[        5      n[         R                  " U[        5      n[         R                  " U[        5      n[        R
                  " U5      R                  SS9R                  [        5      n[        R
                  " U5      R                  SS9R                  [        5      n[        U5      S:”  aX  [         R                  " U5      (       a=  [         R                  " U5      (       a"  [        [         R                  " Xx5      S   5      O[         R                  n	[        S[        [        S[        U5      -  5      5      5      n
[        [         R                  " U* SS9S U
 R!                  5       5      n[        [         R                  " U* SS9S U
 R!                  5       5      n[        U 5      U[        U5      [#        X%5      [#        X&5      [#        X&5      [#        X%5      -
  [        [         R$                  " [         R&                  " Xe-
  5      5      5      [        [         R$                  " Xe-
  5      5      U	[        [        X¼-  5      [        U
S5      -  5      [        [         R$                  " XeS	-   :„  5      5      [        [         R$                  " XeS	-
  :  5      5      US:H  R)                  5       (       a'  [        [         R$                  " Xe-
  US:H     5      5      O[         R                  US
:H  R)                  5       (       a)  [        [         R$                  " Xe-
  US
:H     5      5      S.$ [         R                  S.$ )NÚaverage)Úmethodrd   ©r   rB   rB   r	   rS   rT   ç:Œ0âŽyE>r   )r‰   rŠ   rŒ   Úcommittee_apÚcandidate_apÚdelta_apÚprobability_maeÚprobability_mean_deltaÚ	rank_corrÚtop20_overlapÚincrease_fractionÚdecrease_fractionÚfailure_mean_deltaÚnormal_mean_delta)rC   rD   rE   rH   r¢   ÚSeriesÚrankr¤   rV   ÚstdÚcorrcoefrI   rW   rY   r  r\   ÚtolistrN   rq   r   r~   )r‰   rŠ   rK   rÀ   rö   ÚqrL   ÚrqÚrpr  r`   ÚiqÚips                rM   Úretention_diagnosticsr-  ò   sR  € Ü
�
Š
�1”cÓ€AÜ
�
Š
�9œeÓ$€AÜ
�
Š
�9œeÓ$€AÜ	�Š�1‹×	Ñ	 )Ð	Ð	,×	5Ñ	5´eÓ	<€BÜ	�Š�1‹×	Ñ	 )Ð	Ð	,×	5Ñ	5´eÓ	<€BÜ47¸³F¸Q³JÄ2Ç6Â6È"Ç:Á:ÔRT×RXÒRXÐY[×R\ÑR\””b—k’k "Ó)¨$Ñ/Ô0Ôbd×bhÑbh€IÜˆAŒs”5˜œs 1›v™Ó&Ó'Ó(€AÜ	ŒR�ZŠZ˜˜ Ñ-¨b¨qÐ1×8Ñ8Ó:Ó	;€BÜ	ŒR�ZŠZ˜˜ Ñ-¨b¨qÐ1×8Ñ8Ó:Ó	;€Bä�D“	 g´s¸1³vÜ ›´w¸q³}Ü˜A“M¤G¨A£MÑ1Ü ¤§¢¬¯ª°±«Ó!7Ó8Ü"'¬¯ª°±«Ó"7ØÜœs 2¡7›|¬c°!°Q«iÑ7Ó8Ü"¤2§7¢7¨1°4©x©<Ó#8Ó9Ü"¤2§7¢7¨1°4©x©<Ó#8Ó9ØBCÀqÁ&ÇÁÇÁœe¤B§G¢G¨Q©U°A¸±F©OÓ$<Ô=ÔUW×U[ÑU[ØABÀaÁÇÁÇÁœU¤2§7¢7¨A©E°1¸±6©?Ó#;Ó<ñð ô UW×TZÑTZñð rO   c           	     óÆ  • [         R                  " U[        5      n[         R                  " U[        5      n[         R                  " U[        5      n[         R                  " U[        5      R	                  S5      n	[         R                  " U[        5      n
[        U5      S:”  aV  [         R                  " U5      S:”  a<  [         R                  " U
5      S:”  a"  [        [         R                  " Xz5      S   5      O[         R                  n0 S[        U 5      _SU_S[        U5      _S[        UR	                  5       5      _S	[        [         R                  " U5      5      _S
US:H  R                  5       (       a  [        XrS:H     R	                  5       5      O[         R                  _SUS:H  R                  5       (       a  [        XrS:H     R	                  5       5      O[         R                  _SU_S[        UR	                  5       5      _S[        U	R	                  5       5      _S[        [         R                  " U	5      R	                  5       5      _S[        [         R                  " U	S:„  5      5      _S[        [         R                  " U	S:  5      5      _SUS:H  R                  5       (       a  [        X’S:H     R	                  5       5      O[         R                  _SUS:H  R                  5       (       a  [        X’S:H     R	                  5       5      O[         R                  _SUS:H  R                  5       (       a'  [        [         R                  " X’S:H     S:  5      5      O[         R                  _SUS:H  R                  5       (       a'  [        [         R                  " X’S:H     S:„  5      5      O[         R                  _US:H  R                  5       (       a'  [        [         R                  " X’S:H     S:  5      5      O[         R                  U
S:¬  R                  5       (       a!  [        XŠS:¬     R	                  5       5      S.E$ [         R                  S.E$ )NrB   rd   r   r  r‰   rŠ   rŒ   Úauthority_meanÚauthority_medianÚauthority_failure_meanÚauthority_normal_meanÚauthority_committee_corrÚ
guard_meanÚcorrection_meanÚcorrection_abs_meanÚcorrection_positive_fractionr  Úcorrection_negative_fractiong:Œ0âŽyE¾Úfailure_correction_meanÚnormal_correction_meanÚ$failure_negative_correction_fractionÚ$failure_positive_correction_fractionç      à?)Ú#normal_negative_correction_fractionÚhigh_committee_guard_mean)rC   rD   rE   rH   rq   rV   r%  r&  rI   r  r~   r   )r‰   rŠ   rK   Ú	authorityÚguardÚcorrection_membersrÀ   r  rÛ   rÜ   r(  Úcorr_qs               rM   Úresidual_diagnosticsrD    ss  € Ü
�
Š
�1”cÓ€AÜ
�
Š
�9œeÓ$€AÜ
�
Š
�5œ%Ó €AÜ
�
Š
Ð%¤uÓ-×2Ñ2°1Ó5€AÜ
�
Š
�9œeÓ$€AÜ/2°1«v¸«z¼b¿fºfÀQ»iÈ!»mÔPR×PVÒPVÐWXÓPYÐ\]ÓP]ŒU”2—;’;˜qÓ$ TÑ*Ô+Ôce×ciÑci€FðØ”�D“	ðØ$ gðØ/5´s¸1³vðàœ% §¡£›/ðà+=¼uÄRÇYÂYÈqÃ\Ó?Rðð 	!¸aÀ1¹f¿\¹\¿^¹^¤%¨¨q©&©	¯©Ó(8Ô"9ÔQS×QWÑQWðð 	 ¸QÀ!¹V¿L¹L¿N¹N¤ q¨a©¡y§~¡~Ó'7Ô!8ÔPR×PVÑPVð	ð
 	# Fðð 	”e˜AŸF™F›H“oðð 	œ5 §¡£›?ðð -BÄ5ÌÏÊÐPQËÏÉÓIYÓCZðð 	'¬¬b¯gªg°a¸$±hÓ.?Ó(@ðð 	'¬¬b¯gªg°a¸%±iÓ.@Ó(Aðð 	"¸qÀA¹v¿l¹l¿n¹n¤5¨°©6©¯©Ó)9Ô#:ÔRT×RXÑRXðð 	!¸aÀ1¹f¿\¹\¿^¹^¤%¨¨q©&©	¯©Ó(8Ô"9ÔQS×QWÑQWðð 	/ÐVWÐ[\ÑV\×UaÑUa×UcÑUc´´b·g²g¸aÀQÁ¹iÈ%Ñ>OÓ6PÔ0QÔik×ioÑioðð 	/ÐUVÐZ[ÑU[×T`ÑT`×TbÑTb´´b·g²g¸aÀQÁ¹iÈ$Ñ>NÓ6OÔ0PÔhj×hnÑhnðð VWÐZ[ÑU[×T`ÑT`×TbÑTb¬u´R·W²W¸QÀA¹v¹YÈÑ=NÓ5OÔ/PÔhj×hnÑhnØBCÀrÁ'ÇÁÇÁ¤U¨1°"©W©:¯?©?Ó+<Ó%=òð ô WY×V\ÑV\òð rO   c           	     óâ  • [         R                  " U [        5      n [        X5      n[        X#5      nU S:H  n[        XdS:H  -  US:H  -  R	                  5       5      [        XdS:H  -  US:H  -  R	                  5       5      [        XdS:H  -  US:H  -  R	                  5       5      [        U S:H  US:H  -  US:H  -  R	                  5       5      [        U S:H  US:H  -  US:H  -  R	                  5       5      S.$ )NrB   r   )Úrescued_failuresÚlost_failuresÚretained_failuresÚnew_false_positivesÚremoved_false_positives)rC   rD   rE   rb   rp   )rK   rÀ   rö   r^   ÚqcÚpcÚ	positivess          rM   Útop_budget_transitionsrN  %  sê   € Ü
�
Š
�1”cÓ€AÜ	�iÓ	&€BÜ	�iÓ	&€BØ�Q‘€Iä °A©gÑ!6¸"À¹'Ñ!B× GÑ GÓ IÓJÜ˜i°©7Ñ3°r¸Q±wÑ?×DÑDÓFÓGÜ  )°Q©wÑ"7¸2À¹7Ñ"C×!HÑ!HÓ!JÓKÜ" Q¨!¡V°°a±Ñ$8¸BÀ!¹GÑ$D×#IÑ#IÓ#KÓLÜ#&¨¨a©°B¸!±GÑ(<ÀÀaÁÑ(H×'MÑ'MÓ'OÓ#Pñð rO   c                óê  • [         R                  " U[        5      n[         R                  " U[        5      n[         R                  " U[        5      n[	        U5      S:X  a  [
        R                  " 5       $ [
        R                  " [
        R                  " U5      [        S[        S[	        [         R                  " U5      5      5      5      SSS9n/ n[        [
        R                  " UR                  5       5      5       HÚ  nUR                  5       U:H  n	UR                  [        U 5      U[        U5      [        U	R!                  5       5      [        XY   R#                  5       5      [        X)   R#                  5       5      [%        X)   X9   5      ['        X)   X9   5      [        [         R(                  " X9   X)   -
  5      R#                  5       5      S.	5        MÜ     [
        R                  " U5      $ )Nr   é   rB   FÚdrop)r(  ÚlabelsÚ
duplicates)	r‰   rŠ   Úuncertainty_quartilerŒ   Úmean_uncertaintyrŽ   r�   r�   Úmean_abs_error)rC   rD   rE   rH   rV   r¢   rÔ   Úqcutr#  rX   rW   rF   r  Údropnar¤   rØ   rp   rq   rN   rQ   r   )
r‰   rŠ   rK   r]   ÚuncertaintyÚur(  rŒ   Úbucketr…   s
             rM   Úuncertainty_diagnosticsr\  3  sO  € Ü
�
Š
�1”cÓ€AÜ�JŠJ�uœeÓ$€EÜ
�
Š
�;¤Ó&€AÜ
ˆ1ƒv�ƒ{Ü�|Š|‹~ÐÜ
�Š”—	’	˜!“¤ A¤s¨1¬c´"·)²)¸A³,Ó.?Ó'@Ó AÈ%Ð\bÑc€AØ€DÜœŸš 1§8¡8£:Ó.Ö/ˆØ�J‰J‹L˜FÑ"ˆØ�‰Ü˜“I¨'Ì3ÈvË;Ü˜Ÿ™›“L´e¸A¹D¿I¹I»KÓ6HÜ ¡§	¡	£Ó,´G¸A¹DÀ%Á(Ó4KÜ˜A™D %¡(Ó+¼uÄRÇVÂVÈEÉHÐWXÑW[ÉOÓE\×EaÑEaÓEcÓ?dñ	
ö 	ñ 0ô �<Š<˜ÓÐrO   c                óh  • UR                   S:X  a  UR                  S5      OUn[        R                  " U[        5      n[        R                  " U[
        5      n/ n[        U5       H³  nUS S 2U4   n	U	R                  5       (       d  M#  SU	4SX’S:H  -  4SX’S:H  -  44 Hw  u  p«UR                  5       (       d  M  UR                  [        U 5      X[        U5      [        XkU4   R                  5       5      [        UR                  5       5      S.5        My     Mµ     [        R                  " U5      $ )Né   rB   ÚallÚfailureÚnormalr   )r‰   rŠ   ÚgroupÚsemester_position_oldest0Úmean_attentionÚrows_with_position)Úndimrq   rC   rD   ÚboolrE   r}   r~   rØ   rH   rp   r¢   rÔ   )r‰   rŠ   rK   Ú	attentionÚmaskÚmax_semestersr  rŒ   ÚposÚvalidrÊ   Úgms               rM   Úattention_diagnosticsrn  F  s  € Ø&Ÿ^™^¨qÓ0ˆ	�‰�qÔ°i€AÜ�:Š:�dœDÓ!€DÜ
�
Š
�1”cÓ€AØ€DÜ�]Ö#ˆØ’Q˜�V‘ˆØ�y‰y�{‰{ÙØ  %˜.¨9°eÀA¹vÑ6FÐ*GÈ(ÐTYÐbcÑ]cÑTdÐIeÓf‰IˆEØ�v‰v�x‹xØ—‘Ü ›I°'Ü14°S³Ü&+¨A°#¨g©J¯O©OÓ,=Ó&>ÔVYÐZ\×Z`ÑZ`ÓZbÓVcñö ó gñ	 $ô �<Š<˜ÓÐrO   c                ó¸  • [         R                  " U [        5      n [         R                  " U[        5      n[         R                  " U 5      R
                  S:  a  SS[        U R                  5       5      4$ [        X5      u  p4n[        U5      S:X  a  SS[        U R                  5       5      4$ US S nUS S nX"-  nSU-   U-  U-  [         R                  " X†-  U-   S5      -  n	[         R                  " U	[         R                  " U	5      :H  5      S   n
[        U
S   5      n[        X[   5      nX:¬  nU[        X›   5      [        UR                  5       5      4$ )Nrd   r=  rz   r   éÿÿÿÿrB   re   )rC   rD   rE   rH   rF   rG   rq   r   rV   ÚmaximumÚwhereÚnanmax)rK   r]   rr   rj   rk   Ú
thresholdsrL   rs   rt   ÚfÚbestr„   Ú	thresholdra   s                 rM   Úbest_fbeta_thresholdrx  Y  s%  € Ü
�
Š
�1”cÓ€AÜ�JŠJ�uœeÓ$€EÜ	‡y‚y�ƒ|×Ñ˜1ÓØ�3œ˜aŸf™f›h›Ð'Ð'Ü$:¸1Ó$DÑ!€I�zÜ
ˆ:ƒ˜!ÓØ�3œ˜aŸf™f›h›Ð'Ð'Ø�#�2ˆ€AØˆs�ˆ€AØ	‰€BØ	
ˆR‰�1‰�qÑœ2Ÿ:š: b¡f¨q¡j°%Ó8Ñ8€AÜ�8Š8�AœŸš 1›Ñ%Ó& qÑ)€DäˆD�‰H‹€AÜ�j‘mÓ$€IØÑ€DØ”e˜A™D“k¤5¨¯©«Ó#5Ð5Ð5rO   c                ó&	  • 1 SknU R                    Vs/ s Hl  oUU;  d  M
  UR                  S5      (       a  M"  UR                  S5      (       a  M:  UR                  S5      (       a  MR  UR                  S5      (       a  Mj  UPMn     nn/ n/ n/ n	[        U R                  R	                  5       5       GHo  n
X R                  U
:H     nU GHS  nX»R
                  R                  U5      X¼   R                  5       -     nX»R
                  R                  U5      X¼   R                  5       -     n[        U5      (       a  [        U5      (       d  M†  UR                  R                  [        5      nXÜ   R                  [        5      n[        UUU5      u  nnn[        UR                  5       5      n[        U[        US5      -  5      nUR!                  X¬[        U5      UUUUUS.5        U GH,  nX»R
                  U:H  X¼   R                  5       -     n[        U5      (       d  M:  UR                  R                  [        5      nUU   R                  [        5      nUU:¬  R#                  [$        R&                  5      nX»R
                  US-
  :H     n[        U5      (       a#  [        UR                  R                  5       5      OUn[        [$        R(                  " UU-  S	S
5      5      n[+        UU5      n[-        UUU5      n[-        UUU5      n X¬[        U5      [        U5      [        UR                  5       5      [/        UU5      [1        UU5      UUUS.
n!U!R3                  UR5                  5        V"V#s0 s H  u  n"n#SU" 3U#_M     sn#n"5        U!R3                  U R5                  5        V"V#s0 s H  u  n"n#SU" 3U#_M     sn#n"5        UR!                  U!5        U/ SQ   R7                  5       n$UU$S'   UU$S'   UU$S'   UU$S'   U	R!                  U$5        GM/     GMV     GMr     [8        R:                  " U5      [8        R:                  " U5      U	(       a  [8        R<                  " U	SS94$ [8        R:                  " 5       4$ s  snf s  sn#n"f s  sn#n"f )N>   r‰   r    rŠ   Ú	course_idÚ
student_idÚtarget_semesterÚhistory_semestersÚ_uncertaintyÚ
_authorityÚ_guardÚ_correctionç�íµ ÷Æ°>)rŠ   r‹   Údev_rowsÚdev_prevalencerw  Údev_f2Údev_flag_rateÚwarning_ratiorB   g-Cëâ6?g®Gáz®ï?)
rŠ   r‹   r‰   rŒ   rŽ   r�   r�   rw  Údev_warning_ratioÚ
drift_rateÚfixed_Údrift_)r‰   rŠ   r    r{  rz  r‹   r]   Úwarning_fixedÚwarning_driftT)Úignore_index)r°   Úendswithr  rŠ   rF   r‰   rÕ   ÚnotnarV   r    r¤   rE   rH   rx  rq   rW   rØ   ÚastyperC   r[   r{   rb   ru   rN   rQ   ÚupdaterÆ   Úcopyr¢   rÔ   Úconcat)%rÉ   Ú	dev_foldsÚholdout_foldsrr   ÚmetarÜ   Ú
model_colsÚpoliciesÚ	eval_rowsÚdecision_framesrŠ   Úfsr‹   ÚdevÚholdÚydevÚsdevrw  r…  ro   Údev_prevÚratior‰   rš   rK   ÚsÚ
fixed_predÚprevÚ	prev_prevr‰  Ú
drift_predÚfixedÚdriftr˜   r`   ÚvrÚ   s%                                        rM   Úbuild_operational_evaluationr«  m  s  € Úk€DØ#Ÿ^š^ó  Aš^˜¸©}“!ÀQÇZÁZÐP^×E_“!Ðhi×hrÑhrÐs÷  iA“!ð  JK÷  JTñ  JTð  U]÷  J^“!ð  gh÷  gqñ  gqð  r÷  g@—!™^€Jð  AØ€HØ€IØ€Oä˜&Ÿ.™.×/Ñ/Ó1×2ˆØ—N‘N gÑ-Ñ.ˆÜˆEØ—W‘W—\‘\ )Ó,¨r©y¯©Ó/@Ñ@ÑAˆCØ—g‘g—l‘l =Ó1°B±I·O±OÓ4EÑEÑFˆDÜ�s—8‘8¤3 t§9¡9ÙØ—:‘:×&Ñ&¤sÓ+ˆDØ‘:×&Ñ&¤uÓ-ˆDÜ+?ÀÀdÈDÓ+QÑ(ˆI�v˜yÜ˜TŸY™Y›[Ó)ˆHÜ˜)¤c¨(°DÓ&9Ñ9Ó:ˆEØ�O‰OØ"ÄÀCÃØ"*¸Ø °9Èuñô ô
 &�ØŸ™ 4™¨2©9¯?©?Ó+<Ñ<Ñ=�Ü˜1—v‘vÙØ—H‘H×%Ñ%¤cÓ*�Ø�e‘H×%Ñ%¤eÓ,�Ø 9™n×4Ñ4´R·W±WÓ=�
ØŸ'™' d¨Q¡hÑ/Ñ0�Ü9<¸T¿¹œE $§+¡+×"2Ñ"2Ó"4Ô5È�	Ü"¤2§7¢7¨5°9Ñ+<¸dÀCÓ#HÓI�
Ü)¨!¨ZÓ8�
Ü.¨q°*¸dÓC�Ü.¨q°*¸dÓC�à&ÄÀDÃ	Ü ›F´%¸¿¹»³/Ü! ! Q›-´¸¸A³Ø!*ÀØ",ñ�ð —
‘
¸¿¹¼ÔFº±°°1˜f Q C˜L¨!šO¹ÒFÔGØ—
‘
¸¿¹¼ÔFº±°°1˜f Q C˜L¨!šO¹ÒFÔGØ× Ñ  Ô%ØÒNÑO×TÑTÓV�Ø"��'‘
Ø��'‘
Ø%/��/Ñ"Ø%/��/Ñ"Ø×&Ñ& q×)ô9 &ô  ñ 3ô^ �<Š<˜Ó!¤2§<¢<°	Ó#:Ön}¼R¿YºYÀÐeiÑ=jð  Sð  Sô  DF÷  DPò  DPó  DRð  Sð  Sùòi AùóT GùÛFs-   “	R R¸RÁRÁ(RÂ RÍ7RÎ0Rc                óæ  • U b  [        U 5      (       d  [        R                  " 5       $ / nU R                  SS/5       HQ  u  u  p#nX#[        U5      S.nS H$  n[	        XF   R                  5       5      USU 3'   M&     UR                  U5        MS     [        R                  " U5      n/ nUR                  S5       HŠ  u  p4X4R                  R                  5       S.nUR                   V	s/ s H  o™R                  S5      (       d  M  U	PM     sn	 H   n[	        XF   R                  5       5      XV'   M"     UR                  U5        MŒ     [        R                  " U5      R                  SSS	9$ s  sn	f )
NrŠ   r‹   )rŠ   r‹   rÎ   )
r�   r�   Úfixed_precisionÚfixed_recallÚfixed_fbetaÚfixed_flag_rateÚdrift_precisionÚdrift_recallÚdrift_fbetaÚdrift_flag_rateÚmean_rß   Úmean_apFrë   )rV   r¢   rÔ   r×   rH   rq   rØ   rŠ   rí   r°   Ú
startswithrî   )
Úeval_dfÚfaculty_rowsrŠ   r‹   rÛ   r˜   rÜ   ÚfdfrŒ   r¶   s
             rM   Úoperational_macror»  ¦  s7  € Ø�œc 'Ÿl™lÜ�|Š|‹~ÐØ€LØ&Ÿ™°	¸7Ð/CÖDÑÑˆ˜!Ø!¼CÀ»FÑCˆó
ˆAô  % Q¡T§Y¡Y£[Ó1ˆC�%˜�s�Óñ	
ð
 	×Ñ˜CÖ ñ  Eô �,Š,�|Ó
$€CØ€DØ—K‘K Ö(‰ˆØ¯I©I×,=Ñ,=Ó,?Ñ@ˆØ Ÿ[š[ÓBš[˜¯L©L¸×,A—!™[ÔBˆAÜ˜1™4Ÿ9™9›;Ó'ˆC‹Fñ Cà�‰�CÖñ	 )ô
 �<Š<˜Ó×)Ñ)¨)¸uÐ)ÐEÐEùò Cs   Ã+E.ÄE.c           
     óš  • U b  [        U 5      (       d  [        R                  " 5       $ / nU R                  SS/5       Hu  u  u  p#nUR                  R                  [        5      nS HH  n[        XTSU 3   R                  [        5      S5      nUR                  X#U[        U5      S.UE5        MJ     Mw     [        R                  " U5      $ )NrŠ   r‹   ©r¨  r©  Úwarning_ç       @)rŠ   r‹   ÚpolicyrŒ   ©	rV   r¢   rÔ   r×   r    r¤   rE   ru   rØ   )Ú	decisionsrŒ   rŠ   r‹   rÛ   rK   rÀ  r…   s           rM   Úpooled_confusionsrÃ  ¼  s·   € ØÑ¤ I§¡Ü�|Š|‹~ÐØ€DØ(×0Ñ0°)¸WÐ1EÖFÑÑˆ˜!Ø�H‰H×ÑœcÓ"ˆÛ(ˆFÜ& q¨h°v°hÐ,?Ñ*@×*IÑ*IÌ#Ó*NÐPSÓTˆAØ�K‰K GÀvÔWZÐ[\ÓW]ÑcÐabÐcÖdó )ñ  Gô
 �<Š<˜ÓÐrO   c           	     óŽ  • U b  [        U 5      (       d  [        R                  " 5       $ / nU R                  S5       Hq  u  p#UR                  R                  [        5      nS HG  n[        XCSU 3   R                  [        5      S5      nUR                  X%[        U5      S.UE5        MI     Ms     [        R                  " U5      $ )Nr‹   r½  r¾  r¿  )r‹   rÀ  rŒ   rÁ  )rÂ  rŒ   r‹   rÛ   rK   rÀ  r…   s          rM   Úoverall_pooled_confusionsrÅ  È  s¥   € ØÑ¤ I§¡Ü�|Š|‹~ÐØ€DØ×%Ñ% gÖ.‰ˆØ�H‰H×ÑœcÓ"ˆÛ(ˆFÜ& q¨h°v°hÐ,?Ñ*@×*IÑ*IÌ#Ó*NÐPSÓTˆAØ�K‰K %Ä3ÀqÃ6ÑOÈQÐOÖPó )ñ /ô
 �<Š<˜ÓÐrO   c                ó‚  • Uc  g [        SU -   5        [        S5        [        S5        [        S5        [        S[        US   5      S S[        US   5      S 35        [        S	[        US
   5      S S[        US   5      S 35        [        S5        [        SUS   S SUS   S SUS   S SSUS   -  S S3	5        g )NÚ
zL----------------------------------------------------------------------------z7  Actual / Predicted          Normal >=5     Failure <5z  Normal >=5                 ri   z>10,z     rg   z  Failure <5                 rh   rf   z  Precision=rj   ú.4fz Recall=rk   z F2=rm   z Flag=r•   ro   z.2fÚ%)ÚprintrE   )Útitler˜   s     rM   Úprint_confusionrÌ  Ô  sß   € Ø
�{ØÜ	ˆ$�‰,ÔÜ	ˆ(„OÜ	ÐCÔEÜ	ˆ(„OÜ	Ð)¬#¨c°$©i«.¸Ð)>¸eÄCÈÈDÉ	ÃNÐSWÐCXÐ
YÔZÜ	Ð)¬#¨c°$©i«.¸Ð)>¸eÄCÈÈDÉ	ÃNÐSWÐCXÐ
YÔZÜ	ˆ(„OÜ	Ø
�s˜;Ñ'¨Ð,¨H°S¸±]À3Ð4Gð HØ�'‰l˜3Ð˜v c¨#¨kÑ*:Ñ&:¸3Ð%?¸qð	BõrO   c                ó�  • [        SU -   5        [        S[        S[        S[        U 5      S-   5      5      -  5        Ub  [        U5      (       d  [        S5        g Uc  UO%X Vs/ s H  oDUR                  ;   d  M  UPM     sn   nU(       a  UR                  [        U5      5      n[        UR                  SS S	95        g s  snf )
NrÇ  Ú-é¾   éP   rP  z<empty>Fc                ó
   • U S $ )NrÈ  © )r¶   s    rM   Ú<lambda>Úprint_table.<locals>.<lambda>í  s
   € ¸Q¸s¸G¸*rO   )ÚindexÚfloat_format)rÊ  rX   rW   rV   r°   ÚheadrE   Ú	to_string)rË  r©   ÚcolsÚmax_rowsrÜ   rš   s         rM   Úprint_tablerÛ  ä  s¡   € Ü	ˆ$�‰,ÔÜ	ˆ#”�Cœ˜R¤ U£¨a¡Ó0Ó1Ñ
1Ô2Ø	�zœ˜RŸ™ÜˆiÔØØ‰l‰ ¨tÓ#Gªt¨!¸B¿J¹J±§A©tÑ#GÑ H€AÞØ�F‰F”3�x“=Ó!ˆÜ	ˆ!�+‰+˜EÑ0Dˆ+Ð
EÕFùò $Hs   Á%CÁ<Cc                ó¼   • [         R                  " U [        5      n[         R                  " [	        U5      S[
        5      nSX!S:H  '   SX!S:H  '   SX!S:¬  US:*  -  '   U$ )	Nú4+Ú0r   Ú1rB   ú2-3rd   r^  )rC   rD   rE   ÚfullrV   Úobject)ÚcountrÜ   Úouts      rM   Úsupport_bucketrå  ð  sU   € Ü‡j‚j�”sÓ€AÜ
�Š”�A“�tœFÓ#€CØ€Cˆ1‰�I˜S�3˜!‘t‘9°U #¨!¡t¨a°©d¡mÑ"4Ø€JrO   c                óü  • [        U5      n/ nS GH%  n	Xy:H  n
U
R                  5       (       d  M  UR                  [        U 5      XU	[        U
R	                  5       5      [        [
        R                  " U5      U
   R	                  5       5      [        [
        R                  " U5      U
   R                  5       5      [        [
        R                  " U5      U
   [
        R                  " U5      U
   5      [        [
        R                  " U5      U
   [
        R                  " U5      U
   5      S.	5        GM(     [        R                  " U5      n[        U5      (       a  UR                  UR                  -
  US'   U$ )N)rÞ  rß  rà  rÝ  )	r‰   rŠ   Úsourcerå  rŒ   r�   rŽ   Úbase_apÚrelational_apr  )rå  r~   rØ   rE   rp   rC   rD   rH   rq   rN   r¢   rÔ   rV   ré  rè  )r‰   rŠ   rK   ÚcountsÚ
score_baseÚ	score_relrç  ÚbucketsrŒ   r[  r…   rÚ   s               rM   Úrelational_support_upliftrî  ÷  s  € Ü˜6Ó"€GØ	€DÜ&ˆØ
‰/ˆØ�u‰u�w‰w™Ø�‰Ü�t“9 wÐPVÜ�q—u‘u“w“<¬3¬r¯zªz¸!«}¸QÑ/?×/CÑ/CÓ/EÓ+FÜœrŸzšz¨!›}¨QÑ/×4Ñ4Ó6Ó7ÜœbŸjšj¨›m¨AÑ.¬r¯zªz¸*Ó/EÀaÑ/HÓIÜ#¤B§J¢J¨q£M°!Ñ$4´R·Z²ZÀ	Ó5JÈ1Ñ5MÓNñ
÷ 	ñ 'ô 	‡l‚l�4Ó€AÜ
ˆ1‡v�v˜QŸ_™_¨Q¯Y©YÑ6ˆq�‰}Ø€HrO   c                óž  • [         R                  " U[        5      n[         R                  " U[        5      n/ n[        U5       Hñ  u  p‰US S 2U4   o¦S S 2U4   n[        [         R                  " U
5      5      n[        [         R                  " U5      5      n[        [         R
                  " U
5      5      n[        [         R
                  " U5      5      nUR                  [        U 5      XXÍXï[        XÜ-
  5      [        US5      -  [        [         R                  " US-   US-   -  5      5      S.	5        Mó     [        R                  " U5      $ )Nr‚  )	r‰   rŠ   ÚfeatureÚrecent_meanÚtarget_meanÚ
recent_stdÚ
target_stdÚmean_shift_stdÚscale_log_shift)rC   rD   rH   Ú	enumerateÚnanmeanÚnanstdrØ   rE   r   rW   Úlogr¢   rÔ   )r‰   rŠ   Úfeature_namesÚrecent_xÚtarget_xr  r™   rŒ   ÚjÚnameÚarÚbrÚmu_aÚmu_bÚsd_aÚsd_bs                   rM   Úfeature_drift_rowsr  	  sö   € Ü‡j‚j�œ%Ó €A¤B§J¢J¨x¼Ó$> !Ø	€DÜ˜MÖ*‰ˆØŠQˆqˆS‰6ˆš˜!˜‘f�2Ü”2—:’:˜b“>Ó"ˆ¬¬r¯zªz¸"«~Ó)> DÜ”2—9’9˜R“=Ó!ˆ¬¬b¯iªi¸«mÓ(< 4Ø�‰Ü�t“9 wØ¸tÜ  ¡›^¬C°°T«NÑ:Ü!¤"§&¢&¨$¨t©)°d¸4±iÑ)@Ó"AÓBñ	
ö 	ñ	 +ô �<Š<˜ÓÐrO   c           	     óü  • [         R                  " U [        5      n [         R                  " U[        5      n[         R                  " U[        5      n[	        U 5      S:  d$  [         R
                  " U 5      R                  S:  a1  [         R                  [         R                  [         R                  SS.$ [        X5      [        X5      -
  n[         R                  R                  U5      n/ n	[	        U 5      n
[        [        U5      5       Hm  nUR                  SXª5      n[         R
                  " X   5      R                  S:  a  M=  U	R                  [        X   X\   5      [        X   Xl   5      -
  5        Mo     U	(       d#  U[         R                  [         R                  SS.$ [         R                  " U	SS/5      u  pÞ[        U5      [        U5      [        U5      [        US:„  5      [        US:  5      S.$ )Nrd   F)r  Úci_lowÚci_highÚpositive_cir   gš™™™™™™?g333333ï?)r  r  r	  r
  Únegative_ci)rC   rD   rE   rH   rV   rF   rG   rI   rN   ÚrandomÚdefault_rngr}   ÚintegersrØ   r  rg  )rK   rö   r÷   ÚrepsÚseedr  r™   ÚbaseÚrngÚvalsr_   Ú_ÚixÚloÚhis                  rM   Úpaired_bootstrap_apr    sn  € Ü‡j‚j�”3Ó€Aœ2Ÿ:š: i´Ó6˜¼"¿*º*ÀYÌuÓ:U¸Ü
ˆ1ƒvˆaƒx”2—9’9˜Q“<×$Ñ$ QÓ&¼2¿6¹6Ì2Ï6É6Ô\^×\bÑ\bÐqvÑ/wÐ(wÜ	�‹”g˜a“lÑ	"€DÜ
�	‰	×Ñ˜dÓ#€C¨" T´°A³¨aÜ”3�t“9ÖˆØ�<‰<˜˜!ÓˆÜ�9Š9�Q‘UÓ× Ñ  Ó"¡HØ�‰”G˜A™E !¡%Ó(¬°±°q±uÓ)=Ñ=Ö>ñ ö  D´"·&±&Ä2Ç6Á6ÐX]Ñ^Ð^Ü
�+Š+�d˜D ˜;Ó
'�E€BÜ˜T“{¬E°"«IÄÀbÃ	ÔX\Ð]_Ð`aÑ]aÓXbÔquÐvxÐyzÑvzÓq{Ñ|Ð|rO   )r¿  )r	   r¿  )rˆ   )r	   )rÉ   úpd.DataFrame)r¸  r  )rÂ  r  )NN)éÈ   é*   )-Ú
__future__r   Úpathlibr   ÚnumpyrC   Úpandasr¢   Úsklearn.metricsr   r   r   r   r—   rÅ   rN   rQ   rb   ru   rx   r†   rœ   rª   r²   r¾   rÌ   rÝ   rð   r  r-  rD  rN  r\  rn  rx  r«  r»  rÃ  rÅ  rÌ  rÛ  rå  rî  r  r  rÒ  rO   rM   Ú<module>r!     sµ  ðÝ "å ã Û ÷ó ð €ðØÐSðàÐLðð ÐDðð Ð;ð	ð
 ÐCðð ÐDðð ÐTðð ÐLðð ÐQðð ÐIðð ÐPðð ÐHðð ÐLðð ÐXðð ÐFðð  Ð;ð!ð" Ð;ð#ð$ <ØIØ9Ø;ò+Ð ò2Uò
Kò
ôô&Fôô ò(&ò$6ò)ò&òò$
Kòò0ò2ô4òò&ô&6ö(6SôrFô,	ô	òô 	Gòòò$õ }rO   