ó
    ºRŸjLÏ  ã                  ó  • S r SSKJr  SSKJr  SSKJr  SSKrSSK	r
SSKrSSKJrJr  SS/S	S
/SS/SS// SQSS// SQ/ SQS.rSrSrS r S3S jrS rS rS rS r\" SS9S 5       rS  rS! rS" rS# rS$ rS% rS& r S' r!S(S)S*SSS+S,\RD                  \RD                  \RD                  \RD                  \RD                  \RD                  \RD                  \RD                  \RD                  4S- jr# S3S. jr$S/ r% S3S0 jr&S1 r'S2 r(g)4zŸ
Final Suite19 reporting hardening.

This module layers on top of suite19_reporting.py without changing the
underlying metric definitions used by the project.
é    )Úannotations)Ú	lru_cache©ÚPathN)Úmetric_dictÚprint_tableÚsuite17_hgb_finalÚsuite17_finalÚsuite17_hgb_rawÚsuite17_rawÚsuite18_hgb_finalÚsuite18_finalÚsuite18_hgb_rawÚsuite18_raw)Úsuite16_hgb_finalÚ!suite16_conditional_tree_boostingÚsuite16Úsuite14_authority_safe_finalÚsuite14)Úfailure_probability_lt5Úfailure_probabilityÚstacked_probabilityÚstacked_probÚv2)Úsuite4_competence_councilÚcompetence_councilÚcouncilÚfinalÚcouncil_probability)r
   r   r   r   r   r   r   r   )ÚactualÚactual_failure_lt5ÚyÚlabelÚtarget)r   r   r
   r   r   r   c                óÔ   • [         R                  " U [        5      R                  S5      n [	        [        U 5      =(       a$    [         R                  " U 5      R                  5       5      $ )Néÿÿÿÿ)ÚnpÚasarrayÚfloatÚreshapeÚboolÚlenÚisfiniteÚall)Úss    ÚG/root/disk_3tb/earlywarning/30072026/suite19/suite19_final_reporting.pyÚ_finite_scorer1   P   sB   € Ü
�
Š
�1”eÓ×$Ñ$ RÓ(€AÜ”�A“×/œ2Ÿ;š; q›>×-Ñ-Ó/Ó0Ð0ó    c                óp  • [         R                  " U[        5      n[         R                  " U[        5      n/ n[        U5       GHm  u  p‰Xh:H  n
U
R	                  5       (       d  M!  UR                  5        GH7  u  p¼[         R                  " U[        5      n[        U5      [        U5      :w  d+  [         R                  " XÚ   5      R                  5       (       dP  [         R                  [         R                  S.nS H$  nS H  n[         R                  UU SU 3'   M     M&     SnSnO[        X:   XÚ   U5      nSnSnUR                  [        U 5      U	U[        U
R                  5       5      [        X:   R                  5       5      [        X:   R                  5       5      UUS	.UE5        GM:     GMp     U$ )
N)ÚapÚauc)é
   é   é   é   é   )Ú	precisionÚrecallÚf1ÚfbetaÚ	flag_rateÚ_at_ç        ÚUNAVAILABLEç      ð?ÚOK)ÚfoldÚfacultyÚmodelÚrowsÚfailuresÚ
prevalenceÚscore_available_rateÚstatus)r'   r(   ÚintÚ	enumerateÚanyÚitemsr)   r,   r-   r.   Únanr   ÚappendÚsumÚmean)rE   Úfaculty_namesÚfaculty_idxr"   ÚscoresÚbetaÚfirH   ÚirF   ÚmrG   Úscorer/   ÚmdÚrateÚkeyÚavailabilityrL   s                      r0   Úfaculty_fold_rowsra   U   s~  € ô 	�
Š
�1”cÓ€AÜ	�Š�K¤Ó	%€BØ€Dä ×.‰
ˆØ‰Gˆà�u‰u�w‰wÙà"ŸL™LŸN‰LˆEÜ—
’
˜5¤%Ó(ˆAô �A“œ#˜a›&Ó Ü—{’{ 1¡4Ó(×,Ñ,×.Ñ.ô Ÿ&™&ÜŸ6™6ñ�ó
 1�Dó ˜ô 24·±˜˜c˜U $ t fÐ-Ó.ó ñ 1ð  #�Ø&‘ä Ø‘DØ‘DØó�ð
  #�Ø�à�K‰Kä ›IØ&Ø"Ü §¡£›LÜ # A¡D§H¡H£J£Ü"'Ø™Ÿ	™	›ó#ð -9Ø$ñð ð÷ôC +ñ /ðn €Kr2   c           	     ód  • [        U [        R                  5      (       a  U R                  5       O[        R                  " U 5      n1 SknUR                  (       d   UR                  UR                  5      (       d  [        R                  " / SQS9$ [        U [        R                  5      (       a  U R                  5       O[        R                  " U 5      n/ SQnUU Vs/ s H  owUR                  ;   d  M  UPM     sn   R                  5       nX3R                  U:H     R                  SSSSS	S
.S9nX3R                  U:H     R                  SSSSSS
.S9n	UR                  S/S9nU	R                  S/S9n	UR                  U	SS/SS9n
[        U
5      (       d  U
$ [        R                  " U
R                  5      [        R                  " U
R                  5      -  nU
R                   U   R                  5       n
[        U
5      (       d  U
$ U
R                  U
R                  -
  U
S'   U
R"                  U
R$                  -
  U
S'   U
R&                  U
R(                  -
  U
S'   [        R*                  " U
R,                  S:„  S[        R*                  " U
R,                  S:  SS5      5      U
S'   U
$ s  snf )N>   r4   r5   rE   rH   rG   rF   rI   Úfbeta_at_20)rE   rF   Úcandidate_rowsÚcandidate_failuresÚcandidate_apÚcandidate_aucÚcandidate_f2_at20Ú	base_rowsÚbase_failuresÚbase_apÚbase_aucÚbase_f2_at20Údelta_apÚ	delta_aucÚdelta_f2_at20Úwin©Úcolumns)rE   rF   rH   rI   rG   r4   r5   rc   rd   re   rf   rg   rh   )rH   rI   r4   r5   rc   ri   rj   rk   rl   rm   rG   rE   rF   Úinner©ÚonÚhowrn   ro   rp   r   ÚWINÚLOSSÚTIErq   )Ú
isinstanceÚpdÚ	DataFrameÚcopyÚemptyÚissubsetrs   rG   ÚrenameÚdropÚmerger,   r'   r-   rf   rk   Úlocrg   rl   rh   rm   Úwherern   )ÚmetricsÚ	candidateÚbaseÚdÚrequiredÚdfÚneededÚcÚaÚbÚzÚfinites               r0   Úpaired_tabler’   ›   sŠ  € ô �gœrŸ|™|×,Ñ,ð 	�‰Œä�\Š\˜'Ó"ð ò	€Hð 	‡w‡w�h×'Ñ'¨¯	©	×2Ñ2Ü�|Š|òñ
ð 	
ô0 �gœrŸ|™|×,Ñ,ð 	�‰Œä�\Š\˜'Ó"ð ò	€Fð 	ÙÓ.’Fˆq 2§:¡:™o�‘FÑ.ñ	ç
�dƒfð ð 	
�'‰'�YÑ
Ñ×&Ñ&à$Ø,Ø Ø"Ø.ñ
ð 	'ð 	€Að 	
�'‰'�T‰/Ñ×!Ñ!àØ'ØØØ)ñ
ð 	"ð 	€Að 	
�‰˜�yˆÐ!€AØ	�‰˜�yˆÐ!€Aà	�‰Ø	Ø�IÐØð 	ð 	€Aô ˆq�6‰6Øˆô 	�Š�A—N‘NÓ#Ü
�+Š+�a—i‘iÓ
 ñ	!ð ð
 	
�‰ˆf‰×ÑÓ€Aäˆq�6‰6Øˆð 	
�‰˜Ÿ™Ñ"ð €j�Mð 	
�‰˜!Ÿ*™*Ñ$ð €k�Nð 	
×ÑØ
�.‰.ñ	ð €oÑô
 �xŠxØ	�
‰
�Q‰ØÜ
�ŠØ�J‰J˜‰NØØó	
ó€A€e�Hð €HùòE 	/s   ÃJ-Ã6J-c                 óŽ   • [        [        R                  " 5       SS5      n U R                  (       d   eSU R                  ;   d   eg )NÚcommittee_residualÚ	committeern   )r’   r|   r}   r   rs   )r�   s    r0   Ú%test_paired_table_empty_input_is_safer–     s:   € ÜÜ
�Š‹ØØó	€Að �7�7€Nˆ7Ø˜Ÿ™Ó"Ð"Ñ"r2   c                ó  • [        U UU5      n[        U5      (       dr  UUSSSS[        R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                  S.$ UU[	        [        U5      5      [	        UR
                  S:„  R                  5       5      [	        UR
                  S:  R                  5       5      [	        UR
                  S:H  R                  5       5      [        UR
                  S:„  R                  5       5      [        UR
                  R                  5       5      [        UR
                  R                  5       5      [        UR
                  R                  S5      5      [        UR
                  R                  5       5      [        UR                  R                  5       5      [        UR                  R                  5       5      S.$ )Nr   )r‡   rˆ   ÚcellsÚwinsÚlossesÚtiesÚwin_rateÚmean_delta_apÚmedian_delta_apÚq25_delta_apÚworst_delta_apÚmean_delta_aucÚmean_delta_f2_at20ç      Ð?)r’   r,   r'   rQ   rM   rn   rS   r)   rT   ÚmedianÚquantileÚminro   rp   )r†   r‡   rˆ   r�   s       r0   Úpaired_summaryr§   %  s‹  € ÜØØØó	€Aô ˆq�6‰6à"ØØØØØÜŸ™ÜŸV™VÜ!Ÿv™vÜŸF™FÜ Ÿf™fÜ Ÿf™fÜ"$§&¡&ñ
ð 	
ð" ØÜ”S˜“V“Ü�Q—Z‘Z !‘^×(Ñ(Ó*Ó+Ü�q—z‘z A‘~×*Ñ*Ó,Ó-Ü�Q—Z‘Z 1‘_×)Ñ)Ó+Ó,ÜØ�Z‰Z˜!‰^×!Ñ!Ó#ó
ô Ø�J‰J�O‰OÓó
ô !Ø�J‰J×ÑÓó
ô Ø�J‰J×Ñ Ó%ó
ô  Ø�J‰J�N‰NÓó
ô  Ø�K‰K×ÑÓó
ô $Ø�O‰O× Ñ Ó"ó
ñ3ð r2   c           	     ó†   • [        U 5      nUSU 3-  U S3-  X2 SU S3-  USU 3-  U S3-  X2 SU S3-  X2 SU S3-  /$ )NÚfold_ú_scores.csvÚ_fold_ú_target_scores.csvú_test_results.csvr   )ÚrootrE   rF   Úrs       r0   Ú_direct_candidatesr°   \  s‡   € ÜˆT‹
€Að 	
ˆe�D�6ˆNÑ ˜y¨Ð4Ñ4Ø	ˆi�v˜d˜V ;Ð/Ñ/Ø	ˆe�D�6ˆNÑ ˜yÐ(:Ð;Ñ;Ø	ˆi�v˜d˜VÐ#5Ð6Ñ6Ø	ˆi�v˜d˜VÐ#4Ð5Ñ5ðð r2   i   )Úmaxsizec           	     óò  • [        U 5      n [        U 5      nUR                  5       (       d  / $ / n[        UUU5       H+  nUR                  5       (       d  M  UR	                  U5        M-     SU SU S3SU SU S3SU SU S3SU SU S3SU SU S34nU H3  nUR                  U5       H  nXT;  d  M
  UR	                  U5        M     M5     [        US S	9 Vs/ s H  n[        U5      PM     sn$ s  snf )
Nz**/fold_Ú/rª   z**/r«   r¬   r­   c                óB   • [        U R                  5      [        U 5      4$ ©N)r,   ÚpartsÚstr©Úxs    r0   Ú<lambda>Ú&_discover_candidates.<locals>.<lambda>Œ  s   € Ü�A—G‘G“Ü�A“ñr2   )r_   )r·   r   Úexistsr°   rR   ÚglobÚsorted)r®   rE   rF   r¯   ÚfoundÚpÚpatternsÚpatterns           r0   Ú_discover_candidatesrÃ   h  s)  € äˆt‹9€DÜˆT‹
€Aà�8‰8�:‰:Øˆ	à€EäØ	ØØöˆð
 �8‰8�:‹:Ø�L‰L˜ŽOñð �4�&˜˜'˜ +Ð.Ø
ˆgˆY�f˜T˜F +Ð.Ø
�4�&˜˜'˜Ð"4Ð5Ø
ˆgˆY�f˜T˜FÐ"4Ð5Ø
ˆgˆY�f˜T˜FÐ"3Ð4ð€Hó ˆØ—‘˜–ˆAØ�~Ø—‘˜Q–ó !ñ ô Øñò
ó	ò
ˆAô 	ˆAŽñ
ñ	ð 	ùò 	s   ÃC4c           
     óâ  ^• [        U 5      n[        R                  " USS9m[        U4S j[         5       S 5      nUc  S S[        U5      S.4$ [        R                  " TU   SS9R                  5       n[        T5      [        U5      :w  dq  [        R                  " U5      R                  5       (       aH  [        R                  " UR                  [        5      [        R                  " U[        5      5      (       d  S S[        U5      US	.4$ [        U4S
 jU 5       S 5      nUc9  S S[        U5      USR!                  [#        [
        TR$                  5      5      S.4$ [        R                  " TU   SS9R                  [&        5      n[        U5      [        U5      :w  d)  [        R                  " U5      R                  5       (       d  S S[        U5      UUS.4$ US[        U5      UUS.4$ )NF)Ú
low_memoryc              3  óL   >#   • U  H  nUTR                   ;   d  M  Uv •  M     g 7frµ   rr   ©Ú.0r�   r‹   s     €r0   Ú	<genexpr>Ú&_try_reference_file.<locals>.<genexpr>   s%   øé € ð 	
â#�Ø�B—J‘J‰÷ ‰AÚ#ùó   ƒ$›	$Úactual_column_missing)rL   ÚpathÚcoerce©ÚerrorsÚalignment_mismatch)rL   rÍ   Úactual_columnc              3  óL   >#   • U  H  nUTR                   ;   d  M  Uv •  M     g 7frµ   rr   rÇ   s     €r0   rÉ   rÊ   Â  s%   øé € ð 	
â�Ø�B—J‘J‰÷ ‰AÚùrË   Úscore_column_missingÚ|)rL   rÍ   rÒ   rs   Ú"score_nonfinite_or_length_mismatch)rL   rÍ   rÒ   ÚcolumnÚloaded)r   r|   Úread_csvÚnextÚACTUAL_ALIASESr·   Ú
to_numericÚto_numpyr,   r'   r-   r.   Úarray_equalÚastyperM   r(   ÚjoinÚmaprs   r)   )	rÍ   r"   ÚaliasesrÀ   r    Úavr×   r/   r‹   s	           @r0   Ú_try_reference_filerä   ”  sò  ø€ ô
 	ˆT‹
€AÜ	�ŠØ	Øñ
€Bô
 ô	
å#ó	
ð
 	ó€Fð �~ØØ-Ü˜“Fñ
ð 
ð 	
ô
 
�ŠØ
ˆ6‰
Øñ
÷ �hƒjð ô 	ˆB‹”3�q“6ÓÜ�{Š{˜2‹×"Ñ"×$Ñ$Ü�~Š~Ø�I‰I”c‹NÜ�JŠJ�qœ#Ó÷
ñ 
ð
 Ø*Ü˜“FØ#ñ
ð 
ð 	
ô ô	
áó	
ð
 	ó€Fð �~ØØ,Ü˜“FØ#Ø—x‘xÜ”C˜Ÿ™Ó$óñ	
ð 
ð 	
ô 	�ŠØ
ˆ6‰
Øñ	÷ �hŒuƒoð ô 	ˆA‹”#�a“&ÓÜ�{Š{˜1‹~×!Ñ!×#Ñ#àØ:Ü˜“FØ#Øñ	
ð 
ð 	
ð ØÜ�A“ØØñ	ð ð r2   c                ó†  • 0 n/ nU =(       d    0 R                  5        GH�  u  pgU(       d"  UR                  UU[        U5      SSS.5        M/  [        U5      nUR	                  5       (       d+  UR                  UU[        U5      S[        U5      S.5        Mz  [        R                  UUS/5      n	[        [        U5      [        U5      [        U5      5      n
S n/ nU
 H,  n[        UUU	5      u  pïUR                  U5        Uc  M(  UU4n  O   Ubj  Uu  pïXäU'   UR                  UU[        U5      S[        U5      US   US   US	   [        US   5      [        UUU5      ;   a  S
OS[        U
5      S.
5        GM]  UR                  UU[        U5      S[        U5      [        U
5      [        US S 5      S.5        GM      XE4$ )NÚdisabledÚ )Ú	referencerF   rE   rL   r®   Úroot_missingr   rØ   rÍ   r×   rÒ   ÚdirectÚ	recursive)
rè   rF   rE   rL   r®   rÍ   r×   rÒ   Ú	discoveryÚcandidate_filesÚmissing_or_unalignedé   )rè   rF   rE   rL   r®   rí   Úattempts)rP   rR   rM   r   r¼   r·   ÚREFERENCE_ALIASESÚgetrÃ   rä   r°   r,   )Úreference_dirsrE   rF   r"   ÚoutÚmetadataÚnamer®   r¯   râ   Ú
candidatesrØ   rð   r‡   r\   Úinfos                   r0   Úreference_scores_for_facultyrù   ì  s  € ð €CØ€Hð 	×˜"ß�eƒgó‰
ˆö Ø�O‰Oà!%Ø&Ü ›IØ(Øñôñ ä�‹Jˆà�x‰x�z‰zØ�O‰Oà!%Ø&Ü ›IØ,Ü ›Fñôñ ä#×'Ñ'ØØ�7ˆOó
ˆô
 *Ü�‹FÜ�‹IÜ�‹Ló
ˆ
ð ˆØˆã#ˆIÜ-ØØØó‰KˆEð �O‰O˜DÔ!àÓ àØð�ñ ñ $ð  ÑØ ‰KˆEØ�‰Ià�O‰Oà!%Ø&Ü ›IØ&Ü ›FØ  ™LØ" 8™nØ%)Ø'ñ&ô
    V¡Ó-Ü-ØØ Ø#óóñ !ð )ä'*Ø"ó(ñ+÷ð8 �O‰Oà!%Ø&Ü ›IØ4Ü ›FÜ'*Ø"ó(ô !$Ø   !˜ó!ñ÷ñwðV ˆ=Ðr2   c                óª  • / nU =(       d    0 R                  5        H•  u  p4U(       a  [        U5      OS nUR                  U[        U=(       d    S5      [	        US L=(       a    UR                  5       5      [        U5      [	        US L=(       a    USU 3-  R                  5       5      S.5        M—     [        S[        R                  " U5      SS9  g )Nrç   r©   )rè   r®   Úroot_existsrE   Úfold_dir_existsz[REFERENCE ROOT AUDIT]éd   ©Úmax_rows)
rP   r   rR   r·   r+   r¼   rM   r   r|   r}   )ró   rE   rH   rö   r®   rÀ   s         r0   Úprint_reference_root_auditr   c  s¾   € ð €Dð 	×˜"ß�eƒgò‰
ˆö ŒD�ŒJ Dˆà�‰à!Ü˜DŸJ B›Ü#Ø˜T�M×0 a§h¡h£jó ô ˜D›	Ü#'Ø˜T�M÷ à˜e D 6˜NÑ*ß‘f“hó	$ñö	
ñô, Ø Ü
�Š�TÓØór2   c                óü  • U (       d`  [         R                  * [         R                  [         R                  [         R                  [         R                  [         R                  SSS.$ [        R                  " U 5      n[        R
                  " X!   SS9n[         R                  " U5      nUR                  U   R                  5       n[        U5      (       d`  [         R                  * [         R                  [         R                  [         R                  [         R                  [         R                  SSS.$ X!   R                  [        5      nUR                  S5      U   R                  5       R                  5       n[        UR                  S   5      n[        UR!                  5       5      n[        [         R                  " U5      5      n	[        [         R"                  " U5      5      n
[        [         R$                  " US5      5      n[        [         R                  " US:„  5      5      nS	U	-  S
U
-  -   SU-  -   SU-  -   S['        SU* 5      -  -
  n[        U5      U	U
UUUU[)        [        U5      5      S.$ )NrA   r   )r\   rT   r¤   Úq25Úrecent_meanÚworst_block_meanrœ   r˜   rÎ   rÏ   Úblockr&   r£   gffffffÖ?g333333Ó?gš™™™™™É?g333333Ã?ç      à?)r'   ÚinfrQ   r|   r}   rÜ   r-   r„   r~   r,   rÝ   r)   ÚgroupbyrT   Ú
sort_indexÚilocr¦   r¤   r¥   ÚmaxrM   )r˜   r×   r‹   r�   ÚkeepÚvÚ
block_meanr  Úworst_blockrT   r¤   r  rœ   r\   s                 r0   Ú_path_statsr  †  s  € Þä—f‘f�WÜ—F‘FÜ—f‘fÜ—6‘6ÜŸ6™6Ü "§¡ØØñ	
ð 		
ô 
�Š�eÓ	€Bä
�ŠØ
‰
Øñ	€Aô
 �;Š;�q‹>€DØ	�‰�‰×	Ñ	Ó	€Bäˆr�7‰7ä—f‘f�WÜ—F‘FÜ—f‘fÜ—6‘6ÜŸ6™6Ü "§¡ØØñ	
ð 		
ð 	‰
×ÑœEÓ"€Að 	�
‰
�7Ó˜FÑ#ß	‰‹ß	‰‹ð ô Ø�‰˜Ñó€Kô Ø�‰Óó€Kô ”—’˜“Ó€DÜ”2—9’9˜Q“<Ó €FÜ
Ü
�Š�A�tÓó€Cô Ü
�Š��A‘‹ó€Hð
 	ˆt‰Ø
�‰-ñ	à
�‰*ñ	ð �Ñ
ñ	ð ”ØØˆLó
ñ 
ñ		
ð 
ô �u“ØØØØ"Ø'ØÜ”S˜“V“ñ	ð 	r2   c                ób  • [         R                  " U R                  R                  S5      [        5      n[         R                  " UR
                  [        5      n/ n[        S [         R                  " U R                  U   5       5       5       GHð  n[        U R                  5       GHÒ  u  pxUU R                  U:H  -  U R                  U:H  -  n	Sn
U
 H)  nU	[         R                  " UR                  U   5      -  n	M+     U	R                  5       S:  d1  [         R                  " U R                   U	   5      R"                  S:  a  M   [$        R&                  " U R                   U	   UR                  S   U	   5      [$        R&                  " U R                   U	   X9   5      -
  n[$        R&                  " U R                   U	   UR                  S   U	   5      [$        R&                  " U R                   U	   UR                  S   U	   5      -
  nUR)                  [	        U5      U[+        U	R                  5       5      [+        U R                   U	   R                  5       5      [	        U5      [	        U5      S	.5        GMÕ     GMó     [-        US
5      n[-        US5      nUS   US   :”  a  SnUnOSnUn0 SUS   _SU_SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _SUS   _S US!   _S"US   _S#US   _US   US   US   US   US!   US   S$.E$ )%Né   c              3  ó`   #   • U  H$  n[         R                  " U5      (       d  M   Uv •  M&     g 7frµ   )r'   r-   )rÈ   r¹   s     r0   rÉ   Ú"selection_stats.<locals>.<genexpr>å  s*   é € ð ò
ˆAô �;Š;�q�>÷	 	
‰ò
ùs   ‚.¥	.)r”   Úhgb_xzÚraw_hgbr6   é   r”   r  r  )r  rF   rH   rI   Úconditional_delta_apÚxz_delta_apr  r  r\   Úhgb_xz_vs_raw_hgbÚcommittee_residual_vs_committeeÚselection_basisÚmean_conditional_delta_aprT   Úmedian_conditional_delta_apr¤   Úq25_conditional_delta_apr  Ú recent_mean_conditional_delta_apr  Úmean_xz_delta_aprœ   r˜   Úresidual_scoreÚresidual_mean_delta_apÚresidual_median_delta_apÚresidual_q25_delta_apÚresidual_recent_mean_delta_apÚ"residual_worst_block_mean_delta_apr  Úresidual_win_rateÚxz_score)Úxz_mean_delta_apÚxz_median_delta_apÚxz_q25_delta_apÚxz_recent_mean_delta_apÚxz_worst_block_mean_delta_apÚxz_win_rate)r'   r(   ÚteacherrT   r)   Úhist_availabler+   r¾   ÚuniqueÚblocksrN   rU   rV   r-   Úhist_scoresrS   r"   ÚsizeÚlegacyÚsafe_aprR   rM   r  )ÚhistÚledgerÚprobeÚqÚvalidr˜   r  rY   rF   r[   Úscore_namesrö   Úresidual_deltaÚxz_deltaÚresidualÚxzÚbasisÚprimarys                     r0   Úselection_statsrD  Ø  s  € Ü
�
Š
Ø�‰×Ñ˜!ÓÜó	€Aô
 �JŠJØ×ÑÜó€Eð
 €Eäñ ä—’Ø�K‰K˜Ñô
ó÷ ˆô %Ø×Ñ÷
‰KˆBð Ø—;‘; %Ñ'ñ)ð ×$Ñ$Øññð ðˆKó $�Ø”R—[’[Ø×%Ñ% dÑ+óñ ’ñ $ð —‘“˜"“Ü—9’9Ø—F‘F˜1‘Ióç‘$˜óñ ô —’Ø—F‘F˜1‘IØ×%Ñ%Ø,ñàñóô —.’.Ø—F‘F˜1‘IØ‘Dóñ	ð ô —’Ø—F‘F˜1‘IØ×%Ñ%Ø ñàñóô —.’.Ø—F‘F˜1‘IØ×%Ñ%Ø!ñàñóñð ð �L‰Lä" 5›\Ø&Ü §¡£›LÜ #ØŸ™˜q™	Ÿ™›ó!ô -2Ø&ó-ô $)Ø ó$ñ÷ôy
ñôh ØØó€Hô 
ØØó
€Bð
 
ˆ'�{�X˜gÑ&Ó&Ø#ˆØ‰ð .ð 	ð ˆð.à�˜Ñ!ð.ð 	˜5ð.ð 	$Ø�VÑð.ð 	&Ø�XÑð.ð 	#Ø�U‰Oð.ð 	+Ø�]Ñ#ð.ð 	˜B˜v™Jð.ð 	�G˜JÑ'ð.ð  	�˜Ñ!ð!.ð& 	˜( 7Ñ+ð'.ð( 	!Ø�VÑð+.ð, 	#Ø�XÑð/.ð0 	 Ø�U‰Oð3.ð4 	(Ø�]Ñ#ð7.ð8 	-ØØ"ñð;.ð@ 	Ø�ZÑ ðC.ðH 	�B�w‘KðI.ðJ ˜v™Jàˆx‰LØ˜e™9àˆ}ÑàÐ!Ñ"Ø˜*‘~ò[.ð .r2   c                óŠ   • [        U U R                  [        U5      :H     UU5      R                  S[        R
                  5      $ ©Nr�   )r§   rE   rM   rò   r'   rQ   )r†   rE   r‡   rˆ   s       r0   Ú_fold_deltarG  |  sJ   € ô ØØ�L‰LÜ�4‹yññ	
ð 	Øó÷ 
�cØÜ
�‰óð
r2   c                ó\   • [        X   UU5      R                  S[        R                  5      $ rF  )r§   rò   r'   rQ   )r†   Úfaculty_maskr‡   rˆ   s       r0   Ú_faculty_deltarJ  �  s3   € ô ØÑØØó÷ 
�cØÜ
�‰óð	r2   c                ó¤  • [         R                  " U 5      (       a  [        U 5      O[         R                  * n[         R                  " U5      (       a  [        U5      O[         R                  * n[         R                  " U5      (       a  [        U5      O[         R                  * n[         R                  " U5      (       a  [        U5      OSn	[         R                  " U5      (       a  [        U5      O[         R                  * n
U(       d  US::  a  gUS:¼  a  US:”  a  US:”  a  U	S:¼  a  U
S:¼  a  gUS:¼  a  US:”  a  US:”  a  U	S	:¼  a  U
S:¼  a  g
g)NrA   r   ÚREJECTç{®Gáz„?ç333333ã?ç{®Gázt¿ÚBREAKTHROUGHg{®Gázt?gš™™™™™á?Ú	PROMISINGÚWEAK)r'   r-   r)   r  )r  Úf6_delta_apÚf7_delta_aprœ   Úit_delta_apÚdominated_by_controlr‰   Úf6Úf7ÚwrÚits              r0   Úclassify_verdictr[  Ÿ  s1  € ô �;Š;Ø ÷
ñ 
ô 	Ð"Ô#ô �f‰fˆWð ô �;Š;�{×#Ñ#ô 	ˆkÔä�f‰fˆWð ô �;Š;�{×#Ñ#ô 	ˆkÔä�f‰fˆWð ô �;Š;�x× Ñ ô 	ˆhŒàð ô �;Š;�{×#Ñ#ô 	ˆkÔä�f‰fˆWð ö ˜q A›vØð 	
ˆU‹
Ø�‹FØ�‹FØ�$‹JØ�&‹Làð 	
ˆU‹
Ø�‹FØ�‹FØ�$‹JØ�&‹Lààr2   )é   é   r  r”   Úkhoa_cong_nghe_thong_tin)Ú
kinh_doanhÚdu_lichÚbusinessÚtourismc           	     ó‚  ^• UR                  5       nUUR                  R                  [        U5      5         n[	        USS5      n[	        USU5      nUR                  S[        R                  5      nUR                  S[        R                  5      n[        R                  " U5      (       a*  [        R                  " U5      (       a  UU:”  a	  SnSnUnUnOSnSnSnUn[        U5      S:”  a  [        UUS   UU5      O[        R                  n[        U5      S:”  a  [        UUS   UU5      O[        R                  nUR                  U:H  n[        UUUU5      nUR                  R                  [        5      R                  R                  5       R!                  U4S	 j5      n [        UU UU5      n!UU4 V"s/ s H'  n"U"c  M  [        R                  " U"5      (       d  M%  U"PM)     n#n"[#        US:H  =(       a7    U#=(       a.    [        R                  " U5      =(       a    U[%        U#5      S
-   :*  5      n$['        UR                  S[        R                  5      UUUR                  S[        R                  5      UU$5      n%0 SU _SU_SUR                  S[        R                  5      _SU	_SU_SU
_SU_SU_SU_SU_SU_SU_SU_SUR                  SS5      _SUR                  SS5      _SUR                  S[        R                  5      _SUR                  S[        R                  5      _UU!UUUU$U%S .E$ s  sn"f )!Nr”   r•   r  r�   r  r  r   r  c                ó0   >^ • [        U 4S jT 5       5      $ )Nc              3  ó.   >#   • U  H
  nUT;   v •  M     g 7frµ   © ©rÈ   Úkr/   s     €r0   rÉ   Ú>summarize_encoder_verdict.<locals>.<lambda>.<locals>.<genexpr>P  s   øé € ð â2�Að �Q–Ú2ùó   ƒ)rO   )r/   Úbusiness_faculty_keywordss   `€r0   rº   Ú+summarize_encoder_verdict.<locals>.<lambda>P  s   ù€ ”cô á2óô r2   g-Cëâ6?rœ   ÚencoderÚverdict_basisÚprimary_conditional_delta_apÚdirect_native_apÚnative_metric_apÚlinear_z_apÚhgb_z_apÚ	hgb_xz_apÚraw_hgb_x_apÚdelta_ap_from_xzÚcommittee_residual_delta_apÚholdout_f6_delta_apÚholdout_f7_delta_apÚfaculty_ap_winsr™   Úfaculty_cellsr˜   Úfaculty_win_rateÚf2_at20_deltar¢   )rU  Úbusiness_delta_apÚruntime_secondsÚ
latent_dimÚeffective_rankrV  Úverdict)r~   rE   ÚisinÚlistr§   rò   r'   rQ   r-   r,   rG  rF   rJ  rß   r·   ÚlowerÚapplyr+   r  r[  )&Úencoder_nameÚ
metrics_dfÚholdout_foldsÚ	raw_modelÚcandidate_modelÚpca_summaryÚrandom_summaryÚprimary_facultyrk  Ú	native_apÚ	linear_aprs  rt  Ú
raw_hgb_apÚruntimer€  Úeffective_rank_valuerq  r‰   Úholdr@  rA  ÚrvÚxvrB  r‡   rˆ   rC  rW  rX  ÚitmaskÚit_deltaÚbmaskÚbusiness_deltar¹   ÚcontrolsÚ	dominatedr‚  s&           `                             r0   Úsummarize_encoder_verdictr�  ß  sñ  ø€ ð2 	�‰Ó€AàØ	�‰�‰Ü�Óó	
ñ€Dô ØØØó€Hô 
ØØØó
€Bð 
�‰ØÜ
�‰ó
€Bð 
�‰ØÜ
�‰ó
€Bô 	�Š�B�‰ä—’˜B—‘Ø�B‹wð $ˆØˆ	ØˆØ‰ð .ð 	ð )ˆ	ØˆØˆô ˆ}Ó Ó!ô 	ØØ˜!ÑØØô		
ô �V‰Vð ô$ ˆ}Ó Ó!ô 	ØØ˜!ÑØØô		
ô �V‰Vð ð 	�‰Øñ	ð ô
 ØØØØó	€Hð 	�‰ß	‰”‹ß	‰�U‰U‹Wß	‰ôó

ð	 
ô $ØØØØó	€Nð Øñ
óò
ˆAð ó 	
ô �KŠK˜�N÷ 	
ñ
ð ð ô ØÐ$Ñ$÷ 	 Ø÷	 ä�KŠK˜‹O÷	 ð Üˆx‹=˜4Ññ ó	€Iô Ø�‰ØÜ�F‰Fó	
ð 	Ø
Ø�‰ØÜ�F‰Fó	
ð 	Øó€Gð4Ø�<ð4à˜ð4ð 	'Ø�K‰KØÜ—‘óð	4ð 	˜Ið4ð 	Øð4ð 	�yð4ð 	�Hð4ð 	�Yð4ð 	˜
ð4ð" 	˜Bð#4ð$ 	&Øð'4ð* 	˜rð+4ð, 	˜rð-4ð0 	Ø�K‰K˜ Ó"ð34ð4 	Ø�K‰K˜ Ó#ð74ð8 	Ø�K‰KØÜ—‘óð;4ðD 	Ø�K‰KØ$Ü—‘óðG4ðP  àà"Ø à ð Øòg4ð 4ùòCs   Æ;L<ÇL<Ç#L<c                ó  • [         R                  " U [        5      n [         R                  " U[        5      nU[        U5      :¬  n[        [         R                  " X@S:H  -  5      5      n[        [         R                  " X@S:H  -  5      5      n[        [         R                  " U) U S:H  -  5      5      n[        [         R                  " U) U S:H  -  5      5      nU[        XV-   S5      -  n	U[        XW-   S5      -  n
SU	-  U
-  [        Xš-   S5      -  n[        U5      S-  nSU-   U	-  U
-  [        XÉ-  U
-   S5      -  n[        U5      [        U	5      [        U
5      [        U5      [        U5      [        UR                  5       5      UUUUS.
$ )Nr  r   r  gê-�™—q=)
Ú	thresholdr;   r<   r=   r>   r?   ÚtpÚfpÚfnÚtn)r'   r(   rM   r)   rS   r  rT   )r"   r/   rŸ  rX   Úpredr   r¡  r¢  r£  rÀ   r¯   r=   Úb2Úfbs                 r0   Ú_binary_metricsr§  ¸  s‡  € ô 	�
Š
�1”cÓ€AÜ
�
Š
�1”eÓ€Aà”�iÓ Ñ €Dä	Ü
�Šˆt˜A‘v‰Óó
€Bô 
Ü
�Šˆt˜A‘v‰Óó
€Bô 
Ü
�Š��˜!˜q™&Ñ!Ó"ó
€Bô 
Ü
�Š��˜!˜q™&Ñ!Ó"ó
€Bð 	ŒS�‘˜!‹_Ñ€AØ
ŒS�‘˜!‹_Ñ€Að 	
ˆA‰�‰	Ü
ˆa‰e�UÓ
ñ	ð ô
 
ˆt‹˜Ñ	€Bð 
ˆR‰�1‰�qÑÜ
ˆb‰f�q‰j˜%Ó
 ñ	!ð ô ˜9Ó%Ü˜1“XÜ˜“(Ü�B‹iÜ�r“ÜØ�I‰I‹Kó
ð ØØØñð r2   c                óD  • [         R                  " U [        5      n [         R                  " U[        5      n[	        U 5      S:X  d)  [         R
                  " U5      R                  5       (       d  g[         R                  " [         R                  " U[         R                  " SS[        S[        S[	        U5      5      5      5      5      5      nS nU H?  n[        U UUU5      nUS   US   US	   US
   * 4nUb
  XtS   :”  d  M1  U[        U5      U4nMA     US   US   S
   4$ )Nr   )NNrM  g®Gáz®ï?éÇ   r8   r>   r<   r;   r?   r  r  )r'   r(   rM   r)   r,   r-   r.   r2  r¥   Úlinspacer¦   r  r§  )r"   r/   rX   ÚqsÚbestÚtr[   r_   s           r0   Ú_select_thresholdr®  ï  s#  € Ü
�
Š
�1”cÓ€AÜ
�
Š
�1”eÓ€Aô 	ˆA‹�!‹Ü�{Š{˜1‹~×!Ñ!×#Ñ#àä	�ŠÜ
�ŠØÜ�KŠKØØÜØÜ˜œC ›F“Oóóó
	
ó
€Bð €DãˆÜØØØØó	
ˆð ˆg‰JØˆh‰KØˆk‰NØˆ{‰^ˆOð	
ˆð ‰LØ˜!‘W�}ð Ü�a“ØðŠDñ' ð4 	ˆQ‰ØˆQ‰�Ñðð r2   c                óN  • / n[        U  Vs1 s H  nUS   iM
     sn5      n[        [        5       R                  " U  Vs/ s H  n[        US   5      PM     sn6 5      nU GHˆ  nU  Vs/ s H  nUS   U:X  d  M  UPM     n	nU GH`  n
U	 Vs/ s H  nUS   U;   d  M  X¥S   ;   d  M  UPM     nnU	 Vs/ s H  nUS   U;   d  M  X¥S   ;   d  M  UPM     nnU(       a  U(       d  Mb  [        R                  " U Vs/ s H!  n[        R
                  " US   [        5      PM#     sn5      n[        R                  " U Vs/ s H$  n[        R
                  " US   U
   [        5      PM&     sn5      n[        R                  " U5      R                  5       (       d  UR                  UU
SSSS.5        GM.  [        UUU5      u  nnUc  UR                  UU
SSS	S.5        GM[  / / // / /S
.nSnU GH4  n[        R
                  " US   [        5      n[        R
                  " US   U
   [        5      n[        R                  " U5      R                  5       (       d  SnUR                  UU
US   SSS.5        MŠ  [        [        R                  " U[        S[        SSU-
  5      5      5      5      nSU4SU44 Hg  u  nn[        UUUU5      nUR                  UU
[        US   5      USS.UE5        UU   S   R                  U5        UU   S   R                  U5        Mi     GM7     U(       a  GM«  S
 H¯  nUU   S   (       d  M  [        R                  " UU   S   5      n[        R                  " UU   S   5      nUS:X  a  UnO7[        [        R                  " U[        S[        SSU-
  5      5      5      5      nUR                  UU
SUSS.[        UUUU5      E5        M±     GMc     GM‹     [         R"                  " U5      $ s  snf s  snf s  snf s  snf s  snf s  snf s  snf )NrF   rW   rE   r"   ÚPOOLEDrB   ÚUNAVAILABLE_DEV_SCORE)rF   rG   rE   ÚpolicyrL   ÚUNAVAILABLE_THRESHOLD)ÚfixedÚdriftFTÚNONFINITE_HOLDOUT_SCORErA   rC   r´  rµ  rD   r   r  )r¾   ÚsetÚunionr'   Úconcatenater(   rM   r)   r-   r.   rR   r®  r¥   r  r¦   r§  r|   r}   )ÚrecordsÚ	dev_foldsr‰  rX   rH   r¯   Ú	facultiesÚmodelsrF   ÚrrrG   Údevr”  ÚydÚsdrŸ  Úwarning_rateÚpooledÚunavailabler"   r/   Údrift_thresholdr²  r­  r[   s                            r0   Úoperational_metricsrÆ  (  s  € ð €Däñ ó	
â�ð ˆiŒLÙñ	
ó€Iô Ü‹�Šñ !óâ �Aô �A�h‘KÖ Ù ñð	
ó€Fô ˆñ ó
â�Ø�‰|˜wÑ&÷ Ùð 	ð 
ô ˆEñ óâ�AØ�V‘9 	Ñ)ó ð ˜x™[Ñ(÷ Ùð ð ñ óâ�AØ�V‘9Ø ñ!ó ð ˜x™[Ñ(÷	 Ùð ð ö ždÙä—’ñ !óò
 !˜ô	 —J’JØ˜#™Üöñ !ñóˆBô —’ñ !óò
 !˜ô	 —J’JØ˜(™ EÑ*Üöñ !ñóˆBô —;’;˜r“?×&Ñ&×(Ñ(Ø—‘à#*Ø!&Ø (Ø"/Ø"9ñôò ô "ØØØóñ $ˆI�|ð Ñ Ø—‘à#*Ø!&Ø (Ø"/Ø"9ñôò ð ˜b˜Ø˜b˜ñˆFð
  ˆKä�Ü—J’JØ�c‘FÜó�ô
 —J’JØ�h‘K Ñ&Üó�ô
 —{’{ 1“~×)Ñ)×+Ñ+Ø"&�KØ—K‘Kà'.Ø%*Ø$% f¡IØ&3Ø&?ñôñ ä"'Ü—K’KØÜØÜØ #Ø #Ø".ñ!/óóó
ó#�ð   Ø!ðð
  Ø'ðó	"‘I�F˜Aô (ØØØØó	�Að —K‘Kð !(Ø%*ä # A f¡I£à &Ø&*ñ
ð  ð
ôð ˜6‘N 1Ñ%×,Ñ,Øôð ˜6‘N 1Ñ%×,Ñ,ØöôE	"ñM öX Úó�ð ˜f‘~ a×(Ùä—N’NØ˜6‘N 1Ñ%ó�ô —N’NØ˜6‘N 1Ñ%ó�ð ˜WÓ$Ø!‘AäÜŸšØÜØ #Ü #Ø$'Ø$'Ø&2ñ%3ó!"óó
ó�Að —‘à#*Ø!&Ø (Ø"(Ø"&ñô *ØØØØ ó	ðöô?ô{ ñ ôf �<Š<˜ÓÐùòE	
ùòùò
ùòùòùòùòsF   ŒP¿P	Á)PÁ:PÂPÂPÂ'PÂ3PÃPÃPÃ:(PÄ=+P"c                ó,  • U U R                   R                  [        U5      5      U R                  R                  [        5      -  [
        R                  " U R                  5      -     R                  5       n/ nUR                  SS/5       Hð  u  u  nnnUR                  SSS9R                  S   nUR                  [        U5      UUR                  [        UR                  5      [        UR                  5      [        UR                   5      SR#                  [%        UR                  R'                  [(        5      R+                  5       5      5      [        UR                  R-                  5       5      S.5        Mò     [.        R0                  " U5      $ )	NrE   rF   r4   F©Ú	ascendingr   rÕ   )rE   rF   Úbest_prior_modelÚbest_prior_apÚbest_prior_aucÚbest_prior_f2_at20Úavailable_prior_modelsÚprior_models_available)rE   rƒ  r„  rG   ÚDEPLOYABLE_PRIORSr'   r-   r4   r~   r  Úsort_valuesr
  rR   rM   r)   r5   rc   rà   r¾   rß   r·   r2  Únuniquer|   r}   )Úreference_metricsr‰  ÚrefsrH   rE   rF   Úgr¬  s           r0   Ú_best_prior_enveloperÖ    ss  € ð Ø×Ñ×#Ñ#Ü�Óó	
ð ×
!Ñ
!×
&Ñ
&Üó
ñ	
ô �+Š+Ø× Ñ ó
ñ	
ñ
÷ �dƒfð 	ð €Dð
 �L‰LØ	�Ðöñ		ñ 	ØØØð �}‰}ØØð ð 
÷ ‰$ˆqñˆð
 	�‰ä˜D›	Ø"à—J‘Jä˜$Ÿ'™'“Nä˜$Ÿ(™(“OäØ×(Ñ(óð —H‘HÜØŸG™GŸN™NÜ #óç$™f›hóóô ØŸ™Ÿ™Ó)óñ-ö	
ñôJ �<Š<˜ÓÐr2   c                ó�  ^ • [        U5      nUS-  nUS-  nUR                  5       (       d  [        S5        g [        R                  " U5      nUR                  5       (       a  [        R                  " U5      O[        R
                  " 5       n[        S5        [        S5        [        S5        [        U5      (       ar  UUR                  R                  T R                  5         R                  S5      R                  SS	S
S9R                  5       nUR                  US-  SS9  [        SUSS9  / n[        U5      (       GaÎ  [        UR                   R#                  5       R%                  5       5       GH˜  n	UUR                   U	:H     n
S GH{  nU
U
R&                  U:H     / SQ   R)                  SSSS.S9n[        U5      (       d  M>  [*         GH2  nUUR&                  U:H     / SQ   R)                  SSSS.S9nUR-                  USS/SS9n[.        R0                  " UR2                  5      [.        R0                  " UR4                  5      -  nUR6                  U   R9                  5       n[        U5      (       d  M¯  XŸS '   X¿S!'   XßS"'   UR2                  UR4                  -
  US#'   UR:                  UR<                  -
  US$'   UR>                  UR@                  -
  US%'   URB                  S&:„  US''   URE                  U5        GM5     GM~     GM›     U(       a  [        RF                  " US(S)9O[        R
                  " 5       nUR                  US*-  SS9  [        U5      (       aŽ  UUR                  R                  T R                  5         R                  / S+Q5      R                  S,S	S-S.S#S/ 4S0S1S2S39R                  5       nUR                  US4-  SS9  [        S5URI                  S!S6/S(S/S79S8S9  [        U5      (       a  [K        UT R                  5      O[        R
                  " 5       nUR                  US9-  SS9  / n[        U5      (       Ga«  [        UR                   R#                  5       R%                  5       5       GHu  n	S GHj  nUUR                   U	:H  UR&                  U:H  -  UR                  R                  T R                  5      -     / SQ   R)                  SSSS.S9n[        U5      (       d  Mt  UR-                  USS/SS9nU[.        R0                  " UR2                  5      [.        R0                  " URL                  5      -     R9                  5       n[        U5      (       d  Më  XŸS '   X¿S!'   UR2                  URL                  -
  US:'   UR:                  URN                  -
  US;'   UR>                  URP                  -
  US<'   URR                  S&:„  US''   URE                  U5        GMm     GMx     U(       a  [        RF                  " US(S)9O[        R
                  " 5       nUR                  US=-  SS9  / n[        U5      (       Ga  UR                  S S!/5       GHó  u  u  n	nnUUR                  T R                  S&   :H     RR                  RU                  5       n[        T R                  5      S>:”  a7  UUR                  T R                  S>   :H     RR                  RU                  5       O[.        RV                  nUURX                  T RZ                  :H     RR                  RU                  5       nURX                  R]                  [^        5      R^                  Ra                  5       Rc                  U 4S? j5      nUR6                  US:4   RU                  5       n[e        URR                  RU                  5       5      n[e        URR                  Rg                  5       5      n[e        URR                  Ri                  S@5      5      n[e        URj                  RU                  5       5      nUS&:”  aW  US&:”  aQ  US&:”  aK  USA:¼  aE  [.        R0                  " U5      (       a  USB:¼  a$  [.        R0                  " U5      (       a  USB:¼  a  SCn OUS&:”  a	  USD:¼  a  SEn OSFn URE                  U	U[m        [        U5      5      [m        URX                  Ro                  5       5      UUU[e        URR                  Rq                  5       5      U[e        U5      [e        U5      [e        U5      [e        U5      [e        URr                  RU                  5       5      U SG.5        GMö     [        R
                  " U5      nUR                  USH-  SS9  [        U5      (       a  [        SIURI                  SJSS79SKS9  [        R
                  " SLSMSNSOSPSQSR./5      n!U!R                  USS-  SS9  [        STU!5        [        U5      (       a¥  UURX                  T RZ                  :H  URX                  R]                  [^        5      R^                  Ra                  5       Rc                  U 4SU j5      -     R9                  5       n"U"R                  USV-  SS9  [        SWU"RI                  / SXQ/ SYQS79S8S9  [        SZ5        [        S[5        [        S\5        g )]Nzall_fold_metrics.csvzall_reference_metrics.csvz0[FINAL REPORT WARN] all_fold_metrics.csv missingzµ
====================================================================================================================================================================================z9SUITE19 FINAL CORRECTED REPRESENTATION / PRIOR-SOTA AUDITz´====================================================================================================================================================================================rG   )rF   r5  )rF   rÒ  )rE   rÒ  )r˜   r¼  Úfoldsz"reference_availability_holdout.csvF)Úindexu"   REFERENCE AVAILABILITY â€” HOLDOUTrý   rþ   )r”   r  Úhgb_zÚlinear_zÚdirect_nativeÚnative_metric)rE   rF   r4   r5   rc   Ú
suite19_apÚsuite19_aucÚsuite19_f2_at20)r4   r5   rc   rr   Úprior_apÚ	prior_aucÚprior_f2_at20rE   rF   rt   ru   rm  r‡   Úprior_modelrn   ro   rp   r   rq   T)Úignore_indexzprior_head_to_head_cells.csv)rm  r‡   rä  )rn   r5  )rn   rT   )rn   r¤   c                ó$   • U R                  S5      $ )Nr£   )r¥   r¸   s    r0   rº   Ú%write_final_reports.<locals>.<lambda>?  s   € ØŸ
™
 4Ô(r2   )rn   r¦   )rq   rT   )rp   rT   )r˜   r¼  r�   rž   rŸ   r    rœ   r¢   zprior_head_to_head_summary.csvz1SUITE19 DIRECT HEAD-TO-HEAD VS HISTORICAL SYSTEMSr�   rÈ  i,  zbest_prior_envelope.csvÚdelta_ap_vs_best_priorÚdelta_auc_vs_best_priorÚdelta_f2_vs_best_priorzsuite19_vs_best_prior_cells.csvr  c                óD   >^ • [        U 4S jTR                   5       5      $ )Nc              3  ó.   >#   • U  H
  nUT;   v •  M     g 7frµ   rf  rg  s     €r0   rÉ   Ú8write_final_reports.<locals>.<lambda>.<locals>.<genexpr>  ó!   øé € ð "ò 7ð ð ˜Qžâ6ùrj  ©rO   rk  ©r/   Úargss   `€r0   rº   rç    ó   ù€ œcô "ð ×6Ò6ó"ô r2   r£   rN  rO  ÚNEW_PROJECT_SOTA_CANDIDATEr  ÚCOMPETITIVE_WITH_PRIOR_SOTAÚBELOW_BEST_PRIOR_ENVELOPE)rm  r‡   r˜   r¼  Úmean_delta_ap_vs_best_priorÚmedian_delta_ap_vs_best_priorÚq25_delta_ap_vs_best_priorÚworst_delta_ap_vs_best_priorÚwin_rate_vs_best_priorÚf6_delta_ap_vs_best_priorÚf7_delta_ap_vs_best_priorÚit_delta_ap_vs_best_priorÚbusiness_delta_ap_vs_best_priorÚmean_delta_f2_vs_best_priorÚproject_sota_verdictz suite19_project_sota_verdict.csvu@   SUITE19 PROJECT-SOTA VERDICT â€” VS BEST PRIOR PER FACULTYÃ—FOLDrö  éÈ   zSuite15 V1+V4Ú aggregate_information_audit_onlyé   gˆ…ZÓ¼ãt?é   z1Do not include in deployable best-prior envelope.)rè   ÚtypeÚholdout_cellsÚmean_delta_ap_vs_controlÚwins_vs_controlÚinterpretationz!suite15_information_reference.csvz#SUITE15 INFORMATION-AUDIT REFERENCEc                óD   >^ • [        U 4S jTR                   5       5      $ )Nc              3  ó.   >#   • U  H
  nUT;   v •  M     g 7frµ   rf  rg  s     €r0   rÉ   rí  ©  rî  rj  rï  rð  s   `€r0   rº   rç  ©  rò  r2   z'dedicated_it_business_vs_best_prior.csvz,IT + BUSINESS/TOURISM VS BEST PRIOR ENVELOPE)rF   rE   rè  )TTFz
[FINAL INTERPRETATION RULE]zCRepresentation verdict answers: does z add conditional information?z‹Project-SOTA verdict answers: does any predeclared Suite19 consumer actually beat the strongest prior deployable system faculty-by-faculty?):r   r¼   Úprintr|   rÙ   r}   r,   rE   rƒ  r‰  r  ÚaggÚreset_indexÚto_csvr   r¾   rm  Údropnar2  rG   r�   rÐ  rƒ   r'   r-   rÞ  rá  r„   r~   rß  râ  rà  rã  rn   rR   ÚconcatrÑ  rÖ  rË  rÌ  rÍ  rè  rT   rQ   rF   rŽ  rß   r·   r…  r†  r)   r¤   r¥   rq   rM   rÒ  r¦   rê  )#rñ  rô   Úfold_resultsÚmetrics_pathÚ	refs_pathr†   rÔ  r`   Úprior_cellsrm  Úemr‡   rŽ   Úpriorr�   r�   r‘   Úprior_cells_dfÚprior_summaryÚenvelopeÚenvelope_cellsÚproject_summaryrÕ  rW  rX  rZ  Úbizmaskra  rT   r¤   r  rœ   r‚  Úsuite15_referenceÚ	dedicateds#   `                                  r0   Úwrite_final_reportsr   S  sõ  ø€ ô
 ˆs‹)€Cð 	Ð$Ñ$ð ð 	Ð)Ñ)ð ð ×Ñ× Ñ Üð+ô	
ð 	ä�kŠkØó€Gð ×Ñ×Ñô 	�Š�IÔä�\Š\‹^ð 	ô 
ð	ôô 
ð	,ôô 
Øôô ˆ4‡y�yàØ—	‘	—‘Ø×&Ñ&óñ÷
 ‰W�WÓß‰SØ)ðð *ð ð ÷ ‰[‹]ð 	ð$ 	×ÑØØ2ñ3àð 	ñ 	
ô 	Ø0ØØò	
ð €Kä
ˆ4‡y‚yÜØ�O‰O×"Ñ"Ó$×+Ñ+Ó-÷
ˆGð Ø—‘ 7Ñ*ñˆBô�	ð Ø—H‘H 	Ñ)ñòñ
÷ ‘&ð )à)à-ñð ð 	ð ô* ˜1—v‘vÙç.Ð.�EØØŸ
™
 eÑ+ñòñ
÷ ‘fð !+à +à /ñ!ð ð 	ð ð* Ÿ™Øà"Ø%ðð $ð  ð �Aô ŸšØŸL™Lóô Ÿ+š+ØŸJ™Jóñð ð Ÿ™Øñç‘d“fð ô ˜qŸ6™6Ù à#*�i‘LØ%.�k‘NØ',�mÑ$ð Ÿ™ØŸ*™*ñ%ð �j‘Mð
 Ÿ™ØŸ+™+ñ&ð �k‘Nð
 ×)Ñ)ØŸ/™/ñ*ð �oÑ&ð
 Ÿ
™
 Q™ð �e‘Hð  ×&Ñ& q×)ôG /ôAñ
öb ô	 	�	Š	ØØò	
ô
 �\Š\‹^ð ð ×ÑØØ
(ñ	)àð ñ ô ˆ>×ÑàØ×#Ñ#×(Ñ(Ø×&Ñ&óñ÷
 ‰Wòó÷ ‰SØ*ððð!ð
 ñ)ðð
 ðð$ð7 ð ÷@ ‰[‹]ð[ 	ð` 	×ÑØØ.ñ/àð 	ñ 	
ô 	ð$à×%Ñ%àØ#ðð
 Øðð &ð 	ð ò	
ô0 ˆt�9‰9ô	 	ØØ×Ñô	
ô
 �\Š\‹^ð ð ‡O�OØØ
#ñ	$àð ñ ð €Nä
ˆ8‡}‚}ÜØ�O‰O×"Ñ"Ó$×+Ñ+Ó-÷
ˆGô�	ð Ø—_‘_¨Ñ/àŸ™Ø$ñ%ñð
 —l‘l×'Ñ'Ø×*Ñ*óññ	òñ÷" ‘&ð )à)à-ñð ð 	ð# ô8 ˜1—v‘vÙà—G‘GØàØ!ðð  ð ð �ð Ü—K’KØŸ™óô —k’kØŸ™óññ÷ ‘$“&ð ô ˜1—v‘vÙà&�)‘Ø!*�+‘à—L‘LØ—o‘oñ&ð Ð*Ñ+ð —M‘MØ×&Ñ&ñ'ð Ø-ñð ×%Ñ%Ø×*Ñ*ñ+ð Ø,ñð ×,Ñ,Øñð �%‘ð
 ×%Ñ% a×(ôgñ
öz ô	 	�	Š	ØØò	
ô
 �\Š\‹^ð ð ×ÑØØ
+ñ	,àð ñ ð €Oä
ˆ>×Òð ×&Ñ&Ø˜Ð$÷
ññ 
ØØØð Ø—‘Ø×%Ñ% aÑ(ñ)ñ÷ %Ñ$§T¡T£Vð ô Ø×&Ñ&óàóð	 Ø—F‘FØ×)Ñ)¨!Ñ,ñ-ñ÷ )Ñ(¯©¬ô —V‘Vð ð Ø—	‘	Ø×'Ñ'ñ(ñ÷ %Ñ$§T¡T£Vð ð —	‘	ß‘œ“ß‘—U‘U“Wß‘ôóð	 ð —u‘uØØ(ð*ñ÷ ‰d‹fð ô
 Ø×(Ñ(×-Ñ-Ó/óˆDô Ø×(Ñ(×/Ñ/Ó1óˆFô Ø×(Ñ(×1Ñ1ØóóˆCô
 Ø—‘—
‘
“óˆHð
 �q“Ø˜“FØ˜“FØ Ó$äŸš BŸ™Ø˜V“|ô Ÿš H×-Ñ-Ø 6Ó)ð 1ñ ð �q“Ø Ó$ð 2ñ ð
 0ð ð ×"Ñ"à&Ø!*Ü ¤ Q£›[äØŸI™I×-Ñ-Ó/óð ààäØ×4Ñ4×8Ñ8Ó:óð !ä˜b›	ä˜b›	ä˜b›	ä˜h›äØ×4Ñ4×9Ñ9Ó;óð  ñC"÷$ñm
ôx —l’lØó€Oð ×ÑØØ
,ñ	-àð ñ ô ˆ?×ÑÜð2à×'Ñ'Ø-Øð (ð ð ò	
ô Ÿšð $à6Ø!#àØ#%àGñð	
óÐð" ×ÑØØ
-ñ	.àð ñ ô Ø-Øôô ˆ>×ÑØ"à×&Ñ&Ø×'Ñ'ñ(ð
 ×&Ñ&ß‘œ“ß‘—U‘U“Wß‘ôóññ
÷$ ‰$‹&ð% 	ð( 	×ÑØØ7ñ8àð 	ñ 	
ô 	ð%à×!Ñ!òò
ð "ð ð ò	
ô$ 
Ø'ôô 
ð	.ôô 
ð	0õr2   )g       @))Ú__doc__Ú
__future__r   Ú	functoolsr   Úpathlibr   Únumpyr'   Úpandasr|   Úsuite19_reportingr6  Úsuite19_commonr   r   rñ   rÛ   rÐ  r1   ra   r’   r–   r§   r°   rÃ   rä   rù   r   r  rD  rG  rJ  r[  rQ   r�  r§  r®  rÆ  rÖ  r   rf  r2   r0   Ú<module>r)     sˆ  ðñõ #å Ý ã Û ã "ß 3ð
 	Øðð
 	Øðð
 	Øðð
 	Øðòð 	'ØðòòñC(Ð ðT€ðÐ ò1ð 
ôCòL}ò~#ò4òn	ñ �4Ññ(ó ð(òVUòptòn òFOòdaòHò&ò =ðF ØØ(ØØØ.ðð �f‰fØ�f‰fØ�V‰VØ�f‰fØ�v‰vØ�F‰FØ�v‰vØŸ™Ø—V‘Vô/Vðz 
ô	4òn6ðz 
ô	kò\:ózC
r2   