ó
    Pï‘jy"  ã                  óÄ  • S r SSKJr  SSKJr  SSKJr  SSKJrJ	r	  SSK
r
SSKrSSKrSSKrSSKrSSKJrJr  SSKr\" \R.                  5      r\R0                  r\R2                  r " S S	5      rS(S
 jrS)S jrS)S jrS*S+S jjrS,S jrS-S jr SSSS.S jr!SSSSSSSS. S.S jjr"S/S jr#S r$S0S1S jjr%SSS .S1S! jjr&S2S" jr'S2S# jr(S3S4S$ jjr)S% r*S5S& jr+S6S' jr,g)7a  Shared utilities for AQ breakthrough experiments.

This module intentionally imports the canonical data preparation, base-expert
configuration, temporal cross-fitting, and metrics from ``train_v3.py`` so the
new experiments do not silently change the benchmark committee.
é    )Úannotations)Ú	dataclass)ÚPath)ÚIterableÚSequenceN)Úaverage_precision_scoreÚroc_auc_scorec                  ód   • \ rS rSrSrS r\SS j5       rSS jrSS jr	SS jr
\SS j5       rS	rg
)Ú
OOVEncoderé   zHDeterministic categorical encoder with index 0 reserved for OOV/missing.c                ó   • 0 U l         g ©N©Úmapping_©Úselfs    ÚC/root/disk_3tb/earlywarning/30072026/suite14/breakthrough_common.pyÚ__init__ÚOOVEncoder.__init__   s	   € Ø(*ˆ�ó    c                ó|   • U c  g [         R                  " U 5      (       a  g [        U 5      $ ! [         a     Nf = f)Nú	<MISSING>)ÚpdÚisnaÚ	ExceptionÚstr)Úvs    r   Ú_normÚOOVEncoder._norm!   sC   € à‰9Øð	Ü�wŠw�q�z‰zØ"ð ô �1‹vˆøô ó 	Ùð	ús   †. ®
;º;c                óî   • [        U Vs1 s H*  o R                  U5      S:w  d  M  U R                  U5      iM,     sn5      n[        U5       VVs0 s H
  u  pBX$S-   _M     snnU l        U $ s  snf s  snnf )Nr   é   )Úsortedr   Ú	enumerater   )r   Úvaluesr   ÚvalsÚis        r   ÚfitÚOOVEncoder.fit,   sb   € Ü©fÓUªf¨¿
¹
À1»ÈÑ8T“}�t—z‘z !–}©fÑUÓVˆÜ.7¸¬oÔ>ªo¡d a˜ ™Eš©oÒ>ˆŒØˆùò VùÛ>s   ŠA,¦A,ÁA1c           
     óô   • U R                   c  [        S5      e[        R                  " U Vs/ s H-  o R                   R	                  U R                  U5      S5      PM/     sn[        R                  S9$ s  snf )NzOOVEncoder not fittedr   ©Údtype)r   ÚRuntimeErrorÚnpÚasarrayÚgetr   Úint64)r   r$   r   s      r   Ú	transformÚOOVEncoder.transform1   sY   € Ø�=‰=Ñ ÜÐ6Ó7Ð7Ü�zŠzÉÓOÊÀ1Ÿ=™=×,Ñ,¨T¯Z©Z¸«]¸AÖ>ÉÑOÔWY×W_ÑW_Ñ`Ð`ùÒOs   ­4A5c                óB   • U R                  U5      R                  U5      $ r   )r'   r1   )r   r$   s     r   Úfit_transformÚOOVEncoder.fit_transform6   s   € Ø�x‰x˜Ó×)Ñ)¨&Ó1Ð1r   c                ó2   • [        U R                  5      S-   $ )Nr!   )Úlenr   r   s    r   ÚsizeÚOOVEncoder.size9   s   € ä�4—=‘=Ó! AÑ%Ð%r   r   N)Úreturnr   )r$   r   r:   z'OOVEncoder')r$   r   r:   ú
np.ndarray)r:   Úint)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   Ústaticmethodr   r'   r1   r4   Úpropertyr8   Ú__static_attributes__© r   r   r   r      sB   † ÙRò+ð óó ðôô
aô
2ð ó&ó ó&r   r   c                óB   • [         R                  " [        U 5      5        g r   )ÚcoreÚseed_allr<   )Úseeds    r   rH   rH   >   s   € Ü‡M‚M”#�d“)Õr   c                óN  • [         R                  " U [        5      R                  S5      n [         R                  " U[        5      R                  S5      n[        U 5      S:X  d$  [         R                  " U 5      R                  S:  a  [	        S5      $ [	        [        X5      5      $ ©Néÿÿÿÿr   é   Únan)	r-   r.   r<   ÚreshapeÚfloatr7   Úuniquer8   r   ©ÚyÚscores     r   Úsafe_aprU   B   sr   € Ü
�
Š
�1”cÓ×"Ñ" 2Ó&€AÜ�JŠJ�uœeÓ$×,Ñ,¨RÓ0€EÜ˜q›6 Q›;¬"¯)ª)°A«,×*;Ñ*;¸aÓ*?Œ5�‹<ÐmÄUÔKbÐcdÓKlÓEmÐmr   c                óN  • [         R                  " U [        5      R                  S5      n [         R                  " U[        5      R                  S5      n[        U 5      S:X  d$  [         R                  " U 5      R                  S:  a  [	        S5      $ [	        [        X5      5      $ rK   )	r-   r.   r<   rO   rP   r7   rQ   r8   r	   rR   s     r   Úsafe_aucrW   H   sq   € Ü
�
Š
�1”cÓ×"Ñ" 2Ó&€AÜ�JŠJ�uœeÓ$×,Ñ,¨RÓ0€EÜ˜q›6 Q›;¬"¯)ª)°A«,×*;Ñ*;¸aÓ*?Œ5�‹<ÐcÄUÌ=ÐYZÓKbÓEcÐcr   éô  éú   c                ó0   • [         R                  " XU5      $ r   )rG   Úmake_base_models©rI   Ún_estimatorsÚhist_max_iters      r   Úmake_committee_modelsr_   N   s   € Ü× Ò  °]ÓCÐCr   c                ón  • [         R                  " U [        5      n[        R                  " U5      n[         R
                  " UUS   US   US   US   US   US   US   /5      n[         Vs/ s H  nSU 3PM
     sn[        [        R                  5      -   n[         R                  " US	S	S	S
9U4$ s  snf )NÚcommittee_meanÚcommittee_minÚcommittee_maxÚcommittee_stdÚcommittee_spreadÚcommittee_entropyÚcommittee_vote_fractionÚexpert_probability_g        )rN   ÚposinfÚneginf)
r-   r.   rP   rG   Úcommittee_statisticsÚcolumn_stackÚ
BASE_NAMESÚlistÚUNCERTAINTY_NAMESÚ
nan_to_num)Úexpert_probabilitiesÚpÚstatsÚzÚnÚnamess         r   Úcommittee_meta_featuresrw   R   s¾   € Ü
�
Š
Ð'¬Ó/€AÜ×%Ò% aÓ(€EÜ
�ŠØ	ØÐÑ  Ñ!7¸¸Ñ9OØˆoÑ Ð&8Ñ 9¸5ÐATÑ;UØÐ'Ñ(ð	ó 	€Aõ 1;Ó;²
¨1Ð" 1 #Ó&±
Ñ;¼dÄ4×CYÑCYÓ>ZÑZ€EÜ�=Š=˜ ¨C¸Ñ<¸eÐCÐCùò <s   Á-B2c           	     óÄ  • [         R                  " X[        U5      5      nUc  g Uu  pEpgp‰n
[        R                  " U[
        5      [        :  R                  [        R                  5      n[        R                  " U[
        5      [        :  R                  [        R                  5      n[         R                  " U5      [         R                  " U5      X¼X‰[        U
5      4$ r   )rG   Ú	load_foldr<   r-   r.   rP   ÚFAILURE_BOUNDARYÚastypeÚint8Úclean_Xrn   )Ú
splits_dirÚfacultyÚfoldÚdataÚXtrÚXteÚgtrÚgteÚtrainÚtestrv   ÚytrÚytes                r   Úload_binary_foldrŠ   _   s�   € Ü�>Š>˜*¬s°4«yÓ9€DØ�|ØØ-1Ñ*€Cˆc˜ UÜ�:Š:�cœ5Ó!Ô$4Ñ4×
<Ñ
<¼R¿W¹WÓ
E€CÜ�:Š:�cœ5Ó!Ô$4Ñ4×
<Ñ
<¼R¿W¹WÓ
E€CÜ�<Š<˜ÓœdŸlšl¨3Ó/°¸5ÌÈUËÐSÐSr   é*   r\   c               óV  • [        X4U5      nUR                  5        HB  nUR                  [        R                  " U 5      [
        R                  " U[        5      5        MD     [        R                  " U[        R                  " U5      5      n[        R                  " U5      S   n	XhU	4$ )Nra   )
r_   r$   r'   rG   r}   r-   r.   r<   Úpredict_expertsrk   )
r‚   rˆ   rƒ   rI   r]   r^   ÚmodelsÚmrr   Úqs
             r   Úfit_production_committeer‘   i   s{   € Ü" 4°}ÓE€FØ�]‰]Ž_ˆØ	�‰Œd�lŠl˜3Ó¤§¢¨C´Ó!5Ö6ñ ä×Ò˜V¤T§\¢\°#Ó%6Ó7€AÜ×!Ò! !Ó$Ð%5Ñ6€AØ�aˆ<Ðr   é   éd   gš™™™™™Ù?F)rI   r]   r^   Úcrossfit_foldsÚcrossfit_min_train_rowsÚmin_oof_coverageÚverbosec               ód   • [         R                  " U5      u  p«[         R                  " XX«X4UXgX‰S9$ )N)rI   r]   r^   r”   Úmin_train_rowsr–   r—   )rG   Úextract_temporal_groupsÚgenerate_crossfit_predictions)ÚXrS   ÚframerI   r]   r^   r”   r•   r–   r—   ÚgroupsÚsources               r   Úcrossfit_historical_committeer    r   s9   € ô
 ×1Ò1°%Ó8�N€FÜ×-Ò-Ø	ˆfØ¸MØ%Ø)ñ	ð r   c                óN   • [         R                  " U5      R                  XU5      $ r   )rG   ÚRawFeatureSelectorr'   )rœ   rS   Úfeature_namesÚks       r   Útop_raw_selectorr¥   €   s    € Ü×"Ò" 1Ó%×)Ñ)¨!°Ó>Ð>r   c                ó$   • U R                  U5      $ r   )r1   )Úselectorrœ   s     r   Útop_raw_transformr¨   „   s   € Ø×Ñ˜aÓ Ð r   ç       @c           
     ó*  • [         R                  " U [        5      n [         R                  " U[        5      n[	        U 5      S:X  a]  [         R
                  [         R
                  [         R
                  [         R
                  [         R
                  [         R
                  S.$ [         R                  " U5      n[	        U5      S::  a  UO@[         R                  " [         R                  " U[         R                  " SSS5      5      5      nS nU H_  nX:¬  R                  [        5      n[        R                  " XXr5      nUS   US   US   US	   * 4n	Ub
  X•S   :”  d  MQ  U	[        U5      U4nMa     Uu  p¦nS
U0S Vs0 s H  o»X‹   _M	     snE$ s  snf )Nr   )Ú	thresholdÚ	precisionÚrecallÚf1ÚfbetaÚ	flag_ratei   gü©ñÒMbP?g+‡ÙÎ÷ï?r¯   r­   r¬   r°   r«   )r¬   r­   r®   r¯   r°   )r-   r.   r<   rP   r7   rN   rQ   ÚquantileÚlinspacer{   rG   Úreport)rS   rT   ÚbetarQ   Ú
thresholdsÚbestÚtÚpredr�   ÚkeyÚ_r¤   s               r   Úbinary_fbeta_at_best_thresholdr»   ˆ   sJ  € Ü
�
Š
�1”cÓ€AÜ�JŠJ�uœeÓ$€EÜ
ˆ1ƒv�ƒ{ÜŸV™V´"·&±&ÄBÇFÁFÔRT×RXÑRXÔce×ciÑciÔxz×x~Ñx~ÑÐÜ�YŠY�uÓ€FÜ˜v›;¨#Ó-‘´2·9²9¼R¿[º[ÈÔQS×Q\ÒQ\Ð]bÐdiÐknÓQoÓ=pÓ3q€JØ€DÛˆØ‘
×"Ñ"¤3Ó'ˆÜ�KŠK˜ $Ó-ˆØ�‰z˜1˜X™;¨¨+©¸¸;¹¸ÐHˆØ‰<˜3 a¡�=Øœ˜q› 1Ð%ŠDñ ð �G€Aˆ!Ø˜ÐeÑ0cÓdÒ0c¨1 !¡$šwÑ0cÑdÐeÐeùÒds   Å>FÚ )Úprefixr´   c               ó²  • [        XU5      nU S3[        [        U 5      5      U S3[        [        R                  " U [        5      R                  5       5      U S3[        U 5      (       a  [        [        R                  " U 5      5      O[        R                  U S3[        X5      U S3[        X5      U S3US   U S3US	   U S
3US   U S3US   0	$ )Nru   ÚfailuresÚ
prevalenceÚapÚaucÚbest_f2r¯   Úbest_precisionr¬   Úbest_recallr­   Úbest_flag_rater°   )r»   r<   r7   r-   r.   ÚsumrP   ÚmeanrN   rU   rW   )rS   rT   r½   r´   Úbs        r   Ú	score_rowrÊ   š   s×   € Ü& q°Ó6€Aàˆ(�!ˆ”cœ#˜a›&“kØˆ(�(ÐœS¤§¢¨A¬sÓ!3×!7Ñ!7Ó!9Ó:Øˆ(�*Ð´C¸·F±Fœu¤R§W¢W¨Q£ZÔ0ÄÇÁØˆ(�"ˆ”w˜qÓ(Øˆ(�#ˆœ Ó*Øˆ(�'Ð˜A˜g™JØˆ(�.Ð! 1 [¡>Øˆ(�+Ð  (¡Øˆ(�.Ð! 1 [¡>ð
ð 
r   c                óB  • SU ;   a!  [        U S   R                  [        5      5      O	[        5       nSU ;   a!  [        U S   R                  [        5      5      O	[        5       nSU;   a&  US   R                  [        5      R                  5       O"[        R
                  " S/[        U5      -  5      nSU;   a&  US   R                  [        5      R                  5       O"[        R
                  " S/[        U5      -  5      n[        R                  " U Vs/ s H  ofU;  PM	     sn5      n[        R                  " U Vs/ s H  oˆU;  PM	     sn5      n	[        R                  " [        U5      [        S9n
SX§) U	) -  '   SX§) U	-  '   SX§U	) -  '   SX§U	-  '   U
$ s  snf s  snf )	zYReturn four stable subgroup labels using only identifiers available before target labels.ÚIDSVÚIDMHz<NA>r*   Úreturning_seenÚreturning_new_courseÚnew_student_seen_courseÚnew_student_new_course)
Úsetr{   r   Úto_numpyr-   Úarrayr7   r.   ÚemptyÚobject)Útrain_frameÚtarget_frameÚtr_studentsÚ
tr_coursesÚstudentsÚcoursesÚsÚnew_sÚcÚnew_cÚouts              r   Únovelty_groupsrâ   ©   sx  € à:@ÀKÓ:O”#�k &Ñ)×0Ñ0´Ó5Ô6ÔUXÓUZ€KØ9?À;Ó9N”�[ Ñ(×/Ñ/´Ó4Ô5ÔTWÓTY€JØ>DÈÓ>Tˆ|˜FÑ#×*Ñ*¬3Ó/×8Ñ8Ô:ÔZ\×ZbÒZbÐdjÐckÔnqÐr~ÓnÑcó  [A€HØ=CÀ|Ó=Sˆl˜6Ñ"×)Ñ)¬#Ó.×7Ñ7Ô9ÔY[×YaÒYaÐciÐbjÔmpÐq}Óm~Ñb~ÓY€GÜ�JŠJ±hÓ?²h° Ô,±hÑ?Ó@€EÜ�JŠJ±WÓ=²W° Ô+±WÑ=Ó>€EÜ
�(Š(”3�|Ó$¬FÑ
3€CØ/€Cˆ�U�FÑÑØ2€Cˆ�5ÑÑØ5€C�%�ÑÑØ1€C�‰ÑØ€Jùò @ùÚ=s   ÄFÄ>Fc                óð   • [        X5      n[        R                  " [        R                  " USS/5      R	                  [
        5      [        R                  " USS/5      R	                  [
        5      /5      $ )NrÐ   rÑ   rÏ   )râ   r-   rl   Úisinr{   rP   )r×   rØ   rž   s      r   Únovelty_flagsrå   ¹   sd   € Ü˜KÓ6€FÜ�?Š?Ü
�Š�Ð2Ð4LÐMÓN×UÑUÔV[Ó\Ü
�Š�Ð/Ð1IÐJÓK×RÑRÔSXÓYðó ð r   c                óD  • [         R                  " U [        5      n [         R                  " U[        5      n[         R                  " U[        5      n[         R                  " U[        5      n/ nS GH*  nX:H  nUR                  5       (       d  M  U[        UR                  5       5      [        X   R                  5       5      [	        X   R                  5       5      [        X   X'   5      [        X   X7   5      [         R                  " X   5      R                  S:X  a!  [        X   X7   5      [        X   X'   5      -
  O[         R                  [        X   X'   5      [        X   X7   5      S.	nU(       a  UR                  U5        UR                  U5        GM-     U$ )N)rÎ   rÏ   rÐ   rÑ   rM   )	ÚgroupÚrowsr¿   rÀ   Úcommittee_apÚcandidate_apÚdelta_apÚcommittee_aucÚcandidate_auc)r-   r.   rÖ   r<   rP   ÚanyrÇ   rÈ   rU   rQ   r8   rN   rW   ÚupdateÚappend)	rž   rS   Ú	committeeÚ	candidateÚextrarè   Úgr�   Úrows	            r   Úsubgroup_metric_rowsrö   Á   s=  € Ü�ZŠZ˜¤Ó'€FÜ
�
Š
�1”cÓ€AÜ—
’
˜9¤eÓ,€IÜ—
’
˜9¤eÓ,€IØ€DÜlˆØ‰KˆØ�u‰u�w‰wÙàÜ˜Ÿ™›“LÜ˜A™DŸH™H›J›Ü ¡§	¡	£Ó,Ü# A¡D¨)©,Ó7Ü# A¡D¨)©,Ó7ÜUW×U^ÒU^Ð_`Ñ_cÓUd×UiÑUiÐmnÓUnœ ¡ i¡lÓ3´g¸a¹dÀIÁLÓ6QÒQÔtv×tzÑtzÜ% a¡d¨I©LÓ9Ü% a¡d¨I©LÓ9ñ

ˆö Ø�J‰J�uÔØ�‰�C×ñ# mð$ €Kr   c                ó–   • [        U5      nUR                  R                  SSS9  [        R                  " U 5      R                  USS9  g )NT)ÚparentsÚexist_okF)Úindex)r   ÚparentÚmkdirr   Ú	DataFrameÚto_csv)rè   Úpaths     r   Úsave_csvr   Ü   s>   € Ü�‹:€DØ‡K�K×Ñ˜d¨TÐÑ2Ü‡L‚L�Ó×Ñ˜d¨%ÐÒ0r   c                ó.   • [         R                  " U 5      $ r   )rG   Úparse_folds)rÝ   s    r   r  r  â   s   € Ü×Ò˜AÓÐr   c           	     óÎ   • [        [        U 5      R                  S5       Vs1 s H2  oR                  5       (       d  M  [	        UR                  5       5      iM4     sn5      $ s  snf )NÚ,)r"   r   ÚsplitÚstriprP   )rÝ   Úxs     r   Úparse_float_gridr  æ   sB   € Ü¬S°«V¯\©\¸#Ô->ÓLÒ->¨Ç'Á'Ç)Ó#”5˜Ÿ™›Ö#Ñ->ÑLÓMÐMùÒLs
   ¢A"¾A")rI   r<   r:   ÚNone)r:   rP   )rX   rY   )rI   r<   r]   r<   r^   r<   )rq   r;   r:   ztuple[np.ndarray, list[str]])r   r   r€   r<   )r�   úpd.DataFrame)é0   )r©   )r:   Údict)r×   r
  rØ   r
  r:   r;   r   )ró   zdict | Noner:   z
list[dict])rÝ   r   r:   z	list[int])rÝ   r   r:   zlist[float])-rA   Ú
__future__r   Údataclassesr   Úpathlibr   Útypingr   r   ÚmathÚrandomÚnumpyr-   Úpandasr   ÚtorchÚsklearn.metricsr   r	   Útrain_v3rG   rn   rm   ÚTARGETrz   r   rH   rU   rW   r_   rw   rŠ   r‘   r    r¥   r¨   r»   rÊ   râ   rå   rö   r   r  r  rE   r   r   Ú<module>r     sè   ðñõ #å !Ý ß %Û Û ã Û Û ß Bã á�$—/‘/Ó"€
Ø	�‰€Ø×(Ñ(Ð ÷&ñ &ôDônôdöDô
DôTð 57ÀSÐX[õ ð (*¸È3Ø¨cÀDØñØõô?ò!öfð$ #%¨3÷ ôô öò61ôõNr   