ó
    bwhjåA  ã                  óÒ  • S SK Jr  S SKrS SKJr  S SKJrJrJrJ	r	J
r
Jr  S SKrS SKrSSKJr  SSKJrJr  SSKJrJrJr  SS	KJr  SS
KJrJrJrJrJ r J!r!  SSK"J#r#  Sr$Sr%SS jr&            SS jr'SS jr(          SS jr)              SS jr*SS jr+        SS jr,  S                   SS jjr-  S                   S S jjr.g)!é    )ÚannotationsN)ÚPath)ÚDictÚListÚMappingÚMutableMappingÚSequenceÚTupleé   )ÚSuiteConfig)ÚFoldDataÚload_raw_fold)ÚEXPERIMENTSÚPreparedMatricesÚprepare_experiment_matrices)Ú#build_leak_safe_historical_features)Úaverage_precisionÚbest_f1_thresholdÚconfusion_metricsÚequal_budget_metricsÚroc_aucÚwarn_by_rate)Útrain_predict)gš™™™™™©?gš™™™™™¹?g333333Ã?gš™™™™™É?)ÚblendedÚclassificationc                óî   • [         R                  " U [        S9n[         R                  " [	        U5      S[
        S9nSX!S:  '   SX!S:  '   SX!S:  '   S	X!S
:  '   SX!S:  '   SX!S:  '   SX!S:  '   SX!S:H  '   U$ )N©Údtypez9-10z8-9é	   z7-8é   z6-7é   z5-6é   z4-5é   z3-4é   z0-3é   zgrade=0r   )ÚnpÚasarrayÚfloatÚfullÚlenÚobject)ÚgradesÚgÚlabelss      Úearly_warning_suite/runner.pyÚ_grade_bandr0      s…   € Ü
�
Š
�6¤Ñ'€AÜ�WŠW”S˜“V˜V¬6Ñ2€FØ€Fˆq‰5�MØ€Fˆq‰5�MØ€Fˆq‰5�MØ€Fˆq‰5�MØ€Fˆq‰5�MØ€Fˆq‰5�MØ€Fˆq‰5�MØ€F�‰6�NØ€Mó    c                ó@  • UR                   R                  5       nUR                  SSU 5        UR                  US'   UR                  UR                  :  R                  [        5      US'   [        UR                  5      US'   X%S'   UR                  U-
  US'   X5S'   U$ )	Nr   ÚfoldÚ
true_gradeÚtrue_failureÚ
grade_bandÚpredicted_blended_gradeÚblended_riskÚclassification_score)Úmetadata_testÚcopyÚinsertÚy_testÚeventÚastypeÚintr0   )r3   ÚpreparedÚblended_gradeÚ	cls_scoreÚconfigÚouts         r/   Ú_fold_prediction_framerF   +   s•   € ð ×
 Ñ
 ×
%Ñ
%Ó
'€CØ‡J�Jˆq�&˜$ÔØ Ÿ™€CˆÑØ#Ÿ?™?¨V¯\©\Ñ9×AÑAÄ#ÓF€CˆÑÜ# H§O¡OÓ4€CˆÑØ%2Ð!Ñ"Ø Ÿ,™,¨Ñ6€CˆÑØ"+ÐÑØ€Jr1   c                ó  • [        U R                  R                  5       5      [        U R                  R                  5       5      U R                  R	                  5       R
                  U R                  R	                  5       R
                  U R                  R	                  5       R                  U R                  R	                  5       R                  UR                  UR                  UR                  UR                  UR                  S.$ )N)Ú
train_pathÚ	test_pathÚ
train_sizeÚ	test_sizeÚtrain_mtime_nsÚtest_mtime_nsr>   Úrecent_semestersÚcourse_prior_strengthÚdefault_failure_priorÚlong_duration_semesters)ÚstrrH   ÚresolverI   ÚstatÚst_sizeÚst_mtime_nsr>   rN   rO   rP   rQ   )Ú	fold_datarD   s     r/   Ú_cache_signaturerX   =   sÅ   € ä˜)×.Ñ.×6Ñ6Ó8Ó9Ü˜×,Ñ,×4Ñ4Ó6Ó7Ø×*Ñ*×/Ñ/Ó1×9Ñ9Ø×(Ñ(×-Ñ-Ó/×7Ñ7Ø#×.Ñ.×3Ñ3Ó5×AÑAØ"×,Ñ,×1Ñ1Ó3×?Ñ?Ø—‘Ø"×3Ñ3Ø!'×!=Ñ!=Ø!'×!=Ñ!=Ø#)×#AÑ#Añð r1   c                óâ  • [        U5      S-  U-  SU R                   3-  nUR                  SSS9  US-  nUS-  nUS-  n[        X5      nUR	                  5       (       a€  UR	                  5       (       ak  UR	                  5       (       aV   [
        R                  " UR                  SS	95      n	X˜:X  a,  [        R                  " U5      [        R                  " U5      4$  [        U R                  U R                  U5      u  p«U
R!                  U5        UR!                  U5        UR#                  [
        R$                  " US
S9SS	9  X«4$ ! [        [        [
        R                  4 a     NŒf = f)NÚ_feature_cacheÚfold_T©ÚparentsÚexist_okzhistorical_train.pklzhistorical_test.pklzmanifest.jsonúutf-8©Úencodingé   )Úindent)r   r3   ÚmkdirrX   ÚexistsÚjsonÚloadsÚ	read_textÚpdÚread_pickleÚOSErrorÚ
ValueErrorÚJSONDecodeErrorr   ÚtrainÚtestÚ	to_pickleÚ
write_textÚdumps)rW   ÚfacultyÚoutput_rootrD   Ú	cache_dirrH   rI   Ú	meta_pathÚ	signatureÚcachedÚ
hist_trainÚ	hist_tests               r/   Ú_load_or_build_historical_cacher{   M   s]  € ô �[Ó!Ð$4Ñ4°wÑ>À5ÈÏÉÐHXÐAYÑY€IØ‡O�O˜D¨4€OÑ0ØÐ3Ñ3€JØÐ1Ñ1€IØ˜OÑ+€IÜ  Ó3€Ià×Ñ×Ñ˜y×/Ñ/×1Ñ1°i×6FÑ6F×6HÑ6Hð	Ü—Z’Z 	× 3Ñ 3¸WÐ 3Ð EÓFˆFØÓ"Ü—~’~ jÓ1´2·>²>À)Ó3LÐLÐLð #ô
 @Ø�‰˜Ÿ™¨óÑ€Jð ×Ñ˜Ô$Ø×Ñ˜	Ô"Ø×ÑœŸš I°aÑ8À7ÐÑKØÐ Ð øô œ¤T×%9Ñ%9Ð:ó 	Ùð	ús   ÂAE ÅE.Å-E.c           	     óT  • [         U   n[        XX55      n[        XpXE5      u  p‰[        UR                  UR
                  UU	UU5      n
[        U
R                  U
R                  U
R                  UU SU 3S9n[        UU
UR                  UR                  U5      nUUS   R                  [        S9US   R                  [        S9US   R                  [        S9US.nU[!        U
R"                  5      U
R"                  U
R$                  S.nS	[        U5      0UR&                  EnXÞU4$ )
Nz/fold)Údescr5   r   r8   r9   )r3   Úy_binr   r   Úframe)r3   Úfeature_countÚfeaturesÚdropped_columnsr3   )r   r   r{   r   rn   ro   r   ÚX_trainÚy_trainÚX_testrF   rB   r9   Úto_numpyÚboolr(   r*   Úfeature_namesr‚   Údiagnostics)rs   Úexperiment_namer3   Ú
splits_dirrt   rD   ÚspecrW   ry   rz   rA   Ú
predictionr   ÚrecordÚmanifestr‰   s                   r/   Ú_generate_recordr�   k   sT  € ô �Ñ'€DÜ˜g¨ZÓ@€IÜ;Ø˜KóÑ€Jô +Ø�‰Ø�‰ØØØØó€Hô Ø×ÑØ×ÑØ�‰ØØÐ   d VÐ,ñ€Jô #ØØØ× Ñ Ø×'Ñ'Øó€Eð Ø�~Ñ&×/Ñ/´dÐ/Ð;Ø˜Ñ(×1Ñ1¼Ð1Ð>ØÐ 6Ñ7×@Ñ@ÄuÐ@ÐMØñ!€Fð Ü˜X×3Ñ3Ó4Ø×*Ñ*Ø#×3Ñ3ñ	€Hð œ5 ›;ÐA¨*×*@Ñ*@ÐA€KØ˜[Ð(Ð(r1   c                ó–  • U (       d  [        S5      e[        R                  " U  Vs/ s H  n[        R                  " US   [        S9PM!     sn5      n[        UR                  5       5      n0 n[         HÆ  n[        R                  " U  Vs/ s H  n[        R                  " X   [
        S9PM      sn5      n[        X&5      u  px[        X&5      [        X&5      U[        US   5      [        US   [        US5      -  5      [        US   5      [        US   5      [        US   5      [        US	   5      US
.
XE'   MÈ     U$ s  snf s  snf )zITune thresholds and warning aggressiveness from development records only.z&At least one tuning record is requiredr~   r   Ú	flag_rategê-�™—q=Ú	precisionÚrecallÚf1Úf2)
ÚapÚaucÚ	thresholdr’   Úratior“   r”   r•   r–   Ú
prevalence)rl   r&   Úconcatenater'   r‡   r(   ÚmeanÚSIGNALSr   r   r   Úmax)	ÚrecordsÚrÚyr›   ÚpoliciesÚsignalÚscorer™   Úfixeds	            r/   Útune_policiesr§   Ÿ   s  € æÜÐAÓBÐBÜ
�ŠÁGÓLÂG¸qœŸ
š
 1 W¡:´TÔ:ÁGÑLÓM€AÜ�q—v‘v“x“€JØ,.€HßˆÜ—’ÉGÓTÊGÀq¤§
¢
¨1©9¼EÔ BÉGÑTÓUˆÜ,¨QÓ6Ñˆ	ä# AÓ-Ü˜1Ó$Ø"Ü˜u [Ñ1Ó2Ü˜5 Ñ-´°JÀÓ0FÑFÓGÜ˜u [Ñ1Ó2Ü˜E (™OÓ,Ü˜˜d™Ó$Ü˜˜d™Ó$Ø$ñ
ˆÓñ ð €Oùò% Mùò  Us   §&EÂ%E
c                óÜ  • U Vs0 s HC  n[        US   5      [        [        R                  " US   [        S9R                  5       5      _ME     nn[        [        R                  " U Vs/ s H  n[        R                  " US   [        S9PM!     sn5      R                  5       5      n/ n[         Vs0 s H  nU/ / / / S._M     nn[        U S S9 GHŠ  n	[        U	S   5      n
[        R                  " U	S   [        S9n[         GH6  n[        R                  " X—   [        S9nX   nU[        US   5      :¬  nUR                  U
S-
  U5      n[        [        R                  " [        US	   5      U-  S
S5      5      n[        UU5      n[        X¾5      n[        UU5      nU
U[        U5      [        UR                  5       5      [        X¼5      [        X¼5      [        US   5      US.nUR!                  UR#                  5        VVs0 s H  u  nnSU 3[        U5      _M     snn5        UR!                  UR#                  5        VVs0 s H  u  nnSU 3[        U5      _M     snn5        UR%                  U5        X‡   S   R%                  U5        X‡   S   R%                  U5        X‡   S   R%                  U5        X‡   S   R%                  U5        US:X  d  GMÛ  U	S   n['        U[(        R*                  5      (       d   eUR-                  [         5      US'   UR-                  [         5      US'   UUS'   GM9     [        UR                  5       5      XJ'   GM�     0 n/ n[         GHG  n[        R                  " X‡   S   5      n[        R                  " X‡   S   5      n[        U[        R                  " X‡   S   5      5      n[        U[        R                  " X‡   S   5      5      n[        X¼5      [        X¼5      [        X   S   5      S.UR#                  5        VVs0 s H  u  nnSU 3[        U5      _M     snnEUR#                  5        VVs0 s H  u  nnSU 3[        U5      _M     snnEUU'   U  Vs/ s HH  n[        US   5      [        R                  " US   [        S9[        R                  " X7   [        S9S.PMJ     nn[.         HÕ  n[1        UU5      nUR%                  UUS.UR#                  5        VVs0 s H  u  nnU[        U5      _M     snnE5        US:X  d  M[  U  Ht  n	U	S   n['        U[(        R*                  5      (       d   e[        [        R                  " X—   [        S9U5      R-                  [         5      US[        US-  5       S3'   Mv     M×     GMJ     UUU4$ s  snf s  snf s  snf s  snnf s  snnf s  snnf s  snnf s  snf s  snnf )Nr3   r~   r   )r¢   r¦   Údriftr¥   c                ó   • [        U S   5      $ )Nr3   )r@   )Úxs    r/   Ú<lambda>Ú#_evaluate_records.<locals>.<lambda>Ì   s   € ´°A°f±I´r1   ©Úkeyr™   r   rš   g-Cëâ6?g®Gáz®ï?)r3   r¤   Únr›   r—   r˜   r™   Ú
drift_rateÚfixed_Údrift_r¢   r¥   r¦   r©   r   r   Ú
warn_fixedÚ
warn_driftÚdrift_warning_rate)r—   r˜   r™   )r3   r~   r¥   )r¤   ÚbudgetÚwarn_at_éd   Úpct)r@   r(   r&   r'   r‡   r�   rœ   rž   ÚsortedÚgetÚclipr   r   r*   r   r   ÚupdateÚitemsÚappendÚ
isinstanceri   Ú	DataFramer?   ÚEQUAL_BUDGET_RATESr   )r    r£   Útuning_recordsr¡   Úprevious_prevalenceÚfallback_prevalenceÚ	fold_rowsr¤   Úpooled_datarŽ   r3   r¢   r¥   ÚpolicyÚ
fixed_warnÚreference_prevr±   Ú
drift_warnr¦   r©   ÚrowÚkÚvr   ÚpooledÚbudget_rowsÚbudget_recordsÚrateÚmetricss                                r/   Ú_evaluate_recordsrÕ   ¸   s’  € ñ  ó-âˆAô 	ˆAˆf‰I‹œœbŸjšj¨¨7©¼4Ñ@×EÑEÓGÓHÒHÙð ð -ô  Ü
�ŠÙ9GÓHº°AŒR�ZŠZ˜˜'™
¬$Ô/¹ÑHó	
ç
‰$‹&óÐð
 )+€Iõ ó;âˆFð 	�b 2°¸RÑ@Ò@Ùð ð ;ô
 ˜Ñ&>Õ?ˆÜ�6˜&‘>Ó"ˆÜ�JŠJ�v˜g‘¬dÑ3ˆß�gˆFÜ—J’J˜v™~´UÑ;ˆEØÑ%ˆFØ¤%¨¨{Ñ(;Ó"<Ñ<ˆJØ0×4Ñ4°T¸A±XÐ?RÓSˆNÜÜ—’œ˜f W™oÓ.°Ñ?ÀÀtÓLóˆJô & e¨ZÓ8ˆJÜ% aÓ4ˆEÜ% a¨Ó4ˆEàØ Ü˜“VÜ# A§F¡F£H›oÜ'¨Ó1Ü˜qÓ(Ü" 6¨+Ñ#6Ó7Ø(ñ	%ˆCð �J‰J¸5¿;¹;¼=ÔIº=±4°1°a˜&  ˜¤e¨A£hÒ.¹=ÒIÔJØ�J‰J¸5¿;¹;¼=ÔIº=±4°1°a˜&  ˜¤e¨A£hÒ.¹=ÒIÔJØ×Ñ˜SÔ!àÑ Ñ$×+Ñ+¨AÔ.ØÑ Ñ(×/Ñ/°Ô6ØÑ Ñ(×/Ñ/°
Ô;ØÑ Ñ(×/Ñ/°
Ô;ØÐ)Ö)Ø˜w™�Ü! %¬¯©×6Ñ6Ð6Ð6Ø&0×&7Ñ&7¼Ó&<��lÑ#Ø&0×&7Ñ&7¼Ó&<��lÑ#Ø.8�Ð*Ô+ñE ôF %*¨!¯&©&«(£OÐÔ!ñM @ðP +-€FØ*,€Kß�'ˆÜ�NŠN˜;Ñ.¨sÑ3Ó4ˆÜ—’˜{Ñ2°7Ñ;Ó<ˆÜ! !¤R§^¢^°KÑ4GÈÑ4PÓ%QÓRˆÜ! !¤R§^¢^°KÑ4GÈÑ4PÓ%QÓRˆä# AÓ-Ü˜1Ó$Ü˜xÑ/°Ñ<Ó=ñ
ð 38·+±+´-Ô@²-©$¨!¨Q�˜�sˆ|œU 1›XÒ%±-Ò@ð	
ð
 38·+±+´-Ô@²-©$¨!¨Q�˜�sˆ|œU 1›XÒ%±-Ò@ð
ˆˆv‰ñ ó
ò �ô	 ˜A˜f™I›ÜŸš A g¡J´dÑ;ÜŸš A¡I´UÑ;ôñ
 ð 	ð 
÷ 'ˆDÜ*¨>¸4Ó@ˆGØ×Ñà$Ø"ñð 07¯}©}¬Ô?ª¡t q¨!�qœ% ›(’{©Ò?ðôð Ð)Õ)Û%�FØ" 7™O�EÜ% e¬R¯\©\×:Ñ:Ð:Ð:Ü=IÜŸ
š
 6¡>¼Ñ?Àó>ç‘fœS“kð ˜H¤S¨°©£_Ð$5°SÐ9Ó:ó &ô 'ñ) ðH �f˜kÐ)Ð)ùò{-ùò Iùò;ùó: JùÛIùó4 AùÛ@ùò
ùó @s7   …A
V<Á*&WÂ3WÈ	WÉWÐ'W
ÑW
ÒAW#Ô
W(c	                ó2  ^^• U[         ;  a  [        SU 35      e[         U   n	[        U5      U -  U	R                   SU	R                   3-  n
U
R                  SSS9  Ub  [        U5      O
[        U5      n[        [        U5      [        U5      -  5      n0 n0 m0 nU H6  n[        U UU[        U5      [        U5      U5      u  nnnXüU'   UTU'   UXÞ'   M8     U Vs/ s H  nUU   PM
     nnU Vs/ s H  nUU   PM
     nn[        U5      m[        UTU5      u  nnnU(       aR  U HL  nUS   n[        U[        R                  5      (       d   eUR                  U
S[!        US   5       S3-  S	S
9  MN     [        R                  " U5      R                  U
SU S3-  S	S
9  [        R                  " U5      R                  U
S-  S	S
9  [        R                  " U5      R                  U
SU S3-  S	S
9  [        R                  " U5      R                  U
S-  S	S
9  [        R                  " U Vs/ s H  nUU   PM
     sn5      nUR                  U
SU S3-  S	S
9  UR                  U
S-  S	S
9  U Vs/ s H  nTU   PM
     nnU
SU S3-  R#                  SSS9 n[$        R&                  " UUS	SS9  S S S 5        U
S-  R#                  SSS9 n[$        R&                  " UUS	SS9  S S S 5        [)        [*        U4S jS9nUU	R                  U[        U5      [        U5      UTU[-        U4S jU 5       5      [)        U4S jU 5       5      S.
nU
SU S3-  R#                  SSS9 n[$        R&                  " UUS	SSS9  S S S 5        U
S -  R#                  SSS9 n[$        R&                  " UUS	SSS9  S S S 5        U$ s  snf s  snf s  snf s  snf ! , (       d  f       GN%= f! , (       d  f       GN= f! , (       d  f       N|= f! , (       d  f       U$ = f)!NzUnknown experiment: Ú_Tr\   r   Úpredictions_fold_r3   ú.csvF©ÚindexÚfold_metrics_zfold_metrics.csvÚequal_budget_metrics_zequal_budget_metrics.csvÚtraining_diagnostics_ztraining_diagnostics.csvÚfeature_manifest_z.jsonÚwr_   r`   rb   )Úensure_asciirc   zfeature_manifest.jsonc                óx   >• [         R                  " TU    S   5      (       a  TU    S   $ [         R                  * $ )Nr—   )r&   ÚisfiniteÚinf)r¤   r£   s    €r/   r¬   Ú run_experiment.<locals>.<lambda>j  s>   ø€ Ü�;Š;�x Ñ'¨Ñ-×.Ñ.ð $ FÑ+¨DÑ1ð ä�f‰fˆWðr1   r®   c              3  ó4   >#   • U  H  nTU   S    v •  M     g7f©r€   N© ©Ú.0ÚfÚ	manifestss     €r/   Ú	<genexpr>Ú!run_experiment.<locals>.<genexpr>w  ó   øé € Ð RÊ	À1 ¨1¡¨oÖ!>Ê	ùó   ƒc              3  ó4   >#   • U  H  nTU   S    v •  M     g7frç   rè   ré   s     €r/   rí   rî   x  rï   rð   )
Ú
experimentÚslugÚphaseÚevaluation_foldsÚtuning_foldsÚselected_signalÚtuning_policiesrÐ   Úfeature_count_minÚfeature_count_maxÚsummary_)rá   rc   Ú	allow_nanzsummary.json)r   ÚKeyErrorr   Úcoderó   rd   Úlistr»   Úsetr�   r§   rÕ   rÁ   ri   rÂ   Úto_csvr@   Úopenrf   ÚdumprŸ   rž   Úmin) rs   rŠ   Úfoldsr‹   rt   rD   rô   Úsave_predictionsrö   rŒ   Úexperiment_dirÚ	all_foldsÚ
record_mapr‰   r3   rŽ   r�   Ú
diagnosticrë   rÄ   Úevaluation_recordsrÇ   rÐ   rÑ   r   Údiagnostics_frameÚmanifest_payloadÚhandleÚselectedÚsummaryrì   r£   s                                  @@r/   Úrun_experimentr    sd  ù€ ð œkÓ)ÜÐ-¨oÐ->Ð?Ó@Ð@Ü�Ñ'€DÜ˜+Ó&¨Ñ0°d·i±i°[ÀÀ$Ç)Á)ÀÐ3MÑM€NØ×Ñ °ÐÑ5à)5Ñ)A”4˜Ô%ÄtÈEÃ{€LÜ”s˜5“z¤C¨Ó$5Ñ5Ó6€IØ9;€JØ.0€IØ/1€KÛˆÜ'7ØØØÜ�ÓÜ�ÓØó(
Ñ$ˆ�˜*ð "�4ÑØ"ˆ	�$‰Ø&ˆÓñ ñ .:Ó:ª\¨�j ”m©\€NÐ:Ù16Ó7²¨A˜* Qœ-±ÐÐ7Ü˜^Ó,€HÜ%6Ø˜H nó&Ñ"€Iˆv�{ö Û(ˆFØ˜7‘OˆEÜ˜e¤R§\¡\×2Ñ2Ð2Ð2Ø�L‰LØÐ#4´S¸À¹Ó5HÐ4IÈÐ!NÑNØð ó ñ )ô ‡L‚L�Ó×"Ñ" >°mÀEÀ7È$Ð4OÑ#OÐW\Ð"Ñ]ä‡L‚L�Ó×"Ñ" >Ð4FÑ#FÈeÐ"ÑTÜ‡L‚L�Ó×$Ñ$ØÐ0°°°tÐ<Ñ<ÀEð %ñ ô ‡L‚L�Ó×$Ñ$ØÐ3Ñ3¸5ð %ñ ô Ÿš¹iÓ%Hºi¸ k°!¤n¹iÑ%HÓIÐØ×ÑØÐ0°°°tÐ<Ñ<ÀEð ñ ð ×Ñ˜^Ð.HÑHÐPUÐÑVÙ.7Ó8ªi¨˜	 !œ©iÐÐ8Ø
Ð.¨u¨g°UÐ;Ñ
;×	AÑ	AØ�gð 
Bñ 
à	Ü�	Š	Ð" F¸ÀqÒI÷
ð Ð2Ñ
2×	8Ñ	8Ø�gð 
9ñ 
à	Ü�	Š	Ð" F¸ÀqÒI÷
ô
 Üôñ€Hð &Ø—	‘	ØÜ  ›KÜ˜\Ó*Ø#Ø#ØÜ Ô RÉ	Ó RÓRÜ Ô RÉ	Ó RÓRñ"€Gð ˜X e W¨EÐ2Ñ
2×	8Ñ	8¸ÀwÐ	8Ñ	OÐSYÜ�	Š	�'˜6°¸aÈ4ÒP÷ 
Pà
˜>Ñ
)×	/Ñ	/°¸gÐ	/Ñ	FÈ&Ü�	Š	�'˜6°¸aÈ4ÒP÷ 
Gà€Nùò} ;ùÚ7ùò0 &Iùò
 9÷
ö 
ú÷
ö 
ú÷. 
PÕ	Oúç	FÔ	Fà€NúsH   ÃN>Ã2OÈ0OÉ4OÊOËO$Í'O6ÎPÏ
O!Ï$
O3Ï6
PÐ
Pc	                óü  • / n	U GH  n
[        U U
UUUUUUUS9	nU
US   US   US   US   S.n[        U5      U -  U
 SUS    3-  n[        R                  " USU S	3-  5      n[         H�  nUS
   U   nXîS   U:H     nUS   XÏ S3'   US   XÏ S3'   US   XÏ S3'   US   XÏ S3'   US   XÏ S3'   US   XÏ S3'   US   XÏ S3'   [        US   R                  5       5      XÏ S3'   [        US   R                  5       5      XÏ S3'   MŸ     U	R                  U5        GM     [        R                  " U	5      R                  / SQSS9n[        U5      U -  nUR                  SSS 9  UR                  US!U S	3-  SS"9  U$ )#N)	rs   rŠ   r  r‹   rt   rD   rô   r  rö   ró   r÷   rù   rú   )rò   ró   r÷   rù   rú   r×   rÜ   rÙ   rÐ   r¤   r—   Ú_apr˜   Ú_aucÚfixed_f1Ú	_fixed_f1Úfixed_recallÚ_fixed_recallÚfixed_precisionÚ_fixed_precisionÚdrift_f1Ú	_drift_f1Údrift_recallÚ_drift_recallÚ_mean_fold_apÚ_worst_fold_ap)Úclassification_apÚclassification_mean_fold_apÚclassification_fixed_f1F)Ú	ascendingTr\   Úexperiment_comparison_rÚ   )r  r   ri   Úread_csvrž   r(   r�   r  rÀ   rÂ   Úsort_valuesrd   r  )rs   Úexperiment_namesr  r‹   rt   rD   rô   r  rö   Ú	summariesÚnameÚresultrÍ   r  Úfold_metricsr¤   Úpooled_signalÚsignal_foldsÚ
comparisonÚfaculty_dirs                       r/   Úrun_many_experimentsr1  �  s(  € ð €IÜ ˆÜØØ ØØ!Ø#ØØØ-Ø%ñ

ˆð Ø˜6‘NØ%Ð&7Ñ8Ø!'Ð(;Ñ!<Ø!'Ð(;Ñ!<ñ"
ˆô ˜kÓ*¨WÑ4¸$¸¸qÀÈÁÐ@PÐ7QÑQˆÜ—{’{ >°mÀEÀ7È$Ð4OÑ#OÓPˆßˆFØ" 8Ñ,¨VÑ4ˆMØ'°XÑ(>À&Ñ(HÑIˆLØ"/°Ñ"5ˆC�(˜#�ÑØ#0°Ñ#7ˆC�(˜$�Ñ Ø(5°jÑ(AˆC�(˜)Ð$Ñ%Ø,9¸.Ñ,IˆC�(˜-Ð(Ñ)Ø/<Ð=NÑ/OˆC�(Ð*Ð+Ñ,Ø(5°jÑ(AˆC�(˜)Ð$Ñ%Ø,9¸.Ñ,IˆC�(˜-Ð(Ñ)Ü,1°,¸tÑ2D×2IÑ2IÓ2KÓ,LˆC�(˜-Ð(Ñ)Ü-2°<ÀÑ3E×3IÑ3IÓ3KÓ-LˆC�(˜.Ð)Ó*ñ ð 	×Ñ˜×ñC !ôF —’˜iÓ(×4Ñ4ÚWØð 5ð €Jô �{Ó# gÑ-€KØ×Ñ˜d¨TÐÑ2Ø×Ñ�kÐ&<¸U¸GÀ4Ð$HÑHÐPUÐÑVØÐr1   )r,   ú
np.ndarrayÚreturnr2  )r3   r@   rA   r   rB   r2  rC   r2  rD   r   r3  úpd.DataFrame)rW   r   rD   r   r3  úDict[str, object])
rW   r   rs   rR   rt   r   rD   r   r3  z!Tuple[pd.DataFrame, pd.DataFrame])rs   rR   rŠ   rR   r3   r@   r‹   r   rt   r   rD   r   r3  z=Tuple[Dict[str, object], Dict[str, object], Dict[str, float]])r    úSequence[Mapping[str, object]]r3  zDict[str, Dict[str, float]])r    z%Sequence[MutableMapping[str, object]]r£   z!Mapping[str, Mapping[str, float]]rÄ   r6  r3  zRTuple[List[Dict[str, float]], Dict[str, Dict[str, float]], List[Dict[str, float]]])TN)rs   rR   rŠ   rR   r  úSequence[int]r‹   r   rt   r   rD   r   rô   rR   r  r‡   rö   úSequence[int] | Noner3  r5  )rs   rR   r(  zSequence[str]r  r7  r‹   r   rt   r   rD   r   rô   rR   r  r‡   rö   r8  r3  r4  )/Ú
__future__r   rf   Úpathlibr   Útypingr   r   r   r   r	   r
   Únumpyr&   Úpandasri   rD   r   Údatar   r   Úfeature_setsr   r   r   Ú
historicalr   rÔ   r   r   r   r   r   r   Úmodelr   rÃ   rž   r0   rF   rX   r{   r�   r§   rÕ   r  r1  rè   r1   r/   Ú<module>rB     s#  ðÝ "ã Ý ß G× Gã Û å ß )ß TÑ TÝ ;÷÷ õ !ð .Ð Ø
'€ôðØ
ðàðð ðð ð	ð
 ðð ôô$ð !Øð!àð!ð ð!ð ð	!ð
 'ô!ð<1)Øð1)àð1)ð ð1)ð ð	1)ð
 ð1)ð ð1)ð Cô1)ôhð2b*Ø2ðb*à/ðb*ð 3ðb*ð Xô	b*ðZ "Ø)-ðaØðaàðað ðað ð	að
 ðað ðað ðað ðað 'ðað õaðX "Ø)-ð6Øð6à#ð6ð ð6ð ð	6ð
 ð6ð ð6ð ð6ð ð6ð 'ð6ð ö6r1   