ó
    ¤‡ljÝ  ã                  ó   • S SK Jr  S SKJr  S SKrS SKrSSKJ	r	J
r
JrJr  SSKJr  SSKJr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJrJr  SSKJr  SS jr            SS jr SS jr!          SS jr"g)é    )Úannotations)ÚDictNé   )Údrop_cluster_identifierÚdrop_course_statusÚdrop_time_featuresÚstructural_clean)ÚSuiteConfig)Úbaseline_feature_columns)ÚExperimentSpec)Úprune_correlated_features)Úadd_sparse_history_masksÚcompress_course_status)Úbuild_cross_fitted_risk_bins)Úscore_temporal_stabilityÚ!select_temporally_stable_features)ÚFeatureBuildResultc                óÞ  • [        X5      nX   R                  [        R                  SS9R	                  [
        R                  [
        R                  * /[
        R                  5      nUR                  US9R                  [        R                  SS9R	                  [
        R                  [
        R                  * /[
        R                  5      nUR                  SS9UR                  SS94$ )NÚcoerce©Úerrors©ÚcolumnsT©Údrop)
r   ÚapplyÚpdÚ
to_numericÚreplaceÚnpÚinfÚnanÚreindexÚreset_index)Ú	train_rawÚtest_rawÚconfigr   ÚtrainÚtests         ÚM/root/disk_3tb/earlywarning/experiment/track_b/experiment_b_suite/features.pyÚ_base_framesr+      sÇ   € Ü& yÓ9€GØÑ×$Ñ$¤R§]¡]¸8Ð$ÐD×LÑLÌbÏfÉfÔWY×W]ÑW]ÐV]ÐM^Ô`b×`fÑ`fÓg€EØ×Ñ GÐÐ,×2Ñ2´2·=±=ÈÐ2ÐR×ZÑZÔ\^×\bÑ\bÔeg×ekÑekÐdkÐ[lÔnp×ntÑntÓu€DØ×Ñ $ÐÐ'¨×)9Ñ)9¸tÐ)9Ð)DÐDÐDó    c                ó”  • [         R                  " X#R                     SS9R                  [        S9UR
                  :  R                  [        5      n[         R                  " X#R                     SS9R                  5       n[        XXc5      u  px0 n	Ub  [        U UUU5      u  pyX   R                  5       X   R                  5       X‰4$ )Nr   r   ©Údtype)r   r   Ú
target_colÚto_numpyÚfloatÚeventÚastypeÚintÚsemester_colr   r   Úcopy)
Ú
base_trainÚ	base_testr%   r'   Úcorrelation_thresholdÚyÚ	semestersÚselectedÚreportÚprunings
             r*   Ú_stable_corer@      sÀ   € ô 
�Š�y×!2Ñ!2Ñ3¸HÑ	E×	NÑ	NÔUZÐ	NÐ	[Ð^d×^jÑ^jÑ	j×rÑrÔsvÓw€AÜ—’˜i×(;Ñ(;Ñ<ÀXÑN×WÑWÓY€IÜ8¸È	ÓZÑ€HØ!#€GØÑ(Ü5ØØØØ!ó	
Ñˆð Ñ×$Ñ$Ó&¨	Ñ(;×(@Ñ(@Ó(BÀFÐSÐSr,   c                ó¦  • [         R                  " U R                  SS9UR                  SS9/SS9n[         R                  " UR                  SS9UR                  SS9/SS9nUR                  S S 2UR                  R                  SS9) 4   nUR                  S S 2UR                  R                  SS9) 4   R                  UR                  S9nXE4$ )NTr   r   )ÚaxisÚfirst©Úkeepr   )r   Úconcatr$   Úlocr   Ú
duplicatedr#   )r8   r9   Úextra_trainÚ
extra_testÚtrÚtes         r*   Ú_appendrM   4   sÑ   € Ü	�Š�J×*Ñ*°Ð*Ð5°{×7NÑ7NÐTXÐ7NÐ7YÐZÐabÑ	c€BÜ	�Š�I×)Ñ)¨tÐ)Ð4°j×6LÑ6LÐRVÐ6LÐ6WÐXÐ_`Ñ	a€BØ	�‰’�B—J‘J×)Ñ)¨wÐ)Ð7Ð7Ð7Ñ	8€BØ	�‰’�B—J‘J×)Ñ)¨wÐ)Ð7Ð7Ð7Ñ	8×	@Ñ	@ÈÏÉÐ	@Ð	T€BØˆ6€Mr,   c           
     ó‚
  • [        XU5      u  pEUR                  UR                  [        UR                  5      [        UR                  5      S.n0 nUR                  S:X  a  XEp˜GOÐUR                  S:X  a4  [        XEU5      n
U
R                  U
R                  p˜U
R                  US'   GOŒUR                  S:X  a3  [        XE5      n
U
R                  U
R                  p˜U
R                  US'   GOIUR                  S:X  a4  [        XEU5      n
U
R                  U
R                  p˜U
R                  US'   GOUR                  S:X  a3  [        XE5      n
U
R                  U
R                  p˜U
R                  US'   GOÂUR                  S:X  aË  [        R                  " XR                     S	S
9R!                  ["        S9UR$                  :  R'                  [(        5      n[        R                  " XR*                     S	S
9R!                  5       n[-        XKXÃ5      n[/        U[        UR                  5      UUR0                  5      u  pïXN   X^   p˜X×S'   X÷S'   GOçUR                  S:X  a  [3        XEXS 5      u  p‰pßX×S'   GOÁUR                  S;   Ga˜  [3        UUU UUR4                  5      u  p‰pßX×S'   X÷S'   UR                  S:X  aT  [7        XE5      nUR                  S   n[9        X‰UR                  U   UR                  U   5      u  p‰UR                  US'   GO(UR                  S:X  a¦  [;        XE5      nUR                  S   nUR                   Vs/ s H  nUR=                  S5      (       d  M  UPM     nnUR?                  US9nU	R?                  USS9n	[9        X‰UR                  U   UR                  U   5      u  p‰UR                  US'   OrUR                  S:X  aI  [A        XURB                  U5      n[9        X‰UR                  UR                  5      u  p‰UR                  US'   O[E        SUR                   35      eURF                  S S 2UR                  RI                  SS9) 4   nU	RK                  UR                  S9n	URM                  [        UR                  5      [        UR                  5      UR                   Vs/ s H  nUUR                  ;  d  M  UPM     snUR                   Vs/ s H  nUUR                  ;  d  M  UPM     snURN                  S.5        [Q        X‰URN                  Xg5      $ s  snf s  snf s  snf )N)Ú
experimentÚpipelineÚbaseline_feature_countÚbaseline_featuresÚbaseliner	   Ú	transformr   Ú	drop_timeÚdrop_clusterÚisolated_redundancyr   r   r.   Útemporal_stabilityÚcorrelation_pruningÚ	stability>   Ú	risk_binsÚstable_coreÚsparse_masksÚcourse_status_compressionr]   Úadded_masksr^   Úadded_summary_columnsÚcourse_status_r   Úignore)r   r   r[   zUnknown Experiment B pipeline: rC   rD   )Úfeature_countÚfeaturesÚremoved_from_baselineÚadded_to_baselineÚpreprocessing_mode))r+   ÚcoderP   Úlenr   Úlistr	   r(   r)   Úmanifestr   r   r   r   r   r0   r1   r2   r3   r4   r5   r6   r   r   Úisolated_redundancy_thresholdr@   r:   r   rM   r   Ú
startswithr   r   Úrisk_bin_candidatesÚ
ValueErrorrG   rH   r#   Úupdaterg   r   )r%   r&   Úspecr'   r8   r9   rk   Ú	artifactsr(   r)   Úresultr;   r<   r>   r=   r?   ÚmasksÚ	mask_colsÚ
compressedÚsummary_colsÚcÚstatus_in_coreÚrisks                          r*   Úbuild_experiment_featuresr{   <   s™  € ô )¨¸fÓEÑ€Jà—i‘iØ—M‘MÜ"% j×&8Ñ&8Ó"9Ü! *×"4Ñ"4Ó5ñ	#€Hð $&€Ià‡}�}˜
Ó"Ø ŠtØ	�‰Ð,Ó	,Ü! *¸Ó@ˆØ—l‘l F§K¡KˆtØ &§¡ˆ�ÓØ	�‰Ð.Ó	.Ü# JÓ:ˆØ—l‘l F§K¡KˆtØ &§¡ˆ�ÓØ	�‰˜+Ó	%Ü# J¸6ÓBˆØ—l‘l F§K¡KˆtØ &§¡ˆ�ÓØ	�‰˜.Ó	(Ü(¨Ó?ˆØ—l‘l F§K¡KˆtØ &§¡ˆ�ÓØ	�‰Ð/Ó	/Ü�]Š]˜9×%6Ñ%6Ñ7ÀÑI×RÑRÔY^ÐRÐ_Ðbh×bnÑbnÑn×vÑvÔwzÓ{ˆÜ—M’M )×,?Ñ,?Ñ"@ÈÑR×[Ñ[Ó]ˆ	Ü)¨*¸ÓKˆÜ5ØÜ�×#Ñ#Ó$ØØ×0Ñ0ó	
Ñˆð !Ñ*¨IÑ,?ˆtØ*0Ð&Ñ'Ø+2Ð'Ó(Ø	�‰˜+Ó	%Ü'3°JÈ9Ð^bÓ'cÑ$ˆ�VØ*0Ð&Ó'Ø	�‰ÐcÔ	cÜ'3ØØØØØ×(Ñ(ó(
Ñ$ˆ�Vð +1Ð&Ñ'Ø+2Ð'Ñ(Ø�=‰=˜NÓ*Ü,¨ZÓCˆEØŸ™ }Ñ5ˆIÜ! %¨u¯{©{¸9Ñ/EÀuÇzÁzÐR[ÑG\Ó]‰KˆEØ',§~¡~ˆH�^Ó$Ø�]‰]Ð9Ó9Ü/°
ÓFˆJØ%×.Ñ.Ð/FÑGˆLà).¯ªÓYª A¸!¿,¹,ÐGW×:XŸa©ˆNÐYØ—J‘J ~�JÐ6ˆEØ—9‘9 ^¸H�9ÐEˆDÜ! %¨z×/?Ñ/?ÀÑ/MÈzÏÉÐ_kÑOlÓm‰KˆEØ4>×4GÑ4GˆHÐ0Ò1Ø�]‰]˜kÓ)Ü/°	ÀV×E_ÑE_ÐagÓhˆDÜ! %¨t¯z©z¸4¿9¹9ÓE‰KˆEØ$(§M¡MˆH�[Ñ!øäÐ:¸4¿=¹=¸/ÐJÓKÐKà�I‰I’a˜%Ÿ-™-×2Ñ2¸Ð2Ð@Ð@Ð@ÑA€EØ�<‰< §¡ˆ<Ð.€DØ‡O�Oä  §¡Ó/Ü˜UŸ]™]Ó+Ø1;×1CÒ1CÓ%^Ò1C¨AÀqÐPU×P]ÑP]ÑG]§aÑ1CÑ%^Ø-2¯]ª]Ó!Zª]¨¸aÀz×GYÑGYÑ>Y§!©]Ñ!ZØ"&×"9Ñ"9ñ	
ôô ˜e¨4×+BÑ+BÀHÓXÐXùò/ Zùò$ &_ùÚ!Zs$   ÍT2Í$T2Ò;T7ÓT7Ó)T<ÔT<)r%   úpd.DataFramer&   r|   r'   r
   Úreturnú!tuple[pd.DataFrame, pd.DataFrame])r8   r|   r9   r|   r%   r|   r'   r
   r:   zfloat | Noner}   zBtuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, Dict[str, object]])
r8   r|   r9   r|   rI   r|   rJ   r|   r}   r~   )
r%   r|   r&   r|   rq   r   r'   r
   r}   r   )#Ú
__future__r   Útypingr   Únumpyr    Úpandasr   Úcleaningr   r   r   r	   r'   r
   Údatar   Úexperimentsr   Ú
redundancyr   Úrepresentationr   r   r[   r   rZ   r   r   Útypesr   r+   r@   rM   r{   © r,   r*   Ú<module>rŠ      sÄ   ðÝ "å ã Û ÷ó õ  Ý *Ý 'Ý 1ß LÝ 3ß RÝ %ôEðTØðTàðTð ðTð ð	Tð
 (ðTð HôTô*ð[YØð[Yàð[Yð ð[Yð ð	[Yð
 õ[Yr,   