ó
    ¨räj­p  ã                   óf  • S r SSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
Jr  SSKJs  Jr  SSKrSSKrSSKrSSKJr  \R.                  R1                  \R.                  R3                  \5      5      r\R.                  R9                  \SS5      r\R.                  R9                  \SS5      r\R.                  R9                  \SSS	5      r\R.                  R9                  \S
S5      r \R.                  R9                  \SSS5      r!\R.                  R9                  \SSS5      r"\R.                  R9                  \S
S5      r#Sr$Sr%Sr&Sr'Sr(Sr)S r*S r+ " S S\RX                  5      r-S r. " S S\RX                  5      r/S r0S r1S r2S r3S r4S  r5S! r6S'S" jr7S# r8S$ r9S% r:\;S&:X  a  \:" 5         gg)(aÅ  Causal logit transformer, third round.

Same official metric as ensemble_clf_transformer_v2.py: H is the harmonic mean of
subject-mean accuracy and switch score L from score.score_dataset. Leave-one-dataset-out.
Inner selection (nested mode) uses the acc74 rule on datasets that are neither the
held-out dataset nor the scored inner dataset. Emissions are fit without the scored
dataset. Test labels are read only inside the scorer.

What is new, relative to stem_ms96 (nested family mean H 0.8677, Acc 82.42):
  gate_local   the 24-d multi-scale logit features are 6 groups x 4 classifier seeds.
               Each seed is projected and passed through a causal depthwise conv, then
               the four seeds are mixed by attention at the same time step. A zero-init
               dilated conv stem and local causal attention (relative bias, 6.25 s)
               sit in front of a zero-init head. The linear skip still starts as the
               mean of the four raw logits. Loss matches stem_ms96.
  gate_robust  the same network with label smoothing, cross-window mixup, per-channel
               gain, mild group-DRO, a small CORAL penalty across training datasets,
               and a lighter temporal-smoothness penalty.

Train seed 20270101.

usage:
  SMOKE=1 STEPS=30 FOLD=AVGC python ensemble_clf_transformer_v3.py
  MODE=screen STEPS=1000 python ensemble_clf_transformer_v3.py
  MODE=nested ONLY=gate_local STEPS=1000 python ensemble_clf_transformer_v3.py
é    N)ÚDS_KEYSÚmodelsÚlogit_tf_v3Úlogit_tf_v2ÚresultsÚresearchztransformer_v3_screen.csvÚexperimentsztransformer_v3_screen.logzensemble_clf_transformer_v3.csvz%ensemble_clf_transformer_v3_inner.csvztransformer_v3.log))
Ú
gate_localÚmsé`   Úgateé@   é   ç-Cëâ6J?çš™™™™™¹?g-Cëâ6?F)
Úgate_robustr   r   r   r   r   r   g333333Ã?ga2U0*©3?Té   é   ç{®Gáz”?g{®Gázt?c                 óð   • [        USS9  [        R                  " [        R                  R	                  U 5      SS9  [        U S5       nUR                  US-   5        S S S 5        g ! , (       d  f       g = f)NT©Úflush©Úexist_okÚaÚ
)ÚprintÚosÚmakedirsÚpathÚdirnameÚopenÚwrite)r    ÚtextÚfhs      Ú6/root/disk_3tb/eeg-thao/ensemble_clf_transformer_v3.pyÚlog_liner'   @   sL   € Ü	ˆ$�dÒÜ‡K‚K”—‘—‘ Ó%°Ò5Ü	ˆd�CŒ˜BØ
�‰�˜‘Ô÷ 
�Žús   Á	A'Á'
A5c                  óà  • [         R                  R                  SS5      R                  5       n U (       d  [	        [
        5      $ U R                  S5       Vs1 s H)  oR                  5       (       d  M  UR                  5       iM+     nn[
         Vs/ s H  o3S   U;   d  M  UPM     nnX$ Vs1 s H  o3S   iM	     sn-
  nU(       a  [        S[        U5       35      eU$ s  snf s  snf s  snf )NÚONLYÚ Ú,r   zunknown ONLY names: )	r   ÚenvironÚgetÚstripÚlistÚCONFIGSÚsplitÚ
SystemExitÚsorted)ÚonlyÚnÚnamesÚcÚcfgsÚmissings         r&   Úwanted_configsr:   G   s¸   € Ü�:‰:�>‰>˜& "Ó%×+Ñ+Ó-€DÞÜ”G‹}ÐØ $§
¡
¨3¤Ó=¢˜1·7±7·9‹YˆQ�W‰WŽY¡€EÐ=ÝÓ0’w�! A¡$¨%¡-�A‘w€DÐ0Ø TÓ*¢T ˜”t¡TÑ*Ñ*€GÞÜÐ/´°w³Ð/@ÐAÓBÐBØ€Kùò >ùÚ0ùÚ*s   ÁC!Á4C!ÂC&Â"C&Â.C+c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚLocalCausalBlockéS   zPNorm-first local causal attention plus an FFN. Output projections start at zero.c           	      ó€  >• [         TU ]  5         X-  S:w  a  [        SU SU 35      eX l        X0l        X-  U l        [        R                  " U5      U l        [        R                  " USU-  5      U l
        [        R                  " X5      U l        [        R                  " [        R                  " X#5      5      U l        [        R                   " U5      U l        [        R                  " U5      U l        [        R&                  " [        R                  " USU-  5      [        R(                  " 5       [        R                   " U5      [        R                  " SU-  U5      5      U l        [        R,                  R/                  U R                  R0                  5        [        R,                  R/                  U R                  R2                  5        [        R,                  R/                  U R*                  S   R0                  5        [        R,                  R/                  U R*                  S   R2                  5        g )Nr   zd_model z not divisible by nhead é   r   éÿÿÿÿ)ÚsuperÚ__init__Ú
ValueErrorÚnheadÚspanÚdhÚnnÚ	LayerNormÚnorm1ÚLinearÚqkvÚoutÚ	ParameterÚtorchÚzerosÚrelÚDropoutÚdropÚnorm2Ú
SequentialÚGELUÚffÚinitÚzeros_ÚweightÚbias)ÚselfÚdrD   rE   ÚdropoutÚ	__class__s        €r&   rB   ÚLocalCausalBlock.__init__V   sc  ø€ Ü‰ÑÔØ‰9˜‹>Ü˜x¨ sÐ*BÀ5À'ÐJÓKÐKØŒ
ØŒ	Ø‘*ˆŒÜ—\’\ !“_ˆŒ
Ü—9’9˜Q  A¡Ó&ˆŒÜ—9’9˜Q“?ˆŒÜ—<’<¤§¢¨EÓ 8Ó9ˆŒÜ—J’J˜wÓ'ˆŒ	Ü—\’\ !“_ˆŒ
Ü—-’-Ü�IŠI�a˜˜Q™Ó¤§¢£¬B¯JªJ°wÓ,?ÄÇÂÈ1ÈqÉ5ÐRSÓATóVˆŒä
�‰�‰�t—x‘x—‘Ô'Ü
�‰�‰�t—x‘x—}‘}Ô%Ü
�‰�‰�t—w‘w˜r‘{×)Ñ)Ô*Ü
�‰�‰�t—w‘w˜r‘{×'Ñ'Õ(ó    c                 ó4  • UR                   u  p#nU R                  U5      nU R                  U5      R                  X#SU R                  U R
                  5      R                  SSSSS5      nUS   US   US   p˜nU R                  S-
  n
[        R                  " USSU
S45      n[        R                  " U	SSU
S45      nUR                  SU R                  S5      nUR                  SU R                  S5      nUR                  S5      U-  R                  S5      [        R                  " U R
                  5      -  nXðR                  R                  SU R                  SU R                  5      -   n[         R"                  " U R                  UR$                  S9n[         R"                  " X1R$                  S9nUR                  SU R                  5      U R                  S-
  U-
  R                  US5      :¬  nUR'                  UR                  SSX0R                  5      ) [         R(                  " UR*                  5      R,                  5      nU R/                  [         R0                  " USS	95      nUR                  S5      U-  R                  S5      nUR                  SSSS5      R3                  5       R                  X#U5      nXR/                  U R5                  U5      5      -   nXR/                  U R7                  U R9                  U5      5      5      -   nU$ )
Nr?   r   r   é   r   r@   éþÿÿÿ©Údevice©Údim)ÚshaperI   rK   ÚviewrD   rF   ÚpermuterE   ÚFÚpadÚunfoldÚ	unsqueezeÚsumÚmathÚsqrtrP   rN   Úarangere   Úmasked_fillÚfinfoÚdtypeÚminrR   ÚsoftmaxÚ
contiguousrL   rV   rS   )r[   ÚxÚbÚtr\   ÚhrK   ÚqÚkÚvrl   Úk_padÚv_padÚk_uÚv_uÚscoreÚslotsÚtimesÚvalidrY   Úctxs                        r&   ÚforwardÚLocalCausalBlock.forwardj   sa  € Ø—'‘'‰ˆˆaØ�J‰J�q‹MˆØ�h‰h�q‹k×Ñ˜q Q¨¯
©
°D·G±GÓ<×DÑDÀQÈÈ1ÈaÐQRÓSˆØ�a‘&˜#˜a™& # a¡&ˆaˆØ�i‰i˜!‰mˆÜ—’�a˜!˜Q  Q˜Ó(ˆÜ—’�a˜!˜Q  Q˜Ó(ˆØ�l‰l˜1˜dŸi™i¨Ó+ˆØ�l‰l˜1˜dŸi™i¨Ó+ˆØ—‘˜R“ 3Ñ&×+Ñ+¨BÓ/´$·)²)¸D¿G¹GÓ2DÑDˆØŸ™Ÿ™ a¨¯©°Q¸¿	¹	ÓBÑBˆÜ—’˜TŸY™Y¨q¯x©xÑ8ˆÜ—’˜Q§x¡xÑ0ˆØ—
‘
˜1˜dŸi™iÓ(¨T¯Y©Y¸©]¸UÑ-B×,HÑ,HÈÈAÓ,NÑNˆØ×!Ñ! 5§:¡:¨a°°A·y±yÓ#AÐ"AÄ5Ç;Â;ÈuÏ{É{ÓC[×C_ÑC_Ó`ˆØ—‘œ5Ÿ=š=¨°BÑ7Ó8ˆØ×Ñ Ó# cÑ)×.Ñ.¨rÓ2ˆØ�k‰k˜!˜Q  1Ó%×0Ñ0Ó2×7Ñ7¸¸aÓ@ˆØ—	‘	˜$Ÿ(™( 3›-Ó(Ñ(ˆØ—	‘	˜$Ÿ'™' $§*¡*¨Q£-Ó0Ó1Ñ1ˆØˆr`   )
rF   rR   rV   rD   rI   rS   rL   rK   rP   rE   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rB   r‰   Ú__static_attributes__Ú__classcell__©r^   s   @r&   r<   r<   S   s   ø† ÙZõ)÷(ð r`   r<   c                 ór  • [         R                  " U SS9n[        R                  " USSUS45      S S 2S U R	                  S5      24   n[         R
                  " SU R	                  S5      S-   U R                  S9R                  US9R                  SSS5      nX#-
  UR                  U R                  5      -  $ )Nrb   rf   r   rd   )Úmaxr@   )rN   Úcumsumrk   rl   Úsizerr   re   Úclampri   Útoru   )r|   r~   ÚtotalÚshiftedÚcounts        r&   Úcausal_poolr�   ‚   sš   € Ü�LŠL˜ Ñ"€EÜ�eŠe�E˜A˜q ! Q˜<Ó(ª¨K¨a¯f©f°Q«i¨K¨Ñ8€GÜ�LŠL˜˜AŸF™F 1›I¨™M°!·(±(Ñ;×AÑAÀaÐAÐH×MÑMÈaÐQSÐUVÓW€EØ‰O˜uŸx™x¨¯©Ó0Ñ0Ð0r`   c                   ó:   ^ • \ rS rSrSrSU 4S jjrSS jrSrU =r$ )ÚSeedGateLocalé‰   zUPer-seed causal conv, same-time seed attention, dilated stem, local causal attention.c                 ó  >• [         T
U ]  5         U[        [        -  :w  a  [	        S[        [        -   SU 35      eX l        X0l        SU l        [        R                  " [        U5      U l
        [        R                  " X3U R                  S-   US9U l        [        R                  " X35      U l        [        R                  " X35      U l        [        R                  " X35      U l        [        R                  " X35      U l        [        R"                  " S Vs/ s H  n[$        R&                  " USX†S9PM     sn5      U l        [        R"                  " [+        U5       V	s/ s H  n	[-        X4Xv5      PM     sn	5      U l        [        R                  " US	-  S5      U l        [        R                  " USS
S9U l        [        R4                  R7                  U R                  R8                  5        [        R4                  R7                  U R                  R:                  5        [        R4                  R7                  U R                   R8                  5        [        R4                  R7                  U R                   R:                  5        [        R4                  R7                  U R0                  R8                  5        [        R4                  R7                  U R0                  R:                  5        [        R4                  R7                  U R2                  R8                  5        [<        R>                  " 5          S[        -  U R2                  R8                  SS [        24'   S S S 5        g s  snf s  sn	f ! , (       d  f       g = f)Nz	expected z channels, got é   rb   )Úgroups)rb   r   r   r¢   é   )r~   Údilationr]   r?   F)rZ   ç      ð?r   ) rA   rB   ÚN_SEEDSÚN_GROUPSrC   ÚwindowÚd_modelÚdw_padrG   rJ   Ú	seed_projÚConv1dÚdwr}   r~   r   Úseed_outÚ
ModuleListÚVÚDilatedResidualÚblocksÚranger<   ÚlocalÚheadÚskiprW   rX   rY   rZ   rN   Úno_grad)r[   Úc_inr©   rª   rD   Únlayersr]   rE   ÚdilÚ_r^   s             €r&   rB   ÚSeedGateLocal.__init__Œ   sE  ø€ Ü‰ÑÔØ”7œXÑ%Ó%Ü˜y¬´8Ñ);Ð(<¸OÈDÈ6ÐRÓSÐSØŒØŒØˆŒÜŸš¤8¨WÓ5ˆŒÜ—)’)˜G¨d¯k©k¸A©oÀgÑNˆŒÜ—’˜7Ó,ˆŒÜ—’˜7Ó,ˆŒÜ—’˜7Ó,ˆŒÜŸ	š	 'Ó3ˆŒÜ—m’má#ó%
â#�ô ×Ò˜g¨°SÔJÙ#ñ%
ó ˆŒô —]’]ÜEJÈ7Ä^ó$
ÚESÀÔ˜W¨TÖ;Á^ñ$
ó ˆŒ
ô —I’I˜g¨™k¨1Ó-ˆŒ	Ü—I’I˜d A¨EÑ2ˆŒ	Ü
�‰�‰�t—w‘w—~‘~Ô&Ü
�‰�‰�t—w‘w—|‘|Ô$Ü
�‰�‰�t—}‘}×+Ñ+Ô,Ü
�‰�‰�t—}‘}×)Ñ)Ô*Ü
�‰�‰�t—y‘y×'Ñ'Ô(Ü
�‰�‰�t—y‘y—~‘~Ô&Ü
�‰�‰�t—y‘y×'Ñ'Ô(Ü�]Š]�_Ø,/´'©MˆD�I‰I×Ñ˜Q ¤ ˜[Ñ)÷ ˆ_ùò!%
ùò$
÷ �_ús   Ä M)Å!M.Ì7)M3Í3
Nc                 ó  • UnUR                   u  pEnUR                  5       R                  XE[        [        5      R                  SSSS5      nU R                  U5      nUR                  U[        -  XPR                  5      n	X�R                  [        R                  " U	R                  SS5      U R                  S45      5      R                  SS5      -   n	U	R                  U[        XPR                  5      R                  SSSS5      n[        R                  " [        R                   " [        R                  " U R#                  U5      U R%                  U5      R                  SS5      5      [&        R(                  " U R                  5      -  SS9U R+                  U5      5      n
UR-                  SS9U R/                  U
R-                  SS95      -   nU R0                   H  nU" U5      nM     U R2                   H  nU" U5      nM     [        R4                  " U[7        US5      [7        US	5      /SS9nU R9                  U5      R;                  S5      U R=                  U5      R;                  S5      -   nU(       a  Xû4$ U$ )
Nr   r?   rb   r   r@   rc   rf   r   é   )rh   rx   ri   r¨   r§   rj   r¬   Úreshaperª   r®   rk   rl   Ú	transposer«   rN   Úmatmulrw   r}   r~   rp   rq   r   Úmeanr¯   r³   rµ   Úcatr�   r¶   Úsqueezer·   )r[   ry   Úreturn_hÚrawrz   r{   r¼   Úseedsr|   ÚflatÚmixÚfusedÚblockÚlayerÚpooledÚlogits                   r&   r‰   ÚSeedGateLocal.forward¬   só  € ØˆØ—'‘'‰ˆˆaØ—‘“×#Ñ# A¬(´GÓ<×DÑDÀQÈÈ1ÈaÓPˆØ�N‰N˜5Ó!ˆØ�y‰y˜œW™ a¯©Ó6ˆØ—g‘gœaŸeše D§N¡N°1°aÓ$8¸4¿;¹;ÈÐ:JÓKÓL×VÑVÐWXÐZ[Ó\Ñ\ˆØ�I‰I�aœ !§\¡\Ó2×:Ñ:¸1¸aÀÀAÓFˆÜ�lŠlœ5Ÿ=š=¬¯ª°d·f±f¸Q³iÀÇÁÈÃ×ATÑATÐUWÐY[ÓA\Ó)]Ü,0¯IªI°d·l±lÓ,Cñ*DØIKñMØNRÏfÉfÐUVËióYˆà—‘˜1�� §¡¨c¯h©h¸1¨h¨oÓ >Ñ>ˆØ—[”[ˆEÙ˜%“LŠEñ !à—Z”ZˆEÙ˜%“LŠEñ  ä—’˜E¤;¨u°aÓ#8¼+ÀeÈRÓ:PÐQÐWYÑZˆØ—	‘	˜&Ó!×)Ñ)¨"Ó-°·	±	¸#³×0FÑ0FÀrÓ0JÑJˆÞØ�<ÐØˆr`   )r³   rª   r®   r«   r¶   r~   rµ   r}   r¯   r¬   r·   r   r©   )r   r   r   r   é   )Fr‹   r“   s   @r&   rŸ   rŸ   ‰   s   ø† Ù_÷:÷@ò r`   rŸ   c           
      óJ   • U u
  p#pEpgp‰p«US:w  a  [        U5      e[        XXgU	S9$ )Nr   )rª   rº   r]   )rC   rŸ   )Úcfgr¹   Ú_nameÚ_familyr©   Úkindrª   rº   Ú_lrr]   Ú_wdÚ_robusts               r&   Úbuild_modelrÚ   Â   s4   € ØQTÑN€E�F '°CÀ#Øˆvƒ~Ü˜ÓÐÜ˜¨wÐQXÑYÐYr`   c           	      ó@  • U R                  S5      n/ n[        R                  " UR                  S5      UR                  S9n[        U5       Hb  nX$U-  U:H     nXfR                  SSS9-
  nUR                  UR                  SS5      U-  [        UR                  S5      S-
  S5      -  5        Md     UR                  S5      nSn[        U5       HG  n	[        U	S-   U5       H1  n
XsU	   X:   -
  R                  S5      R                  5       -   nUS-  nM3     MI     Xx-  $ )Nrb   r   rd   T)Úkeepdim© r   )rÃ   rN   rr   r—   re   r´   ÚappendrÁ   r•   Ú	new_zerosÚpow)r|   Ún_gÚzÚcovsÚindexÚgiÚzgÚlossÚpairsÚiÚjs              r&   Ú
coral_lossrë   É   s	  € Ø	�‰ˆq‹	€AØ€DÜ�LŠL˜Ÿ™ ›¨1¯8©8Ñ4€EÜ�CŽjˆØ�s‰{˜bÑ Ñ!ˆØ—'‘'˜! T�'Ð*Ñ*ˆØ�‰�B—L‘L  AÓ&¨Ñ+¬c°"·'±'¸!³*¸q±.À!Ó.DÑDÖEñ ð �;‰;�r‹?€DØ€EÜ�3ŽZˆÜ�q˜1‘u˜cÖ"ˆAØ ™7 T¡WÑ,×1Ñ1°!Ó4×9Ñ9Ó;Ñ;ˆDØ�Q‰JŠEó #ñ ð ‰<Ðr`   c                 ó@  • [         R                  " U 5      n [        R                  " U R	                  S5      SU R	                  S5      U R
                  S9R                  [        R                  " S5      [        R                  " S5      5      R                  5       nX-  $ )Nr   rb   r   rd   g      è?g      ô?)
r±   ÚaugmentrN   Úemptyr—   re   Úuniform_rp   ÚlogÚexp)ry   Úgains     r&   Úrobust_augmentró   Ú   sj   € Ü	�	Š	�!‹€AÜ�;Š;�q—v‘v˜a“y ! Q§V¡V¨A£Y°q·x±xÑ@×IÑIÌ$Ï(Ê(ÐSWË.ÔZ^×ZbÒZbÐcgÓZhÓi×mÑmÓo€DØ‰8€Or`   c                 ó®  • U u
  pVpxpšp¼pÞ[         R                  " X&   U5      u  nn[         R                  " X&   U5      u  nn[        UR	                  5       5      n[
        R                  " SU-
  [        US5      -  /US9nUS   S   S   S   R                  S   n[
        R                  " [         R                  5        UR                  S:X  a-  [
        R                  R                  [         R                  5        [        U U5      R                  U5      n[         R"                  " U5      R%                  5       n[
        R&                  R)                  UR+                  5       X½S9n[,        R.                  R1                  [         R                  5      n[3        U5      n[
        R4                  " UUS9nUR7                  5         [8        R8                  " 5       nS n[;        U5       GH{  n[         R<                  " UUXK5        [         R>                  " UX÷[         R@                  UU5      u  nn n![
        RB                  " U5      R                  US	S
9n"[
        RB                  " U 5      R                  US	S
9n#[
        RB                  " U!5      R                  US	S
9n$U(       a¤  [E        U"5      n"[
        RF                  " SUS9S:  ai  [        URI                  SS5      5      n%[
        RJ                  " U"RM                  S5      US9n&U%U"-  SU%-
  U"U&   -  -   n"U%U#-  SU%-
  U#U&   -  -   n#U$U$U&   -  n$U#S[N        -
  -  S[N        -  -   n'O[         RP                  " U"5      n"U#n'URS                  S	S9  U(       a  U" U"S	S9u  n(n)O
U" U"5      n(S n)[         RT                  " U#U$5      n*[V        RX                  " U(U'USS9n+U(       a  [Z        OSn,U(       Ga/  U+U*-  U$-  R]                  S5      U*U$-  R]                  S5      R_                  S5      -  n-/ n.[
        R`                  " U-RM                  S5      US9n/[;        U5       H+  n0U.Rc                  U-U/U-  U0:H     R	                  5       5        M-     [
        Rd                  " U.5      n.[
        Rf                  " UUR	                  5       -
  S-  SS9n1U1U.-  R]                  5       [h        [k        U)U5      -  -   n2URm                  S5      Ro                  U.Rq                  5       SS9  O;U*U$-  R]                  5       R_                  S5      n3U+U*-  U$-  R]                  5       U3-  n2U(S S 2SS 24   U(S S 2S S24   -
  Rs                  5       n4U$S S 2SS 24   U$S S 2S S24   -  n5U2U,U4U5-  R]                  5       -  U5R]                  5       R_                  S5      -  -   n2[
        Rt                  " U25      (       d  [w        SU SU 35      eU2Ry                  5         [
        Rz                  R|                  R                  UR+                  5       S5        UR�                  5         [         R‚                  " UUS5        [        U2R…                  5       5      nGM~     UR%                  5         [         R†                  " UX÷UUU5      n6[‰        SU SSR‹                  U5       SUS S U6S S![8        R8                  " 5       U-
  S" S#3S	S$9  UUU4$ )%Nr¦   gü©ñÒMbP?rd   r   Útrialsrb   Úcuda)ÚlrÚweight_decayT)Únon_blockingrÝ   g      à?gš™™™™™Ù?)Úset_to_none)rÆ   Únone)Ú
pos_weightÚ	reductiongš™™™™™©?r   rf   çÍÌÌÌÌÌì?)Úalphar@   znon-finite loss at step z for g×£p=
×ï?z
  trained z on Ú+z  loss ú.3fz  frame-acc z  (ú.0fús)r   )Fr±   Útrial_groupsÚ
norm_statsÚfloatrÃ   rN   Útensorr•   rh   Úmanual_seedÚ
TRAIN_SEEDÚtyperö   Úmanual_seed_allrÚ   r™   ÚcopyÚdeepcopyÚevalÚoptimÚAdamWÚ
parametersÚnpÚrandomÚdefault_rngÚlenrO   ÚtrainÚtimer´   Úset_lrÚsample_batchÚBATCHÚ
from_numpyró   ÚrandÚbetaÚrandpermr—   Ú
SMOOTH_EPSrí   Ú	zero_gradÚswitch_weightsrk   Ú binary_cross_entropy_with_logitsÚ	TV_ROBUSTro   Ú	clamp_minrr   rÞ   Ústackrw   ÚCORAL_Wrë   Úmul_Úadd_ÚdetachÚabsÚisfiniteÚRuntimeErrorÚbackwardrG   ÚutilsÚclip_grad_norm_ÚstepÚ
ema_updateÚitemÚtrain_frame_accr   Újoin)7rÓ   ÚdatasetsÚfeatre   ÚstepsÚnameÚfamilyr©   Ú_kindÚ_d_modelÚ_nlayersr÷   Ú_dropoutÚwdÚrobustr£   Úy_allÚmuÚsdÚposrü   r¹   ÚmodelÚemaÚoptÚrngrá   ÚdroÚt0Úlastr0  ÚxbÚybÚmbry   ÚyÚmÚlamÚpermÚy_hatrÏ   ÚhiddenÚswÚbceÚtv_wÚperÚgroupedÚrowsrå   r}   rç   ÚdenomÚdvÚm2Úaccs7                                                          r&   Útrain_modelr^  à   sÀ  € ØPSÑM€D�& °RÀ2Ü—N’N 4¡<°Ó:�M€FˆEÜ�\Š\˜$™,¨Ó1�F€BˆÜ
�—
‘
“Ó
€CÜ—’  c¡	¬S°°d«^Ñ;Ð<ÀVÑL€JØ�!‰9�XÑ˜qÑ! !Ñ$×*Ñ*¨1Ñ-€DÜ	×Ò”a—l‘lÔ#Ø‡{�{�fÓÜ�
‰
×"Ñ"¤1§<¡<Ô0Ü˜˜TÓ"×%Ñ% fÓ-€EÜ
�-Š-˜Ó
×
#Ñ
#Ó
%€CÜ
�+‰+×
Ñ
˜E×,Ñ,Ó.°2Ð
Ð
G€CÜ
�)‰)×
Ñ
¤§¡Ó
-€CÜ
ˆf‹+€CÜ
�+Š+�c &Ñ
)€CØ	‡K�K„MÜ	�Š‹€BØ€DÜ�e—ˆÜ	�Š��d˜EÔ&Ü—^’^ C¨¼¿¹À"ÀbÓI‰
ˆˆB�Ü×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÜ×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÜ×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÞÜ˜qÓ!ˆAÜ�zŠz˜" VÑ,¨sÓ2Ü˜CŸH™H S¨#Ó.Ó/�Ü—~’~ a§f¡f¨Q£i¸Ñ?�Ø˜!‘G˜s S™y¨A¨d©GÑ3Ñ3�Ø˜!‘G˜s S™y¨A¨d©GÑ3Ñ3�Ø˜˜$™‘K�Ø˜œzÑ)Ñ*¨S´:Ñ-=Ñ=‰Eä—	’	˜!“ˆAØˆEØ�‰ $ˆÑ'ÞÙ! !¨dÑ3‰MˆE‘6á˜!“HˆEØˆFÜ×Ò˜a Ó#ˆÜ×0Ò0°¸È*Ð`fÑgˆÞ"�y¨ˆßØ˜‘8˜a‘<×$Ñ$ QÓ'¨2°©6¯,©,°q«/×*CÑ*CÀCÓ*HÑHˆCØˆGÜ—<’< §¡¨£°FÑ;ˆDÜ˜C–j�Ø—‘˜s 4¨#¡:°Ñ#3Ñ4×9Ñ9Ó;Ö<ñ !ä—k’k 'Ó*ˆGÜ—’˜s S§X¡X£ZÑ/°3Ñ6¸AÑ>ˆAØ˜‘K×$Ñ$Ó&¬´:¸fÀcÓ3JÑ)JÑJˆDØ�H‰H�S‹M×Ñ˜wŸ~™~Ó/°sÐÒ;à˜!‘V—L‘L“N×,Ñ,¨SÓ1ˆEØ˜"‘H˜q‘L×%Ñ%Ó'¨%Ñ/ˆDØ’A�q‘r�E‰l˜U¢1 c r c 6™]Ñ*×/Ñ/Ó1ˆØŠq�!‘"ˆu‰X˜š!˜S˜b˜S˜&™	Ñ!ˆØ�d˜b 2™gŸ]™]›_Ñ,¨r¯v©v«x×/AÑ/AÀ#Ó/FÑFÑFˆÜ�~Š~˜d×#Ñ#ÜÐ!9¸$¸¸uÀTÀFÐKÓLÐLØ�‰ŒÜ�‰�‰×&Ñ& u×'7Ñ'7Ó'9¸3Ô?Ø�‰Œ
Ü	�Š�U˜C Ô'Ü�T—Y‘Y“[Ó!‹ñc ðd ‡H�H„JÜ
×
Ò
˜C °°R¸Ó
@€CÜ	ˆJ�t�f˜D §¡¨(Ó!3Ð 4°G¸DÀ¸:À\ÐRUÐVYÐQZð [Ü�iŠi‹k˜BÑ˜sÐ# 2ð'Ø.2ò4à��Bˆ;Ðr`   c                 ó>  • [         R                  " S5        [        [        [        -  SSSS9R                  U 5      R                  5       n[         R                  " SS[        [        -  U S9n[         R                  " 5          U" U5      nUR                  5       n[         R                  " S[        [        -  U S9US S 2S4'   U" U5      nUR                  5       n[         R                  " S[        [        -  U S9US S 2S4'   U" U5      nS S S 5        [         R                  " WS S 2S S24   WS S 2S S24   S	S
9(       d  [        S5      e[         R                  " US S 2S S24   WS S 2S S24   S	S
9(       d  [        S5      e[        SSS9  g ! , (       d  f       N‰= f)Nr   é    rb   )rª   rº   r   rd   r@   é
   gñhãˆµøä>)Úatolz!seed-gate readout sees the futurez-seed-gate readout is not causal at mid-windowzcausality check passedTr   )rN   r  rŸ   r§   r¨   r™   r  Úrandnr¸   ÚcloneÚallcloser,  r   )re   rD  ry   ÚbaseÚlateÚy_lateÚmidÚy_mids           r&   Úassert_causalrk  ,  sL  € Ü	×Ò�aÔÜœ'¤HÑ,¨b¸"ÀaÑH×KÑKÈFÓS×XÑXÓZ€EÜ�Š�A�rœ7¤XÑ-°fÑ=€AÜ	�Š�Ù�Q‹xˆØ�w‰w‹yˆÜ—k’k !¤W¬xÑ%7ÀÑGˆŠQ�ˆU‰Ù�t“ˆØ�g‰g‹iˆÜ—[’[ ¤G¬hÑ$6¸vÑFˆŠAˆrˆE‰
Ù�c“
ˆ÷ 
ô �>Š>˜$šq # 2 #˜v™,¨ªq°#°2°#¨v©¸T×BÜÐ>Ó?Ð?Ü�>Š>˜$šq # 2 #˜v™,¨ªa°°"°¨f©¸D×AÜÐJÓKÐKÜ	Ð
"¨$Ó/÷ 
�ús   ÂB	FÆ
Fc           
      óÔ  • [         R                  R                  [        SU  S35      n[         R                  R	                  U5      (       d  [        SU 35      e[        R                  " USSS9nUS   nUS   [        S	      [        [        US   [        S	      5      5         R                  S
   n[        R                  " XeS   US   US   [        US   5      S9R                  U5      nUR!                  US   5        UR#                  5         [        R$                  " XqS   US   US   US   U5      nAUR&                  S:X  a  [        R(                  R+                  5         U$ )NÚstem_ms96__ú.ptzmissing stem checkpoint ÚcpuF©Úmap_locationÚweights_onlyrÓ   r   r   rb   r   r?   r   r¤   ©rª   rº   ÚstemÚstaterA  rB  rö   )r   r    r4  Ú	STEM_CKPTÚexistsr,  rN   Úloadr   ÚnextÚiterrh   r±   ÚLogitTransformerÚboolr™   Úload_state_dictr  Ú	infer_allr
  rö   Úempty_cache)	Útagr6  re   r    ÚblobrÓ   r¹   rD  Ústreamss	            r&   Úload_stem_streamsrƒ  ?  s9  € Ü�7‰7�<‰<œ	 [°°°SÐ#9Ó:€DÜ�7‰7�>‰>˜$×ÑÜÐ5°d°VÐ<Ó=Ð=Ü�:Š:�d¨¸UÑC€DØ
ˆu‰+€CØ�‰:”g˜a‘jÑ!¤$¤t¨D°©J´w¸q±zÑ,BÓ'CÓ"DÑE×KÑKÈAÑN€DÜ×Ò˜t¨¡V°S¸±VÀSÈÁVÔRVÐWZÐ[\ÑW]ÓR^Ñ_×bÑbÐciÓj€EØ	×Ñ˜$˜w™-Ô(Ø	‡J�J„LÜ�kŠk˜% d¡¨T°$©Z¸¸d¹ÀSÈÁVÈVÓT€GØØ‡{�{�fÓÜ�
‰
×ÑÔ Ø€Nr`   c                 ó|  • / n[         R                   GHr  n[         R                   GHY  n/ nUR                  5        H2  u  p‰UR	                  [
        R                  " X€U   X#U-  XY5      5        M4     UR	                  [        XV[        [        R                  " U V
s/ s H  oªS   PM	     sn
5      5      [        [        R                  " U V
s/ s H  oªS   PM	     sn
5      5      [        [        R                  " U V
s/ s H  oªS   PM	     sn
5      5      [        [        R                  " U V
s/ s H  oªS   PM	     sn
5      5      [        [        R                  " U V
s/ s H  oªS   PM	     sn
5      5      S95        GM\     GMu     [
        R                  " U5      u  p¼X¼4$ s  sn
f s  sn
f s  sn
f s  sn
f s  sn
f )NÚHÚAccÚSDTÚLÚacc1)Úp_stayÚscaler…  r†  r‡  rˆ  r‰  )r±   ÚP_STAYÚSCALESÚitemsrÞ   ÚIMGÚevaluate_keysÚdictr  r  rÃ   Úpick_setting)r‚  Úval_keysrA  rB  rY  Úpr‹  ÚmergedÚdsÚkeysr   ÚbestÚreacheds                r&   Úselect_on_valrš  P  sN  € Ø€DÜ�X�XˆÜ—X•XˆEØˆFØ$ŸN™NÖ,‘�Ø—‘œc×/Ò/°¸B±KÀÈ%ÁZÐQRÓYÖZñ -à�K‰Kœ AÜ$¤R§W¢W¹fÓ-Eºf¸°¬f¹fÑ-EÓ%FÓGÜ!&¤r§w¢wÁ&Ó/IÂ&¸Q°%´Á&Ñ/IÓ'JÓ!KÜ!&¤r§w¢wÁ&Ó/IÂ&¸Q°%´Á&Ñ/IÓ'JÓ!KÜ$¤R§W¢W¹fÓ-Eºf¸°¬f¹fÑ-EÓ%FÓGÜ"'¬¯ªÁFÓ0KÂF¸q°6´ÁFÑ0KÓ(LÓ"MñO÷ Pô	 ñ ô ×$Ò$ TÓ*�M€DØˆ=Ðùò .FùÚ/IùÚ/IùÚ-EùÚ0Ks   ÂF%ÃF*Ã?F/ Ä0F4"Å!F9$c                 ó$  • [         R                  " U SSS9nUS   nUS   US   US   [        US   5      4u  pgp‰X   [        S	      [	        [        X   [        S	      5      5         R                  S
   n
[        R                  " X¦XxU	S9R                  U5      nUR                  US   5        UR                  5         [        R                  " X±U   US   US   Xb5      nAUR                  S:X  a  [         R                  R                  5         U$ )Nro  Frp  rÓ   r   r?   r   r¤   r   rb   rs  ru  rA  rB  rö   )rN   rx  r|  r   ry  rz  rh   r±   r{  r™   r}  r  r~  r
  rö   r  )r    r6  re   r9  r�  rÓ   r©   rª   rº   rt  r¹   rD  r‚  s                r&   Úload_streamsrœ  a  sú   € Ü�:Š:�d¨¸UÑC€DØ
ˆu‰+€CØ%(¨¡V¨S°©V°S¸±V¼TÀ#ÀaÁ&»\Ð%IÑ"€F�WØ‰<œ ™
Ñ#¤D¬¨d©l¼7À1¹:Ñ.FÓ)GÓ$HÑI×OÑOÐPQÑR€DÜ×Ò˜t°WÐTXÑY×\Ñ\Ð]cÓd€EØ	×Ñ˜$˜w™-Ô(Ø	‡J�J„LÜ�kŠk˜% f¡¨t°D©z¸4À¹:ÀvÓV€GØØ‡{�{�fÓÜ�
‰
×ÑÔ Ø€Nr`   c                 óP  • [        [        R                  R                  SS5      5      n[        R                  R                  SS5      R	                  5       nUR                  S5       Vs/ s H)  oUR	                  5       (       d  M  UR	                  5       PM+     sn=(       d    [        [        5      nU Vs/ s H  oU[        ;  d  M  UPM     nnU(       a  [        SU 35      e[        5       n/ n	U GH  n
[        R                  " U
5      u  p¼n[        R                  " X«5        [        R                  " X5      n/ nU GHe  n[        R                  " 5       n[        U[        U5      XàU5      u  nnn[        R                  R                  S5      (       dg  [        R                   R#                  [$        US    S	U
 S
35      n[&        R(                  " UR+                  5       UUUU[,        R.                  S.U5        [,        R0                  " UUUS      UUUS   U 5      nAU R2                  S:X  a  [&        R4                  R7                  5         [        R8                  " UU5      u  nn[;        UUUU5      u  nn[        R<                  " UU
UUUR>                  UR@                  5      n[C        SUS   U
[E        UR>                  5      [E        UR@                  5      US   US   US   US   US   [E        URF                  5      [E        URH                  5      [K        U5      [        R                  " 5       U-
  SS9nU	RM                  U5        [O        USUS    SU
 SUR>                   SUR@                   SUS   S SUS   S SUS   S S US   S! S"URF                  S S#URH                  S S$US%   S& S'35        URM                  U5        [        R                  R                  S(5      S):X  Ga†  [        R                   R#                  [P        S*S+S,U
 S
35      n[,        RR                  " U[U        UXUS   5      /5      n[        R8                  " UU5      u  nn [;        UUUU 5      u  n!n"[        R<                  " UU
UU U!R>                  U!R@                  5      n#U	RM                  [C        SS-US    S.3U
[E        U!R>                  5      [E        U!R@                  5      U#S   U#S   U#S   U#S   U#S   [E        U!RF                  5      [E        U!RH                  5      [K        U"5      S/SS95        [O        US0US    S1U
 SU!R>                   SU!R@                   SU#S   S SU#S   S SU#S   S S U#S   S! 35        [        R                  R                  S5      (       d  GMb  U	s  s  $    [W        U5      S:¼  d  GMÉ  [,        RR                  " U5      n$[        R8                  " U$U5      u  nn[;        U$UUU5      u  nn[        R<                  " U$U
UUUR>                  UR@                  5      nU	RM                  [C        SS2U
[E        UR>                  5      [E        UR@                  5      US   US   US   US   US   [E        URF                  5      [E        URH                  5      [K        U5      S/SS95        [O        US3U
 SUR>                   SUR@                   SUS   S SUS   S SUS   S S US   S! 35        GM     [X        RZ                  " U	5      n%[        R                  R                  S5      (       dá  [        R\                  " [        R                   R_                  [`        5      S4S59  U%Rc                  [`        S6S79  [O        US85        U%Re                  S95       Hc  u  nn&U&/ S:Q   Rg                  5       n'[O        US;US< SU'RF                  S S=U'RH                  S S>U'Rh                  S S?U'Rj                  S! 3
5        Me     [O        US@[`         35        U	$ s  snf s  snf )ANÚSTEPSÚ1000ÚFOLDr*   r+   zunknown FOLD ÚSMOKEr   z	__screen-rn  )ru  rA  rB  rÓ   r7  Úseedrb   r   rö   Úimage_screenr…  r†  r‡  rˆ  r‰  Úofficial)ÚprotocolrD  r–  rŠ  r‹  r…  r†  r‡  rˆ  r‰  Úval_HÚval_AccÚval_reached_74ÚsecondsÚscorerzscreen Ú ú p ú sc ú  H ú.4fú Acc ú.2fú SDT ú L r  z valH z valAcc z (r©  r  r  ÚENS_IMAGE_STEMÚ1r   Ústem_ms96_image_splitzstem_ms96__held-Úens_Ú_stemg        zscreen ens_z_stem Ú
ens_screenzscreen ens_screen Tr   F©rä   zscreen means:rD  )r…  r†  r‡  rˆ  ú  Ú16sú  Acc ú  SDT ú  L úwrote )6Úintr   r,   r-   r.   r1   r/   r   r2   r:   r�  Úcompetition_splitÚassert_matches_saved_splitÚfilter_featr  r^  r    r4  ÚCKPTrN   ÚsaveÚ
state_dictr±   r	  r~  r
  rö   r  Úemis_onrš  Úofficial_testrŠ  r‹  r‘  r  r…  r†  r|  rÞ   r'   ÚEXPÚaverage_streamsrœ  r  ÚpdÚ	DataFramer   r!   Ú
SCREEN_OUTÚto_csvÚgroupbyrÃ   r‡  rˆ  )(re   r6  rð   r7  r4   r|   ÚheldsÚunknownr8   rY  ÚheldÚ	_val_subjÚ
train_keysr“  Ú
train_featÚfold_streamsrÓ   rI  rD  rA  rB  r    r‚  Úmu_eÚsd_er˜  r™  ÚoffÚrecÚ	stem_pathrÊ   Úmu_mÚsd_mÚbest_mÚ	reached_mÚoff_mÚavgÚdfÚsubrO  s(                                           r&   Ú
run_screenrå  p  sZ  € Ü”—
‘
—‘˜w¨Ó/Ó0€EÜ�:‰:�>‰>˜& "Ó%×+Ñ+Ó-€DØ $§
¡
¨3¤Ó=¢˜1·7±7·9‹YˆQ�W‰WŽY¡Ñ=×NÄÄgÃ€EÙÓ4š%�Q¬GÑ#3�q™%€GÐ4ÞÜ˜=¨¨	Ð2Ó3Ð3ÜÓ€DØ€DÜˆÜ*-×*?Ò*?ÀÓ*EÑ'ˆ	˜xÜ×&Ò& tÔ7Ü—_’_ TÓ6ˆ
ØˆÜˆCÜ—’“ˆBÜ'¨¬T°*Ó-=¸zÐSXÓY‰MˆE�2�rÜ—:‘:—>‘> '×*Ñ*Ü—w‘w—|‘|¤D¨S°©V¨H°I¸d¸VÀ3Ð*GÓH�Ü—
’
 U×%5Ñ%5Ó%7¸rÈÐTWØ%*´A·L±LñBØCGôIä—k’k %¨¨c°!©f©°r¸2¸sÀ1¹vÀvÓNˆGØØ�{‰{˜fÓ$Ü—
‘
×&Ñ&Ô(ÜŸš W¨jÓ9‰JˆD�$Ü)¨'°8¸TÀ4ÓH‰MˆD�'Ü×#Ò# G¨T°4¸¸t¿{¹{ÈDÏJÉJÓWˆCÜ °c¸!±fÀÌeÐTX×T_ÑT_ÓN`Ü" 4§:¡:Ó.°#°c±(ÀÀEÁ
ÐPSÐTYÑPZÐ^aÐbeÑ^fØ ™K¬u°T·V±V«}ÄeÈDÏHÉHÃoÜ&*¨7£m¼T¿YºY»[È2Ñ=MÐV`ñbˆCð �K‰K˜ÔÜ�S˜G C¨¡F 8¨1¨T¨F°#°d·k±k°]À$ÀtÇzÁzÀlð SØ" 3™x¨˜n¨E°#°e±*¸SÐ1AÀÀsÈ5ÁzÐRUÐFVÐVYÐZ]Ð^aÑZbÐcfÐYgð h"Ø"&§&¡&¨ ¨X°d·h±h¸s°^À2ÀcÈ)ÁnÐUXÐEYÐY[ð]ô ^ð ×Ñ Ô(Ü�z‰z�~‰~Ð.Ó/°3Ô6ÜŸG™GŸL™L¬¨hÐ8OÐScÐdhÐciÐilÐQmÓn�	Ü×'Ò'¨´,¸yÈ$ÐX[Ð\]ÑX^Ó2_Ð(`Óa�Ü Ÿ[š[¨¨jÓ9‘
��dÜ$1°#°xÀÀtÓ$LÑ!�˜	Ü×)Ò)¨#¨t°T¸4ÀÇÁÐPV×P\ÑP\Ó]�Ø—‘œD¨.À$ÀsÈ1ÁvÀhÈeÐ@TÐY]Ü(-¨f¯m©mÓ(<ÄEÈ&Ï,É,ÓDWØ#(¨¡:°5¸±<ÀUÈ5Á\ÐUZÐ[^ÑU_Ø&+¨F¡m¼5ÀÇÁ»?ÔTYÐZ`×ZdÑZdÓTeÜ04°Y³ÈÐU_ñ	aô bô
 ˜ ¨C°©F¨8°6¸$¸¸sÀ6Ç=Á=À/ÐQUÐV\×VbÑVbÐUcð d#Ø#(¨¡:¨cÐ"2°%¸¸e¹ÀSÐ7IÈÈuÐUZÉ|Ð\_ÐN`Ð`cÐdiÐjmÑdnÐorÐcsðuô vä�z‰z�~‰~˜g×&Ô&Ø”ñK ôL ˆ|Ó Ö!Ü×#Ò# LÓ1ˆCÜŸš S¨*Ó5‰JˆD�$Ü)¨#¨x¸¸tÓD‰MˆD�'Ü×#Ò# C¨¨t°T¸4¿;¹;ÈÏ
É
ÓSˆCØ�K‰Kœ n¸LÈTÔZ_Ð`d×`kÑ`kÓZlÜ#(¨¯©Ó#4¸¸C¹ÀcÈ%ÁjÐVYÐZ_ÑV`ÐdgÐhkÑdlØ"% f¡+´U¸4¿6¹6³]ÌEÐRV×RZÑRZËOÜ,0°«MÀ3Èzñ[ô \ô �SÐ.¨t¨f°C¸¿¹°}ÀDÈÏÉÈð UØ" 3™x¨˜n¨E°#°e±*¸SÐ1AÀÀsÈ5ÁzÐRUÐFVÐVYÐZ]Ð^aÑZbÐcfÐYgði÷ jñi ôl 
�Š�dÓ	€BÜ�:‰:�>‰>˜'×"Ñ"Ü
�Š”B—G‘G—O‘O¤JÓ/¸$Ò?Ø
�	‰	”* Eˆ	Ñ*Ü��oÔ&ØŸ*™* WÖ-‰JˆE�3ØÒ,Ñ-×2Ñ2Ó4ˆAÜ�S˜B˜u S˜k¨¨a¯c©c°#¨Y°f¸Q¿U¹UÀ3¸KÀvÈaÏeÉeÐTWÈ[ÐX\Ð]^×]`Ñ]`ÐadÐ\eÐfÖgñ .ô 	�˜œz˜lÐ+Ô,Ø€KùòK >ùÚ4s   Á+`Â`Â8`#Ã	`#c                 óä  ^ ^^^/^0^1^2^3^4^5^6^7^8• [        [        R                  R                  SS5      5      m7[	        5       m1[        R                  R                  SS5      S:g  n0 m/0 m6U UU64S jm0/ m5/ n[        [        5       H*  u  pV[        US-   S   H  nUR                  Xg45        M     M,     U/UU54S jnT1 GH  n	U	S	   U	S   U	S
   pËn
U Hü  u  pg[         Vs/ s H  oÝXg4;  d  M  UPM     nn[        XžTT T75      u  nnn[        R                  R                  [        U
 S[        R                  " Xg45       S35      n[        R                  " UR!                  5       UUXžT7S.U5        [        R"                  " UTU   UUUT 5      nUT/X¦U4'   UT/X§U4'   AT R$                  S:X  d  MÞ  [        R&                  R)                  5         Mþ     U" U
5        GM     T1 Vs/ s H  nUS	   PM
     snm2/ n[+        T25      S
:¼  a^  U H?  u  pg[        R,                  " T2 Vs/ s H
  nT/UXg4   PM     sn5      nUT/SXg4'   UT/SXv4'   MA     UR                  S5        U" S5        U(       Ga  U H«  u  pgT0" [        R                  " Xg45      5      nT2 H4  n
[        R,                  " T/X¦U4   U/5      nSU
 S3nUT/UXg4'   UT/UXv4'   M6     [+        T25      S
:¼  d  Mm  [        R,                  " T2 Vs/ s H
  nT/UXg4   PM     snU/-   5      nUT/SXg4'   UT/SXv4'   M­     T2 H$  n
UR                  SU
 S35        U" SU
 S35        M&     [+        T25      S
:¼  a  UR                  S5        U" S5        T2U-   n[.        R0                  " T55      n[        R2                  " [        R                  R5                  [6        5      SS9  UR9                  [6        SS9  [;        TS[6         35        UR=                  / SQ5      / SQ   R?                  5       RA                  5       n0 m4U0U1U UU2U4U7U84S jm8/ m3UU3U84S jn[        RB                  " U[        RD                  5      u  nn[;        TSU S[        RD                  S SURF                  R?                  5       S  S!URH                  R?                  5       S" 35        URK                  5        Hb  u  n n[;        TS#URL                   S$URN                   S%URP                   S&URR                   SURF                  S  S!URH                  S" 35        Md     U" S'U5        U H“  n
UURN                  U
:H     n![        RB                  " U![        RD                  5      u  n"n#[;        TS(U
 S)U# S*U"RF                  R?                  5       S  S!U"RH                  R?                  5       S" 35        U" S+U
 3U"5        M•     [.        R0                  " T35      n$U$R9                  [T        SS9  [;        TS,5        U$R=                  S-5       Hq  u  n%n!U!/ SQ   R?                  5       n&[;        TS#U%S. S/U&RF                  S  S0U&RH                  S" S1U&RV                  S" S2U&RX                  S3 S4U&RZ                  S3 35        Ms     U$U$R\                  S':H     n'[_        U'RF                  R?                  5       5      [_        U'RH                  R?                  5       5      n)n([;        TS5U(S  S0U)S" 35        S'U(U)n,n+n*U$R=                  S-5       H~  u  n%n![_        U!RF                  R?                  5       5      [_        U!RH                  R?                  5       5      n.n-U-U+:”  d&  [`        Rb                  " U-U+5      (       d  Mp  U.U,:”  d  Mx  U%U-U.n,n+n*M€     [;        TS6U* S/U+S  S0U,S" 35        [;        TS7[e        U+S8:„  =(       a    U,S9:„  5       35        [;        TS[T         35        U$$ s  snf s  snf s  snf s  snf ):Nrž  rŸ  ÚENS_STEMrµ  Ú0c                 ó:   >• U T;  a  [        U TT5      TU '   TU    $ )N)rƒ  )r€  re   r6  Ú
stem_caches    €€€r&   Úcached_stemÚrun_nested.<locals>.cached_stemÂ  s'   ø€ Ø�jÓ Ü/°°T¸6ÓBˆJ�s‰OØ˜#‰Ðr`   rb   c                 ó¢  >• [          HÒ  n[          HÅ  nX!:X  a  M
  [          Vs/ s H  o3X4;  d  M  UPM     nnTXU4   n[        R                  " U Vs0 s H  o3XS   _M	     snU5      u  pg[        R                   HV  n[        R
                   H?  n	[        R                  " X%U   XgU	-  U5      n
TR                  [        SXXU	S.U
D65        MA     MX     MÇ     MÔ     [        R                  " T V
s/ s H  oªS   U :X  d  M  U
PM     sn
5      nUR                  / SQ5      / SQ   R                  5       R                  5       n[        R                  " UR                  U S9[        R                   5      u  pÞ[#        TSU  SU SUR$                  R                  5       S	 S
UR&                  R                  5       S 35        g s  snf s  snf s  sn
f )N)ÚouterÚinnerrD  rŠ  r‹  rD  )rî  rŠ  r‹  )r…  r†  r‡  )rD  zinner ú: z/5 Acc>=74  H r¯  r°  r±  rÝ   )r   ÚNÚemisr±   rŒ  r�  ÚevaluaterÞ   r‘  rÌ  rÍ  rÐ  rÃ   Úreset_indexÚselect_ruleÚassignÚTHRr'   r…  r†  )r8  rî  Úinner_dsr\   Útrain_dsr‚  rA  rB  r”  r‹  Úrrä  Úgrouped_oneÚpicksÚn_okÚcacherð   rY  s                  €€€r&   Ú
score_nameÚrun_nested.<locals>.score_nameÌ  s�  ø€ ßˆEß#�ØÓ$ÙÝ'.ÓM¢w !¸EÐ;LÑ2LŸA¡w�ÐMØ ¨hÐ 7Ñ8�ÜŸš¹Ó Aº°1 G¡J¢¹Ñ AÀ8ÓL‘�ÜŸœ�AÜ!"§¤˜ÜŸJšJ x¸Ñ1BÀBÈUÉ
ÐTUÓV˜ØŸ™¤DÐ$m¨uÈDÐbgÑ$mÐklÑ$mÖnó "*ó "ó $ñ ô �lŠl¡tÓB¢t !°©z¸TÑ/AŸA¡tÑBÓCˆØ—k‘kÒ">Ó?Ò@SÑT×YÑYÓ[×gÑgÓiˆÜ—m’m K×$6Ñ$6¸TÐ$6Ð$BÄAÇEÁEÓJ‰ˆÜ�˜˜t˜f B t f¨N¸5¿7¹7¿<¹<»>È#Ð:NÈeÐTY×T]ÑT]×TbÑTbÓTdÐehÐSiÐjÕkùò Nùâ Aùò
 Cs   £
G±GÁGÃ2GÄGr   r   Ú__rn  ©ru  rA  rB  rÓ   r  r7  rö   Úens_newr·  r¸  Úens_all_stemTr   Frº  rÀ  )rî  rD  rŠ  r‹  )r…  r†  r‡  rˆ  r‰  c           	      ó  >^ • T U4nUT;   a  TU   $ T S:X  a5  [         R                  " T Vs/ s H  nT" X15      PM     sn5      TU'   TU   $ T S:X  aB  [         R                  " T Vs/ s H  nT" X15      PM     snT" SU 35      /-   5      TU'   TU   $ T R                  S5      (       a]  T R                  S5      (       aG  T [	        S5      [	        S5      *  n[         R                  " T" XA5      T" SU 35      /5      TU'   TU   $ [        U 4S jT 5       5      n[         Vs/ s H  ofU:w  d  M
  UPM     nn[        XWTTT5      u  p‰n
[        R                  R                  [        T  SU S35      n[        R                  " UR                  5       XšXWTS	.U5        [         R                  " UTUS
      XšUS   T5      TU'   ATR                   S:X  a  [        R"                  R%                  5         TU   $ s  snf s  snf s  snf )Nr  r  zheld-r·  r¸  c              3   ó<   >#   • U  H  oS    T:X  d  M  Uv •  M     g7f)r   NrÝ   )Ú.0r7   r8  s     €r&   Ú	<genexpr>Ú2run_nested.<locals>.streams_for.<locals>.<genexpr>#  s   øé € Ð3šd˜¨¡d¨d¡l—1‘1šdùs   ƒ“	z__held-rn  r  rb   r   rö   )r±   rË  Ú
startswithÚendswithr  ry  r   r^  r   r    r4  rÅ  rN   rÆ  rÇ  r~  r
  rö   r  )r8  rî  Úkeyr5   rf  rÓ   r\   rù  rD  rA  rB  r    rë  r8   re   r6  r6   Úrefitr7  Ústreams_fors   `           €€€€€€€€r&   r  Úrun_nested.<locals>.streams_for  sü  ù€ Ø�UˆmˆØ�%‹<Ø˜‘:ÐØ�9ÓÜ×*Ò*É5Ó+QÊ5Àa©K¸Ö,AÉ5Ñ+QÓRˆE�#‰JØ˜‘:ÐØ�>Ó!Ü×*Ò*É5Ó+QÊ5Àa©K¸Ö,AÉ5Ñ+QÙ.9¸EÀ%À¸/Ó.JÐ-Kñ,Ló MˆE�#‰Jà˜‘:ÐØ�?‰?˜6×"Ñ" t§}¡}°W×'=Ñ'=Øœ˜F›¤S¨£\ MÐ2ˆDÜ×*Ò*Ù˜DÓ(Ù˜e E 7˜OÓ,ð,ó ˆE�#‰Jð ˜‘:ÐÜÔ3™dÓ3Ó3ˆÝ&Ó5šw˜!¨u©*—A™wˆÐ5Ü# C°4¸ÀÓG‰ˆ�2Ü�w‰w�|‰|œD T F¨'°%°¸Ð"<Ó=ˆÜ�
Š
˜U×-Ñ-Ó/°rÈCØ"ñ$Ø%)ô	+ä—[’[ ¨¨S°©V©°b¸cÀ!¹fÀfÓMˆˆc‰
ØØ�;‰;˜&Ó Ü�J‰J×"Ñ"Ô$Ø�S‰zÐùò/ ,Rùò ,Rùò 6s   ¬G9Á'G>Ä%	HÄ2Hc                 óÂ  >• UR                  5        GHI  u  p#T" UR                  UR                  5      n[        R                  " XCR                  UR
                  UR                  5      nTR                  [        XR                  UR                  [        UR
                  5      [        UR                  5      US   US   US   US   US   [        UR                  5      [        UR                  5      SS95        [        TU  SUR                   SUR                   S	UR
                   S
UR                   SUS   S SUS   S SUS   S SUS   S 35        GML     g )Nr…  r†  r‡  rˆ  r‰  r¤  )Úroler–  rD  rŠ  r‹  r…  r†  r‡  rˆ  r‰  Úinner_HÚ	inner_Accrª  r«  r¬  r­  r®  r¯  r°  r±  r²  r³  r  )ÚiterrowsrD  rî  r±   Ú
score_pickrŠ  r‹  rÞ   r‘  r  r…  r†  r'   )	r  rü  r¼   rz   r‚  rÚ  rð   Úout_rowsr  s	         €€€r&   ÚemitÚrun_nested.<locals>.emit1  s:  ø€ Ø—N‘N×$‰DˆAÙ! !§'¡'¨1¯7©7Ó3ˆGÜ—,’,˜w¯©°·±¸1¿7¹7ÓCˆCØ�O‰OœD d¯w©w¸a¿g¹gÌeÐTU×T\ÑT\ËoÜ',¨Q¯W©W£~¸¸S¹ÀsÈ5ÁzÐWZÐ[`ÑWaØ#& s¡8°#°f±+ÄuÈQÏSÉSÃzÔ]bÐcd×chÑchÓ]iØ(2ñ4ô 5ô �S˜T˜F ! A§G¡G 9¨A¨a¯g©g¨Y°c¸!¿(¹(¸À4ÈÏÉÀyð QØ" 3™x¨˜n¨E°#°e±*¸SÐ1AÀÀsÈ5ÁzÐRUÐFVÐVYÐZ]Ð^aÑZbÐcfÐYgði÷ jò %r`   zselect acc74: z/5 folds reached inner Acc>=Úgz
  inner H r¯  r°  r±  r»  r«  r¬  r­  Úacc74zfamily rð  z/5 inner Acc>=74  inner H Úfamily_zmeans:r  Ú28sr®  r½  r¾  r¿  r  z  acc1 zPRIMARY acc74  H z
BEST role zgoal H>0.9 and Acc>85: rþ   éU   )3rÁ  r   r,   r-   r:   Ú	enumerater   rÞ   r^  r    r4  rÅ  r±   Útriplet_keyrN   rÆ  rÇ  r~  r
  rö   r  r  rË  rÌ  rÍ  r   r!   Ú
NEST_INNERrÏ  r'   rÐ  rÃ   rô  rõ  r÷  r…  r†  r  rî  rD  rŠ  r‹  ÚNEST_OUTr‡  rˆ  r‰  r  r  rp   Úiscloser|  )9re   r6  rð   Úens_stemÚexcluded_pairsré   r   rz   rÿ  rÓ   r8  r9  r©   r\   rù  rD  rA  rB  r    r‚  r7   Úextrar5   râ  rt  r€  Úmodel_namesrï  rX  r  rü  rý  r¼   rä  ÚfpÚn_familyrã  r  rO  Úprimr|   r]  Ú	best_roleÚbest_hÚbest_accÚhhÚaarþ  rë  r8   r6   r  r  rY  rê  r7  r  s9   ```                                            @@@@@@@@@@r&   Ú
run_nestedr/  »  sU  ÿü€ Ü”—
‘
—‘˜w¨Ó/Ó0€EÜÓ€DÜ�z‰z�~‰~˜j¨#Ó.°#Ñ5€HØ€EØ€J÷ð €DØ€NÜœ'Ö"‰ˆÜ˜˜Q™˜“ˆAØ×!Ñ! 1 &Ö)ó !ñ #÷lô" ˆØ" 1™v s¨1¡v¨s°1©v�fˆÛ"‰DˆAÝ#*Ó>¢7˜a¸°v©oŸ¡7ˆHÐ>Ü'¨°t¸VÀUÓK‰MˆE�2�rÜ—7‘7—<‘<¤¨¨¨b´·²À¸vÓ1FÐ0GÀsÐ&KÓLˆDÜ�JŠJ ×!1Ñ!1Ó!3¸2ÀRÐPSØ!&ñ(Ø)-ô/ä—k’k %¨¨f©°r¸2¸vÀvÓNˆGØ")ˆE�4˜A�,ÑØ")ˆE�4˜A�,ÑØØ�{‰{˜fÕ$Ü—
‘
×&Ñ&Ö(ñ #ñ 	�4×ñ ñ   Ó š4�aˆQˆqŒT™4Ñ €EØ€EÜ
ˆ5ƒz�QƒÛ"‰DˆAÜ×#Ò#¹uÓ$Eºu¸! U¨A¨q¨9Ô%5¹uÑ$EÓFˆCØ'*ˆE�9˜aÐ#Ñ$Ø'*ˆE�9˜aÐ#Ó$ñ #ð 	�‰�YÔÙ�9ÔßÛ"‰DˆAÙœqŸ}š}¨a¨VÓ4Ó5ˆDÛ�Ü×'Ò'¨°¸¨|Ñ)<¸dÐ(CÓD�Ø˜T˜F %Ð(�Ø%(��s˜A�kÑ"Ø%(��s˜A�kÓ"ñ	 ô
 �5‹z˜Q�Ü×'Ò'Á5Ó(IÂ5¸a¨°°1¨yÔ)9Á5Ñ(IÈTÈFÑ(RÓS�Ø03��~ qÐ,Ñ-Ø03��~ qÐ,Ó-ñ #ó ˆDØ�L‰L˜4 ˜v UÐ+Ô,Ù˜˜d˜V 5Ð)Ö*ñ ô ˆu‹:˜‹?Ø�L‰L˜Ô(Ù�~Ô&à˜%‘-€KÜ�LŠL˜Ó€EÜ‡K‚K”—‘—‘¤
Ó+°dÒ;Ø	‡L�L” 5€LÑ)ÜˆS�Fœ:˜,Ð'Ô(Ø�m‰mÒAÓBÒCcÑd×iÑiÓk×wÑwÓy€GØ€E÷ô ð< €H÷	jô —-’- ¬¯©Ó/�K€Eˆ4ÜˆS�N 4 &Ð(DÄQÇUÁUÈ1ÀIð NØ"ŸW™WŸ\™\›^¨CÐ0°°e·i±i·n±nÓ6FÀsÐ5KðMô Nà—‘Ö ‰ˆˆ1Ü�˜˜1Ÿ7™7˜) 1 Q§W¡W I¨S°·±°
¸$¸q¿w¹w¸iÀzÐRS×RUÑRUÐVYÐQZÐZ_Ð`a×`eÑ`eÐfiÐ_jÐkÖlñ !áˆ�%ÔÛˆØ�g—m‘m tÑ+Ñ,ˆÜ—}’} S¬!¯%©%Ó0‰ˆˆHÜ�˜ ˜v R¨ zÐ1KÈBÏDÉDÏIÉIËKÐX[ÐK\Ð\aÐbd×bhÑbh×bmÑbmÓboÐpsÐatÐuÔvÙˆw�t�fÐ˜rÖ"ñ	 ô 
�Š�hÓ	€BØ‡I�IŒh˜e€IÑ$ÜˆS�(ÔØ—Z‘Z Ö'‰	ˆˆcØÒ0Ñ1×6Ñ6Ó8ˆÜ�˜˜4 ˜* D¨¯©¨S¨	°¸¿¹¸c°{À&ÈÏÉÈsÈÐSWÐXY×X[ÑX[Ð\_ÐW`Ð`gÐhi×hnÑhnÐorÐgsÐtÖuñ (ð ˆb�g‰g˜Ñ Ñ!€DÜ�4—6‘6—;‘;“=Ó!¤5¨¯©¯©«Ó#9€s€AÜˆSÐ% a¨ W¨F°3°s°)Ð<Ô=Ø")¨1¨c�xˆv€IØ—Z‘Z Ö'‰	ˆˆcÜ�s—u‘u—z‘z“|Ó$¤e¨C¯G©G¯L©L«NÓ&;ˆBˆØ�‹KœTŸ\š\¨"¨f×5Ó5¸"¸x½-Ø*.°°B˜x�vˆI™xñ (ô ˆS�J˜y˜k¨¨f°S¨\¸ÀÈ¸~ÐNÔOÜˆSÐ+¬D°¸#±×1OÀ(ÈRÁ-Ó,PÐ+QÐRÔSÜˆS�Fœ8˜*Ð%Ô&Ø€Iùòs ?ùò !ùò %Fùò )Js   Ã
_Ã*_Ç _#È_(
Ë,_-
c                  óô  • [         R                  R                  SS5      R                  5       R	                  5       n U S:X  a  [
        O[        n[         R                  R                  S5      (       a  [        US5      R                  5         OL[         R                  " [         R                  R                  U5      SS9  [        US5      R                  5         [        R                  " [        R                  R                  5       (       a  S	OS
5      n[!        [         R                  R                  SS5      5      n[#        USU SU  SU S[$        R&                   35        S[        R(                  R                  R*                  l        S[        R(                  R.                  l        [0        R2                  " 5         [0        R4                  " 5       n[7        [8        R:                  " U Vs/ s H
  oUS   S   PM     sn5      5      n[7        [8        R:                  " U Vs/ s H
  oUS   S   PM     sn5      5      n[#        USUS SUS 35        [=        US-
  5      S:”  d  [=        US-
  5      S:”  a  [?        S5      e[A        U5        [$        RB                  " 5       n[         R                  " [D        SS9  U S:X  a  [G        X(U5        g U S:X  a  [I        X(U5        g [K        SU  35      es  snf s  snf )NÚMODEÚscreenÚnestedr¡  r   Tr   Úwrö   ro  rž  rŸ  zdevice z  mode z  steps z  batch rb   r…  r†  zreproduced lda4 @ 0.998  H r¯  r½  r±  g°rh‘í|é?gü©ñÒMb@?gq=
×£@R@r   z>baseline reproduction does not match the published LDA numberszunknown MODE )&r   r,   r-   r.   ÚlowerÚNEST_LOGÚ
SCREEN_LOGr"   Úcloser   r    r!   rN   re   rö   Úis_availablerÁ  r'   r±   r  ÚbackendsrÂ   Ú
allow_tf32Úcudnnrñ  Úload_allÚcheck_fast_matches_officialr  r  rÃ   r*  r,  rk  Úbuild_featuresrÅ  rå  r/  r2   )	Úmoderð   re   r7  Úchecksr7   Úbase_hÚbase_accr6  s	            r&   ÚmainrD  \  s'  € Ü�:‰:�>‰>˜& (Ó+×1Ñ1Ó3×9Ñ9Ó;€DØ˜hÓ&�(¬J€CÜ	‡z�z‡~�~�g×ÑÜˆS�#‹×ÑÕä
�Š”B—G‘G—O‘O CÓ(°4Ò8ÜˆS�#‹×ÑÔÜ�\Š\¤E§J¡J×$;Ñ$;×$=Ñ$=™&À5ÓI€FÜ”—
‘
—‘˜w¨Ó/Ó0€EÜˆS�G˜F˜8 7¨4¨&°¸¸¸xÌÏÉÀyÐQÔRØ,0„E‡N�N×Ñ×ÑÔ)Ø&*„E‡N�N×ÑÔ#Ü‡J‚J„LÜ×*Ò*Ó,€FÜ”2—7’7©vÓ6ªv¨!˜a™D œI©vÑ6Ó7Ó8€FÜ”R—W’W±6Ó:²6¨a ™d 5œk±6Ñ:Ó;Ó<€HÜˆSÐ/°°s¨|¸6À(È3ÀÐPÔQÜ
ˆ6�F‰?Ó˜dÓ"¤c¨(°UÑ*:Ó&;¸dÓ&BÜÐ[Ó\Ð\Ü�&ÔÜ×ÒÓ€DÜ‡K‚K”˜tÒ$ØˆxÓÜ�6 Õ%Ø	�Ó	Ü�6 Õ%ä˜=¨¨Ð/Ó0Ð0ùò 7ùÚ:s   Ç/K0È$K5Ú__main__)r   )<r�   r  rp   r   r  Únumpyr  ÚpandasrÌ  rN   Útorch.nnrG   Útorch.nn.functionalÚ
functionalrk   Úensemble_clf_nestedrñ  Úensemble_clf_transformer_v2r±   r¶  r�  Úcommonr   r    r!   ÚabspathÚ__file__rÊ  r4  rÅ  rv  rÎ  r7  r!  r   r6  r0   r§   r¨   r#  r  r&  r'   r:   ÚModuler<   r�   rŸ   rÚ   rë   ró   r^  rk  rƒ  rš  rœ  rå  r/  rD  rŒ   rÝ   r`   r&   Ú<module>rQ     sŸ  ðñó4 Û Û 	Û ã Û Û Ý ß Ð ã Û 'Û #Ý à‡g�g‡o�o�b—g‘g—o‘o hÓ/Ó0€Ø	‡w�w‡|�|�C˜ =Ó1€Ø�G‰G�L‰L˜˜h¨Ó6€	Ø�W‰W�\‰\˜#˜y¨*Ð6QÓR€
Ø�W‰W�\‰\˜#˜}Ð.IÓJ€
Ø�7‰7�<‰<˜˜Y¨
Ð4UÓV€Ø�W‰W�\‰\˜#˜y¨*Ð6]Ó^€
Ø�7‰7�<‰<˜˜]Ð,@ÓA€ð€ð €Ø€Ø€	Ø€
Ø
€òò	ô,�r—y‘yô ,ò^1ô6�B—I‘Iô 6òrZòò"òIòX0ò&ò"ô"òHòV^òB1ð> ˆzÓÙ…Fð r`   