ó
    ù·âjîU  ã                   ód  • S r SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
Jr  SSKJs  Jr  SSKrSSKrSSKrSSKrSSKJr  \R2                  R5                  \R2                  R7                  \5      5      r\R2                  R=                  \SS5      r\R2                  R=                  \SS5      r \R2                  R=                  \SS5      r!\R2                  R=                  \SS	5      r"\R2                  R=                  \S
SS5      r#Sr$Sr%\&" \RN                  RQ                  SS5      5      r)\&" \RN                  RQ                  SS5      5      r*\&" \RN                  RQ                  SS5      5      r+\," \RN                  RQ                  SS5      5      r-Sr.Sr/Sr0Sr1SSSSSS\/4r2Sr3S r4S!r5S" r6S# r7\5SS4S$ jr8S% r9 " S& S'\Rt                  5      r; " S( S)\Rt                  5      r<S* r=S+ r>S, r?S- r@S. rAS/ rB " S0 S1\C5      rDS:S2 jrE S;S3 jrFS4 rGS5 rHS6 rIS7 rJS8 rK\LS9:X  a  \K" 5         gg)<uì  RoPE (rotary) Q/K fine-tune of stem_ms96 (replace absolute pos).

Removes absolute `self.pos` embeddings from LogitTransformer and applies
rotary embeddings to projected Q/K inside a custom causal TransformerEncoder
(2 layers, d=64, window 96, ms features, dilated stem kept).

NOT ALiBi. NOT v3/v4 local relative-bias span 6.25s. NOT env_xattn / TFDirect.

Init from models/stem_ms96_swa/{held-*,all5}.pt (drop `pos`; remap
TransformerEncoderLayer MHA â†’ RoPE qkv/out_proj; restore causal mask).
EMA 0.995 eval. Score LODO 5 + AASD + Carta at HMM p_stay=0.994 scale=0.8;
also report linear_hp30 on logits.

Early abort: if after EARLY_STEPS the train loss is still ~chance (â‰ˆ0.69), stop.

usage:
  STEPS=1000 BATCH=512 python stem_ms96_rope.py
é    N)ÚDS_KEYSÚmodelsÚstem_ms96_swaÚlogit_tf_v2Ústem_ms96_ropeÚexperimentszstem_ms96_rope.logÚresultsÚresearchzstem_ms96_rope.jsonzb/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/experiments/decoder_lodo/results/research)ÚAASDÚ	Carta2026ÚSTEPSÚ1000ÚBATCHÚ512ÚEARLY_STEPSÚ80ÚEARLY_ABORT_LOSSz0.65g×£p=
×ï?g-Cëâ6J?g+‡ÙÎï?gš™™™™™é?Úmsé`   é@   é   Té   g     ˆÃ@c                 ó   • [        U SS9  [        R                  " [        R                  R	                  [
        5      SS9  [        [
        S5       nUR                  U S-   5        S S S 5        g ! , (       d  f       g = f)NT)Úflush©Úexist_okÚaÚ
)ÚprintÚosÚmakedirsÚpathÚdirnameÚLOGÚopenÚwrite)ÚtextÚfhs     Ú)/root/disk_3tb/eeg-thao/stem_ms96_rope.pyÚlog_liner*   <   sL   € Ü	ˆ$�dÒÜ‡K‚K”—‘—‘¤Ó$¨tÒ4Ü	Œc�3Œ˜2Ø
�‰�˜‘Ô÷ 
�Žús   ÁA/Á/
A=c                 óV   • U R                  SSS9u  p[        R                  " U* U4SS9$ )Nr   éÿÿÿÿ©Údim)ÚchunkÚtorchÚcat)ÚxÚx1Úx2s      r)   Úrotate_halfr5   C   s-   € Ø�W‰W�Q˜BˆWÐ�F€BÜ�9Š9�r�c˜2�Y BÑ'Ð'ó    c           	      ó   • US-  S:w  a  [        SU 35      eSU[        R                  " SUS[        R                  S9U-  -  -  n[        R                  " U [        R                  S9n[        R                  " Xe5      n[        R
                  " Xw4SS9nUR                  5       SSSS2SS24   n	UR                  5       SSSS2SS24   n
Ub  U	R                  US	9n	U
R                  US	9n
Ub  U	R                  US9n	U
R                  US9n
Xš4$ )
zEcos/sin tables shaped (1, 1, T, Dh) for broadcast over (B, H, T, Dh).r   r   z RoPE head_dim must be even, got ç      ð?©Údtyper,   r-   N©Údevice)	Ú
ValueErrorr0   ÚarangeÚfloat32Úouterr1   ÚcosÚsinÚto)Úseq_lenÚhead_dimÚbaser<   r:   Úinv_freqÚtÚfreqsÚembrA   rB   s              r)   Úbuild_rope_cacherK   H   s  € à�!�|�qÓÜÐ;¸H¸:ÐFÓGÐGØ�dœuŸ|š|¨A¨x¸Ä%Ç-Á-ÑPÐS[Ñ[Ñ\Ñ]€HÜ�Š�W¤E§M¡MÑ2€AÜ�KŠK˜Ó$€EÜ
�)Š)�U�N¨Ñ
+€CØ
�'‰'‹)�D˜$¢¢1Ð$Ñ
%€CØ
�'‰'‹)�D˜$¢¢1Ð$Ñ
%€CØÑØ�f‰f˜FˆfÐ#ˆØ�f‰f˜FˆfÐ#ˆØÑØ�f‰f˜5ˆfÐ!ˆØ�f‰f˜5ˆfÐ!ˆØˆ8€Or6   c                 ór   • U R                  S5      nXSSU2SS24   -  [        U 5      USSU2SS24   -  -   $ )zx, cos, sin: (..., T, Dh).éþÿÿÿ.N)Úsizer5   )r2   rA   rB   rH   s       r)   Ú
apply_roperO   [   sB   € à	�‰ˆr‹
€AØ�3˜˜˜šA�:‰Ñ¤¨Q£°#°c¸2¸A¸2ºq°j±/Ñ!AÑAÐAr6   c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )ÚRopeEncoderLayeréa   zNPre-norm causal attention with RoPE on Q/K + GELU FFN (matches stem TF layer).c                 ó*  >• [         TU ]  5         X-  S:w  a  [        S5      eXl        X l        X-  U l        [        R                  " U5      U l        [        R                  " U5      U l	        [        R                  " USU-  5      U l        [        R                  " X5      U l        [        R                  " X5      U l        [        R                  " X15      U l        [        R                  " U5      U l        [        R                  " U5      U l        g )Nr   z"d_model must be divisible by nheadé   )ÚsuperÚ__init__r=   Úd_modelÚnheadrE   ÚnnÚ	LayerNormÚnorm1Únorm2ÚLinearÚqkvÚout_projÚlinear1Úlinear2ÚDropoutÚdropoutÚ	attn_drop)ÚselfrW   rX   Údim_feedforwardrc   Ú	__class__s        €r)   rV   ÚRopeEncoderLayer.__init__d   s¾   ø€ Ü‰ÑÔØ‰?˜aÓÜÐAÓBÐBØŒØŒ
ØÑ(ˆŒÜ—\’\ 'Ó*ˆŒ
Ü—\’\ 'Ó*ˆŒ
Ü—9’9˜W a¨'¡kÓ2ˆŒÜŸ	š	 'Ó3ˆŒÜ—y’y Ó:ˆŒÜ—y’y Ó:ˆŒÜ—z’z 'Ó*ˆŒÜŸš GÓ,ˆ�r6   c           	      óî  • UR                   u  pVnU R                  U5      nU R                  U5      R                  XVSU R                  U R
                  5      n	U	R                  SSSSS5      n	U	S   U	S   U	S   pËn
[        X¢U5      n
[        X²U5      nU R
                  S-  n[        R                  " X«R                  SS5      5      U-  nXäS U2S U24   R                  UR                  UR                  S	9-   nU R                  [        R                  " USS
95      n[        R                  " Xì5      nUR                  SS5      R!                  5       R                  XVU R"                  5      nXR%                  U R'                  U5      5      -   nU R)                  U5      nU R+                  U R%                  [,        R.                  " U R1                  U5      5      5      5      nXR%                  U5      -   $ )NrT   r   r   é   r   g      à¿rM   r,   )r:   r<   r-   )Úshaper[   r^   ÚviewrX   rE   ÚpermuterO   r0   ÚmatmulÚ	transposerC   r:   r<   rd   ÚsoftmaxÚ
contiguousrW   rc   r_   r\   ra   ÚFÚgelur`   )re   r2   rA   rB   Úcausal_maskÚbrH   Ú_Úhr^   ÚqÚkÚvÚscaleÚattnÚouts                   r)   ÚforwardÚRopeEncoderLayer.forwardt   s•  € à—'‘'‰ˆˆaØ�J‰J�q‹MˆØ�h‰h�q‹k×Ñ˜q Q¨¯
©
°D·M±MÓBˆØ�k‰k˜!˜Q  1 aÓ(ˆØ�a‘&˜#˜a™& # a¡&ˆaˆÜ�q˜sÓ#ˆÜ�q˜sÓ#ˆØ—‘ Ñ%ˆÜ�|Š|˜AŸ{™{¨2¨rÓ2Ó3°eÑ;ˆØ " 1 " b q b &Ñ)×,Ñ,°4·:±:ÀdÇkÁkÐ,ÐRÑRˆØ�~‰~œeŸmšm¨D°bÑ9Ó:ˆÜ�lŠl˜4Ó#ˆØ�m‰m˜A˜qÓ!×,Ñ,Ó.×3Ñ3°A¸$¿,¹,ÓGˆØ—‘˜TŸ]™]¨3Ó/Ó0Ñ0ˆØ�J‰J�q‹MˆØ�L‰L˜Ÿ™¤a§f¢f¨T¯\©\¸!«_Ó&=Ó>Ó?ˆØ—<‘< “?Ñ"Ð"r6   )rd   rW   rc   rE   r`   ra   rX   r[   r\   r_   r^   )çš™™™™™¹?©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rV   r~   Ú__static_attributes__Ú__classcell__©rg   s   @r)   rQ   rQ   a   s   ø† ÙX÷-÷ #ð #r6   rQ   c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )ÚLogitTransformerRoPEé‰   zJstem_ms96 LogitTransformer with absolute pos removed and RoPE on attn Q/K.c                 ó  >• [         TU ]  5         U(       d  [        S5      eX l        X@l        X0l        X4-  U l        SU l        [        R                  " XU R                  S-   US9U l
        [        R                  R                  U R                  R                  5        [        R                  R                  U R                  R                  5        [        R                  " X5      U l        [        R"                  " S Vs/ s H  n[$        R&                  " USX†S9PM     sn5      U l        [        R"                  " [+        U5       V	s/ s H  n	[-        X4US-  US	9PM     sn	5      U l        [        R                  " US5      U l        [        R                  " USS
S9U l        [        R4                  R7                  U5      n
U R9                  SU
5        [;        X R                  5      u  p¼U R9                  SU5        U R9                  SU5        [        R                  R                  U R0                  R                  5        [        R                  R                  U R0                  R                  5        [        R                  R                  U R2                  R                  5        [<        R>                  " 5          SU R2                  R                  SS [A        SU5      24'   S S S 5        g s  snf s  sn	f ! , (       d  f       g = f)NzRoPE script expects stem=Trueé   rj   )Úgroups)rj   r   r   rŽ   é   )ry   Údilationrc   r   )rc   F)ÚbiasÚmaskÚrope_cosÚrope_sing      Ð?r   r   )!rU   rV   r=   ÚwindowrX   rW   rE   Údw_padrY   ÚConv1dÚdwÚinitÚzeros_Úweightr’   r]   ÚprojÚ
ModuleListÚVÚDilatedResidualÚblocksÚrangerQ   ÚlayersÚheadÚskipÚTransformerÚgenerate_square_subsequent_maskÚregister_bufferrK   r0   Úno_gradÚmin)re   Úc_inr–   rW   rX   Únlayersrc   ÚstemÚdrv   r“   rA   rB   rg   s                €r)   rV   ÚLogitTransformerRoPE.__init__Œ   s  ø€ Ü‰ÑÔÞÜÐ<Ó=Ð=ØŒØŒ
ØŒØÑ(ˆŒØˆŒÜ—)’)˜D¨¯©°a©ÀÑEˆŒÜ
�‰�‰�t—w‘w—~‘~Ô&Ü
�‰�‰�t—w‘w—|‘|Ô$Ü—I’I˜dÓ,ˆŒ	Ü—m’má!ó%
â!�ô ×Ò˜g¨°QÔHÙ!ñ%
ó ˆŒô
 —m’mä˜7”^ó%
â#�ô ˜W¨W°q©[À'ÔJÙ#ñ%
ó ˆŒô —I’I˜g qÓ)ˆŒ	Ü—I’I˜d A¨EÑ2ˆŒ	Ü�~‰~×=Ñ=¸fÓEˆØ×Ñ˜V TÔ*Ü# F¯M©MÓ:‰ˆØ×Ñ˜Z¨Ô-Ø×Ñ˜Z¨Ô-Ü
�‰�‰�t—y‘y×'Ñ'Ô(Ü
�‰�‰�t—y‘y—~‘~Ô&Ü
�‰�‰�t—y‘y×'Ñ'Ô(Ü�]Š]�_Ø2;ˆD�I‰I×Ñ˜Q ¤# a¨£, Ð.Ñ/÷ ˆ_ùò'%
ùò
%
÷ �_ús   Ä K0Å
K5Ê?(K:Ë:
Lc           	      óª  • UnXR                  [        R                  " UR                  SS5      U R                  S45      5      R                  SS5      -   nU R                  U5      nU R                   H  nU" U5      nM     UR                  S5      nU R                  SS U2S S 24   R                  UR                  S9nU R                  SS U2S S 24   R                  UR                  S9nU R                   H  nU" X6XpR                  5      nM     U R                  U5      R                  S5      U R!                  U5      R                  S5      -   $ )Nrj   r   r   .r9   r,   )r™   rr   Úpadro   r—   r�   r¡   rN   r”   rC   r:   r•   r£   r“   r¤   Úsqueezer¥   )	re   r2   Úrawrw   ÚblockrH   rA   rB   Úlayers	            r)   r~   ÚLogitTransformerRoPE.forward¯   s  € ØˆØ—‘œŸš˜aŸk™k¨!¨QÓ/°$·+±+¸qÐ1AÓBÓC×MÑMÈaÐQRÓSÑSˆØ�I‰I�a‹LˆØ—[”[ˆEÙ�a“ŠAñ !à�F‰F�1‹IˆØ�m‰m˜C  ! ¢Q˜JÑ'×*Ñ*°·±Ð*Ð9ˆØ�m‰m˜C  ! ¢Q˜JÑ'×*Ñ*°·±Ð*Ð9ˆØ—[”[ˆEÙ�a˜c§9¡9Ó-ŠAñ !à�y‰y˜‹|×#Ñ# BÓ'¨$¯)©)°C«.×*@Ñ*@ÀÓ*DÑDÐDr6   )r¡   rW   r™   r—   r¤   rE   r£   rX   r�   r¥   r–   )r   r   r   r€   Tr�   r‰   s   @r)   r‹   r‹   ‰   s   ø† ÙT÷!<÷FEð Er6   r‹   c                 ól  • 0 nU R                  5        GH  u  p#US:X  a  M  UR                  S5      (       aÕ  U[        S5      S nUR                  SS5      n[        U5      S:w  a  MU  Uu  pgUS:X  a	  X1SU S	3'   O�US
:X  a	  X1SU S3'   O€US:X  a	  X1SU S3'   OqUS:X  a	  X1SU S3'   ObUR                  S5      (       dB  UR                  S5      (       d,  UR                  S5      (       d  UR                  S5      (       a
  X1SU SU 3'   Mù  US;   d  UR                  S5      (       a  GM  X1U'   GM     U$ )u»   Map stem SWA / nn.TransformerEncoder keys onto RoPE module names.

Drops absolute `pos`. Remaps enc.layers.i.self_attn.{in_proj,out_proj}
and enc.layers.i.{linear*,norm*} â†’ layers.i.*.
Úposzenc.layers.NÚ.rj   r   zself_attn.in_proj_weightzlayers.z.qkv.weightzself_attn.in_proj_biasz	.qkv.biaszself_attn.out_proj.weightz.out_proj.weightzself_attn.out_proj.biasz.out_proj.biaszlinear1.zlinear2.znorm1.znorm2.)r“   zenc.)ÚitemsÚ
startswithÚlenÚsplit)Ústater}   ry   rz   ÚrestÚpartsÚidxÚtails           r)   Úremap_swa_staterÃ   ½   sL  € ð €CØ—‘—‰ˆØ�‹:ÙØ�<‰<˜×&Ñ&à”S˜Ó'Ð(Ð)ˆDØ—J‘J˜s AÓ&ˆEÜ�5‹z˜Q‹ÙØ‰IˆCØÐ1Ó1Ø23�g˜c˜U +Ð.Ò/ØÐ1Ó1Ø01�g˜c˜U )Ð,Ò-ØÐ4Ó4Ø78�g˜c˜UÐ"2Ð3Ò4ØÐ2Ó2Ø56�g˜c˜U .Ð1Ò2Ø—‘ ×,Ñ,°·±À
×0KÑ0KØ—‘ x×0Ñ0°D·O±OÀH×4MÑ4MØ./�g˜c˜U ! D 6Ð*Ñ+áØ�	‹>˜QŸ\™\¨&×1Ñ1ÚØˆAŒñ3 ð4 €Jr6   c                 ó  • [         R                  " S5        [        SSSSSS9R                  U 5      R	                  5       n[         R
                  " SSSU S	9nUR                  5       n[         R
                  " SSU S	9US S 2S
4'   [         R                  " 5          U" U5      nU" U5      nS S S 5        [         R                  " WS S 2S S
24   WS S 2S S
24   SS9(       d  [        S5      e[        S5        g ! , (       d  f       NR= f)Nr   rŽ   é   é    r   rj   )rW   rX   r¬   r   r;   r,   çñhãˆµøä>)Úatolz&RoPE transformer readout is not causalzRoPE causality check passed)r0   Úmanual_seedr‹   rC   ÚevalÚrandnÚcloner©   ÚallcloseÚRuntimeErrorr*   )r<   Úmodelr2   r4   Úy1Úy2s         r)   Úassert_causal_roperÒ   á   sÖ   € Ü	×Ò�aÔÜ   B°¸!ÀQÑG×JÑJÈ6ÓR×WÑWÓY€EÜ�Š�A�r˜1 VÑ,€AØ	
�‰‹€BÜ—’˜A˜q¨Ñ0€B‚qˆ"€u�IÜ	�Š�Ù�1‹XˆÙ�2‹Yˆ÷ 
ô �>Š>˜"šQ   ˜V™* bª¨C¨R¨C¨¡j°t×<ÜÐCÓDÐDÜÐ*Õ+÷ 
�ús   ÂC7Ã7
Dc                 ó:  • U S:X  aU  [         R                  R                  [        S5      n[         R                  R	                  U5      (       a  U$ [        U5      e[         R                  R                  [        SU  S35      n[         R                  R	                  U5      (       a  U$ [         R                  R                  [        SU  S35      n[         R                  R	                  U5      (       a  [        SU  SU 35        U$ [        SU  S	U S
U 35      e)z9Prefer SWA held-*/all5; fall back to published stem_ms96.Úall5úall5.ptúheld-ú.ptzstem_ms96__held-z  SWA missing for z; falling back to zno init for z: tried z and )r    r"   ÚjoinÚSWA_DIRÚisfileÚFileNotFoundErrorÚSTEM_DIRr*   )ÚheldÚswar­   s      r)   Úresolve_initrß   ï   sà   € àˆvƒ~Ü�g‰g�l‰lœ7 IÓ.ˆÜ�7‰7�>‰>˜#×ÑØˆJÜ Ó$Ð$Ü
�'‰'�,‰,”w %¨ v¨SÐ 1Ó
2€CÜ	‡w�w‡~�~�c×ÑØˆ
Ü�7‰7�<‰<œÐ$4°T°F¸#Ð">Ó?€DÜ	‡w�w‡~�~�d×ÑÜÐ% d VÐ+=¸d¸VÐDÔEØˆÜ
˜l¨4¨&°¸¸¸UÀ4À&ÐIÓ
JÐJr6   c           	      óv  • [         R                  " U SSS9nUS   u  p4pVpxn	U(       a1  [        U5      [        :w  d  [        U5      S:w  d  [        U5      S:w  a  [	        SUS    SU  35      e[        [
        R                  " US	   5      R                  S
   5      n
[        U
[        U[        USS9nUR                  R                  5       R                  5       nUR                  R                  5       R                  5       nUR                  R                  5       R                  5       n[        US   5      nUR!                  USS9u  nnU Vs1 s H  nUR#                  S5      (       a  M  UiM     nnU(       a  [%        S['        U5       35        U(       a  [%        S['        U5       35        UR                  R)                  U5        UR                  R)                  U5        UR                  R)                  U5        [         R*                  " UR                  5      R-                  5       (       a  [	        SU  35      eUR/                  U5      n[
        R                  " US	   [
        R0                  5      n[
        R                  " US   [
        R0                  5      nUUU[3        US   5      4$ s  snf )NÚcpuF)Úmap_locationÚweights_onlyÚcfgr   r   zunexpected stem cfg z from Úmur   T)rW   rX   r¬   r­   r¾   )Ústrict)r”   r•   r“   z&  warn missing keys after RoPE remap: z  warn unexpected keys: z-causal mask still has nan after restore from ÚsdÚtrain)r0   ÚloadÚintÚWINDOWrÎ   ÚnpÚasarrayrk   r‹   ÚNHEADr“   ÚdetachrÌ   r”   r•   rÃ   Úload_state_dictr»   r*   ÚsortedÚcopy_ÚisnanÚanyrC   r?   Úlist)r"   r<   ÚblobÚnameÚfamilyr–   rW   r¬   r­   Ú_lrr«   ÚnetÚ	good_maskÚgood_cosÚgood_sinÚremappedÚmissingÚ
unexpectedÚmÚmiss_okrå   rç   s                         r)   Úload_rope_from_swar     s(  € Ü�:Š:�d¨¸UÑC€DØ8<¸U¹Ñ5€D�& 7°#Þ”3�v“;¤&Ó(¬C°«L¸BÓ,>Ä#ÀgÃ,ÐRSÓBSÜÐ1°$°u±+°¸fÀTÀFÐKÓLÐLÜŒr�zŠz˜$˜t™*Ó%×+Ñ+¨AÑ.Ó/€DÜ
ØŒf˜g¬U¸GÈ$ñP€Cà—‘—‘Ó!×'Ñ'Ó)€IØ�|‰|×"Ñ"Ó$×*Ñ*Ó,€HØ�|‰|×"Ñ"Ó$×*Ñ*Ó,€HÜ˜t G™}Ó-€HØ×-Ñ-¨h¸uÐ-ÐEÑ€GˆZÙ!ÓXš'�Q¨¯©Ð6V×)W�q™'€GÐXæÜÐ9¼&À»/Ð9JÐKÔLÞÜÐ+¬F°:Ó,>Ð+?Ð@ÔAØ‡H�H‡N�N�9ÔØ‡L�L×Ñ�xÔ Ø‡L�L×Ñ�xÔ Ü‡{‚{�3—8‘8Ó× Ñ ×"Ñ"ÜÐJÈ4È&ÐQÓRÐRØ
�&‰&�‹.€CÜ	�Š�D˜‘J¤§
¡
Ó	+€BÜ	�Š�D˜‘J¤§
¡
Ó	+€BØ��Bœ˜T '™]Ó+Ð+Ð+ùò Ys   ÅJ6Å-J6c                 ó¸  • [         R                  " X5      n[        R                  " XUSS9nXB-  R	                  5       R                  S5      nXT-  U-  R	                  5       U-  nU S S 2SS 24   U S S 2S S24   -
  R                  5       nUS S 2SS 24   US S 2S S24   -  n	USX‰-  R	                  5       -  U	R	                  5       R                  S5      -  -   nU$ )NÚnone)Ú
pos_weightÚ	reductionr8   rj   r,   gš™™™™™©?)rŸ   Úswitch_weightsrr   Ú binary_cross_entropy_with_logitsÚsumÚ	clamp_minÚabs)
ÚlogitÚyr  r  ÚswÚbceÚdenomÚlossÚdvÚm2s
             r)   Úweighted_bcer    sÓ   € Ü	
×	Ò	˜!Ó	€BÜ
×
,Ò
,¨UÀ*ÐX^Ñ
_€CØ‰V�L‰L‹N×$Ñ$ SÓ)€EØ‰H�q‰L×ÑÓ %Ñ'€DØ
’�1‘2�‰,˜šq # 2 #˜v™Ñ
&×	+Ñ	+Ó	-€BØ	
Š1ˆa‰bˆ5‰�A’a˜˜"˜�f‘IÑ	€BØ�$˜"™'Ÿ™›Ñ(¨2¯6©6«8×+=Ñ+=¸cÓ+BÑBÑB€DØ€Kr6   c                 ó  • [        S[        US-  S5      5      nX:  a  X1S-   -  U-  nOIX-
  [        X$-
  S5      -  nSSUS-
  -  S[        R                  " [        R                  U-  5      -   -  -   nU R
                   H  nXWS'   M	     U$ )Né(   é
   rj   rÇ   g      à?r8   Úlr)rª   ÚmaxÚmathrA   ÚpiÚparam_groups)ÚoptÚstepÚtotalrF   Úwarmr  ÚpÚgroups           r)   Úset_lrr$  )  sŽ   € Üˆr”3�u ‘{ AÓ&Ó'€DØƒ{Ø˜A‘XÑ Ñ%‰à‰[œC ¡¨aÓ0Ñ0ˆØ�C˜4 $™;Ñ'¨3´·²¼$¿'¹'ÀA¹+Ó1FÑ+FÑGÑGˆØ×!Ô!ˆØˆd‹ñ "à€Ir6   c                   ó   • \ rS rSrSrg)ÚChanceLikeAborti5  © N)r‚   rƒ   r„   r…   r‡   r'  r6   r)   r&  r&  5  s   † Úr6   r&  c
                 óŽ  • [        UR                  5       5      n
[        R                  " SU
-
  [	        U
S5      -  /US9n[        R
                  R                  U R                  5       [        SS9n[        R                  R                  [        R                  5      n[        R                  " U 5      R!                  5       nU R#                  5         [$        R$                  " 5       nS n/ n['        U5       GHb  n[)        UUU[        5        [        R*                  " XÑ[,        XsU5      u  nnn[        R.                  " U5      R1                  USS9n[        R.                  " U5      R1                  USS9n[        R.                  " U5      R1                  USS9n[        R2                  " U5      nUR5                  SS9  U " U5      n[7        UUUU5      n[        R8                  " U5      (       d  [;        S	U S
U 35      eUR=                  5         [        R>                  R@                  RC                  U R                  5       S5        URE                  5         [        RF                  " X[H        5        [        URK                  5       5      nU[L        :  a  URO                  U5        U	(       aq  US-   [L        :X  ad  [        [        R                  " USS  5      5      n[Q        SU S[L         SUS S[R         S3	5        U[R        :¼  a  [U        SUS S[L         S35      eUS:X  d  US-   S-  S:X  d  US-   U:X  d  GMG  [Q        SU SUS-    SU SUS 35        GMe     UR!                  5         [        RV                  " Xá[,        X4U5      n[Q        SU SUS SUS S[$        R$                  " 5       U-
  S S 3	5        U[Y        UUS!94$ )"Nr8   gü©ñÒMbP?r;   g-Cëâ6?)r  Úweight_decayT)Únon_blocking)Úset_to_noneznon-finite loss at step z for rj   iØÿÿÿz    z early@z mean_loss ú.4fz (abort if >= Ú)zchance-like loss z after z stepsr   éÈ   z step Ú/z loss z
  trained z frame-acc ú.3fz (z.0fzs))r  Ú	frame_acc)-ÚfloatÚmeanr0   Útensorr  ÚoptimÚAdamWÚ
parametersÚLRrì   ÚrandomÚdefault_rngrŸ   Ú
TRAIN_SEEDÚcopyÚdeepcopyrÊ   rè   Útimer¢   r$  Úsample_batchrë   Ú
from_numpyrC   ÚaugmentÚ	zero_gradr  ÚisfiniterÎ   ÚbackwardrY   ÚutilsÚclip_grad_norm_r  Ú
ema_updateÚ	EMA_DECAYÚitemr   Úappendr*   r   r&  Útrain_frame_accÚdict)rÏ   r�   Úy_allrå   rç   r<   ÚstepsÚbatchÚtagÚearly_abortr¸   r  r  ÚrngÚemaÚt0ÚlastÚearly_lossesr  ÚxbÚybÚmbr2   r  r  r  r  Ú
mean_earlyÚaccs                                r)   Ú
train_roper\  9  s7  € Ü
�—
‘
“Ó
€CÜ—’  c¡	¬S°°d«^Ñ;Ð<ÀVÑL€JÜ
�+‰+×
Ñ
˜E×,Ñ,Ó.´2ÀDÐ
Ð
I€CÜ
�)‰)×
Ñ
¤§¡Ó
-€CÜ
�-Š-˜Ó
×
#Ñ
#Ó
%€CØ	‡K�K„MÜ	�Š‹€BØ€DØ€LÜ�e—ˆÜˆs�D˜%¤Ô$Ü—^’^ C´¸ÀBÓG‰
ˆˆB�Ü×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÜ×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÜ×Ò˜RÓ ×#Ñ# F¸Ð#Ð>ˆÜ�IŠI�a‹LˆØ�‰ $ˆÑ'Ù�a“ˆÜ˜E 1 a¨Ó4ˆÜ�~Š~˜d×#Ñ#ÜÐ!9¸$¸¸uÀSÀEÐJÓKÐKØ�‰ŒÜ�‰�‰×&Ñ& u×'7Ñ'7Ó'9¸3Ô?Ø�‰Œ
Ü	�Š�U¤Ô+Ü�T—Y‘Y“[Ó!ˆØ”+ÓØ×Ñ Ô%Þ˜4 !™8¤{Ó2ÜœrŸwšw |°C°DÐ'9Ó:Ó;ˆJÜ�t˜C˜5 ¬ }°KÀ
È3Ð?Oð P%Ü%5Ð$6°að9ô :àÔ-Ó-Ü%Ø'¨
°3Ð'7°w¼{¸mÈ6ÐRóTð Tà�1‹9˜ ™ SÑ(¨AÓ-°¸±¸UÖ1BÜ�t˜C˜5  t¨a¡x j°°%°¸¸tÀC¸jÐI×Jñ7 ð8 ‡H�H„JÜ
×
Ò
˜C¬°¸Ó
@€CÜˆz˜#˜˜f T¨# J¨k¸#¸c¸À"ÄTÇYÂYÃ[ÐSUÑEUÐVYÐDZÐZ\Ð]Ô^Ø”˜$¨#Ñ.Ð.Ð.r6   c	                 óF  • [        X5      u  pšp¼ [        X‘X*X³XEUUS9
u  pÞXÚXµU4$ ! [        R                  R                   a]    U(       a  e [        SU SU S35        A	UR                  S:X  a  [        R                  R                  5         [        XX#XES-  UUSS9	s $ f = f)	N©rQ  z	  OOM on z
 at batch z; halving onceÚcudar   T)rQ  Úhalved)	r  r\  r0   r_  ÚOutOfMemoryErrorr*   ÚtypeÚempty_cacheÚfit_with_oom)Ú	init_pathr�   rM  r<   rN  rO  rP  rQ  r`  rÏ   rå   rç   Ú_trainrS  Ústatss                  r)   rd  rd  e  s³   € ä.¨yÓAÑ€Eˆrð2ÜØ˜5 b°%ÀØ#ñ%‰
ˆð ˜ 5Ð(Ð(øÜ�:‰:×&Ñ&ó 	2ÞØÜ�9˜S˜E ¨E¨7°.ÐAÔBØØ�;‰;˜&Ó Ü�J‰J×"Ñ"Ô$ÜØ˜u¨e¸a±ZÀØ#¨Dñ2ò 	2ð	2ús   �% ¥A8B ÂB c                 ó  • 0 nU R                  5        Hs  u  p#[        R                  " [        R                  " U[        R
                  5      S S 2S 4   5      R                  S5      R                  [        R
                  5      X'   Mu     U$ )Nr,   )rº   ÚV4Úhighpassrì   rí   Úfloat64ÚreshapeÚastype)Ústreamr}   Úkeyr®   s       r)   Úhp30_streamrp  y  sc   € Ø
€CØ—,‘,–.‰ˆÜ—;’;œrŸzšz¨!¬R¯Z©ZÓ8º¸D¸ÑAÓB×JÑJÈ2ÓN×UÑUÔVX×V`ÑV`Óaˆ‹ñ !à€Jr6   c                 ó2  • [         R                  " XX#[        U5      n[        R                  " U5      n[        R
                  " XG[        [        5      nU V	s0 s H  o™[        Xi   5      _M     n
n	[        R                  " U
5      n[        R
                  " XK[        [        5      n[        SU S[         S[         SUS   S SUS   S	 S
US   S SUS   S	 35        [        SU S[         S[         SUS   S SUS   S	 3
5        [        XŒS9U4$ s  sn	f )NzLODO ú p ú sc ú  H ÚHr,  ú Acc ÚAccú.2fú L ÚLr0  ú SDT ÚSDTz
LODO_hp30 ©ÚropeÚhp30)rŸ   Ú	infer_allrë   ÚNÚto_evÚofficialÚP_STAYÚSCALErp  r*   rL  )rÏ   Úfeatrå   rç   rÝ   r<   ÚstreamsÚevÚoffÚdsÚhpÚev_hpÚoff_hps                r)   Ú
score_lodorŽ  €  s  € Ü�kŠk˜% r¬v°vÓ>€GÜ	
�Š�Ó	€BÜ
�*Š*�Tœv¤uÓ
-€CÙ18Ó	9²¨2Œk˜'™+Ó&Ò
&±€BÐ	9Ü�GŠG�B‹K€EÜ�ZŠZ˜¤V¬UÓ3€FÜˆu�T�F˜#œf˜X T¬%¨°°S¸±X¸c°NÀ%ÈÈEÉ
ÐSVÐGWð XØ�c‘(˜3�˜u S¨¡Z°Ð$4ð6ô 7äˆz˜$˜˜s¤6 (¨$¬u¨g°T¸&À¹+ÀcÐ9JÈ%ÐPVÐW\ÑP]Ð^aÐObÐcÔdÜ�SÑ&¨Ð/Ð/ùò 
:s   ÁDc                 óF  • / nUR                  5        Ht  u  pšXi   nU
R                  5        HW  u  pÍUR                  [        R                  " U[        R                  5      [        R
                  " X¼   S:H  SS5      -  5        MY     Mv     [        [        R                  " U5      R                  5       5      n[        [        R                  " U5      R                  5       5      n[        SUS SUS S[         35        U V	s0 s H  o™[        XY   5      _M     nn	/ nUR                  5        Ht  u  pšXi   nU
R                  5        HW  u  pÍUR                  [        R                  " U[        R                  5      [        R
                  " X¼   S:H  SS5      -  5        MY     Mv     [        [        R                  " U5      R                  5       5      n[        [        R                  " U5      R                  5       5      n[        SUS SUS S[         35        0 n[         GHõ  n	[        R                  " X	X   0X4[         U5      U	   n0 nUR                  5        Ho  u  pÍX)   U   n[        R                  " U[        R                  5      n[#        U[        R$                  " ['        U5      5      US	   US
   US   US   US   S9UU'   Mq     [(        R*                  " UXï[        -  [,        S5      n[        U5      n0 nUR                  5        Ho  u  pÍX)   U   n[        R                  " U[        R                  5      n[#        U[        R$                  " ['        U5      5      US	   US
   US   US   US   S9UU'   Mq     [(        R*                  " UUU[        -  [,        S5      n[#        UUS9UU	'   [        SU	 S[,         S[         SUS   S SUS   S SUS   S SUS   S 35        [        SU	 S[,         S[         SUS   S SUS   S 3
5        GMø     U$ s  sn	f )Nr   r8   g      ð¿z  external emissions mu r,  z sd z scale z  external hp30 emissions mu ÚattÚt_endÚatt128Ún128Úsubject)Úr0Úr1r�  r‘  r’  r“  r”  g      è?r}  zEXT rr  rs  rt  ru  rv  rw  rx  ry  rz  r0  r{  r|  z	EXT_hp30 )rº   rJ  rì   rí   rk  Úwherer2  Úconcatenater3  Ústdr*   r…  rp  Ú
TEST_NAMESrŸ   r€  rë   rL  Úzerosr¼   ÚEÚscorer„  )rÏ   Úfeat_extÚmetaÚmu_inÚsd_inÚcorpus_streamsÚ
corpus_attr<   rÀ   rŠ  ÚtableÚlabelsro  r®   Úmu_eÚsd_eÚ	hp_corpusÚparts_hpÚmu_hpÚsd_hpÚrowsrn  rˆ  Úmeta_kÚrÚ	stream_hprŒ  Úr_hps                               r)   Úscore_externalr±  �  s¸  € Ø€EØ#×)Ñ)Ö+‰	ˆØ‘ˆØ—k‘k–m‰FˆCØ�L‰LœŸš A¤r§z¡zÓ2´R·X²X¸f¹kÈQÑ>NÐPSÐUYÓ5ZÑZÖ[ó $ñ ,ô ”—’ Ó&×+Ñ+Ó-Ó.€DÜ”—’ Ó&×*Ñ*Ó,Ó-€DÜÐ'¨¨S z°°d¸3°Z¸wÄuÀgÐNÔOá?MÓNº~¸”[ Ñ!3Ó4Ò4¹~€IÐNØ€HØ—_‘_Ö&‰	ˆØ‘ˆØ—k‘k–m‰FˆCØ�O‰OœBŸJšJ q¬"¯*©*Ó5¼¿ºÀÁÐPQÑAQÐSVÐX\Ó8]Ñ]Ö^ó $ñ 'ô ”"—.’. Ó*×/Ñ/Ó1Ó2€EÜ”"—.’. Ó*×.Ñ.Ó0Ó1€EÜÐ,¨U°3¨K°t¸EÀ#¸;ÀgÌeÈWÐUÔVà€DßˆjˆÜ—’˜U¨©Ð$6¸ÄfÈfÓUÐVXÑYˆØˆØ—l‘l–n‰FˆCØ‘X˜c‘]ˆFÜ—
’
˜1œbŸj™jÓ)ˆAÜ˜a¤B§H¢H¬S°«VÓ$4¸&À¹-ÈvÐV]ÉØ"(¨Ñ"2¸À¹ÐQWÐXaÑQbñdˆBˆs‹Gñ %ô
 �GŠG�B˜¤U™l¬F°DÓ9ˆÜ Ó'ˆ	ØˆØ—o‘oÖ'‰FˆCØ‘X˜c‘]ˆFÜ—
’
˜1œbŸj™jÓ)ˆAÜ ¤r§x¢x´°A³Ó'7¸VÀE¹]ÐRXÐY`ÑRaØ%+¨HÑ%5¸FÀ6¹NÐTZÐ[dÑTeñgˆE�#‹Jñ (ô
 �wŠw�u˜e U¬U¡]´F¸DÓAˆÜ˜Q TÑ*ˆˆR‰Ü�4˜�t˜3œv˜h d¬5¨'°°a¸±f¸S°\ÀÀqÈÁxÐPSÀnð UØ˜‘f˜S�\  q¨¡x° nð6ô 	7ä�9˜R˜D ¤F 8¨4´¨w°d¸4À¹9ÀS¸/ÈÈtÐTYÉ{Ð[^ÐN_Ð`×añ) ð* €KùòA Os   ÄPc                  ó®  • [         R                  R                  S/ SQS9  0 0 p[         GH�  n/ n[	        S5       Hh  n[
        R                  R                  [        SU SU S35      n[        US5       nUR                  [        R                  " U5      5        S S S 5        Mj     0 0 sX'   X'   US	    Há  nU Vs/ s H/  oˆU   S
   X‡   S   -
  R                  [        R                  5      PM1     n	nUS	   U   n
[        R                   " U	S5      X   U'   [#        [        R$                  " U
S   5      [        R$                  " U
S   [        R&                  5      [        R$                  " U
S   5      [)        U
S   5      [)        U
S   5      S9X   U'   Mã     [+        SU S[-        X   5       S35        GM�     X4$ ! , (       d  f       GM”  = fs  snf )NÚ
eaP_n5_t60)ÚenvÚonsetÚdom)ÚprepÚtargetsr   Úext_clf_Ú_w10b3sw2P_evidence_sz.pklÚrbr   r•  r–  rj   r�  r‘  r’  r“  r”  )r�  r‘  r’  r“  r”  z  external z: z trials)rœ  ÚCONFÚupdaterš  r¢   r    r"   rØ   ÚEVIDr%   rJ  Úpickleré   rm  rì   r?   ÚstackrL  rí   rk  rê   r*   r¼   )ÚlogitsrŸ  rŠ  ÚevsÚir"   r(   ro  rˆ  ÚcolsÚe0s              r)   Úload_externalrÆ  º  s˜  € Ü‡F�F‡M�M�|Ò-D€MÑEØ�rˆDßˆjˆØˆÜ�q–ˆAÜ—7‘7—<‘<¤¨°°Ð4IÈ!ÈÈDÐ&QÓRˆDÜ�d˜DÔ! RØ—
‘
œ6Ÿ;š; r›?Ô+÷ "Ñ!ñ ð  " 2Ðˆ‰
�D‘HØ�q”6ˆCÙQTÓUÒQTÈ2˜‘W˜T‘] R¡W¨T¡]Ñ2×:Ñ:¼2¿:¹:ÖFÑQTˆDÐUØ�Q‘˜‘ˆBÜ Ÿhšh t¨QÓ/ˆF‰J�s‰OÜ Ü—J’J˜r %™yÓ)´·²¸B¸w¹KÌÏÉÓ1TÜ—z’z " X¡,Ó/´c¸"¸V¹*³oÌsÐSUÐV_ÑS`ÓOañcˆD‰H�S‹Mñ	 ô 	�;˜r˜d "¤S¨©£_Ð$5°WÐ=×>ñ ð ˆ<Ð÷ "×!üò Vs   Á5&F?Â<6GÆ?
Gc                  ó8  • [         R                  " [         R                  R                  [        5      SS9  [        [        S5      R                  5         [        R                  " [        R                  R                  5       (       a  SOS5      n [        SU  S[         S[         S	[         S
[         S[          S["         35        S[        R$                  R                  R&                  l        S[        R$                  R*                  l        [-        U 5        [.        R0                  " 5         [2        R4                  " 5       S   n[         R                  " [6        SS9  0 n[9        [:        5       GH'  u  p4[=        U5      n[:         Vs/ s H  ofU:w  d  M
  UPM     nn[2        R>                  " X5      u  p‰[        SU SU 35         [A        XXX�[        [        SU 3US:H  S9u  p«pÍn[        RR                  " U
RU                  5       X¼UU[        UU[V        [        S.
[         R                  RY                  [6        SU S35      5        [[        X¡X¼X@5      u  nn[E        UXíS9X$'   A
U R\                  S:X  d  GM	  [        R                  R_                  5         GM*     [=        S5      n[        SU 35        [2        R>                  " U[a        [:        5      5      u  p‰[A        UX‰U [        [        SSS9u  p«pÍn[        RR                  " U
RU                  5       X¼U[a        [:        5      [        UU[V        [        S.
[         R                  RY                  [6        S5      5        [2        Rb                  " X¡X¼[d        U 5      n[g        5       u  nn[h        Rj                  " U5      S   n[m        U
UUX¼U[.        Rn                  S   U 5      n[O        [p        Rr                  " [:         Vs/ s H  obU   S    S!   S"   PM     sn5      5      n[O        [p        Rr                  " [:         Vs/ s H  obU   S    S!   S#   PM     sn5      5      n[        S$US% S&US' 35        [E        [t        [v        [        [        [        [:         Vs0 s H
  ofX&   S    _M     sn[:         Vs0 s H
  ofX&   S(   _M     sn[E        UUS)9UUS*9
n[         R                  " [         R                  R                  [H        5      SS9  [        [H        S5       n[J        RL                  " UUS[N        S9  S S S 5        [        S+[H         35        g s  snf ! [B         a®  n[        SU 35        [E        S[G        U5      US9n[         R                  " [         R                  R                  [H        5      SS9  [        [H        S5       n[J        RL                  " UUS[N        S9  S S S 5        O! , (       d  f       O= f[Q        S5      eS nAff = fs  snf s  snf s  snf s  snf ! , (       d  f       Nõ= f),NTr   Úwr_  rá   zdevice z steps z batch z EMA z RoPE base z early_abort@z>=r   zfold held-out=z init rÖ   r   r^  zABANDONED: )Ú	abandonedÚreasonrÝ   rj   )ÚindentÚdefaultr   )
r¾   rå   rç   rš   rè   rN  rO  rg  rä   Ú	rope_baser×   )Úscoresrg  rO  rÔ   z.all-5 RoPE fine-tune for AASD and Carta, init FrÕ   r�  rÎ  r~  ru  rw  zLODO mean H r,  rv  rx  rg  )ru  rw  )
Úp_stayr{   rN  rO  rÍ  ÚlodoÚ
lodo_statsÚ	lodo_meanÚexternalÚ
all5_statszwrote )<r    r!   r"   r#   r$   r%   Úcloser0   r<   r_  Úis_availabler*   r   r   rH  Ú	ROPE_BASEr   r   Úbackendsrn   Ú
allow_tf32ÚcudnnrÒ   r�  Úload_allrŸ   Úbuild_featuresÚCKPT_DIRÚ	enumerater   rß   Útrial_groupsrd  r&  rL  ÚstrÚOUTÚjsonÚdumpr2  Ú
SystemExitÚsaveÚ
state_dictÚCFGrØ   rŽ  rb  rc  rõ   r€  rë   rÆ  ri  Úbuild_features_fromr±  ÚGrì   r3  r„  r…  )r<   Úfeat_allÚperrÃ  rÝ   r"   r®   Útrain_dsr�   rM  rS  rå   rç   Úusedrg  Úexcr}   r(   rÎ  Ú_streamsÚall_pathr¢  Ú
ext_logitsÚext_metaÚext_featÚextÚmean_hÚmean_accs                               r)   Úmainr÷  Ï  sº  € Ü‡K‚K”—‘—‘¤Ó$¨tÒ4ÜŒˆcƒN×ÑÔÜ�\Š\¤E§J¡J×$;Ñ$;×$=Ñ$=™&À5ÓI€FÜˆw�v�h˜g¤e W¨G´E°7¸%Ä	¸{ð KÜ#˜ M´+°¸bÔAQÐ@RðTô Uà,0„E‡N�N×Ñ×ÑÔ)Ø&*„E‡N�N×ÑÔ#Ü�vÔÜ‡J‚J„LÜ×ÒÓ! $Ñ'€HÜ‡K‚K” 4Ò(à
€CÜœW×%‰ˆÜ˜DÓ!ˆÝ&Ó4šw˜!¨t©)—A™wˆÐ4ÜŸš xÓ:‰ˆÜ�> $  v¨d¨VÐ4Ô5ð
	 Ü'3Ø˜e¬U´E¸UÀ4À&¸>Ø !™Vñ(&Ñ$ˆC�R˜uô 	�
Š
˜SŸ^™^Ó-°RÈ4Ø%´ÀØ"¬3¼YñHô —7‘7—<‘<¤¨E°$°°sÐ*;Ó<ô	>ô & c°R¸TÓJÑˆ�Ü ¨eÑ@ˆ‰	ØØ�;‰;˜&Ö Ü�J‰J×"Ñ"×$ñ1 &ô4 ˜FÓ#€HÜÐ=¸h¸ZÐHÔIÜ—N’N 8¬T´'«]Ó;�M€FÜ+Ø�& ¬´°vÈ5ñ RÑ€CˆR�uä	‡J‚J˜Ÿ™Ó)°ÀxÜœg›´ÀØ¤s¼ñDô �w‰w�|‰|œH iÓ0ô2ô —[’[ °¼ÀÓG€NÜ(›?Ñ€J�Ü×%Ò% jÓ1°$Ñ7€HÜ
˜˜h¨°"¸.Ì!Ï#É#ÈeÉ*ÐV\Ó
]€Cä”2—7’7Å7ÓKÂ7¸a ™F 8Ñ,¨VÑ4°SÔ9Á7ÑKÓLÓM€FÜ”R—W’WÅwÓOÂwÀ! !™f XÑ.¨vÑ6°uÔ=ÁwÑOÓPÓQ€HÜˆ|˜F 3˜< u¨X°c¨NÐ;Ô<ä
ÜœU¬%´uÜÝ+2Ó3ª7 a�‘˜Ñ!Ò!©7Ñ3Ý07Ó8²¨1�s‘v˜g‘Ò&±Ñ8Ü˜ XÑ.ØØñ€Cô ‡K‚K”—‘—‘¤Ó$¨tÒ4Ü	Œc�3Œ˜2Ü�	Š	�#�r !¬UÒ3÷ 
äˆv”c�Uˆ^Õùòm 5øô ó 	 Ü�{ 3 %Ð(Ô)Ü ¬c°#«h¸TÑBˆCÜ�KŠKœŸ™Ÿ™¬Ó,°tÒ<Ü”c˜3” 2Ü—	’	˜#˜r¨!´UÒ;÷  —–úä˜Q“-Ðûð	 üò> LùÚOùò 4ùÚ8÷ 
�úsa   Æ	T7ÆT7Ç
 T<Ï$W7Ð#W<Ñ>XÒXÔ XÔ<
W4ÕA(W/Ö.W×
	W/×
W!×W/×/W4Ø
XÚ__main__)F)FF)Mr†   r<  râ  r  r    r¿  r>  Únumpyrì   r0   Útorch.nnrY   Útorch.nn.functionalÚ
functionalrr   Úensemble_clf_nestedr�  Úensemble_clf_transformer_v2rŸ   Úensemble_clf_transformer_v4ri  Úeval_externalrœ  Úcommonr   r"   r#   ÚabspathÚ__file__ÚEXPrØ   rÙ   rÜ   rÝ  r$   rá  r¾  rš  rê   ÚenvironÚgetr   r   r   r2  r   rH  r8  r„  r…  rç  rë   rî   r×  r*   r5   rK   rO   ÚModulerQ   r‹   rÃ   rÒ   rß   r  r  r$  rÎ   r&  r\  rd  rp  rŽ  r±  rÆ  r÷  r‚   r'  r6   r)   Ú<module>r     s#  ðñó$ Û Û Û 	Û Û ã Û Ý ß Ð ã Û 'Û (Û Ý à‡g�g‡o�o�b—g‘g—o‘o hÓ/Ó0€Ø
�'‰'�,‰,�s˜H oÓ
6€Ø�7‰7�<‰<˜˜X }Ó5€Ø�7‰7�<‰<˜˜XÐ'7Ó8€Ø‡g�g‡l�l�3˜Ð';Ó<€Ø‡g�g‡l�l�3˜	 :Ð/DÓE€Øk€Ø"€
áˆB�J‰J�N‰N˜7 FÓ+Ó,€ÙˆB�J‰J�N‰N˜7 EÓ*Ó+€Ù�"—*‘*—.‘. °Ó5Ó6€Ù˜Ÿ™Ÿ™Ð(:¸FÓCÓDÐ Ø€	Ø	€Ø	€Ø€Ø˜˜r 2 q¨$°Ð3€Ø	€Ø	€Ø€	òò(ð
 .7¸tÈ4ô ò&Bô%#�r—y‘yô %#ôP1E˜2Ÿ9™9ô 1Eòh!òH,òKò",ò<ò	ô	�lô 	ô)/ðZ ,1ô2ò(ò
0ò*òZò*FðR ˆzÓÙ…Fð r6   