ó
    y‡ÛjæE  ã                   ód  • S r SSKrSSKJs  Jr  SSKrSSKrSSK	J
r
  SSKJr  SSKrSSKJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJr  SSKJ r J!r!  SSK"J#r#   " S S\
RH                  5      r%S	 r&S
 r'S r(S r)S r*S r+S r,S r-S r.S r/S r0S r1S r2S r3S r4S r5S r6S r7S r8S r9S r:S r;g)zVFeedback decoder: null slots stay on the F12 path, and stage 2 opens the encoder tail.é    N©Ú	Namespace)ÚREASON_FALSE_CRACKLEÚREASON_FALSE_WHEEZEÚREASON_MISSED_BOTHÚREASON_MISSED_CRACKLEÚREASON_MISSED_WHEEZEÚS1_TAU_CÚS1_TAU_WÚW_MISSED_BOTHÚFeedbackDecoderÚFeedbackRefinerÚSelfFeedbackÚadapt_legacy_feedback_stateÚattach_stage2Úerror_sample_weightsÚload_feedback_state_dictÚreason_targets_from_train)ÚFrameASDHeadÚFrameASDLoss)Úload_checkpointc                   ó,   ^ • \ rS rSrSU 4S jjrSrU =r$ )ÚDummyEncoderé    c                 ód   >• [         TU ]  5         Xl        [        R                  " X5      U l        g ©N)ÚsuperÚ__init__Úencoder_dimÚnnÚLinearÚproj)ÚselfÚdimÚ	__class__s     €Ú</root/disk_3tb/ICBHI_2017_F12/tests/test_feedback_decoder.pyr   ÚDummyEncoder.__init__!   s$   ø€ Ü‰ÑÔØÔÜ—I’I˜cÓ'ˆ�	ó    )r   r"   )r   ©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú__static_attributes__Ú__classcell__©r%   s   @r&   r   r       s   ø† ÷(õ (r(   r   c                  ó"   • [        SSSSSSSSS9$ )	Né   é   çš™™™™™É?é   é   Fr   )Úfeedback_slotsÚfeedback_layersÚfeedback_dropÚreason_codesÚmil_topkÚfeedback_stageÚresume_backbone_onlyÚencoder_tail_layersr   © r(   r&   Ú_argsr@   '   s'   € ÜØØØØØØØ"Øñ	ð 	r(   c            	      ó6  • [        SSS9n [        R                  " SSS5      n[        R                  " SSS5      nU " X5      u  p4[        R                  oU" X15      of(       Gd8  SS[        R
                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      S[        R
                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS	[        R
                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS	[        R                  " U5      S
.-  n[        [        R                  " U5      5      eS =pV[        R                  " 5          U R                  5        H(  nUR                  [        R                  " U5      5        M*     S S S 5        U " U[        R                  " U5      5      u  p”[        R                  nSoe" X‘US9oª(       GdM  SS[        R
                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      S[        R
                  " 5       ;   d  [        R                  " U	5      (       a  [        R                  " U	5      OSS	[        R
                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS	[        R                  " U5      [        R                  " U
5      S.-  n[        [        R                  " U5      5      eS =n=pjg ! , (       d  f       GN = f)Nr   r3   )r$   Úlayersé   r2   úSassert %(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py3)s, %(py4)s)
}ÚtorchÚrefinedÚframes©Úpy0Úpy2Úpy3Úpy4Úpy6çñhãˆµøä>©Úatolúaassert %(py8)s
{%(py8)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py3)s, %(py4)s, atol=%(py6)s)
}Úrefined_zero©rI   rJ   rK   rL   rM   Úpy8)r   rE   ÚrandnÚallcloseÚ@py_builtinsÚlocalsÚ
@pytest_arÚ_should_repr_global_nameÚ	_safereprÚAssertionErrorÚ_format_explanationÚno_gradÚ
parametersÚadd_Ú
randn_likeÚ
zeros_like)ÚdecoderrG   ÚslotsrF   Ú_Ú@py_assert1Ú@py_assert5Ú@py_format7ÚparamrR   Ú@py_assert7Ú@py_format9s               r&   Ú-test_decoder_zero_slots_and_init_match_framesrl   4   s÷  € Ü "¨QÑ/€GÜ�[Š[˜˜A˜rÓ"€FÜ�KŠK˜˜1˜bÓ!€EÙ˜Ó'�J€GÜ�>‰>Ð*ˆ>˜'Ó*Ð*Ö*×*×*×*×*×*Ñ*Œ5×*×*×*Ñ*Œ5×*×*Õ*ˆ>×*×*×*×*×*Ô*˜'×*×*×*Ñ*˜'×*×*×*×*×*Ö* 6×*×*×*Ñ* 6×*×*Õ*Ð*×*×*×*×*×*Ò*ä	�Š�Ø×'Ñ'Ö)ˆEØ�J‰J”u×'Ò'¨Ó.Ö/ñ *÷ 
ñ ˜f¤e×&6Ò&6°uÓ&=Ó>�O€LÜ�>‰>Ð:°TÐ:ˆ>˜,°TÑ:Ð:Ö:×:×:×:×:×:Ñ:Œ5×:×:×:Ñ:Œ5×:×:Õ:ˆ>×:×:×:×:×:Ô:˜,×:×:×:Ñ:˜,×:×:×:×:×:Ö:¨×:×:×:Ñ:¨×:×:Õ:°T×:×:Ó:Ð:×:×:×:×:×:Õ:÷	 
Žús   Æ3=N	Î	
Nc            	      ót  • [         R                  " S5        [        SSSSS9n U R                  5         [	        S5      n[	        S5      n[         R
                  " SSS5      n[         R
                  " SSS5      nU " X1X$[         R                  " S5      S9nUS	   ofR                  otR                  o‡U:H  o™(       dÏ  [        R                  " S
U	4SXx45      [        R                  " U5      [        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      S.-  n
SSU
0-  n[        [        R                  " U5      5      eS =n=n=p˜[         R                   oÅS	   oÜ" XÔ5      oˆ(       Gd  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      [        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      S.-  n
[        [        R                  " U
5      5      eS =n=pØg )Nr   r   r2   r3   ç        ©r$   rd   rB   Údrop_prC   )Úexternal_slotsÚenergyrd   ©ú==)zH%(py3)s
{%(py3)s = %(py1)s.shape
} == %(py7)s
{%(py7)s = %(py5)s.shape
}Úexternal©Úpy1rK   Úpy5Úpy7úassert %(py9)sÚpy9zSassert %(py7)s
{%(py7)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py4)s, %(py5)s)
}rE   )rI   rJ   rL   rx   ry   )rE   Úmanual_seedr   Úevalr   rU   ÚonesÚshaperY   Ú_call_reprcomparer[   rW   rX   rZ   r\   r]   rV   )ÚrefinerÚhead1Úhead2rG   ru   ÚoutÚ@py_assert0Ú@py_assert2Ú@py_assert6Ú@py_assert4Ú@py_format8Ú@py_format10rf   Ú@py_assert3s                 r&   Ú)test_external_slots_replace_self_feedbackrŒ   B   s•  € Ü	×Ò�aÔÜ "¨A°aÀÑD€GØ‡L�L„NÜ˜Ó€EÜ˜Ó€EÜ�[Š[˜˜A˜rÓ"€FÜ�{Š{˜1˜a Ó$€HÙ
�& ÌÏ
Ê
ÐSTËÑ
V€CØˆw‰<Ð/×ÑÐ/§¡Ð/ Ñ/×/×/×/Ò/Ð×/×/Ô/ˆ<×/×/Ó/Ð×/×/×/×/×/Ô/ ×/×/×/Ñ/ ×/×/Õ/ ×/×/×/×/×/×/Õ/Ü�>‰>Ð1˜g™,Ð1ˆ>˜,Ó1Ð1Ö1×1×1×1×1×1Ñ1Œ5×1×1×1Ñ1Œ5×1×1Õ1ˆ>×1×1Ó1˜,×1×1×1×1×1Ô1¨×1×1×1Ñ1¨×1×1Õ1Ð1×1×1×1×1×1Õ1r(   c                  ó4  • [         R                  " S5        [        SSSSS9n U R                  5         [	        S5      n[	        S5      n[         R
                  " SSS5      nU " X1U[         R                  " S5      S9n[         R                  oTS	   oe" U5      nSo‡U:H  o™(       dì  [        R                  " S
U	4SXx45      S[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n
SSU
0-  n[        [        R                  " U5      5      eS =n=n=n=p˜g )Nr   r   r2   r3   ç      ð?ro   rC   ©rr   rd   rs   )zS%(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.count_nonzero
}(%(py4)s)
} == %(py9)srE   ©rI   rJ   rL   rM   r{   úassert %(py11)sÚpy11)rE   r|   r   Útrainr   rU   r~   Úcount_nonzerorY   r€   rW   rX   rZ   r[   r\   r]   )r�   r‚   rƒ   rG   r„   rf   r‹   rg   Ú@py_assert8rj   rŠ   Ú@py_format12s               r&   Ú test_slot_dropout_can_zero_slotsr—   O   s  € Ü	×Ò�aÔÜ "¨A°aÀÑD€GØ‡M�M„OÜ˜Ó€EÜ˜Ó€EÜ�[Š[˜˜A˜rÓ"€FÙ
�& ¬u¯zªz¸!«}Ñ
=€CÜ×ÑÐ1 7™|Ð1Ð˜|Ó,Ð1°Ð1°Ñ1×1×1×1Ò1Ð,×1×1×1×1×1Õ1Œ5×1×1×1Ñ1Œ5×1×1Õ1Ð×1×1Ó1˜|×1×1Ó1Ð,×1×1Ó1°×1×1×1×1×1×1×1Ð1r(   c            	      óZ  • [         R                  " S5        [        SSSSS9n U R                  5         [	        S5      n[	        S5      n[         R
                  " SSS5      nU " X1U[         R                  " S5      S9nUS	   oUR                  nS
ovU:H  oˆ(       d„  [        R                  " SU4SXg45      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n	SSU	0-  n
[        [        R                  " U
5      5      eS =n=n=p‡US   oUR                  nS
ovU:H  oˆ(       d„  [        R                  " SU4SXg45      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n	SSU	0-  n
[        [        R                  " U
5      5      eS =n=n=p‡US   oUR                  nSovU:H  oˆ(       d„  [        R                  " SU4SXg45      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n	SSU	0-  n
[        [        R                  " U
5      5      eS =n=n=p‡US   o»R                  oÌ" 5       n[        U5      nSoíU:  oÿ(       Gd  [        R                  " SU4SXÞ45      S[        R                   " 5       ;   d  [        R"                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n=n=n=pþg )Nr   r   r2   r3   rn   ro   rC   r�   Úlogit_c©r3   rC   rs   ©z-%(py3)s
{%(py3)s = %(py1)s.shape
} == %(py6)s©rw   rK   rM   úassert %(py8)srT   Úlogit_wÚreason_target©r3   é   )Ú<)z`%(py8)s
{%(py8)s = %(py0)s(%(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s.max
}()
})
} < %(py11)sÚint©rI   rJ   rL   rM   rT   r’   úassert %(py13)sÚpy13)rE   r|   r   r}   r   rU   r~   r   rY   r€   r[   r\   r]   Úmaxr£   rW   rX   rZ   )r�   r‚   rƒ   rG   r„   r…   r†   rg   rˆ   rh   rk   rf   r‹   rj   Ú@py_assert10Ú@py_assert9r–   Ú@py_format14s                     r&   Ú!test_forward_does_not_need_labelsr«   Z   sF  € Ü	×Ò�aÔÜ "¨A°aÀÑD€GØ‡L�L„NÜ˜Ó€EÜ˜Ó€EÜ�[Š[˜˜A˜rÓ"€FÙ
�& ¬u¯zªz¸!«}Ñ
=€CØˆy‰>Ð)×ÑÐ) 6Ð) 6Ñ)×)×)×)Ò)Ð×)×)Ô)ˆ>×)×)Ó)Ð×)×)Ó) 6×)×)×)×)×)×)Õ)Øˆy‰>Ð)×ÑÐ) 6Ð) 6Ñ)×)×)×)Ò)Ð×)×)Ô)ˆ>×)×)Ó)Ð×)×)Ó) 6×)×)×)×)×)×)Õ)ØˆÑÐ-×%Ñ%Ð-¨Ð-¨Ñ-×-×-×-Ò-Ð%×-×-Ô-Ð×-×-Ó-Ð%×-×-Ó-¨×-×-×-×-×-×-Õ-Ø�?Ñ#Ð.×'Ñ'Ð.Ð'Ó)Ð.Œ3Ð)Ó*Ð.¨QÐ.¨QÑ.×.×.×.Ó.Ð*×.×.×.×.×.Õ.Œ3×.×.×.Ñ.Œ3×.×.Õ.Ð#×.×.Ó.Ð'×.×.Ó.Ð)×.×.Ó.Ð*×.×.Ó.¨Q×.×.×.×.×.×.×.Ò.r(   c            	      ój  • [         R                  " S5         " S S[        R                  5      n U " 5       n[	        S5      n[        5       nSUl        [        XU[         R                  " S5      5      nS UR                  R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR&                  R(                  R*                  S   R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR&                  R(                  R*                  S   R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR&                  R(                  R*                  S   R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR&                  R(                  R*                  S   R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pVS UR,                  R                  5        5       n[        U5      of(       d¢  SS	[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                   " [        5      OS	[        R                   " U5      [        R                   " U5      S
.-  n[#        [        R$                  " U5      5      eS =pV[         R.                  " SSSSS9nUn	UR&                  R(                  R*                   H  n
U
" U	5      n	M     U" X’UR,                  [         R0                  " S5      S9nUS   R3                  5       R5                  5         UR&                  R(                  R*                  S   oÌR6                  oÝR8                  nS oþUL nU(       d™  [        R:                  " SU4SXï45      [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S.-  nSSU0-  n[#        [        R$                  " U5      5      eS =n=n=n=nnUR&                  R(                  R*                  S   oÌR6                  oÝR8                  nS oþUL nU(       d™  [        R:                  " SU4SXï45      [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S.-  nSSU0-  n[#        [        R$                  " U5      5      eS =n=n=n=nnUR&                  R(                  R*                  S   oÌR6                  oÝR8                  nS oþULnU(       d™  [        R:                  " SU4SXï45      [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S.-  nSSU0-  n[#        [        R$                  " U5      5      eS =n=n=n=nnUR&                  R(                  R*                  S   oÌR6                  oÝR8                  nS oþULnU(       d™  [        R:                  " SU4SXï45      [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S.-  nSSU0-  n[#        [        R$                  " U5      5      eS =n=n=n=nnUR<                  oUR6                  ofR8                  nS nUUL oÿ(       då  [        R:                  " SU4SUU45      S [        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                   " U5      OS [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S!.-  nS"S#U0-  n[#        [        R$                  " U5      5      eS =n=n=n=nnUR,                  oUR<                  ofR6                  nUR8                  nS nUULnU(       dú  [        R:                  " SU4S$UU45      S%[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                   " U5      OS%[        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      [        R                   " U5      S&.-  nS'S(U0-  n[#        [        R$                  " U5      5      eS =n=n=n=n=nng ))Nr   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚCtest_attach_stage2_unfreezes_only_the_last_two_blocks.<locals>._Encék   c           	      óª  >• [         TU ]  5         SU l        [        R                  " SS5      U l        [        R                  " 5       U l        [        R                  " 5       U R                  l        [        R                  " [        S5       Vs/ s H  n[        R                  " SS5      PM     sn5      U R                  R                  l        g s  snf ©Nr   r2   ©r   r   r   r    r!   ÚstemÚModuleÚbeatsÚencoderÚ
ModuleListÚrangerB   ©r#   re   r%   s     €r&   r   ÚLtest_attach_stage2_unfreezes_only_the_last_two_blocks.<locals>._Enc.__init__l   ó†   ø€ Ü‰GÑÔØ!ˆDÔÜŸ	š	 " bÓ)ˆDŒIÜŸš›ˆDŒJÜ!#§¢£ˆD�J‰JÔÜ(*¯ªÜ,1°!¬HÓ5ªH q”—’˜2˜rÖ"©HÑ5ó)7ˆD�J‰J×ÑÕ%ùÚ5ó   Â!C©rµ   r   r³   r)   r0   s   @r&   Ú_Encr®   k   ó   ø† ÷	7ó 	7r(   r¾   r   r3   Úcpuc              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   ©Úrequires_grad©Ú.0ri   s     r&   Ú	<genexpr>ÚHtest_attach_stage2_unfreezes_only_the_last_two_blocks.<locals>.<genexpr>z   s   é € ÐLÒ4K¨5×&Ñ&×&Ñ&Ò4Kùó   ‚ú,assert %(py4)s
{%(py4)s = %(py0)s(%(py2)s)
}Úall©rI   rJ   rL   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ   {   ó   é € Ð_Ò4^¨5×&Ñ&×&Ñ&Ò4^ùrÈ   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ   |   rÍ   rÈ   é   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ   }   ó   é € Ð[Ò0Z u×"Ö"Ò0Zùó   ‚c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ   ~   rÑ   rÒ   r6   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ      ó   é € ÐFÒ4E¨5×&Ñ&×&Ñ&Ò4EùrÈ   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rÇ   €   ó   é € ÐKÒ0J u×"Ö"Ò0JùrÒ   rC   TrÂ   r�   r™   ©Úis)zH%(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.weight
}.grad
} is %(py8)s©rw   rK   rx   rT   úassert %(py10)sÚpy10©úis not)zL%(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.weight
}.grad
} is not %(py8)s©ze%(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.crackle
}.weight
}.grad
} is %(py9)sÚheadr�   r‘   r’   ©z…%(py8)s
{%(py8)s = %(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.head2
}.crackle
}.weight
}.grad
} is not %(py11)sr�   r¤   r¥   r¦   )rE   r|   r    r´   r   r@   r>   r   Údevicer³   r_   rÊ   rW   rX   rY   rZ   r[   r\   r]   rµ   r¶   rB   rƒ   rU   r~   ÚsumÚbackwardÚweightÚgradr€   Úcrackle)r¾   Úmodelrà   Úargsr�   rf   r‹   Ú@py_format5rG   ÚhiddenÚlayerr„   r…   r†   rˆ   rj   r‡   rk   Ú@py_format11rg   r•   rŠ   r–   r¨   r©   rª   s                             r&   Ú5test_attach_stage2_unfreezes_only_the_last_two_blocksrî   h   s´  € Ü	×Ò�aÔô7Œr�y‰yô 7ñ ‹F€EÜ˜Ó€DÜ‹7€DØ €DÔÜ˜E¨¬u¯|ª|¸EÓ/BÓC€GÙL°E·J±J×4IÑ4IÔ4KÓLÐLŒ3ÐLÓLÐLÕL×L×L×L×L×LÑLŒ3×L×L×LÑLŒ3×L×LÕLÐL×L×LÓLÐL×L×L×L×L×LÒLÙ_°E·K±K×4GÑ4G×4NÑ4NÈqÑ4Q×4\Ñ4\Ô4^Ó_Ð_Œ3Ð_Ó_Ð_Õ_×_×_×_×_×_Ñ_Œ3×_×_×_Ñ_Œ3×_×_Õ_Ð_×_×_Ó_Ð_×_×_×_×_×_Ò_Ù_°E·K±K×4GÑ4G×4NÑ4NÈqÑ4Q×4\Ñ4\Ô4^Ó_Ð_Œ3Ð_Ó_Ð_Õ_×_×_×_×_×_Ñ_Œ3×_×_×_Ñ_Œ3×_×_Õ_Ð_×_×_Ó_Ð_×_×_×_×_×_Ò_Ù[°·±×0CÑ0C×0JÑ0JÈ1Ñ0M×0XÑ0XÔ0ZÓ[Ð[Œ3Ð[Ó[Ð[Õ[×[×[×[×[×[Ñ[Œ3×[×[×[Ñ[Œ3×[×[Õ[Ð[×[×[Ó[Ð[×[×[×[×[×[Ò[Ù[°·±×0CÑ0C×0JÑ0JÈ1Ñ0M×0XÑ0XÔ0ZÓ[Ð[Œ3Ð[Ó[Ð[Õ[×[×[×[×[×[Ñ[Œ3×[×[×[Ñ[Œ3×[×[Õ[Ð[×[×[Ó[Ð[×[×[×[×[×[Ò[ÙF°D·O±OÔ4EÓFÐFŒ3ÐFÓFÐFÕF×F×F×F×F×FÑFŒ3×F×F×FÑFŒ3×F×FÕFÐF×F×FÓFÐF×F×F×F×F×FÒFÙK°·±×0HÑ0HÔ0JÓKÐKŒ3ÐKÓKÐKÕK×K×K×K×K×KÑKŒ3×K×K×KÑKŒ3×K×KÕKÐK×K×KÓKÐK×K×K×K×K×KÒKÜ�[Š[˜˜A˜r°Ñ6€FØ€FØ—‘×$Ñ$×+Ô+ˆÙ�v“Šñ ,á
�& §¡´e·j²jÀ³mÑ
D€CØˆ	�N×ÑÓ×!Ñ!Ô#Ø�;‰;×Ñ×%Ñ% aÑ(Ð<×/Ñ/Ð<×4Ñ4Ð<¸Ð<¸Ð<×<×<×<Ó<Ð4×<×<Ô<Ð(×<×<Ó<Ð/×<×<Ó<Ð4×<×<Ó<¸×<×<×<×<×<×<×<Ð<Ø�;‰;×Ñ×%Ñ% aÑ(Ð<×/Ñ/Ð<×4Ñ4Ð<¸Ð<¸Ð<×<×<×<Ó<Ð4×<×<Ô<Ð(×<×<Ó<Ð/×<×<Ó<Ð4×<×<Ó<¸×<×<×<×<×<×<×<Ð<Ø�;‰;×Ñ×%Ñ% aÑ(Ð@×/Ñ/Ð@×4Ñ4Ð@¸DÐ@¸DÐ@×@×@×@Ó@Ð4×@×@Ô@Ð(×@×@Ó@Ð/×@×@Ó@Ð4×@×@Ó@¸D×@×@×@×@×@×@×@Ð@Ø�;‰;×Ñ×%Ñ% aÑ(Ð@×/Ñ/Ð@×4Ñ4Ð@¸DÐ@¸DÐ@×@×@×@Ó@Ð4×@×@Ô@Ð(×@×@Ó@Ð/×@×@Ó@Ð4×@×@Ó@¸D×@×@×@×@×@×@×@Ð@Ø�<‰<Ð+×ÑÐ+×#Ñ#Ð+ tÐ+Ð# tÐ+×+×+×+Ò+Ð# t×+×+×+×+×+Õ+ˆ4×+×+×+Ñ+ˆ4×+×+Õ+ˆ<×+×+Ó+Ð×+×+Ó+Ð#×+×+Ó+ t×+×+×+×+×+×+×+Ð+Ø�=‰=Ð8× Ñ Ð8×'Ñ'Ð8Ð'×,Ñ,Ð8°DÐ8Ð,°DÐ8×8×8×8Ó8Ð,°D×8×8×8×8×8Õ8ˆ7×8×8×8Ñ8ˆ7×8×8Õ8ˆ=×8×8Ó8Ð ×8×8Ó8Ð'×8×8Ó8Ð,×8×8Ó8°D×8×8×8×8×8×8×8Ó8r(   c                  ó  • [         R                  " SS5      n SU S S 2S4'   Su  U S'   U S'   Su  U S	'   U S
'   Su  U S'   U S'   Su  U S'   U S'   Su  U S'   U S'   [         R                  " / SQ5      n[        X5      nUR                  o3" 5       n[
        [        [        [        [        /oTU:H  of(       dÏ  [        R                  " SU4SXE45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[!        [        R"                  " U5      5      eS =n=n=peg )NrC   é	   rŽ   é   )gÍÌÌÌÌÌì?çš™™™™™¹?)r   r   )r   rÏ   )r4   rò   )rÏ   r   )rÏ   rÏ   )rò   r4   )r3   r   )r3   rÏ   )rò   çš™™™™™é?)r6   r   )r6   rÏ   )ró   rò   )r2   r   )r2   rÏ   )r6   rÏ   r3   r   r   rs   ©zE%(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.tolist
}()
} == %(py7)sÚtarget©rI   rJ   rL   ry   rz   r{   )rE   ÚzerosÚtensorr   Útolistr   r   r	   r   r   rY   r€   rW   rX   rZ   r[   r\   r]   )	ÚstatsÚlabelsrõ   rf   r‹   r‡   rg   r‰   rŠ   s	            r&   Ú9test_reason_targets_from_train_marks_the_four_sign_errorsrü   �   s  € Ü�KŠK˜˜1Ó€EØ€EŠ!ˆQˆ$�KØ'Ñ€Eˆ$�K��t‘Ø'Ñ€Eˆ$�K��t‘Ø'Ñ€Eˆ$�K��t‘Ø'Ñ€Eˆ$�K��t‘Ø'Ñ€Eˆ$�K��t‘Ü�\Š\š/Ó*€FÜ& vÓ5€FØ�=‰=ð ˆ=‹?ð ÜÜÜÜÜðð ð ñ ÷ ó ïé ôˆ?÷ ò ïé ÷õ ïé ðˆ6÷ ö ïé ðˆ6÷ ò ïé ðˆ=÷ ð ïé ðˆ?÷ ð ïé ðð ÷ ÷ ÷ ñ ïé ÷÷ ò r(   c                  ó  • [         R                  " S5        [        SSSSS9n U R                  5         [	        S5      n[	        S5      n[         R
                  " SSS5      n[         R                  " S5      n[         R                  " SS/5      nU " X1X$S	9nU " X1X$US
9nUS   oˆR                  n	So©U
:H  o»(       d„  [        R                  " SU4SXš45      [        R                  " U5      [        R                  " U	5      [        R                  " U
5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n	=pºUS   oˆR                  n	So©U
:H  o»(       d„  [        R                  " SU4SXš45      [        R                  " U5      [        R                  " U	5      [        R                  " U
5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n	=pº[         R                  oæS   o÷S   o®" Xú5      nU(       dÌ  SS[        R                   " 5       ;   d  [        R"                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      [        R                  " U5      [        R                  " U
5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =n=n=n
nUS   oˆR$                  n	So©U
:H  o»(       d„  [        R                  " SU4SXš45      [        R                  " U5      [        R                  " U	5      [        R                  " U
5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n	=pºg )Nr   r   r2   r3   rn   ro   rC   r6   r�   )rr   rû   r™   rš   rs   r›   rœ   r�   rT   rŸ   r    rd   zPassert %(py8)s
{%(py8)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py4)s, %(py6)s)
}rE   )rI   rJ   rL   rM   rT   Úreason_loss)z,%(py3)s
{%(py3)s = %(py1)s.ndim
} == %(py6)s)rE   r|   r   r}   r   rU   r~   rø   r   rY   r€   r[   r\   r]   ÚequalrW   rX   rZ   Úndim)r�   r‚   rƒ   rG   rr   rû   ÚplainÚlabeledr…   r†   rg   rˆ   rh   rk   rf   r‹   rj   s                    r&   Ú>test_forward_without_labels_still_runs_and_slots_ignore_labelsr  ¢   s?  € Ü	×Ò�aÔÜ "¨A°aÀÑD€GØ‡L�L„NÜ˜Ó€EÜ˜Ó€EÜ�[Š[˜˜A˜rÓ"€FÜ�ZŠZ˜‹]€FÜ�\Š\˜1˜a˜&Ó!€FÙ�F 5Ñ8€EÙ�f UÀ&ÑI€GØ�ÑÐ+×!Ñ!Ð+ VÐ+ VÑ+×+×+×+Ò+Ð!×+×+Ô+Ð×+×+Ó+Ð!×+×+Ó+ V×+×+×+×+×+×+Õ+Ø�Ñ!Ð/×'Ñ'Ð/¨4Ð/¨4Ñ/×/×/×/Ò/Ð'×/×/Ô/Ð!×/×/Ó/Ð'×/×/Ó/¨4×/×/×/×/×/×/Õ/Ü�;‰;Ð8˜W‘~Ð8¨wÑ'7Ð8ˆ;�~Ó8Ð8Ö8×8×8×8×8×8Ñ8Œ5×8×8×8Ñ8Œ5×8×8Õ8ˆ;×8×8Ó8�~×8×8Ó8Ð'7×8×8Ó8Ð8×8×8×8×8×8×8Ø�=Ñ!Ð+×&Ñ&Ð+¨!Ð+¨!Ñ+×+×+×+Ò+Ð&×+×+Ô+Ð!×+×+Ó+Ð&×+×+Ó+¨!×+×+×+×+×+×+Ö+r(   c                  óŽ  •  " S S[         R                  5      n U " 5       n[        S5      n[        X[	        5       [
        R                  " S5      5      nS UR                  5        5       n[        U5      oU(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =pES	 UR                  5        5       n[        U5      oU(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =pES
 UR                   R                  5        5       n[        U5      oU(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =pEg )Nc                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚGtest_attach_stage2_keeps_encoder_frozen_when_tail_is_zero.<locals>._Encé´   c           	      óª  >• [         TU ]  5         SU l        [        R                  " SS5      U l        [        R                  " 5       U l        [        R                  " 5       U R                  l        [        R                  " [        S5       Vs/ s H  n[        R                  " SS5      PM     sn5      U R                  R                  l        g s  snf r±   r²   r¹   s     €r&   r   ÚPtest_attach_stage2_keeps_encoder_frozen_when_tail_is_zero.<locals>._Enc.__init__µ   r»   r¼   r½   r)   r0   s   @r&   r¾   r  ´   r¿   r(   r¾   r   rÀ   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   ÚLtest_attach_stage2_keeps_encoder_frozen_when_tail_is_zero.<locals>.<genexpr>Á   ó   é € ÐGÒ4F¨5×&Ñ&×&Ñ&Ò4FùrÈ   rÉ   rÊ   rË   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   r  Â   rÕ   rÈ   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   r  Ã   r×   rÒ   )r    r´   r   r   r@   rE   râ   r_   rÊ   rW   rX   rY   rZ   r[   r\   r]   rƒ   )r¾   rè   rà   r�   rf   r‹   rê   s          r&   Ú9test_attach_stage2_keeps_encoder_frozen_when_tail_is_zeror  ³   sŠ  € ô7Œr�y‰yô 7ñ ‹F€EÜ˜Ó€DÜ˜E¬«´%·,²,¸uÓ2EÓF€GÙG°E×4DÑ4DÔ4FÓGÐGŒ3ÐGÓGÐGÕG×G×G×G×G×GÑGŒ3×G×G×GÑGŒ3×G×GÕGÐG×G×GÓGÐG×G×G×G×G×GÒGÙF°D·O±OÔ4EÓFÐFŒ3ÐFÓFÐFÕF×F×F×F×F×FÑFŒ3×F×F×FÑFŒ3×F×FÕFÐF×F×FÓFÐF×F×F×F×F×FÒFÙK°·±×0HÑ0HÔ0JÓKÐKŒ3ÐKÓKÐKÕK×K×K×K×K×KÑKŒ3×K×K×KÑKŒ3×K×KÕKÐK×K×KÓKÐK×K×K×K×K×KÓKr(   c                 óÄ   • SS K nSS KnU R                  US/ SQ5        [        R                  " [
        SS9   UR                  5         S S S 5        g ! , (       d  f       g = f)Nr   Úargv)ztrain.pyz--feedback_stageÚ2Útrain_feedback©Úmatch)Úsysr“   ÚsetattrÚpytestÚraisesÚ
SystemExitÚmain)Úmonkeypatchr  r“   s      r&   Ú(test_train_py_rejects_feedback_stage_twor  Æ   s?   € ÛÛØ×Ñ˜˜VÒ%JÔKÜ	�Š”zÐ)9Ó	:Ø�
‰
Œ÷ 
;×	:Ö	:ús   ·AÁ
Ac                  ó(  • [         R                  " S5        [        5       n [        S5      n[	        X[        5       [         R                  " S5      5      nS U R                  5        5       n[        U5      oD(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =p4S UR                  5        5       n[        U5      oD(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =p4S	 UR                   R                  5        5       n[        U5      oD(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =p4UR                  5        Vs/ s H  ofR"                  (       d  M  UPM     nnU(       dp  S
SS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS0-  n[        [        R                  " U5      5      eS U 5       n[        U5      oD(       d¢  SS[        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =p4[%        U R                  5       5      [%        UR                  5       5      -    Vs1 s H  n['        U5      iM     n	nU	R(                  o7 Vs1 s H  n['        U5      iM     nnU" U5      oª(       d¯  SS[        R                  " 5       ;   d  [        R                  " U	5      (       a  [        R                  " U	5      OS[        R                  " U5      [        R                  " U5      [        R                  " U
5      S.-  n[        [        R                  " U5      5      eS =n=pJ[         R*                  " SSS5      nU" XÁUR                   [         R,                  " S5      S9nUS   R/                  5       US   -   US   -   R1                  5         UR2                  o3R4                  oDR6                  n
S oêUL oÿ(       dä  [        R8                  " SU4SX®45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      [        R                  " U
5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n=n
=pþU R:                  o3R4                  oDR6                  n
S oêUL oÿ(       dä  [        R8                  " SU4SX®45      S[        R                  " 5       ;   d  [        R                  " U 5      (       a  [        R                  " U 5      OS[        R                  " U5      [        R                  " U5      [        R                  " U
5      [        R                  " U5      S.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n=n
=pþUR                   o3R2                  oDR4                  oªR6                  nS nUULnU(       dú  [        R8                  " SU4S UU45      S![        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS![        R                  " U5      [        R                  " U5      [        R                  " U
5      [        R                  " U5      [        R                  " U5      S".-  nS#S$U0-  n[        [        R                  " U5      5      eS =n=n=n
=n=nnUR                   o3R2                  oDR4                  oªR6                  oÿR<                  nU" 5       nUR.                  nU" 5       nSnUU:„  nU(       GdN  [        R8                  " S%U4S&UU45      S![        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS![        R                  " U5      [        R                  " U5      [        R                  " U
5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S'.
-  nS(S)U0-  n[        [        R                  " U5      5      eS =n=n=n
=n=n=n=n=n=nng s  snf s  snf s  snf )*Nr   r   rÀ   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   Ú8test_stage2_freezes_encoder_and_head1.<locals>.<genexpr>Ó   r  rÈ   rÉ   rÊ   rË   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   r   Ô   rÕ   rÈ   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   r   Õ   r×   rÒ   zassert %(py0)srI   Ú	trainablec              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   r   Ø   s   é € Ð:²	 u×"Ö"²	ùrÒ   zLassert %(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.isdisjoint
}(%(py4)s)
}Ú
frozen_ids)rI   rJ   rL   rM   r3   rC   r�   r™   Úconsistencyrþ   rØ   rß   rà   r�   r‘   r’   )zb%(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.proj
}.weight
}.grad
} is %(py9)srè   rÝ   rá   r�   r¤   r¥   r¦   ©Ú>)zè%(py16)s
{%(py16)s = %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s
{%(py10)s = %(py8)s
{%(py8)s = %(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.head2
}.crackle
}.weight
}.grad
}.abs
}()
}.sum
}()
} > %(py19)s)
rI   rJ   rL   rM   rT   rÜ   Úpy12Úpy14Úpy16Úpy19zassert %(py21)sÚpy21)rE   r|   r   r   r   r@   râ   r_   rÊ   rW   rX   rY   rZ   r[   r\   r]   rƒ   rÃ   ÚlistÚidÚ
isdisjointrU   r~   rã   rä   rç   rå   ræ   r€   r"   Úabs)rè   rà   r�   rf   r‹   rê   ri   r#  Ú@py_format1r%  rg   rh   rG   r„   r•   rj   rŠ   r–   r¨   r©   rª   Ú@py_assert11Ú@py_assert13Ú@py_assert15Ú@py_assert18Ú@py_assert17Ú@py_format20Ú@py_format22s                               r&   Ú%test_stage2_freezes_encoder_and_head1r:  Î   so  € Ü	×Ò�aÔÜ‹N€EÜ˜Ó€DÜ˜E¬«´%·,²,¸uÓ2EÓF€GÙG°E×4DÑ4DÔ4FÓGÐGŒ3ÐGÓGÐGÕG×G×G×G×G×GÑGŒ3×G×G×GÑGŒ3×G×GÕGÐG×G×GÓGÐG×G×G×G×G×GÒGÙF°D·O±OÔ4EÓFÐFŒ3ÐFÓFÐFÕF×F×F×F×F×FÑFŒ3×F×F×FÑFŒ3×F×FÕFÐF×F×FÓFÐF×F×F×F×F×FÒFÙK°·±×0HÑ0HÔ0JÓKÐKŒ3ÐKÓKÐKÕK×K×K×K×K×KÑKŒ3×K×K×KÑKŒ3×K×KÕKÐK×K×KÓKÐK×K×K×K×K×KÒKØ$+×$6Ñ$6Ô$8ÓPÒ$8˜5×<OÕ<O—Ñ$8€IÐPÞ×××××Òˆ9×××Ñˆ9×××××ÐÙ:±	Ó:Ð:Œ3Ð:Ó:Ð:Õ:×:×:×:×:×:Ñ:Œ3×:×:×:Ñ:Œ3×:×:Õ:Ð:×:×:Ó:Ð:×:×:×:×:×:Ò:Ü)-¨e×.>Ñ.>Ó.@Ó)AÄDÈÏÉÓIZÓD[Ò)[Ó\Ò)[ ”"�U–)Ñ)[€JÐ\Ø× Ñ ÐD¸Ó!Cº°¤" U¦)¹ÐDÐ!CÑ Ð!CÓDÐDÕD×D×D×D×D×DÑDˆ:×D×D×DÑDˆ:×D×DÕDÐ ×D×DÓDÐ!C×D×DÓDÐD×D×D×D×D×DÔDä�[Š[˜˜A˜rÓ"€FÙ
�& §¡´e·j²jÀ³mÑ
D€CØˆ�^×ÑÓ˜C Ñ.Ñ.°°]Ñ1CÑC×MÑMÔOØ�<‰<Ð+×ÑÐ+×#Ñ#Ð+ tÐ+ tÐ+×+×+×+Ò+Ð#×+×+×+×+×+Õ+ˆ4×+×+×+Ñ+ˆ4×+×+Õ+ˆ<×+×+Ó+Ð×+×+Ó+Ð#×+×+Ó+ t×+×+×+×+×+×+×+Ø�:‰:Ð)×ÑÐ)×!Ñ!Ð) TÐ) TÐ)×)×)×)Ò)Ð!×)×)×)×)×)Õ)ˆ5×)×)×)Ñ)ˆ5×)×)Õ)ˆ:×)×)Ó)Ð×)×)Ó)Ð!×)×)Ó) T×)×)×)×)×)×)×)Ø�=‰=Ð8× Ñ Ð8×'Ñ'Ð8×,Ñ,Ð8°DÐ8Ð,°DÐ8×8×8×8Ó8Ð,°D×8×8×8×8×8Õ8ˆ7×8×8×8Ñ8ˆ7×8×8Õ8ˆ=×8×8Ó8Ð ×8×8Ó8Ð'×8×8Ó8Ð,×8×8Ó8°D×8×8×8×8×8×8×8Ò8Ø�=‰=Ð<× Ñ Ð<×'Ñ'Ð<×,Ñ,Ð<×0Ñ0Ð<Ñ0Ó2Ð<Ð2×6Ñ6Ð<Ñ6Ó8Ð<¸1Ð<Ð8¸1Ñ<×<×<×<Ô<Ð8¸1×<×<×<×<×<Õ<ˆ7×<×<×<Ñ<ˆ7×<×<Õ<ˆ=×<×<Ó<Ð ×<×<Ó<Ð'×<×<Ó<Ð,×<×<Ó<Ð0×<×<Ó<Ð2×<×<Ó<Ð6×<×<Ó<Ð8×<×<Ó<¸1×<×<×<×<×<×<×<×<Ó<ùò Qùò ]ùÚ!Cs   ËlË0lÑ#l
Ò	lc                 ó¾  • [         R                  " S5        [        5       n[        S5      n[	        5       n[        XU[         R                  " S5      5      n[         R                  " 5          UR                  R                  R                  S5        UR                  R                  R                  S5        S S S 5        U S-  n[         R                  " SUR                  5       UR                  5       SSS	00/ S
.S.U5        [         R                  R!                  UR#                  5        Vs/ s H  ofR$                  (       d  M  UPM     snSS9n['        U5      Ul        [+        X1X'[         R                  " S5      US9nSo˜U	:H  oª(       d¥  [,        R.                  " SU
4SX‰45      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U	5      S.-  nSSU0-  n[9        [,        R:                  " U5      5      eS =p©[         R<                  o¤R>                  oÝR                  oîR                  oòR                  nUR                  nU
" UU5      nU(       Gd¡  SS[0        R2                  " 5       ;   d  [,        R4                  " [         5      (       a  [,        R6                  " [         5      OS[,        R6                  " U
5      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U5      [,        R6                  " U5      [,        R6                  " U5      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U5      [,        R6                  " U5      [,        R6                  " U5      S.
-  n[9        [,        R:                  " U5      5      eS =n
=n=n=n=n=nn[         R<                  o¤R>                  oÝR                  oîR                  oòR                  nUR                  nU
" UU5      nU(       Gd¡  SS[0        R2                  " 5       ;   d  [,        R4                  " [         5      (       a  [,        R6                  " [         5      OS[,        R6                  " U
5      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U5      [,        R6                  " U5      [,        R6                  " U5      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U5      [,        R6                  " U5      [,        R6                  " U5      S.
-  n[9        [,        R:                  " U5      5      eS =n
=n=n=n=n=nnUR@                  n
0 oÚU:H  nU(       dº  [,        R.                  " SU4SX­45      S[0        R2                  " 5       ;   d  [,        R4                  " U5      (       a  [,        R6                  " U5      OS[,        R6                  " U
5      [,        R6                  " U5      S.-  nSSU0-  n[9        [,        R:                  " U5      5      eS =n
=nnS  UR#                  5        5       n
[C        U
5      nU(       d¢  S!S"[0        R2                  " 5       ;   d  [,        R4                  " [B        5      (       a  [,        R6                  " [B        5      OS"[,        R6                  " U
5      [,        R6                  " U5      S#.-  n[9        [,        R:                  " U5      5      eS =n
nS$ UR>                  R#                  5        5       n
[C        U
5      nU(       d¢  S!S"[0        R2                  " 5       ;   d  [,        R4                  " [B        5      (       a  [,        R6                  " [B        5      OS"[,        R6                  " U
5      [,        R6                  " U5      S#.-  n[9        [,        R:                  " U5      5      eS =n
ng ! , (       d  f       GN= fs  snf )%Nr   r   rÀ   g      ô?g      à¿zf12.pthr2   ÚsentinelrÏ   )ÚstateÚparam_groups)Úepochrè   Ú
classifierÚ	optimizerg-Cëâ6?)Úlr)r�   rs   )z%(py0)s == %(py3)sr?  )rI   rK   úassert %(py5)srx   zçassert %(py16)s
{%(py16)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.head2
}.crackle
}.weight
}, %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.crackle
}.weight
})
}rE   r�   rà   )
rI   rJ   rK   rx   ry   r{   rÜ   r)  r*  r+  záassert %(py16)s
{%(py16)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.head2
}.wheeze
}.bias
}, %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.wheeze
}.bias
})
})z-%(py2)s
{%(py2)s = %(py0)s.state
} == %(py5)srA  )rI   rJ   rx   zassert %(py7)sry   c              3   óB   #   • U  H  oR                   (       + v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   ÚItest_stage2_checkpoint_copies_head_and_skips_optimizer.<locals>.<genexpr>ý   r  rÈ   rÉ   rÊ   rË   c              3   ó8   #   • U  H  oR                   v •  M     g 7fr   rÂ   rÄ   s     r&   rÆ   rE  þ   r×   rÒ   )"rE   r|   r   r   r@   r   râ   r^   rç   rå   Úfill_ÚwheezeÚbiasÚsaveÚ
state_dictÚoptimÚAdamr_   rÃ   ÚstrÚresumer   rY   r€   rW   rX   rZ   r[   r\   r]   rV   rƒ   r=  rÊ   )Útmp_pathrè   rà   ré   r�   Ú	ckpt_pathri   rA  r?  r†   rf   Ú@py_format4Ú@py_format6rˆ   r‡   r•   r3  r4  r5  Ú@py_format17r‹   r‰   rê   s                          r&   Ú6test_stage2_checkpoint_copies_head_and_skips_optimizerrU  å   s	  € Ü	×Ò�aÔÜ‹N€EÜ˜Ó€DÜ‹7€DÜ˜E¨¬u¯|ª|¸EÓ/BÓC€GÜ	�Š�Ø�‰×Ñ×!Ñ! $Ô'Ø�‰×Ñ×Ñ˜tÔ$÷ 
ð ˜9Ñ$€IÜ	‡J‚JØØ×!Ñ!Ó#Ø—o‘oÓ'Ø ! J° ?Ð3ÀRÑHñ	ð
 ôô —‘× Ñ Ø#×.Ñ.Ô0ÓHÒ0�5×4GÕ4G�Ñ0ÑHÈTð !ð S€Iä�i“.€D„KÜ˜D¨¼%¿,º,ÀuÓ:MÐW^Ñ_€EØÐ�A‰:×××Òˆ5×××××Õˆ5×××Ñˆ5××Õ�A××××××ÑÜ�>‰>ÐLŸ-™-ÐL×/Ñ/ÐL×6Ñ6ÐL¿¹ÐL¸×8KÑ8KÐL‰>Ð6Ð8KÓLÐL×L×L×L×L×L×LÑLŒ5×L×L×LÑLŒ5×L×LÕLˆ>×L×L×L×L×LÔL˜'×L×L×LÑL˜'×L×LÕL˜-×L×LÓLÐ/×L×LÓLÐ6×L×L×L×L×LÔL¸×L×L×LÑL¸×L×LÕL¸×L×LÓLÐ8K×L×LÓLÐL×L×L×L×L×L×LÕLÜ�>‰>ÐFŸ-™-ÐF×.Ñ.ÐF×3Ñ3ÐF·[±[ÐF°[×5EÑ5EÐF‰>Ð3Ð5EÓFÐF×F×F×F×F×F×FÑFŒ5×F×F×FÑFŒ5×F×FÕFˆ>×F×F×F×F×FÔF˜'×F×F×FÑF˜'×F×FÕF˜-×F×FÓFÐ.×F×FÓFÐ3×F×F×F×F×FÔF°T×F×F×FÑF°T×F×FÕF°[×F×FÓFÐ5E×F×FÓFÐF×F×F×F×F×F×FÕFØ�?‰?Ð ˜bÐ ˜bÑ × × × Ó ˆ?× × × × × Õ ˆ9× × × Ñ ˆ9× × Õ ˆ?× × Ó ˜b× × × × × × Ô ÙG°E×4DÑ4DÔ4FÓGÐGŒ3ÐGÓGÐGÖG×G×G×G×G×GÑGŒ3×G×G×GÑGŒ3×G×GÕGÐG×G×GÓGÐG×G×G×G×G×GÓGÙK°·±×0HÑ0HÔ0JÓKÐKŒ3ÐKÓKÐKÖK×K×K×K×K×KÑKŒ3×K×K×KÑKŒ3×K×KÕKÐK×K×KÓKÐK×K×K×K×K×KÔK÷' 
Žüò 	Is   Á,AcÄ.cÅcã
cc            	      ór  ^^• [         R                  " S5        [        SSSS9n U R                  5         [         R                  " SSS5      n[         R                  " SS5      n[         R                  " SS5      n[         R                  " SSS5      n[         R                  " SSS5      nU " XX4S9u  pgU " XX5S9u  p‡[         R
                  o™" Xh5      oª(       + o»(       Gd8  S	S
[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS
[        R                  " U	5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U
5      S.-  n[        [        R                  " U5      5      eS =n	=p«[        SSSSS9nUR                  5         [        S5      n[        S5      n0 mUR                  mUU4S jnUUl        U" XXôS9  [         R
                  n	TS   nSnU	" UUUS9nU(       Gd  SS
[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS
[        R                  " U	5      [        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =n	=n=nng )Nr   r   r2   rn   ©r$   rd   Úkey_dropr3   rC   )Úslot_framesúWassert not %(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py3)s, %(py4)s)
}rE   Úslots_aÚslots_brH   rÏ   ro   c                 óV   >• U R                  5       R                  5       TS'   T" U 5      $ ©NrG   ©ÚdetachÚclone)rë   ÚoriginalÚseens    €€r&   Ú_captureÚLtest_slot_frames_change_the_keys_and_not_the_residual_init.<locals>._capture  ó&   ø€ ØŸ™›×.Ñ.Ó0ˆˆX‰Ù˜ÓÐr(   rG   rN   rO   úaassert %(py9)s
{%(py9)s = %(py2)s
{%(py2)s = %(py0)s.allclose
}(%(py4)s, %(py5)s, atol=%(py7)s)
}©rI   rJ   rL   rx   ry   r{   )rE   r|   r   r}   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r   r   Úforward)ÚfeedbackrG   r™   rž   Úkeys_aÚkeys_br[  re   r\  rf   rg   rj   r‰   r�   r‚   rƒ   rd  r‹   r‡   r•   rŠ   rb  rc  s                        @@r&   Ú:test_slot_frames_change_the_keys_and_not_the_residual_initrm    sH  ù€ Ü	×Ò�aÔÜ ¨!°cÑ:€HØ‡M�M„OÜ�[Š[˜˜A˜rÓ"€FÜ�kŠk˜!˜QÓ€GÜ�kŠk˜!˜QÓ€GÜ�[Š[˜˜A˜rÓ"€FÜ�[Š[˜˜A˜rÓ"€FÙ˜&¨7ÑG�J€GÙ˜&¨7ÑG�J€GÜ�~‰~Ð/ˆ~˜gÓ/Ð/Ô/Ð/Ö/×/×/×/×/×/Ñ/Œu×/×/×/Ñ/Œu×/×/Õ/ˆ~×/×/×/×/×/Ô/˜g×/×/×/Ñ/˜g×/×/×/×/×/Ö/ w×/×/×/Ñ/ w×/×/Õ/Ð/×/×/×/×/×/Ô/ä "¨A°aÀÑD€GØ‡L�L„NÜ˜Ó€EÜ˜Ó€EØ€DØ�}‰}€Hö ð €E„MÙˆF˜5Ò5Ü�>‰>Ð<˜$˜x™.Ð<°tÐ<‰>˜.¨&°tÑ<Ð<×<×<×<×<×<×<Ñ<Œ5×<×<×<Ñ<Œ5×<×<Õ<ˆ>×<×<Ó<˜.×<×<×<×<×<Ô<¨&×<×<×<Ñ<¨&×<×<Õ<°t×<×<Ó<Ð<×<×<×<×<×<×<Ð<r(   c            
      óL  • [         R                  " S5        [        SSSS9n U R                  5         [         R                  oR
                  o"R                  o1" U5      nSoTU:H  of(       GdL  [        R                  " SU4SXE45      S[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      S	[        R                  " 5       ;   d  [        R                  " U 5      (       a  [        R                  " U 5      OS	[        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S
.-  nSSU0-  n[        [        R                  " U5      5      eS =n=n=n=n=pe[         R                  " SSS5      n	[         R                  " SS5      n
[         R                  " SS5      n[         R                   " SS5      nU " XšX¼S9u  pÞ[         R"                  " 5          U R
                  R                  R%                  S5        S S S 5        U " XšX¼S9u  pþ[         R&                  o" Xß5      nU(       + nU(       Gd8  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      S.-  n[        [        R                  " U5      5      eS =n=nn[         R"                  " 5          U R
                  R                  R)                  5         S S S 5        U " XšX¼S9u  nn[         R&                  o" UU5      nU(       Gd8  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      S.-  n[        [        R                  " U5      5      eS =nng ! , (       d  f       GN%= f! , (       d  f       GNˆ= f)Nr   r   r2   rn   rW  rs   )zŒ%(py9)s
{%(py9)s = %(py2)s
{%(py2)s = %(py0)s.count_nonzero
}(%(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.timbre
}.weight
})
} == %(py12)srE   rj  )rI   rJ   rK   rx   ry   r{   r)  úassert %(py14)sr*  r3   rC   r�   rò   rZ  rd   ÚshiftedrH   rD   Úrestored)rE   r|   r   r}   r”   Útimbrerå   rY   r€   rW   rX   rZ   r[   r\   r]   rU   Úrandr^   rG  rV   Úzero_)rj  rf   rˆ   r‡   r•   r3  r¨   Ú@py_format13Ú@py_format15rG   r™   rž   rr   rd   re   rp  rg   rj   r‰   rq  rh   s                        r&   Ú,test_zero_init_timbre_leaves_slots_unchangedrw    s  € Ü	×Ò�aÔÜ ¨!°cÑ:€HØ‡M�M„OÜ×ÑÐ;Ÿ™Ð;×5Ñ5Ð;ÐÐ5Ó6Ð;¸!Ð;¸!Ñ;×;×;×;Ó;Ð6×;×;×;×;×;Õ;Œ5×;×;×;Ñ;Œ5×;×;Õ;Ð×;×;×;×;×;Ô;˜x×;×;×;Ñ;˜x×;×;Õ;˜×;×;Ó;Ð5×;×;Ó;Ð6×;×;Ó;¸!×;×;×;×;×;×;×;Ñ;Ü�[Š[˜˜A˜rÓ"€FÜ�kŠk˜!˜QÓ€GÜ�kŠk˜!˜QÓ€GÜ�ZŠZ˜˜1Ó€FÙ˜¨Ñ@�H€EÜ	�Š�Ø�‰×Ñ×$Ñ$ SÔ)÷ 
á˜&¨7ÑB�J€GÜ�~‰~Ð-ˆ~˜eÓ-Ð-Ð-Ô-Ð-×-×-×-×-×-×-Ñ-Œu×-×-×-Ñ-Œu×-×-Õ-ˆ~×-×-×-×-×-Ô-˜e×-×-×-Ñ-˜e×-×-×-×-×-Ö- W×-×-×-Ñ- W×-×-Õ-Ð-×-×-×-×-×-Õ-Ü	�Š�Ø�‰×Ñ×$Ñ$Ô&÷ 
á˜6¨GÑC�K€HˆaÜ�>‰>Ð*ˆ>˜% Ó*Ð*×*×*×*×*×*×*Ñ*Œ5×*×*×*Ñ*Œ5×*×*Õ*ˆ>×*×*×*×*×*Ô*˜%×*×*×*Ñ*˜%×*×*×*×*×*Ö* ×*×*×*Ñ* ×*×*Õ*Ð*×*×*×*×*×*Ô*÷ 
Žú÷ 
Žús   È>&VÏ.%VÖ
VÖ
V#c                  ó‚  • [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        XU5      n/ nSn[        U:H  ofov(       a  Sn[        U:H  o™nU(       Gd�  [
        R                  " SU4S[        U45      S[        R                  " 5       ;   d  [
        R                  " [        5      (       a  [
        R                  " [        5      OS[
        R                  " U5      S	.-  n
S
SU
0-  o´R                  U5        U(       a£  [
        R                  " SW	4S[        W45      S[        R                  " 5       ;   d  [
        R                  " [        5      (       a  [
        R                  " [        5      OS[
        R                  " U5      S.-  nSSU0-  oÔR                  U5        [
        R                  " US5      0 -  nSSU0-  n[        [
        R                  " U5      5      eS =n=n=n=n=p˜US   owR                  nU" 5       oU[         :H  nU(       dÜ  [
        R                  " SU4SU[         45      [
        R                  " U5      [
        R                  " U5      [
        R                  " U5      S[        R                  " 5       ;   d  [
        R                  " [         5      (       a  [
        R                  " [         5      OSS.-  nSSU0-  n[        [
        R                  " U5      5      eS =n=n=nnUS   osS   ogU:„  nU(       do  [
        R                  " SU4SXv45      [
        R                  " U5      [
        R                  " U5      S.-  nSSU0-  n[        [
        R                  " U5      5      eS =n=nnUR"                  oD" 5       n/ SQnUU:H  nU(       dÐ  [
        R                  " SU4S UU45      S![        R                  " 5       ;   d  [
        R                  " U5      (       a  [
        R                  " U5      OS![
        R                  " U5      [
        R                  " U5      [
        R                  " U5      S".-  nSSU0-  n[        [
        R                  " U5      5      eS =n=n=nn[        XUS#S#S$9nUS%   owR                  nU" 5       nS&nUU:H  nU(       dš  [
        R                  " SU4S'UU45      [
        R                  " U5      [
        R                  " U5      [
        R                  " U5      [
        R                  " U5      S(.-  nS)S*U0-  n[        [
        R                  " U5      5      eS =n=n=n=nnUS%   owR                  nU" 5       nS+nUU:H  nU(       dš  [
        R                  " SU4S'UU45      [
        R                  " U5      [
        R                  " U5      [
        R                  " U5      [
        R                  " U5      S(.-  nS)S*U0-  n[        [
        R                  " U5      5      eS =n=n=n=nng ),N)r6   r   rÏ   r3   )ç      à?rò   g333333ã?r4   )ry  r4   rò   gš™™™™™Ù?g…ëQ¸í?g      è?rs   )z%(py2)s == %(py5)sr
   )rJ   rx   z%(py7)sry   )z%(py9)s == %(py12)sr   )r{   r)  z%(py14)sr*  r   zassert %(py17)sÚpy17)zC%(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.item
}()
} == %(py7)sr   rv   rz   r{   rÏ   r'  )z%(py1)s > %(py4)s)rw   rL   zassert %(py6)srM   )ç      @rŽ   ç       @g      ø?rô   rå   rö   ry  )Útau_cÚtau_wr3   rŽ   )zC%(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.item
}()
} == %(py8)srÚ   rÛ   rÜ   r|  )rE   rø   r   r
   r   rY   r€   rW   rX   rZ   r[   ÚappendÚ_format_boolopr\   r]   Úitemr   rù   )rû   Úp_cÚp_wrå   rf   rˆ   r‹   r…   r3  r¨   rS  r‰   ru  rv  Ú@py_format16Ú@py_format18r†   r‡   rŠ   rê   rh   rg   Úat_halfrj   rk   rí   s                             r&   Ú=test_error_weights_use_stage1_thresholds_and_rank_missed_bothr‡  2  ss  € Ü�\Š\š,Ó'€FÜ
�,Š,Ò/Ó
0€CÜ
�,Š,Ò/Ó
0€CÜ! &¨sÓ3€FÙ0�tÐ0Œ8�tÑ×0¨DÐ0¤¨DÑ 0×0×0×0Õ0Œ8�t×0×0×0×0×0Õ0Œ8×0×0×0Ñ0Œ8×0×0Õ0�t×0×0×0×0×0×0×0Ò0¤¨D×0×0×0×0×0Õ0¤×0×0×0Ñ0¤×0×0Õ0¨D×0×0×0×0×0×0×0×0×0×0×0×0×0Ò0Ø�!‰9Ð,�>‰>Ð,‰>ÓÐ,œ}Ñ,×,×,×,Ó,Ðœ}×,×,Ô,ˆ9×,×,Ó,ˆ>×,×,Ó,Ð×,×,×,×,×,Ô,œ}×,×,×,Ñ,œ}×,×,×,×,×,×,×,Ð,Ø�!‰9Ð ˜a‘yÐ �yÑ × × × Ó ˆ9× × Ô ˆ9× × Ó �y× × × × × × Ô Ø�=‰=Ð2ˆ=‹?Ð2Ò2Ð2ˆ?Ð2Ñ2×2×2×2Ó2ˆ?Ð2×2×2×2×2×2Õ2ˆ6×2×2×2Ñ2ˆ6×2×2Õ2ˆ=×2×2Ó2ˆ?×2×2Ó2Ð2×2×2×2×2×2×2Ö2Ü" 6°¸3ÀcÑJ€GØ�1‰:Ð#�?‰?Ð#‰?ÓÐ# Ð#Ð Ñ#×#×#×#Ó#Ð ×#×#Ô#ˆ:×#×#Ó#ˆ?×#×#Ó#Ð×#×#Ó# ×#×#×#×#×#×#×#Ð#Ø�!‰9Ð"�>‰>Ð"‰>ÓÐ"˜sÐ"Ð˜sÑ"×"×"×"Ó"Ð˜s×"×"Ô"ˆ9×"×"Ó"ˆ>×"×"Ó"Ð×"×"Ó"˜s×"×"×"×"×"×"×"Ñ"r(   c            
      ó
  • [         R                  " S5        [        SSSSS9n [         R                  " SS5      n[         R                  " SS5      n[         R                  " / S	Q5      n[         R                  " / S
Q5      nU " XX45      u  pVU " XX4[         R
                  " S5      S9u  pv[         R                  oˆ" XW5      o™(       Gd8  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U	5      S.-  n
[        [        R                  " U
5      5      eS =p‰U " XX4[         R                  " / SQ5      S9u  p¶[         R                  oˆ" X[5      o™(       + oÌ(       Gd8  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U	5      S.-  n[        [        R                  " U5      5      eS =n=pœg )Nr   gìQ¸…ëù?gR¸…ëQ@Útopkr6   )Úpos_weight_crackleÚpos_weight_wheezeÚmil_poolr;   r2   é   )rŽ   rn   rŽ   rn   )rn   rŽ   rŽ   rn   )Úsample_weightrD   rE   r  ÚweightedrH   )r{  rŽ   rŽ   rŽ   rZ  Úboosted)rE   r|   r   rU   rø   r~   rV   rW   rX   rY   rZ   r[   r\   r]   )Ú	criterionr™   rž   Úy_cÚy_wr  re   r�  rf   rg   rh   r�  rj   r‰   s                 r&   Ú.test_sample_weight_ones_matches_unweighted_bcer”  @  só  € Ü	×Ò�aÔÜ°ÈÐW]ÐhiÑj€IÜ�kŠk˜!˜QÓ€GÜ�kŠk˜!˜QÓ€GÜ
�,Š,Ò+Ó
,€CÜ
�,Š,Ò+Ó
,€CÙ˜¨3Ó4�H€EÙ˜G¨cÄeÇjÂjÐQRÃmÑT�K€HÜ�>‰>Ð*ˆ>˜%Ó*Ð*Ö*×*×*×*×*×*Ñ*Œ5×*×*×*Ñ*Œ5×*×*Õ*ˆ>×*×*×*×*×*Ô*˜%×*×*×*Ñ*˜%×*×*×*×*×*Ö* ×*×*×*Ñ* ×*×*Õ*Ð*×*×*×*×*×*Ò*Ù˜7¨SÄUÇ\Â\ÒRfÓEgÑh�J€GÜ�~‰~Ð-ˆ~˜eÓ-Ð-Ô-Ð-Ö-×-×-×-×-×-Ñ-Œu×-×-×-Ñ-Œu×-×-Õ-ˆ~×-×-×-×-×-Ô-˜e×-×-×-Ñ-˜e×-×-×-×-×-Ö- W×-×-×-Ñ- W×-×-Õ-Ð-×-×-×-×-×-Õ-r(   c            	      óä  ^^• [         R                  " S5        [        SSSSS9n U R                  5         SU R                  l        [        S5      n[        S5      n0 mUR                  mUU4S jnX2l        [         R                  " SS	S5      nU " XAU[         R                  " SS	5      S
9  [         R                  nTS   nSou" XdUS9oˆ(       Gd  SS[        R                  " 5       ;   d  [        R                  " [         5      (       a  [        R                  " [         5      OS[        R                  " U5      [        R                  " U5      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  n	[!        [        R"                  " U	5      5      eS =n=n=pxTS   oªR$                  o»" 5       oÌR&                  ow" 5       nSoØU:„  oî(       dÃ  [        R(                  " SU4SX�45      [        R                  " U
5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  nSSU0-  n[!        [        R"                  " U5      5      eS =n
=n=n=n=n=píg )Nr   r   r2   r3   rn   ro   rŽ   c                 óV   >• U R                  5       R                  5       TS'   T" U 5      $ r^  r_  )rG   rb  rc  s    €€r&   rd  ÚBtest_key_dropout_does_not_zero_frames_into_head2.<locals>._captureX  rf  r(   rC   r�   rG   rN   rO   rg  rE   rh  r'  )zr%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.abs
}()
}.sum
}()
} > %(py12)s)rw   rK   rx   ry   r{   r)  ro  r*  )rE   r|   r   r“   Úself_feedbackrX  r   ri  rU   rs  rV   rW   rX   rY   rZ   r[   r\   r]   r1  rã   r€   )r�   r‚   rƒ   rd  rG   rf   r‹   r‡   r•   rŠ   r…   r†   rˆ   r3  r¨   ru  rv  rb  rc  s                    @@r&   Ú0test_key_dropout_does_not_zero_frames_into_head2r™  N  sÞ  ù€ Ü	×Ò�aÔÜ "¨A°aÀÑD€GØ‡M�M„OØ%(€G×ÑÔ"Ü˜Ó€EÜ˜Ó€EØ€DØ�}‰}€Hö ð „MÜ�[Š[˜˜A˜rÓ"€FÙˆF˜5¬¯ª°A°qÓ)9Ò:Ü�>‰>Ð<˜$˜x™.Ð<°tÐ<ˆ>˜.°tÑ<Ð<Ö<×<×<×<×<×<Ñ<Œ5×<×<×<Ñ<Œ5×<×<Õ<ˆ>×<×<Ó<˜.×<×<×<×<×<Ô<¨&×<×<×<Ñ<¨&×<×<Õ<°t×<×<Ó<Ð<×<×<×<×<×<Ö<Ø�‰>Ð)×ÑÐ)ÐÓÐ)×#Ñ#Ð)Ð#Ó%Ð)¨Ð)¨Ñ)×)×)×)Ò)Ð%×)×)Ô)ˆ>×)×)Ó)Ð×)×)Ó)Ð×)×)Ó)Ð#×)×)Ó)Ð%×)×)Ó)¨×)×)×)×)×)×)×)Ô)r(   c                 óú   • [         R                  " 5          S HF  n[        X5      nUR                  R	                  SS5        UR
                  R	                  SS5        MH     S S S 5        g ! , (       d  f       g = f)N)ÚqÚkÚvr„   r   gš™™™™™©?)rE   r^   Úgetattrrå   Únormal_rI  )ÚmoduleÚnameÚlinears      r&   Ú_fill_attentionr£  c  sQ   € Ü	�Š�Û*ˆDÜ˜VÓ*ˆFØ�M‰M×!Ñ! ! TÔ*Ø�K‰K×Ñ  4Ö(ñ +÷ 
�Žús   –AA,Á,
A:c                 ó(  • U R                  5       R                  5        VVs0 s H#  u  pXR                  5       R                  5       _M%     nnn[        R
                  " UR                  S5      UR                  S5      UR                  S5      /SS9n[        R
                  " UR                  S5      UR                  S5      UR                  S5      /SS9nXCS	'   XSS
'   UR                  S5      US'   UR                  S5      US'   US	 U$ s  snnf )zNState dict shaped like feedback_s2/best.pth: packed attn, no timbre, no q/k/v.úself_feedback.q.weightzself_feedback.k.weightzself_feedback.v.weightr   )r$   zself_feedback.q.biaszself_feedback.k.biaszself_feedback.v.biasú!self_feedback.attn.in_proj_weightzself_feedback.attn.in_proj_biaszself_feedback.out.weightú"self_feedback.attn.out_proj.weightzself_feedback.out.biasz self_feedback.attn.out_proj.biaszself_feedback.timbre.weight)rK  Úitemsr`  ra  rE   ÚcatÚpop)r�   ÚkeyÚvaluer=  Úin_wÚin_bs         r&   Ú_legacy_blobr¯  k  s  € à;B×;MÑ;MÓ;O×;UÑ;UÔ;WÔXÒ;W©Z¨SˆS—,‘,“.×&Ñ&Ó(Ò(Ñ;W€EÑXÜ�9Š9Ø�	‰	Ð*Ó+Ø�	‰	Ð*Ó+Ø�	‰	Ð*Ó+ðð ñ	€Dô
 �9Š9Ø�	‰	Ð(Ó)Ø�	‰	Ð(Ó)Ø�	‰	Ð(Ó)ðð ñ	€Dð
 26Ð
-Ñ.Ø/3Ð
+Ñ,Ø27·)±)Ð<VÓ2W€EÐ
.Ñ/Ø05·	±	Ð:RÓ0S€EÐ
,Ñ-ØÐ+Ð,Ø€Lùó! Ys   ¢*Dc                  ó*'  • [         R                  " S5        Su  p[        U SSUSS9n[        U5        SUl        [         R
                  " SSU 5      n[         R
                  " SSU 5      n[        R                  " XS	S
9n[         R                  " 5          UR                  R                  [         R                  " UR                  R                  UR                  R                  UR                  R                  /S5      5        UR                   R                  [         R                  " UR                  R"                  UR                  R"                  UR                  R"                  /S5      5        UR$                  R                  R                  UR&                  R                  5        UR$                  R"                  R                  UR&                  R"                  5        S S S 5        UR)                  5         UR)                  5         UR+                  X4S 5      nU" X4USS9u  px[         R,                  n	So©" XgU
S9o»(       GdM  SS[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OSS[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS[2        R6                  " U
5      [2        R6                  " U5      S.-  n[9        [2        R:                  " U5      5      eS =n	=p«[=        U SSSSSUS9n[?        U 5      Ul         [        URB                  5        [E        U5      nSoÿU;   nU(       d¥  [2        RF                  " SU4SXþ45      [2        R6                  " U5      S[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OSS.-  nSSU0-  n[9        [2        R:                  " U5      5      eS =nnSoÿU;  nU(       d¥  [2        RF                  " SU4SXþ45      [2        R6                  " U5      S[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OSS.-  nSSU0-  n[9        [2        R:                  " U5      5      eS =nn[=        U SSSSSUS9n[?        U 5      Ul         [I        UU5        [         RJ                  n	URB                  nUR                  nUR                  nURB                  nUR                  nUR                  nU	" UU5      nU(       Gd¶  S S[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S"[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS"[2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S#.-  n[9        [2        R:                  " U5      5      eS =n	=n=n=n=n=n=nn[         RJ                  n	URB                  nUR                  nUR                  nURB                  nUR                  nUR                  nU	" UU5      nU(       Gd¶  S$S[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S"[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS"[2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S#.-  n[9        [2        R:                  " U5      5      eS =n	=n=n=n=n=n=nn[         RJ                  n	URB                  nUR                  nUR                  nURB                  nUR                  nUR                  nU	" UU5      nU(       Gd¶  S%S[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S"[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS"[2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S#.-  n[9        [2        R:                  " U5      5      eS =n	=n=n=n=n=n=nn[         RJ                  n	URB                  nUR&                  nUR                  nURB                  nUR&                  nUR                  nU	" UU5      nU(       Gd¶  S&S[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S"[.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS"[2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      S#.-  n[9        [2        R:                  " U5      5      eS =n	=n=n=n=n=n=nnURB                  o™R                  nSnUUL oª(       dÐ  [2        RF                  " S'U
4S(UU45      S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U	5      [2        R6                  " U5      [2        R6                  " U5      S).-  nS*S+U0-  n[9        [2        R:                  " U5      5      eS =n	=n=n
nURB                  nURL                  nUR                  nURN                  nU" 5       nURP                  n U " 5       n![S        U!5      n"Sn#U"U#:H  n$U$(       Gd¡  [2        RF                  " S,U$4S-U"U#45      S.[.        R0                  " 5       ;   d  [2        R4                  " [R        5      (       a  [2        R6                  " [R        5      OS.S![.        R0                  " 5       ;   d  [2        R4                  " U5      (       a  [2        R6                  " U5      OS![2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U5      [2        R6                  " U 5      [2        R6                  " U!5      [2        R6                  " U"5      [2        R6                  " U#5      S/.-  n%S0S1U%0-  n&[9        [2        R:                  " U&5      5      eS =n=n=n=n=n=n =n!=n"=n$n#UR)                  5         [         R
                  " SS2U 5      n'[         R
                  " SS25      n([         R
                  " SS25      n)URC                  U'U(U)5      u  n*n[         R                  " 5          URB                  R                  R                  RU                  5         S S S 5        URC                  U'U(U)5      u  n+n[         R,                  o™" U*U+5      oª(       + o»(       Gd8  S3S[.        R0                  " 5       ;   d  [2        R4                  " [         5      (       a  [2        R6                  " [         5      OS[2        R6                  " U	5      S4[.        R0                  " 5       ;   d  [2        R4                  " U*5      (       a  [2        R6                  " U*5      OS4S5[.        R0                  " 5       ;   d  [2        R4                  " U+5      (       a  [2        R6                  " U+5      OS5[2        R6                  " U
5      S6.-  n[9        [2        R:                  " U5      5      eS =n	=p«g ! , (       d  f       GN“= f! , (       d  f       GNš= f)7Nr   )r   r2   r3   r6   rn   )rd   r‰  ÚheadsrX  FrC   T)Úbatch_first)Úneed_weightsrN   rO   rQ   rE   ÚdeltaÚmha_outrS   rÏ   r5   )rd   rB   rp   r:   r;   r±  r¦  )Úin)z%(py1)s in %(py3)sÚblob)rw   rK   rC  rx   r¥  )únot in)z%(py1)s not in %(py3)sa  assert %(py18)s
{%(py18)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.q
}.weight
}, %(py16)s
{%(py16)s = %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.self_feedback
}.q
}.weight
})
}Úfreshr�   )rI   rJ   rK   rx   ry   r{   rÜ   r)  r*  r+  Úpy18a  assert %(py18)s
{%(py18)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.k
}.weight
}, %(py16)s
{%(py16)s = %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.self_feedback
}.k
}.weight
})
}a  assert %(py18)s
{%(py18)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.v
}.weight
}, %(py16)s
{%(py16)s = %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.self_feedback
}.v
}.weight
})
}a	  assert %(py18)s
{%(py18)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.out
}.weight
}, %(py16)s
{%(py16)s = %(py14)s
{%(py14)s = %(py12)s
{%(py12)s = %(py10)s.self_feedback
}.out
}.weight
})
}rØ   ©zU%(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.self_feedback
}.use_timbre
} is %(py7)srö   rz   r{   rs   )zô%(py17)s
{%(py17)s = %(py0)s(%(py15)s
{%(py15)s = %(py13)s
{%(py13)s = %(py11)s
{%(py11)s = %(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s
{%(py3)s = %(py1)s.self_feedback
}.timbre
}.weight
}.abs
}()
}.sum
}()
})
} == %(py20)sÚfloat)rI   rw   rK   rx   ry   r{   r’   r¦   Úpy15rz  Úpy20zassert %(py22)sÚpy22r�  rZ  rd   Ú
slots_zerorH   )+rE   r|   r   r£  Ú
use_timbrerU   r    ÚMultiheadAttentionr^   Úin_proj_weightÚcopy_r©  r›  rå   rœ  r�  Úin_proj_biasrI  Úout_projr„   r}   Ú_attendrV   rW   rX   rY   rZ   r[   r\   r]   r   r   rƒ   r˜  r¯  r€   r   rÿ   rr  r1  rã   r¼  rt  ),r$   r±  ÚsrcÚqueryÚkeysÚmhar´  rµ  re   rf   rg   rj   rk   r�   r·  r…   r†   rR  rS  r¹  rˆ   r‡   r•   r3  r4  r5  r7  Ú@py_format19r‹   r‰   rŠ   r¨   Ú@py_assert12Ú@py_assert14Ú@py_assert16Ú@py_assert19r6  Ú@py_format21Ú@py_format23rG   r™   rž   rd   rÀ  s,                                               r&   ÚBtest_legacy_in_proj_matches_multihead_attention_and_is_not_droppedrÓ  €  sZ  € Ü	×Ò�aÔØ�J€CÜ
�s !¨!°5À3Ñ
G€CÜ�CÔØ€C„NÜ�KŠK˜˜1˜cÓ"€EÜ�;Š;�q˜!˜SÓ!€DÜ
×
Ò
 ¸Ñ
=€CÜ	�Š�Ø×Ñ× Ñ ¤§¢¨C¯E©E¯L©L¸#¿%¹%¿,¹,ÈÏÉÏÉÐ+UÐWXÓ!YÔZØ×Ñ×ÑœuŸyšy¨#¯%©%¯*©*°c·e±e·j±jÀ#Ç%Á%Ç*Á*Ð)MÈqÓQÔRØ�‰×Ñ×!Ñ! #§'¡'§.¡.Ô1Ø�‰×Ñ×Ñ §¡§¡Ô-÷	 
ð
 ‡H�H„JØ‡H�H„JØ�K‰K˜ TÓ*€EÙ�U $°UÑ;�J€GÜ�>‰>Ð4¨tÐ4ˆ>˜%¨tÑ4Ð4Ö4×4×4×4×4×4Ñ4Œ5×4×4×4Ñ4Œ5×4×4Õ4ˆ>×4×4×4×4×4Ô4˜%×4×4×4Ñ4˜%×4×4×4×4×4Ö4 ×4×4×4Ñ4 ×4×4Õ4¨t×4×4Ó4Ð4×4×4×4×4×4Ô4ä˜c¨°1¸SÈrÐ\]ÐejÑk€GÜ  Ó%€G„MÜ�G×)Ñ)Ô*Ü˜Ó €DØ.Ð6°$Ñ6×6×6×6Ó6Ð.×6×6Ô6Ð.×6×6×6×6×6Ô6°$×6×6×6Ñ6°$×6×6×6×6×6×6Ô6Ø#Ð/¨4Ñ/×/×/×/Ó/Ð#×/×/Ô/Ð#×/×/×/×/×/Ô/¨4×/×/×/Ñ/¨4×/×/×/×/×/×/Ô/Ü˜C q°¸3ÈRÐZ[ÐchÑi€EÜ˜sÓ#€E„KÜ˜U DÔ)Ü�;‰;ÐT�u×*Ñ*ÐTÐ*×,Ñ,ÐTÐ,×3Ñ3ÐT°W×5JÑ5JÐTÐ5J×5LÑ5LÐTÐ5L×5SÑ5SÐT‰;Ð3Ð5SÓTÐT×T×T×T×T×T×TÑTŒ5×T×T×TÑTŒ5×T×TÕTˆ;×T×T×T×T×TÔT�u×T×T×TÑT�u×T×TÕTÐ*×T×TÓTÐ,×T×TÓTÐ3×T×T×T×T×TÔT°W×T×T×TÑT°W×T×TÕTÐ5J×T×TÓTÐ5L×T×TÓTÐ5S×T×TÓTÐT×T×T×T×T×T×T×TÜ�;‰;ÐT�u×*Ñ*ÐTÐ*×,Ñ,ÐTÐ,×3Ñ3ÐT°W×5JÑ5JÐTÐ5J×5LÑ5LÐTÐ5L×5SÑ5SÐT‰;Ð3Ð5SÓTÐT×T×T×T×T×T×TÑTŒ5×T×T×TÑTŒ5×T×TÕTˆ;×T×T×T×T×TÔT�u×T×T×TÑT�u×T×TÕTÐ*×T×TÓTÐ,×T×TÓTÐ3×T×T×T×T×TÔT°W×T×T×TÑT°W×T×TÕTÐ5J×T×TÓTÐ5L×T×TÓTÐ5S×T×TÓTÐT×T×T×T×T×T×T×TÜ�;‰;ÐT�u×*Ñ*ÐTÐ*×,Ñ,ÐTÐ,×3Ñ3ÐT°W×5JÑ5JÐTÐ5J×5LÑ5LÐTÐ5L×5SÑ5SÐT‰;Ð3Ð5SÓTÐT×T×T×T×T×T×TÑTŒ5×T×T×TÑTŒ5×T×TÕTˆ;×T×T×T×T×TÔT�u×T×T×TÑT�u×T×TÕTÐ*×T×TÓTÐ,×T×TÓTÐ3×T×T×T×T×TÔT°W×T×T×TÑT°W×T×TÕTÐ5J×T×TÓTÐ5L×T×TÓTÐ5S×T×TÓTÐT×T×T×T×T×T×T×TÜ�;‰;ÐX�u×*Ñ*ÐXÐ*×.Ñ.ÐXÐ.×5Ñ5ÐX°w×7LÑ7LÐXÐ7L×7PÑ7PÐXÐ7P×7WÑ7WÐX‰;Ð5Ð7WÓXÐX×X×X×X×X×X×XÑXŒ5×X×X×XÑXŒ5×X×XÕXˆ;×X×X×X×X×XÔX�u×X×X×XÑX�u×X×XÕXÐ*×X×XÓXÐ.×X×XÓXÐ5×X×X×X×X×XÔX°w×X×X×XÑX°w×X×XÕXÐ7L×X×XÓXÐ7P×X×XÓXÐ7W×X×XÓXÐX×X×X×X×X×X×X×XØ×ÑÐ2×)Ñ)Ð2¨UÐ2Ð)¨UÐ2×2×2×2Ò2Ð)¨U×2×2×2×2×2Õ2ˆ5×2×2×2Ñ2ˆ5×2×2Õ2Ð×2×2Ó2Ð)×2×2Ó2¨U×2×2×2×2×2×2Ö2Ø×$Ñ$ÐFÐ$×+Ñ+ÐFÐ+×2Ñ2ÐFÐ2×6Ñ6ÐFÑ6Ó8ÐFÐ8×<Ñ<ÐFÑ<Ó>ÐFŒ5Ð>Ó?ÐFÀ3ÐFÐ?À3ÑF×F×F×FÔFÐ?À3×F×F×F×F×FÕFŒ5×F×F×FÑFŒ5×F×F×F×F×FÖF�×F×F×FÑF�×F×FÕFÐ$×F×FÓFÐ+×F×FÓFÐ2×F×FÓFÐ6×F×FÓFÐ8×F×FÓFÐ<×F×FÓFÐ>×F×FÓFÐ?×F×FÓFÀ3×F×F×F×F×F×F×F×FÒFà	‡J�J„LÜ�[Š[˜˜A˜sÓ#€FÜ�kŠk˜!˜QÓ€GÜ�kŠk˜!˜QÓ€GØ×"Ñ" 6¨7°GÓ<�H€Eˆ1Ü	�Š�Ø×Ñ×Ñ×$Ñ$×*Ñ*Ô,÷ 
à×'Ñ'¨°¸ÓA�M€J�Ü�~‰~Ð0ˆ~˜e ZÓ0Ð0Ô0Ð0Ö0×0×0×0×0×0Ñ0Œu×0×0×0Ñ0Œu×0×0Õ0ˆ~×0×0×0×0×0Ô0˜e×0×0×0Ñ0˜e×0×0×0×0×0Ö0 Z×0×0×0Ñ0 Z×0×0Õ0Ð0×0×0×0×0×0Õ0÷G 
Žú÷@ 
Žús    ÂEAM1ÁG/ANÁM1
AN ÁN
ANc                  ó²   • S[         R                  " SS5      0n [        R                  " [        SS9   [        U 5        S S S 5        g ! , (       d  f       g = f)Nr¦  r�  r3   Ú
incompleter  )rE   r÷   r  r  ÚRuntimeErrorr   )r=  s    r&   Ú,test_legacy_attention_rejects_a_partial_blobr×  ¯  s9   € Ø0´%·+²+¸aÀÓ2CÐD€EÜ	�Š”|¨<Ó	8Ü# EÔ*÷ 
9×	8Ö	8ús   ³AÁ
Ac                  ól  • [         R                  R                  SSS5      n [         R                  R                  U 5      (       d  [        R
                  " S5        [        R                  " U SSS9nUS   nUS	   n[        S
SSSSSS9n[        S
5      Ul
        [        XB5        UR                  S5      n[        R                  odR                  owR                  oˆR                   o“S U o¦" Xš5      o»(       GdA  SS["        R$                  " 5       ;   d  [&        R(                  " [        5      (       a  [&        R*                  " [        5      OS[&        R*                  " U5      S["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS[&        R*                  " U5      [&        R*                  " U5      [&        R*                  " U	5      [&        R*                  " U
5      [&        R*                  " U5      S.-  n[-        [&        R.                  " U5      5      eS =n=n=n=n	=p«[        R                  odR                  owR0                  oˆR                   o“USU-   o¦" Xš5      o»(       GdA  SS["        R$                  " 5       ;   d  [&        R(                  " [        5      (       a  [&        R*                  " [        5      OS[&        R*                  " U5      S["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS[&        R*                  " U5      [&        R*                  " U5      [&        R*                  " U	5      [&        R*                  " U
5      [&        R*                  " U5      S.-  n[-        [&        R.                  " U5      5      eS =n=n=n=n	=p«[        R                  odR                  owR2                  oˆR                   o“SU-  S  o¦" Xš5      o»(       GdA  SS["        R$                  " 5       ;   d  [&        R(                  " [        5      (       a  [&        R*                  " [        5      OS[&        R*                  " U5      S["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS[&        R*                  " U5      [&        R*                  " U5      [&        R*                  " U	5      [&        R*                  " U
5      [&        R*                  " U5      S.-  n[-        [&        R.                  " U5      5      eS =n=n=n=n	=p«[        R                  odR                  owR4                  oˆR                   o’S   o¦" Xš5      o»(       GdA  SS["        R$                  " 5       ;   d  [&        R(                  " [        5      (       a  [&        R*                  " [        5      OS[&        R*                  " U5      S["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS[&        R*                  " U5      [&        R*                  " U5      [&        R*                  " U	5      [&        R*                  " U
5      [&        R*                  " U5      S.-  n[-        [&        R.                  " U5      5      eS =n=n=n=n	=p«UR                  ofR6                  nSo�UL oî(       dÏ  [&        R8                  " SU4SXØ45      S["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS[&        R*                  " U5      [&        R*                  " U5      [&        R*                  " U5      S.-  nSSU0-  n[-        [&        R.                  " U5      5      eS =n=n=pèUR;                  5         [        R<                  " SSS
5      n[        R>                  " SS5      n[        R>                  " SS5      nUR                  UUU5      u  nn[        R@                  " 5          UR                  R4                  R                   RC                  S5        S S S 5        UR                  UUU5      u  nn[        RD                  of" UU5      oî(       + nU(       Gd8  S S["        R$                  " 5       ;   d  [&        R(                  " [        5      (       a  [&        R*                  " [        5      OS[&        R*                  " U5      S!["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS!S"["        R$                  " 5       ;   d  [&        R(                  " U5      (       a  [&        R*                  " U5      OS"[&        R*                  " U5      S#.-  n[-        [&        R.                  " U5      5      eS =n=nng ! , (       d  f       GNŠ= f)$NrJ  Ú/icbhi_beats_frame_asd_frame_asd_F12_feedback_s2zbest.pthz%feedback_s2 checkpoint is not on diskrÀ   F)Úmap_locationÚweights_onlyrj  r¦  i   r2   r3   r4   r5   r6   )rd   rB   rp   r:   r;   rÏ   z©assert %(py13)s
{%(py13)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.q
}.weight
}, %(py11)s)
}rE   r�   )rI   rJ   rK   rx   ry   r{   r’   r¦   z©assert %(py13)s
{%(py13)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.k
}.weight
}, %(py11)s)
}z©assert %(py13)s
{%(py13)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.v
}.weight
}, %(py11)s)
}r§  z«assert %(py13)s
{%(py13)s = %(py2)s
{%(py2)s = %(py0)s.equal
}(%(py9)s
{%(py9)s = %(py7)s
{%(py7)s = %(py5)s
{%(py5)s = %(py3)s.self_feedback
}.out
}.weight
}, %(py11)s)
}rØ   r»  rö   rz   r{   r   rZ  rd   rÀ  rH   )#ÚosÚpathÚjoinÚisfiler  ÚskiprE   Úloadr   r   rƒ   r   Úsizerÿ   r˜  r›  rå   rW   rX   rY   rZ   r[   r\   r]   rœ  r�  r„   rÁ  r€   r}   rU   r÷   r^   Úmul_rV   )rÝ  Úckptr·  r­  r�   r$   rf   rˆ   r‡   r•   r¨   rÍ  rª   r‹   rg   r‰   rŠ   rG   r™   rž   rd   re   rÀ  rj   s                           r&   Ú/test_saved_feedback_s2_attention_loads_into_qkvrå  µ  sè  € Ü�7‰7�<‰<ØÐAÀ:óO€Dä�7‰7�>‰>˜$×ÑÜ�ŠÐ;Ô<Ü�:Š:�d¨¸UÑC€DØ�
Ñ€DØÐ3Ñ4€DÜ˜c¨°1¸SÈrÐ\]Ñ^€GÜ  Ó%€G„MÜ˜WÔ+Ø
�)‰)�A‹,€CÜ�;‰;ÐB×,Ñ,ÐB×.Ñ.ÐB×5Ñ5ÐB¸D¸S°zÐBˆ;Ð5ÓBÐBÖB×B×B×B×B×BÑBŒ5×B×B×BÑBŒ5×B×BÕBˆ;×B×B×B×B×BÔB�w×B×B×BÑB�w×B×BÕBÐ,×B×BÓBÐ.×B×BÓBÐ5×B×BÓB°z×B×BÓBÐB×B×B×B×B×B×BÒBÜ�;‰;ÐI×,Ñ,ÐI×.Ñ.ÐI×5Ñ5ÐI¸CÀÀCÁÐ7HÐIˆ;Ð5ÓIÐIÖI×I×I×I×I×IÑIŒ5×I×I×IÑIŒ5×I×IÕIˆ;×I×I×I×I×IÔI�w×I×I×IÑI�w×I×IÕIÐ,×I×IÓIÐ.×I×IÓIÐ5×I×IÓIÐ7H×I×IÓIÐI×I×I×I×I×I×IÒIÜ�;‰;ÐF×,Ñ,ÐF×.Ñ.ÐF×5Ñ5ÐF¸AÀ¹G¸H°~ÐFˆ;Ð5ÓFÐFÖF×F×F×F×F×FÑFŒ5×F×F×FÑFŒ5×F×FÕFˆ;×F×F×F×F×FÔF�w×F×F×FÑF�w×F×FÕFÐ,×F×FÓFÐ.×F×FÓFÐ5×F×FÓF°~×F×FÓFÐF×F×F×F×F×F×FÒFÜ�;‰;Ðd×,Ñ,Ðd×0Ñ0Ðd×7Ñ7ÐdÐ>bÑ9cÐdˆ;Ð7ÓdÐdÖd×d×d×d×d×dÑdŒ5×d×d×dÑdŒ5×d×dÕdˆ;×d×d×d×d×dÔd�w×d×d×dÑd�w×d×dÕdÐ,×d×dÓdÐ0×d×dÓdÐ7×d×dÓdÐ9c×d×dÓdÐd×d×d×d×d×d×dÒdØ× Ñ Ð4×+Ñ+Ð4¨uÐ4¨uÐ4×4×4×4Ò4Ð+×4×4×4×4×4Õ4ˆ7×4×4×4Ñ4ˆ7×4×4Õ4Ð ×4×4Ó4Ð+×4×4Ó4¨u×4×4×4×4×4×4Õ4Ø‡L�L„NÜ�[Š[˜˜A˜sÓ#€FÜ�kŠk˜!˜QÓ€GÜ�kŠk˜!˜QÓ€GØ×$Ñ$ V¨W°gÓ>�H€Eˆ1Ü	�Š�Ø×Ñ×!Ñ!×(Ñ(×-Ñ-¨aÔ0÷ 
à×)Ñ)¨&°'¸7ÓC�M€J�Ü�~‰~Ð0ˆ~˜e ZÓ0Ð0Ô0Ð0×0×0×0×0×0×0Ñ0Œu×0×0×0Ñ0Œu×0×0Õ0ˆ~×0×0×0×0×0Ô0˜e×0×0×0Ñ0˜e×0×0×0×0×0Ö0 Z×0×0×0Ñ0 Z×0×0Õ0Ð0×0×0×0×0×0Ö0÷ 
Žús   á10h$è$
h3)<Ú__doc__ÚbuiltinsrW   Ú_pytest.assertion.rewriteÚ	assertionÚrewriterY   r  rE   Útorch.nnr    Úargparser   rÜ  Úmethod.feedback_decoderr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   Úmethod.frame_asdr   r   Úutil.frame_asd_trainr   r´   r   r@   rl   rŒ   r—   r«   rî   rü   r  r  r  r:  rU  rm  rw  r‡  r”  r™  r£  r¯  rÓ  r×  rå  r?   r(   r&   Ú<module>rð     s¾   ðÙ \ç  „ ƒÛ Ý Ý ã 	÷÷ ÷ ÷ ó ÷$ 8Ý 0ô(�2—9‘9ô (ò
ò;ò
2ò2ò/ò$9òNò&,ò"Lò&ò=ò.Lò8=ò:+ò(#ò.ò*ò*)òò*,1ò^+ó1r(   