ó
    Å Öjxy  ã                   ó  • S SK r S SKrS SKJrJrJr  S SKrS SKJrJ	r	  S SK
J	s  Jr  S SKJrJr  S SKJrJrJrJrJr   " S S\	R,                  5      r " S S	\	R,                  5      r " S
 S\	R,                  5      rS rg)é    N)ÚDictÚOptionalÚTuple)ÚTensorÚnn)Ú	LayerNormÚ	Parameter)ÚGradMultiplyÚSamePadÚget_activation_fnÚ
GLU_LinearÚquant_noisec                   ó<   ^ • \ rS rSrU 4S jrSS jrSS jrSrU =r$ )ÚTransformerEncoderé   c                 óx  >• [         TU ]  5         UR                  U l        UR                  U l        [
        R                  " U R                  U R                  UR                  UR                  S-  UR                  S9U l	        Sn[        R                  " SSU-
  -  UR                  U R                  -  -  5      n[
        R                  R                  U R                  R                  SUS9  [
        R                  R                  U R                  R                   S5        [
        R"                  R%                  U R                  SSS9U l	        [
        R&                  " U R                  [)        UR                  5      [
        R*                  " 5       5      U l	        [-        US	5      (       a4  UR.                  U l        UR0                  U l        UR2                  U l        OS
U l        SU l        SU l        [
        R4                  " [7        UR8                  5       Vs/ s H¥  n[;        U R                  UR<                  UR>                  U R                  UR@                  URB                  URD                  URF                  URH                  U R.                  U R0                  U R2                  URJ                  UR8                  S9PM§     sn5      U l&        U R.                  (       au  [7        SUR8                  5       H[  nU RL                  U   RN                  ?(U RL                  S   RN                  RP                  U RL                  U   RN                  l(        M]     URF                  U l#        [S        U R                  5      U l*        URV                  U l,        U R[                  [\        5        URH                  (       Gaä  [        R^                  " SUR8                  -  S5      n[7        UR8                  5       GH¦  n[
        R                  Ra                  U RL                  U   RN                  Rb                  R                  SS9  [
        R                  Ra                  U RL                  U   RN                  Rd                  R                  US9  [
        R                  Ra                  U RL                  U   RN                  Rf                  R                  SS9  [
        R                  Ra                  U RL                  U   RN                  Rh                  R                  US9  [
        R                  Ra                  U RL                  U   Rj                  R                  US9  [
        R                  Ra                  U RL                  U   Rl                  R                  US9  GM©     [o        USS5      U l8        g s  snf )Né   )Úkernel_sizeÚpaddingÚgroupsr   é   ç      ð?©ÚmeanÚstdÚweight)ÚnameÚdimÚrelative_position_embeddingF)Úembedding_dimÚffn_embedding_dimÚnum_attention_headsÚdropoutÚattention_dropoutÚactivation_dropoutÚactivation_fnÚlayer_norm_firstÚ	deep_normÚhas_relative_attention_biasÚnum_bucketsÚmax_distanceÚgru_rel_posÚencoder_layersé   é   g      Ð¿©ÚgainÚlayer_wise_gradient_decay_ratio)9ÚsuperÚ__init__r#   Úencoder_embed_dimr    r   ÚConv1dÚconv_posÚconv_pos_groupsÚpos_convÚmathÚsqrtÚinitÚnormal_r   Ú	constant_ÚbiasÚutilsÚweight_normÚ
Sequentialr   ÚGELUÚhasattrr   r*   r+   Ú
ModuleListÚranger-   ÚTransformerSentenceEncoderLayerÚencoder_ffn_embed_dimÚencoder_attention_headsr$   r%   r&   r'   r(   r,   ÚlayersÚ	self_attnÚrelative_attention_biasr   Ú
layer_normÚencoder_layerdropÚ	layerdropÚapplyÚinit_bert_paramsÚpowÚxavier_normal_Úk_projÚv_projÚq_projÚout_projÚfc1Úfc2Úgetattrr2   )ÚselfÚargsr#   r   ÚiÚdeep_norm_betaÚ	__class__s         €Ú//root/disk_3tb/ICBHI_2017_F12/BEATs/backbone.pyr4   ÚTransformerEncoder.__init__   sK  ø€ Ü‰ÑÔà—|‘|ˆŒØ!×3Ñ3ˆÔäŸ	š	Ø×ÑØ×ÑØŸ™Ø—M‘M QÑ&Ø×'Ñ'ñ
ˆŒð ˆÜ�iŠi˜˜c G™mÑ,°·±À×ASÑASÑ1SÑTÓUˆÜ
�‰�‰˜Ÿ™×,Ñ,°1¸#ˆÑ>Ü
�‰×Ñ˜$Ÿ-™-×,Ñ,¨aÔ0äŸ™×,Ñ,¨T¯]©]ÀÈqÐ,ÐQˆŒÜŸš d§m¡m´W¸T¿]¹]Ó5KÌRÏWÊWËYÓWˆŒä�4Ð6×7Ñ7Ø/3×/OÑ/OˆDÔ,Ø#×/Ñ/ˆDÔØ $× 1Ñ 1ˆDÕà/4ˆDÔ,Ø ˆDÔØ !ˆDÔä—m’mô$ ˜t×2Ñ2Ô3ó#ò" 4�Aô! 0Ø"&×"4Ñ"4Ø&*×&@Ñ&@Ø(,×(DÑ(DØ ŸL™LØ&*×&<Ñ&<Ø'+×'>Ñ'>Ø"&×"4Ñ"4Ø%)×%:Ñ%:Ø"Ÿn™nØ04×0PÑ0PØ $× 0Ñ 0Ø!%×!2Ñ!2Ø $× 0Ñ 0Ø#'×#6Ñ#6ôñ  4ñ#ó
ˆŒð* ×+×+Ü˜1˜d×1Ñ1Ö2�Ø—K‘K ‘N×,Ñ,ÐDØCGÇ;Á;ØñDç‘)×3Ñ3ð —‘˜A‘×(Ñ(Ö@ñ 3ð !%× 5Ñ 5ˆÔÜ# D×$6Ñ$6Ó7ˆŒØ×/Ñ/ˆŒà�
‰
Ô#Ô$à�>�>ˆ>Ü!ŸXšX a¨$×*=Ñ*=Ñ&=¸vÓFˆNÜ˜4×.Ñ.×/�Ü—‘×&Ñ& t§{¡{°1¡~×'?Ñ'?×'FÑ'F×'MÑ'MÐTUÐ&ÑVÜ—‘×&Ñ&Ø—K‘K ‘N×,Ñ,×3Ñ3×:Ñ:Àð 'ñ ô —‘×&Ñ& t§{¡{°1¡~×'?Ñ'?×'FÑ'F×'MÑ'MÐTUÐ&ÑVÜ—‘×&Ñ&Ø—K‘K ‘N×,Ñ,×5Ñ5×<Ñ<À>ð 'ñ ô —‘×&Ñ& t§{¡{°1¡~×'9Ñ'9×'@Ñ'@À~Ð&ÑVÜ—‘×&Ñ& t§{¡{°1¡~×'9Ñ'9×'@Ñ'@À~Ð&ÔVñ 0ô 07ØÐ3°Qó0
ˆÕ,ùò_s   ÈB,V7c                 óz   • U R                  XU5      u  pU R                  (       a  Uc  U R                  U5      nX4$ ©N)Úextract_featuresr'   rM   )r[   ÚxÚpadding_maskÚlayerÚlayer_resultss        r`   ÚforwardÚTransformerEncoder.forwardm   s;   € Ø×0Ñ0°À%ÓHÑˆà× ×  U¡]Ø—‘ Ó"ˆAàÐÐó    c                 ó0  • Ub  SX'   U R                  UR                  SS5      5      nUR                  SS5      nUR                  5       U-   nU R                  (       d  U R	                  U5      n[
        R                  " XR                  U R                  S9nUR                  SS5      n/ nS nUb  UR                  X45        S nS n[        U R                  5       H�  u  pšU R                  S:w  a   [        R                  " XR                  5      n[        R                  R                  5       nU R                  (       a  X°R                   :”  a  U
" UUSUS9u  pnUb  UR                  X45        X“:X  d  M›  Un  O   Ub  UnUR                  SS5      nX4$ )Nr   r.   r   )ÚpÚtrainingr   F)Úself_attn_padding_maskÚneed_weightsÚpos_bias)r9   Ú	transposeÚcloner'   rM   ÚFr#   rn   ÚappendÚ	enumeraterJ   r2   r
   rP   ÚnpÚrandomrO   )r[   re   rf   Ú	tgt_layerÚx_convrh   ÚzÚrrq   r]   rg   Údropout_probabilitys               r`   rd   Ú#TransformerEncoder.extract_featuresu   sz  € ØÑ#ØˆA‰Oà—‘˜qŸ{™{¨1¨aÓ0Ó1ˆØ×!Ñ! ! QÓ'ˆð �G‰G‹I˜Ñˆà×$×$Ø—‘ Ó"ˆAä�IŠI�aŸ<™<°$·-±-Ñ@ˆð �K‰K˜˜1ÓˆàˆØˆØÑ Ø× Ñ  ! Ô(ØˆØˆÜ! $§+¡+Ö.‰HˆAØ×3Ñ3°sÓ:Ü ×&Ò& q×*NÑ*NÓO�Ü"$§)¡)×"2Ñ"2Ó"4ÐØ—=—=Ð%8¿>¹>Ó%IÙ!&ØØ+7Ø!&Ø%ñ	"‘��hð Ñ$Ø×$Ñ$ a VÔ,Ø�~Ø�Ùñ /ð" ‰=ØˆAð �K‰K˜˜1ÓˆàÐÐrk   )r#   r    rM   r'   r2   rO   rJ   r+   r*   r9   r   )NN)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r4   ri   rd   Ú__static_attributes__Ú__classcell__©r_   s   @r`   r   r      s   ø† õP
ôd ÷/ ò / rk   r   c            !       óð   ^ • \ rS rSr               SS\S\S\S\S\S\S\S	\S
\S\S\S\S\S\S\SS4 U 4S jjjr    SS\	R                  S\	R                  S\	R                  S\4S jjrSrU =r$ )rG   é§   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Úrescale_initr,   r-   ÚreturnNc                 óH  >• [         TU ]  5         Xl        X@l        X`l        Xpl        [        U5      U l        [        U R                  UUSU
UUUUS9	U l	        [        R                  " U5      U l        [        R                  " U R                  5      U l        [        R                  " U5      U l        X€l        [!        U R                  5      U l        U R
                  S:X  a  [%        U R                  US5      U l        O&[        R(                  " U R                  U5      U l        [        R(                  " X R                  5      U l        [!        U R                  5      U l        X�l        U R.                  (       a   [0        R2                  " SU-  S5      U l        g SU l        g )NT)r#   Úself_attentionr)   r*   r+   rˆ   r,   ÚgluÚswishr   g      Ð?r.   )r3   r4   r    r#   r%   Úactivation_namer   r&   ÚMultiheadAttentionrK   r   ÚDropoutÚdropout1Údropout2Údropout3r'   r   Úself_attn_layer_normr   rX   ÚLinearrY   Úfinal_layer_normr(   r:   rR   Údeep_norm_alpha)r[   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   rˆ   r,   r-   r_   s                   €r`   r4   Ú(TransformerSentenceEncoderLayer.__init__¨   sF  ø€ ô$ 	‰ÑÔØ*ÔØŒØ"4Ôà,ÔÜ.¨}Ó=ˆÔÜ+Ø×ÑØØ%ØØ(CØ#Ø%Ø%Ø#ñ

ˆŒô Ÿ
š
 7Ó+ˆŒÜŸ
š
 4×#:Ñ#:Ó;ˆŒÜŸ
š
 7Ó+ˆŒà 0Ôä$-¨d×.@Ñ.@Ó$AˆÔ!à×Ñ 5Ó(Ü! $×"4Ñ"4Ð6GÈÓQˆD�Hä—y’y ×!3Ñ!3Ð5FÓGˆDŒHÜ—9’9Ð.×0BÑ0BÓCˆŒä )¨$×*<Ñ*<Ó =ˆÔà"ŒØ�>�>Ü#'§8¢8¨A°Ñ,>ÀÓ#FˆDÕ à#$ˆDÕ rk   re   Úself_attn_maskro   rp   c           
      óŽ  • UnU R                   (       aË  U R                  U5      nU R                  UUUUSUUS9u  pnU R                  U5      nXa-   nUnU R	                  U5      nU R
                  S:X  a  U R                  U5      nO U R                  U R                  U5      5      nU R                  U5      nU R                  U5      nU R                  U5      nXa-   nOäU R                  UUUUUUUS9u  pnU R                  U5      nX`R                  -  U-   nU R                  U5      nUnU R
                  S:X  a  U R                  U5      nO U R                  U R                  U5      5      nU R                  U5      nU R                  U5      nU R                  U5      nX`R                  -  U-   nU R	                  U5      nXU4$ )NF)ÚqueryÚkeyÚvalueÚkey_padding_maskrp   Ú	attn_maskÚposition_biasrŒ   )r'   r”   rK   r‘   r–   rŽ   rX   r&   r’   rY   r“   r—   )r[   re   r™   ro   rp   rq   ÚresidualÚattns           r`   ri   Ú'TransformerSentenceEncoderLayer.forwardã   sÍ  € ð ˆà× × Ø×)Ñ)¨!Ó,ˆAØ $§¡ØØØØ!7Ø"Ø(Ø&ð !/ð !ÑˆA�Xð —‘˜aÓ ˆAØ‘ˆAàˆHØ×%Ñ% aÓ(ˆAØ×#Ñ# uÓ,Ø—H‘H˜Q“K‘à×&Ñ& t§x¡x°£{Ó3�Ø—‘˜aÓ ˆAØ—‘˜“ˆAØ—‘˜aÓ ˆAØ‘‰Aà $§¡ØØØØ!7Ø)Ø(Ø&ð !/ð !ÑˆA�Xð —‘˜aÓ ˆAØ×/Ñ/Ñ/°!Ñ3ˆAà×)Ñ)¨!Ó,ˆAàˆHØ×#Ñ# uÓ,Ø—H‘H˜Q“K‘à×&Ñ& t§x¡x°£{Ó3�Ø—‘˜aÓ ˆAØ—‘˜“ˆAØ—‘˜aÓ ˆAØ×/Ñ/Ñ/°!Ñ3ˆAØ×%Ñ% aÓ(ˆAà˜Ð Ð rk   )r%   r&   rŽ   r(   r—   r#   r‘   r’   r“   r    rX   rY   r–   r'   rK   r”   )i   i   r/   çš™™™™™¹?r¤   r¤   ÚreluFFFr   r   FFr   )NNFN)r   r€   r�   r‚   ÚfloatÚstrÚboolÚintr4   Útorchr   ri   rƒ   r„   r…   s   @r`   rG   rG   §   s%  ø† ð  #Ø#'Ø%&ØØ#&Ø$'Ø#Ø!&ØØ,1ØØØ"Ø!Øñ!9%àð9%ð !ð9%ð #ð	9%ð
 ð9%ð !ð9%ð "ð9%ð ð9%ð ð9%ð ð9%ð &*ð9%ð ð9%ð ð9%ð ð9%ð ð9%ð  ð!9%ð" 
÷#9%ð 9%ð| (,Ø/3Ø"Øñ=!à�<‰<ð=!ð Ÿ™ð=!ð !&§¡ð	=!ð
 ÷=!ó =!rk   rG   c                   óÖ  ^ • \ rS rSrSr               SU 4S jjrS rSS jrS r        S S\	\
   S\	\
   S	\	\
   S
\	\\\\\	\
   4   4      S\S\S\	\
   S\S\S\	\
   S\\
\	\
   \	\
   4   4S jjr\S	\	\
   S\	\
   S\S\S\S\	\
   4S j5       rS
\	\\\\\	\
   4   4      S\\\	\
   4   4S jrS
\\\\\	\
   4   4   S\\\	\
   4   4S jrS\S\S\4S jrSrU =r$ )!r�   i#  zKMulti-headed attention.

See "Attention Is All You Need" for more details.
c                 óÚ  >• [         TU ]  5         Xl        Ub  UOUU l        Ub  UOUU l        U R                  U:H  =(       a    U R                  U:H  U l        X l        [        R                  " U5      U l	        XÐl
        Xàl        Xðl        U R                  (       a  [        R                  " Xâ5      U l        X-  U l        U R                  U l        U R                  U l        U R                  U-  U R                  :X  d   S5       eU R                  S-  U l        X�l        X l        U R&                  (       a  U R
                  (       d   S5       eSnU(       a  SnUnUn[+        [        R,                  " U R                  UUS9X¼5      U l        [+        [        R,                  " U R                  XS9X¼5      U l        [+        [        R,                  " UUUS9X¼5      U l        [+        [        R,                  " XUS9X¼5      U l        U(       aM  [7        [8        R:                  " SSU5      5      U l        [7        [8        R:                  " SSU5      5      U l        OS =U l        U l        X€l         UU l!        U RB                  (       aX  [        R,                  " U R                   S5      U l"        [        R6                  " [8        RF                  " SUSS5      5      U l$        U RK                  5         g )	Nz(embed_dim must be divisible by num_headsg      à¿zCSelf-attention requires query, key and value to be of the same sizeTF)r?   r.   r/   )&r3   r4   Ú	embed_dimÚkdimÚvdimÚqkv_same_dimÚ	num_headsr   r�   Údropout_moduler)   r*   r+   Ú	EmbeddingrL   Úhead_dimÚ
q_head_dimÚ
k_head_dimÚscalingr‹   Úencoder_decoder_attentionr   r•   rT   rU   rV   rW   r	   rª   r   Úbias_kÚbias_vÚadd_zero_attnr,   Úgrep_linearÚonesÚgrep_aÚreset_parameters)r[   r­   r±   r®   r¯   r#   r?   Úadd_bias_kvr»   r‹   r¸   Úq_noiseÚqn_block_sizer)   r*   r+   r,   rˆ   Úk_biasÚk_embed_dimÚq_embed_dimr_   s                        €r`   r4   ÚMultiheadAttention.__init__)  sM  ø€ ô( 	‰ÑÔØ"ŒØ Ñ,‘D°)ˆŒ	Ø Ñ,‘D°)ˆŒ	Ø ŸI™I¨Ñ2×M°t·y±yÀIÑ7MˆÔà"ŒÜ Ÿjšj¨Ó1ˆÔà+FÔ(Ø&ÔØ(ÔØ×+×+Ü+-¯<ª<¸Ó+OˆDÔ(à!Ñ.ˆŒØŸ-™-ˆŒØŸ-™-ˆŒà�M‰M˜IÑ%¨¯©Ó7ð	6à5ó	6Ø7à—}‘} dÑ*ˆŒà,ÔØ)BÔ&à×&×&¨$×*;×*;ð 	
ØTó	
Ð;ð ˆÞØˆFàˆØˆä!Ü�IŠI�d—i‘i °6Ñ:¸Gó
ˆŒô "Ü�IŠI�d—i‘i Ñ6¸ó
ˆŒô "Ü�IŠI�i °4Ñ8¸'ó
ˆŒô $Ü�IŠI�i°Ñ6¸ó
ˆŒö Ü#¤E§L¢L°°A°yÓ$AÓBˆDŒKÜ#¤E§L¢L°°A°yÓ$AÓBˆD�Kà(,Ð,ˆDŒK˜$œ+à*Ôà&ˆÔØ××Ü!Ÿyšy¨¯©¸!Ó<ˆDÔÜŸ,š,¤u§z¢z°!°YÀÀ1Ó'EÓFˆDŒKà×ÑÕrk   c                 ón  • U R                   (       aÜ  [        R                  R                  U R                  R
                  S[        R                  " S5      -  S9  [        R                  R                  U R                  R
                  S[        R                  " S5      -  S9  [        R                  R                  U R                  R
                  S[        R                  " S5      -  S9  O™[        R                  R                  U R                  R
                  5        [        R                  R                  U R                  R
                  5        [        R                  R                  U R                  R
                  5        [        R                  R                  U R                  R
                  5        U R                  R                  b4  [        R                  R                  U R                  R                  S5        U R                  b)  [        R                  R                  U R                  5        U R                  b)  [        R                  R                  U R                  5        U R                   (       a4  [        R                  R                  U R"                  R
                  5        g g )Nr.   r   r0   ç        )r°   r   r<   Úxavier_uniform_rT   r   r:   r;   rU   rV   rW   r?   r>   r¹   rS   rº   r)   rL   )r[   s    r`   r¿   Ú#MultiheadAttention.reset_parameters  s¡  € Ø××ô �G‰G×#Ñ# D§K¡K×$6Ñ$6¸QÄÇÂÈ1ÃÑ=MÐ#ÑNÜ�G‰G×#Ñ# D§K¡K×$6Ñ$6¸QÄÇÂÈ1ÃÑ=MÐ#ÑNÜ�G‰G×#Ñ# D§K¡K×$6Ñ$6¸QÄÇÂÈ1ÃÑ=MÐ#ÒNä�G‰G×#Ñ# D§K¡K×$6Ñ$6Ô7Ü�G‰G×#Ñ# D§K¡K×$6Ñ$6Ô7Ü�G‰G×#Ñ# D§K¡K×$6Ñ$6Ô7ä
�‰×Ñ §¡× 4Ñ 4Ô5Ø�=‰=×ÑÑ)Ü�G‰G×Ñ˜dŸm™m×0Ñ0°#Ô6Ø�;‰;Ñ"Ü�G‰G×"Ñ" 4§;¡;Ô/Ø�;‰;Ñ"Ü�G‰G×"Ñ" 4§;¡;Ô/Ø×+×+Ü�G‰G×"Ñ" 4×#?Ñ#?×#FÑ#FÕGð ,rk   c                 ó’  • U R                   nU R                  nSnU(       aC  US-  nXQS:„  R                  [        R                  5      U-  -  n[        R
                  " U5      nO,[        R                  " U[        R                  " U5      5      * nUS-  nX:  nU[        R                  " UR                  5       U-  5      [        R                  " XF-  5      -  X6-
  -  R                  [        R                  5      -   n[        R                  " U[        R                  " XƒS-
  5      5      nU[        R                  " XqU5      -  nU$ )Nr   r   r.   )r*   r+   Útorª   ÚlongÚabsÚminÚ
zeros_likeÚlogr¦   r:   Ú	full_likeÚwhere)	r[   Úrelative_positionsÚbidirectionalr*   r+   Úrelative_bucketsÚ	max_exactÚis_smallÚrelative_postion_if_larges	            r`   Ú_relative_positions_bucketÚ-MultiheadAttention._relative_positions_bucket•  s5  € Ø×&Ñ&ˆØ×(Ñ(ˆØÐæØ%¨Ñ*ˆKØ°aÑ!7× ;Ñ ;¼E¿J¹JÓ GÈ+Ñ UÑUÐÜ!&§¢Ð+=Ó!>Ñä"'§)¢)Ø"¤E×$4Ò$4Ð5GÓ$Hó#ð "Ðð   1Ñ$ˆ	Ø%Ñ1ˆà$-Ü�IŠIÐ(×.Ñ.Ó0°9Ñ<Ó=Ü�hŠh�|Ñ/Ó0ñ1àÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	%Ð!ô
 %*§I¢IØ%Ü�OŠOÐ5ÀQ±ÓGó%
Ð!ð
 	œEŸKšKØÐ*Có
ñ 	
Ðð  Ðrk   c                 ó|  • [         R                  " U[         R                  S9S S 2S 4   n[         R                  " U[         R                  S9S S S 24   nXC-
  nU R                  USS9nUR	                  U R
                  R                  R                  5      nU R                  U5      nUR                  / SQ5      nU$ )N)ÚdtypeT)rÕ   )r   r   r.   )	rª   ÚarangerÍ   rÚ   rÌ   rL   r   ÚdeviceÚpermute)r[   Úquery_lengthÚ
key_lengthÚcontext_positionÚmemory_positionÚrelative_positionÚrelative_position_bucketÚvaluess           r`   Úcompute_biasÚMultiheadAttention.compute_biasµ  s°   € Ü Ÿ<š<¨¼E¿J¹JÑGÊÈ4ÈÑPÐÜŸ,š, z¼¿¹ÑDÀTÊ1ÀWÑMˆØ+Ñ>ÐØ#'×#BÑ#BØ¨Tð $Cð $
Ð ð $<×#>Ñ#>Ø×(Ñ(×/Ñ/×6Ñ6ó$
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*ˆØˆrk   rœ   r�   rž   Úincremental_staterp   Ú	static_kvrŸ   Úbefore_softmaxÚneed_head_weightsr    r‰   c           	      ó  • U
(       a  SnUR                   R                  S:H  nUR                  5       u  pÞnUnXðR                  :X  d   e[	        UR                  5       5      XÞU/:X  d   eUba  UR                  5       u  nnn[
        R                  R                  5       (       d*  UU:X  d   eUc   eU(       d   XãR                  SS :H  5       eU R                  (       aU  UcR  U R                  UU5      nUR                  S5      R                  USSS5      R                  XàR                  -  UU5      nUbI  U R                  U5      nUb4  SU;   a.  U(       a'  U R                   (       a  U R"                  (       a   eS=p#OSnU R"                  (       a4  U R%                  U5      nU R'                  U5      nU R)                  U5      nO�U R                   (       aA  U R%                  U5      nUc
  Ub   eS=nnO^U R'                  U5      nU R)                  U5      nO;Ub  Uc   eU R%                  U5      nU R'                  U5      nU R)                  U5      nUU R*                  -  nSnUSU-  -  nU R,                  bæ  U R.                  c   e[
        R0                  " UU R,                  R                  SUS5      /5      n[
        R0                  " UU R.                  R                  SUS5      /5      nUb5  [
        R0                  " XˆR3                  UR                  S5      S5      /SS	9nUb6  [
        R0                  " UUR3                  UR                  S5      S5      /SS	9nUR5                  5       R                  XÞU R                  -  U R6                  5      R9                  SS5      nUbG  UR5                  5       R                  S
XàR                  -  U R:                  5      R9                  SS5      nUbG  UR5                  5       R                  S
XàR                  -  U R<                  5      R9                  SS5      nUGbz  SU;   aj  US   nUc   eUR                  XàR                  -  S
U R<                  5      nU(       a  UnOUc   e[
        R0                  " UU/SS	9nUR                  S5      nSU;   aY  US   nUc   eUR                  XàR                  -  S
U R<                  5      nU(       a  UnOUc   e[
        R0                  " UU/SS	9nSnSU;   a  US   nUb  Uc   e[>        RA                  UUUUR                  S5      US9nUR                  XàR                  S
U R<                  5      US'   UR                  XàR                  S
U R<                  5      US'   UUS'   Uc   eU RC                  UU5      nUc   eUR                  S5      U:X  d   eUb  URE                  5       S:X  a  SnUb.  UR                  S5      U:X  d   eUR                  S5      U:X  d   eU RF                  (       Ga%  Uc   eUS-  n[
        R0                  " UUR3                  UR                  S5      S4UR                  5       SS -   5      /SS	9n[
        R0                  " UUR3                  UR                  S5      S4UR                  5       SS -   5      /SS	9nUb5  [
        R0                  " XˆR3                  UR                  S5      S5      /SS	9nUbJ  [
        R0                  " U[
        RH                  " UR                  S5      S5      RK                  U5      /SS	9n[
        RL                  " UUR9                  SS5      5      nUURO                  S
SS9S   -
  U-  nU RQ                  UUUU5      n[	        UR                  5       5      XàR                  -  UU/:X  d   eUb  UR                  S5      nUU-  nUbÙ  UR                  XàR                  UU5      nU(       dW  URS                  UR                  S5      R                  S5      RU                  [
        RV                  5      [Y        S5      5      nO?UR9                  SS5      nURS                  U[Y        S5      5      nUR9                  SS5      nUR                  XàR                  -  UU5      nU	(       a  UUU4$ UGb  UnU RZ                  S:X  aÜ  UR                  XàR                  XÐR6                  5      U-  U R*                  -  nUR                  5       u  n n!n"n#[
        R\                  " U R_                  U5      R                  U U!U"SS5      Ra                  S
SS95      Rc                  SS
S	9u  n$n%U$U%U Rd                  -  S-
  -  S-   n&U&R                  XàR                  -  US5      U-  nUR                  UR                  5       5      nUU-   n[f        Rh                  " US
S	9n'U'RK                  U5      nU Rk                  U5      n(Uc   e[
        RL                  " U(U5      n)[	        U)R                  5       5      XàR                  -  XÐR<                  /:X  d   eU)R9                  SS5      R5                  5       R                  XÞU5      n)U Rm                  U)5      n)SnU(       aC  U'R                  XàR                  UU5      R9                  SS5      nU
(       d  URo                  SS	9nU)UU4$ )a-  Input shape: Time x Batch x Channel

Args:
    key_padding_mask (ByteTensor, optional): mask to exclude
        keys that are pads, of shape `(batch, src_len)`, where
        padding elements are indicated by 1s.
    need_weights (bool, optional): return the attention weights,
        averaged over heads (default: False).
    attn_mask (ByteTensor, optional): typically used to
        implement causal attention, where the mask prevents the
        attention from looking forward in time (default: None).
    before_softmax (bool, optional): return the raw attention
        weights and values before the attention softmax.
    need_head_weights (bool, optional): return the attention
        weights for each head. Implies *need_weights*. Default:
        return the average attention weights over all heads.
TÚxlaNr   r   r.   Úprev_keyé    ©r   éÿÿÿÿÚ
prev_valueÚprev_key_padding_mask)rž   rõ   Ú
batch_sizeÚsrc_lenrë   )r   Úkeepdimz-infr   F)rø   r   g       @)8rß   ÚtypeÚsizer­   Úlistrª   ÚjitÚis_scriptingÚshaper)   rè   Ú	unsqueezeÚrepeatÚviewr±   Ú_get_input_bufferr¸   r‹   rV   rT   rU   r·   r¹   rº   ÚcatÚ	new_zerosÚ
contiguousrµ   rr   r¶   r´   r�   Ú_append_prev_key_padding_maskÚ_set_input_bufferr   r»   ÚzerosÚtype_asÚbmmÚmaxÚapply_sparse_maskÚmasked_fillrÌ   r¨   r¦   r,   Úsigmoidr¼   ÚsumÚchunkr¾   rt   Úsoftmaxr²   rW   r   )*r[   r›   rœ   r�   rž   rê   rp   rë   rŸ   rì   rí   r    Úis_tpuÚtgt_lenÚbszr­   r÷   Úkey_bszÚ_Úsaved_stateÚqÚkÚvÚalphaÚ	_prev_keyrð   Ú_prev_valuerô   rõ   Úattn_weightsÚattn_mask_rel_posÚquery_layerÚ_BÚ_HÚ_LÚ__Úgate_aÚgate_bÚgate_a_1Úattn_weights_floatÚ
attn_probsr¢   s*                                             r`   ri   ÚMultiheadAttention.forwardÃ  sn
  € ö> ØˆLà—‘×"Ñ" eÑ+ˆà"'§*¡*£,Ñˆ�iØˆØŸN™NÓ*Ð*Ð*Ü�E—J‘J“LÓ! g°IÐ%>Ó>Ð>Ð>Ø‰?Ø"%§(¡(£*ÑˆG�W˜aÜ—9‘9×)Ñ)×+Ñ+Ø #“~Ð%�~ØÑ(Ð(Ð(ÞÐ6 §{¡{°2°A Ñ 6Ó6�wà×+×+°Ñ0EØ ×-Ñ-¨g°wÓ?ˆMà×'Ñ'¨Ó*ß‘˜˜Q  1Ó%ß‘�cŸN™NÑ*¨G°WÓ=ð ð Ñ(Ø×0Ñ0Ð1BÓCˆKØÑ&¨:¸Ó+Dö Ø×9×9À$×BU×BUÐUÐUØ"&Ð&�CøàˆKà××Ø—‘˜EÓ"ˆAØ—‘˜EÓ"ˆAØ—‘˜EÓ"‰AØ×+×+à—‘˜EÓ"ˆAØ‰{Ø‘}Ð$�}Ø��‘Aà—K‘K Ó$�Ø—K‘K Ó$‘ð ‘? uÑ'8Ð8Ð8Ø—‘˜EÓ"ˆAØ—‘˜CÓ ˆAØ—‘˜EÓ"ˆAØ	ˆT�\‰\ÑˆØˆØ	ˆQ�‰Y‰ˆà�;‰;Ñ"Ø—;‘;Ñ*Ð*Ð*Ü—	’	˜1˜dŸk™k×0Ñ0°°C¸Ó;Ð<Ó=ˆAÜ—	’	˜1˜dŸk™k×0Ñ0°°C¸Ó;Ð<Ó=ˆAØÑ$Ü!ŸIšIØ× 3Ñ 3°I·N±NÀ1Ó4EÀqÓ IÐJÐPQñ�	ð  Ñ+Ü#(§9¢9à(Ø(×2Ñ2Ð3C×3HÑ3HÈÓ3KÈQÓOðð ñ$Ð ð �L‰L‹Nß‰T�' §¡Ñ/°·±ÓAß‰Y�q˜!‹_ð 	
ð
 ‰=à—‘“ß‘�b˜#§¡Ñ.°·±Ó@ß‘˜1˜a“ð ð
 ‰=à—‘“ß‘�b˜#§¡Ñ.°·±Ó>ß‘˜1˜a“ð ð Ò"à˜[Ó(Ø'¨
Ñ3�	Ø Ñ,Ð,Ð,Ø$Ÿ>™>¨#·±Ñ*>ÀÀDÇMÁMÓR�ÞØ ‘Aà™=Ð(˜=ÜŸ	š	 8¨Q -°QÑ7�AØŸ&™& ›)�Ø˜{Ó*Ø)¨,Ñ7�Ø"Ñ.Ð.Ð.Ø(×-Ñ-¨c·N±NÑ.BÀBÈÏÉÓV�
ÞØ"‘Aà™=Ð(˜=ÜŸ	š	 :¨q /°qÑ9�AØ6:Ð!Ø&¨+Ó5Ø(3Ð4KÑ(LÐ%Ø‘= Q¡]Ð2Ð2Ü1×OÑOØ!1Ø&;ØØŸ™˜q›	Ø#ð  Pð  Ðð '(§f¡f¨S·.±.À"ÀdÇmÁmÓ&TˆK˜
Ñ#Ø()¯©¨s·N±NÀBÈÏÉÓ(VˆK˜Ñ%Ø3CˆKÐ/Ñ0à$Ñ0Ð0Ð0Ø $× 6Ñ 6Ð7HÈ+Ó VÐØ‰}Ðˆ}Ø�v‰v�a‹y˜GÓ#Ð#Ð#ð Ñ'Ð,<×,@Ñ,@Ó,BÀaÓ,GØ#ÐàÑ'Ø#×(Ñ(¨Ó+¨sÓ2Ð2Ð2Ø#×(Ñ(¨Ó+¨wÓ6Ð6Ð6à××ÐØ‘=Ð �=Ø�q‰LˆGÜ—	’	˜1˜aŸk™k¨1¯6©6°!«9°a¨.¸1¿6¹6»8ÀAÀB¸<Ñ*GÓHÐIÈqÑQˆAÜ—	’	˜1˜aŸk™k¨1¯6©6°!«9°a¨.¸1¿6¹6»8ÀAÀB¸<Ñ*GÓHÐIÈqÑQˆAØÑ$Ü!ŸIšIØ× 3Ñ 3°I·N±NÀ1Ó4EÀqÓ IÐJÐPQñ�	ð  Ñ+Ü#(§9¢9à(ÜŸšÐ$4×$9Ñ$9¸!Ó$<¸aÓ@×HÑHØ,óðð ñ$Ð ô —y’y  A§K¡K°°1Ó$5Ó6ˆà˜<×+Ñ+°¸DÐ+ÐAÀ!ÑDÑDØñˆð ×-Ñ-¨l¸GÀWÈcÓRˆä�L×%Ñ%Ó'Ó(¨S·>±>Ñ-AÀ7ÈGÐ,TÓTÐTÐTàÑ Ø!×+Ñ+¨AÓ.ˆIØ˜IÑ%ˆLàÑ'à'×,Ñ,¨S·.±.À'È7ÓSˆLÞØ+×7Ñ7Ø$×.Ñ.¨qÓ1×;Ñ;¸AÓ>×AÑAÄ%Ç*Á*ÓMÜ˜&“Mó ‘ð
  ,×5Ñ5°a¸Ó;�Ø+×7Ñ7Ð8HÌ%ÐPVË-ÓX�Ø+×5Ñ5°a¸Ó;�Ø'×,Ñ,¨S·>±>Ñ-AÀ7ÈGÓTˆLæØ  MÐ1Ð1àÒ$Ø -ÐØ×Ñ 1Ó$à—F‘F˜3§¡°¿¹ÓIØñà—l‘lñ#ð ð
 "-×!1Ñ!1Ó!3‘��B˜˜BÜ!&§¢Ø×$Ñ$ [Ó1ß‘T˜"˜b " a¨Ó+ß‘S˜ U�SÐ+ó"÷ ‘%˜˜r�%Ð"ñ	 �˜ð
 " V¨d¯k©kÑ%9¸CÑ%?Ñ@À3ÑF�à—M‘M #¯©Ñ"6¸ÀÓCÀmÑSð "ð !2× 6Ñ 6°|×7HÑ7HÓ7JÓ KÐà'Ð*;Ñ;ˆLäŸYšY |¸Ñ<ÐØ)×1Ñ1°,Ó?ˆØ×(Ñ(¨Ó6ˆ
à‰}Ðˆ}Ü�yŠy˜ QÓ'ˆÜ�D—I‘I“KÓ  S¯>©>Ñ%9¸7ÇMÁMÐ$RÓRÐRÐRØ�~‰~˜a Ó#×.Ñ.Ó0×5Ñ5°gÀIÓNˆØ�}‰}˜TÓ"ˆØ)-ˆÞØ-×2Ñ2Ø—^‘^ W¨góç‰i˜˜1‹oð ö %à+×0Ñ0°QÐ0Ð7�à�\ =Ð0Ð0rk   rõ   rö   r÷   c                 óà  • Ub  U(       a  UnU$ Ub8  U b5  [         R                  " UR                  5       U R                  5       /SS9nU$ UbŽ  X1R                  S5      :”  ag  [         R                  " X#UR                  S5      -
  4UR
                  S9n[         R                  " UR                  5       UR                  5       /SS9nU$ UR                  5       n U$ U bŽ  X0R                  S5      :”  ag  [         R                  " X#U R                  S5      -
  4U R
                  S9n[         R                  " UR                  5       U R                  5       /SS9nU$ U R                  5       n U$ UnU$ )Nr.   rò   )rß   )rª   r  r¦   rú   r  rß   )rž   rõ   rö   r÷   rë   Únew_key_padding_maskÚfillers          r`   r  Ú0MultiheadAttention._append_prev_key_padding_maskÎ  s�  € ð !Ñ,¶Ø#8Ð ð@ $Ð#ð? #Ñ.Ð3CÑ3OÜ#(§9¢9Ø&×,Ñ,Ó.Ð0@×0FÑ0FÓ0HÐIÈqñ$Ð ð< $Ð#ð1 #Ñ.Ø×3Ñ3°AÓ6Ó6ÜŸšØÐ+@×+EÑ+EÀaÓ+HÑ!HÐIØ0×7Ñ7ñ�ô (-§y¢yØ*×0Ñ0Ó2°F·L±L³NÐCÈñ(Ð$ð$ $Ð#ð (=×'BÑ'BÓ'DÑ$ð $Ð#ð Ñ)Ø×.Ñ.¨qÓ1Ó1ÜŸšØÐ+;×+@Ñ+@ÀÓ+CÑ!CÐDØ+×2Ñ2ñ�ô (-§y¢yØ—\‘\“^Ð%5×%;Ñ%;Ó%=Ð>ÀAñ(Ð$ð $Ð#ð (8×'=Ñ'=Ó'?Ñ$ð $Ð#ð $9Ð Ø#Ð#rk   c                 ó8   • U R                  US5      nUb  U$ 0 nU$ ©NÚ
attn_state)Úget_incremental_state)r[   rê   ÚresultÚempty_results       r`   r  Ú$MultiheadAttention._get_input_bufferú  s-   € ð ×+Ñ+Ð,=¸|ÓLˆØÑØˆMà8:ˆLØÐrk   Úbufferc                 ó(   • U R                  USU5      $ r0  )Úset_incremental_state)r[   rê   r6  s      r`   r  Ú$MultiheadAttention._set_input_buffer  s   € ð
 ×)Ñ)Ð*;¸\È6ÓRÐRrk   r  r  c                 ó   • U$ rc   © )r[   r  r  r÷   r  s        r`   r  Ú$MultiheadAttention.apply_sparse_mask  s   € ØÐrk   )r»   r¹   rº   r²   r­   r¸   r¾   r¼   r,   r)   r´   r¶   rT   r®   r+   r*   r±   rW   rµ   rV   r°   rL   r·   r‹   rU   r¯   )NNrÈ   TFFFFrÈ   r/   Frñ   é€   FF)T)NNTFNFFN)r   r€   r�   r‚   Ú__doc__r4   r¿   rÚ   rè   r   r   r   r§   r¨   r   ri   Ústaticmethodr©   r  r  r  r  rƒ   r„   r…   s   @r`   r�   r�   #  s5  ø† ñð ØØØØØØØ"'ØØØ$)ØØØØ÷%T òlHô, ò@ð& .2ØNRØ!ØØ&*Ø$Ø"'Ø*.ñI1ð �fÑðI1ð ˜Ñð	I1ð
 # 6Ñ*ðI1ð $ D¨¨d°3¸ÀÑ8HÐ3HÑ.IÐ)IÑ$JÑKðI1ð ðI1ð ðI1ð ˜FÑ#ðI1ð ðI1ð  ðI1ð   Ñ'ðI1ð 
ˆv�x Ñ'¨°&Ñ)9Ð9Ñ	:õI1ðV ð)$Ø" 6Ñ*ð)$à'¨Ñ/ð)$ð ð)$ð ð	)$ð
 ð)$ð 
�&Ñ	ó)$ó ð)$ðV Ø!)¨$¨s°D¸¸hÀvÑ>NÐ9NÑ4OÐ/OÑ*PÑ!Qð à	ˆc�8˜FÑ#Ð#Ñ	$ô ðSà  T¨#¨x¸Ñ/?Ð*?Ñ%@Ð @ÑAðSð �S˜( 6Ñ*Ð*Ñ+ôSð°sð ÀSð Ès÷ ò rk   r�   c                 óä  • S n[        U [        R                  5      (       aM  U" U R                  R                  5        U R
                  b$  U R
                  R                  R                  5         [        U [        R                  5      (       aZ  U" U R                  R                  5        U R                  b1  U R                  R                  U R                     R                  5         [        U [        5      (       as  U" U R                  R                  R                  5        U" U R                  R                  R                  5        U" U R                  R                  R                  5        gg)an  
Initialize the weights specific to the BERT Model.
This overrides the default initializations depending on the specified arguments.
    1. If normal_init_linear_weights is set then weights of linear
       layer will be initialized using the normal distribution and
       bais will be set to the specified value.
    2. If normal_init_embed_weights is set then weights of embedding
       layer will be initialized using the normal distribution.
    3. If normal_init_proj_weights is set then weights of
       in_project_weight for MultiHeadAttention initialized using
       the normal distribution (to be validated).
c                 ó�   • U R                  U R                  5       R                  SSS9R                  U R                  5      5        g )NrÈ   g{®Gáz”?r   )Úcopy_Úcpur=   rÌ   rß   )Údatas    r`   r=   Ú!init_bert_params.<locals>.normal_  s5   € ð 	�
‰
�4—8‘8“:×%Ñ%¨3°DÐ%Ð9×<Ñ<¸T¿[¹[ÓIÕJrk   N)Ú
isinstancer   r•   r   rD  r?   Úzero_r³   Úpadding_idxr�   rV   rT   rU   )Úmoduler=   s     r`   rQ   rQ     sø   € òKô
 �&œ"Ÿ)™)×$Ñ$Ù�—‘×"Ñ"Ô#Ø�;‰;Ñ"Ø�K‰K×Ñ×"Ñ"Ô$Ü�&œ"Ÿ,™,×'Ñ'Ù�—‘×"Ñ"Ô#Ø×ÑÑ)Ø�M‰M×Ñ˜v×1Ñ1Ñ2×8Ñ8Ô:Ü�&Ô,×-Ñ-Ù�—‘×$Ñ$×)Ñ)Ô*Ù�—‘×$Ñ$×)Ñ)Ô*Ù�—‘×$Ñ$×)Ñ)Õ*ð .rk   )r:   Únumpyrw   Útypingr   r   r   rª   r   r   Útorch.nn.functionalÚ
functionalrt   Útorch.nnr   r	   ÚBEATs.modulesr
   r   r   r   r   ÚModuler   rG   r�   rQ   r;  rk   r`   Ú<module>rQ     sj   ðó Û ß (Ñ (Û ß ß Ð ß )÷õ ôJ ˜Ÿ™ô J ôZy! b§i¡iô y!ôxi˜Ÿ™ô ióX+rk   