ó
    Å Öj  ã                   óN  • S SK r S SKrS SKrS SKJrJr  S SKJs  Jr   " S S\R                  R                  5      r
 " S S\R                  5      r " S S\R                  5      r " S	 S
\R                  5      rS rS\R                  S\R                  4S jrS\4S jrS rg)é    N)ÚTensorÚnnc                   ó4   • \ rS rSr\S 5       r\S 5       rSrg)ÚGradMultiplyé   c                 ó4   • X l         UR                  U5      nU$ ©N)ÚscaleÚnew)ÚctxÚxr
   Úress       Ú./root/disk_3tb/ICBHI_2017_F12/BEATs/modules.pyÚforwardÚGradMultiply.forward   s   € àŒ	Ø�e‰e�A‹hˆØˆ
ó    c                 ó"   • XR                   -  S 4$ r	   )r
   )r   Úgrads     r   ÚbackwardÚGradMultiply.backward   s   € à—i‘iÑ Ð%Ð%r   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr   r   Ú__static_attributes__r   r   r   r   r      s(   † Øñó ðð
 ñ&ó ó&r   r   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚSamePadé   c                 ój   >• [         TU ]  5         U(       a  US-
  U l        g US-  S:X  a  SOSU l        g )Né   é   r   )ÚsuperÚ__init__Úremove)ÚselfÚkernel_sizeÚcausalÚ	__class__s      €r   r%   ÚSamePad.__init__   s0   ø€ Ü‰ÑÔÞØ%¨™/ˆD�Kà*¨Q™°!Ó3™!¸ˆD�Kr   c                 óX   • U R                   S:”  a  US S 2S S 2S U R                   * 24   nU$ )Nr   ©r&   ©r'   r   s     r   r   ÚSamePad.forward%   s-   € Ø�;‰;˜‹?Ø’!’Q˜˜4Ÿ;™;˜,˜Ð&Ñ'ˆAØˆr   r-   )F©r   r   r   r   r%   r   r   Ú__classcell__©r*   s   @r   r   r      s   ø† ÷;÷ð r   r   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚSwishé+   c                 óp   >• [         [        U ]  5         [        R                  R                  5       U l        g r	   )r$   r4   r%   Útorchr   ÚSigmoidÚact)r'   r*   s    €r   r%   ÚSwish.__init__,   s$   ø€ ÜŒe�TÑ#Ô%Ü—8‘8×#Ñ#Ó%ˆ�r   c                 ó(   • XR                  U5      -  $ r	   ©r9   r.   s     r   r   ÚSwish.forward0   s   € Ø—8‘8˜A“;‰Ðr   r<   r0   r2   s   @r   r4   r4   +   s   ø† õ&÷ð r   r4   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )Ú
GLU_Linearé4   c                 óô  >• [         [        U ]  5         X0l        X l        US:X  a$  [
        R                  R                  5       U l        OiUS:X  a  [        5       U l        OSUS:X  a$  [
        R                  R                  5       U l        O)US:X  a#  [
        R                  R                  5       U l        U(       a   [        R                  " XS-  S5      U l        g [        R                  " XS-  S5      U l        g )NÚsigmoidÚswishÚreluÚgelur#   TF)r$   r?   r%   Úglu_typeÚ
output_dimr7   r   r8   Úglu_actr4   ÚReLUÚGELUÚLinearÚlinear)r'   Ú	input_dimrG   rF   Úbias_in_glur*   s        €r   r%   ÚGLU_Linear.__init__5   s¨   ø€ ÜŒj˜$Ñ(Ô*à ŒØ$Œà�yÓ Ü Ÿ8™8×+Ñ+Ó-ˆD�LØ˜Ó Ü ›7ˆD�LØ˜ÓÜ Ÿ8™8Ÿ=™=›?ˆD�LØ˜ÓÜ Ÿ8™8Ÿ=™=›?ˆDŒLæÜŸ)š) I¸A©~¸tÓDˆD�KäŸ)š) I¸A©~¸uÓEˆD�Kr   c           	      ób  • U R                  U5      nU R                  S:X  a@  US S 2S S 2SU R                  24   US S 2S S 2U R                  U R                  S-  24   -  nU$ US S 2S S 2SU R                  24   U R                  US S 2S S 2U R                  U R                  S-  24   5      -  nU$ )NÚbilinearr   r#   )rL   rF   rG   rH   r.   s     r   r   ÚGLU_Linear.forwardI   sº   € à�K‰K˜‹Nˆà�=‰=˜JÓ&à’!’Q˜˜DŸO™OÐ+Ð+Ñ,Ø’A’q˜$Ÿ/™/¨D¯O©O¸aÑ,?Ð?Ð?Ñ@ñAð ð ˆð	 ’!’Q˜˜DŸO™OÐ+Ð+Ñ,¨t¯|©|Ø’!’Q˜Ÿ™¨$¯/©/¸AÑ*=Ð=Ð=Ñ>ó0ñ ˆAð ˆr   )rH   rF   rL   rG   )rB   Tr0   r2   s   @r   r?   r?   4   s   ø† ÷F÷(ð r   r?   c                 ó"  • [        [        S5      (       d0  [        R                  " S[        R                  -  5      [        l        SU -  S[        R                  " [        R
                  U S[        R                  " U S5      -  -   -  5      -   -  $ )NÚ_ar#   g      à?r"   g÷Hmâä¦?é   )	ÚhasattrÚgelu_accurateÚmathÚsqrtÚpirT   r7   ÚtanhÚpow©r   s    r   rW   rW   Z   sg   € Ü”= $×'Ñ'ÜŸ9š9 Q¬¯©¡[Ó1ŒÔàˆa‰�1”u—z’z¤-×"2Ñ"2°a¸(ÄUÇYÂYÈqÐRSÃ_Ñ:TÑ6TÑ"UÓVÑVÑWðr   r   Úreturnc                 óŽ   • [         R                  R                  R                  U R	                  5       5      R                  U 5      $ r	   )r7   r   Ú
functionalrE   ÚfloatÚtype_asr]   s    r   rE   rE   b   s.   € Ü�8‰8×Ñ×#Ñ# A§G¡G£IÓ.×6Ñ6°qÓ9Ð9r   Ú
activationc                 ó&  • U S:X  a  [         R                  $ U S:X  a  [        $ U S:X  a  [        R                  " S5        [
        $ U S:X  a  [
        $ U S:X  a  [        R                  $ U S:X  a  S $ U S	:X  a  S
 $ [        SR                  U 5      5      e)z=Returns the activation function corresponding to `activation`rD   rE   Ú	gelu_fastz;--activation-fn=gelu_fast has been renamed to gelu_accuraterW   r[   rL   c                 ó   • U $ r	   r   r]   s    r   Ú<lambda>Ú#get_activation_fn.<locals>.<lambda>u   ó   € ™r   Úgluc                 ó   • U $ r	   r   r]   s    r   rg   rh   w   ri   r   z --activation-fn {} not supported)
ÚFrD   rE   ÚwarningsÚwarnrW   r7   r[   ÚRuntimeErrorÚformat)rc   s    r   Úget_activation_fnrq   f   s�   € ð �VÓÜ�v‰vˆØ	�vÓ	ÜˆØ	�{Ó	"Ü�ŠÐSÔTÜÐØ	�Ó	&ÜÐØ	�vÓ	Ü�z‰zÐØ	�xÓ	ÙÐØ	�uÓ	ÙÐäÐ=×DÑDÀZÓPÓQÐQr   c                 ó  ^^^• TS::  a  U $ [        U [        R                  [        R                  [        R                  45      (       d   eU R
                  R                  S:H  mT(       d*  U R
                  R                  S5      T-  S:X  d   S5       eOZU R                  S:X  a  U R                  T-  S:X  d   S5       eO/U R                  S   U R                  S   -  nUT-  S:X  d   S5       eUUU4S jnU R                  U5        U $ )	a  
Wraps modules and applies quantization noise to the weights for
subsequent quantization with Iterative Product Quantization as
described in "Training with Quantization Noise for Extreme Model Compression"

Args:
    - module: nn.Module
    - p: amount of Quantization Noise
    - block_size: size of the blocks for subsequent quantization with iPQ

Remarks:
    - Module weights must have the right sizes wrt the block size
    - Only Linear, Embedding and Conv2d modules are supported for the moment
    - For more detail on how to quantize by blocks with convolutional weights,
      see "And the Bit Goes Down: Revisiting the Quantization of Neural Networks"
    - We implement the simplest form of noise here as stated in the paper
      which consists in randomly dropping blocks
r   é   r"   z0Input features must be a multiple of block sizes©r"   r"   z0Input channels must be a multiple of block sizesz,Kernel size must be a multiple of block sizec                 ó@  >• U R                   (       Ga  T
(       dˆ  U R                  nUR                  S5      nUR                  S5      n[        R                  " UT	-  U-  UR
                  S9nUR                  T5        UR                  T	S5      R                  SU5      nGO1U R                  nU R                  nU R                  nU R                  S:X  ab  [        R                  " [        UT	-  U-  5      UR
                  S9nUR                  T5        UR                  T	S5      R                  SU5      nO›[        R                  " UR                  S5      UR                  S5      UR
                  S9nUR                  T5        UR                  S5      R                  S5      R                  SSU R                  S   U R                  S   5      nUR                  [        R                   5      nSST-
  -  nX‚R#                  US5      -  U R                  l        g g )Nr"   r   )Údeviceéÿÿÿÿrt   r#   rU   )ÚtrainingÚweightÚsizer7   Úzerosrv   Ú
bernoulli_Úrepeat_interleaveÚviewÚin_channelsÚout_channelsr(   ÚintÚ	unsqueezeÚrepeatÚtoÚboolÚmasked_fillÚdata)ÚmodÚinputry   Úin_featuresÚout_featuresÚmaskr   r€   ÚsÚ
block_sizeÚis_convÚps            €€€r   Ú_forward_pre_hookÚ&quant_noise.<locals>._forward_pre_hook¬   s½  ø€ à�<�<ˆ<ÞàŸ™�Ø$Ÿk™k¨!›n�Ø%Ÿ{™{¨1›~�ô —{’{Ø :Ñ-°Ñ<ÀVÇ]Á]ñ�ð —‘ Ô"Ø×-Ñ-¨j¸"Ó=×BÑBÀ2À{ÓS’ð Ÿ™�Ø!Ÿo™o�Ø"×/Ñ/�ð —?‘? fÓ,Ü Ÿ;š;Ü˜K¨:Ñ5¸ÑDÓEØ%Ÿ}™}ñ�Dð —O‘O AÔ&Ø×1Ñ1°*¸bÓA×FÑFÀrÈ;ÓW‘Dä Ÿ;š;ØŸ™ A›¨¯©°A«¸v¿}¹}ñ�Dð —O‘O AÔ&àŸ™ qÓ)ß"™ 1›ß™  1 c§o¡o°aÑ&8¸#¿/¹/È!Ñ:LÓMð ð —7‘7Ü—
‘
óˆDð �Q˜‘U‘ˆAØ×"4Ñ"4°T¸1Ó"=Ñ=ˆC�J‰J�OðY r   )Ú
isinstancer   rK   Ú	EmbeddingÚConv2dry   Úndimrz   r(   r   Úregister_forward_pre_hook)Úmoduler�   rŽ   Úkr‘   r�   s    ``  @r   Úquant_noiserš   |   s  ú€ ð* 	ˆAƒvØˆô �fœrŸy™y¬"¯,©,¼¿	¹	ÐB×CÑCÐCÐCð �m‰m× Ñ  AÑ%€Gö à�M‰M×Ñ˜qÓ! JÑ.°!Ó3ð	>à=ó	>Ù3ð ×Ñ Ó'à×"Ñ" ZÑ/°1Ó4ðBàAóBÙ4ð ×"Ñ" 1Ñ%¨×(:Ñ(:¸1Ñ(=Ñ=ˆAØ�z‘> QÓ&ÐVÐ(VÓVÐ&÷.>ð` ×$Ñ$Ð%6Ô7Ø€Mr   )rX   rm   r7   r   r   Útorch.nn.functionalr`   rl   ÚautogradÚFunctionr   ÚModuler   r4   r?   rW   rE   Ústrrq   rš   r   r   r   Ú<module>r       s”   ðó Û Û ß ß Ð ô	&�5—>‘>×*Ñ*ô 	&ôˆb�i‰iô ôˆB�I‰Iô ô#�—‘ô #òLð:ˆE�L‰Lð :˜UŸ\™\ô :ðR #ô Ró,ar   