ó
    ÁJ®j<N  ã            
       óx  • S SK r S SKJr  S SKJs  Jr  S SKJrJr  S SK	J
r
  S SKJrJr   " S S\R                  5      r " S S\R                  5      r " S	 S
\R                  5      rSSSSSSSSSS.	r\
S+S j5       r\
S+S j5       r\
S+S j5       r\
S+S j5       r\
S+S j5       r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      rS S KJrJr  S S!KJr  S"r  " S# S\R                  5      r " S$ S%\R                  5      r!\"S&:X  a  \!" / S'Q/ S(QS)S*9r#\$" \#5        gg),é    N)Útrunc_normal_ÚDropPath)Úregister_model)ÚDecoderÚQuery2Labelc                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )ÚBlocké   añ  ConvNeXt Block. There are two equivalent implementations:
(1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W)
(2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back
We use (2) as we find it slightly faster in PyTorch

Args:
    dim (int): Number of input channels.
    drop_path (float): Stochastic depth rate. Default: 0.0
    layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.
c                 óü  >• [         TU ]  5         [        R                  " XSSUS9U l        [        USS9U l        [        R                  " USU-  5      U l        [        R                  " 5       U l
        [        R                  " SU-  U5      U l        US:”  a,  [        R                  " U[        R                  " U5      -  SS	9OS U l        US
:”  a  [!        U5      U l        g [        R"                  " 5       U l        g ©Né   é   )Úkernel_sizeÚpaddingÚgroupsç�íµ ÷Æ°>©Úepsé   r   T)Úrequires_gradç        ©ÚsuperÚ__init__ÚnnÚConv2dÚdwconvÚ	LayerNormÚnormÚLinearÚpwconv1ÚGELUÚactÚpwconv2Ú	ParameterÚtorchÚonesÚgammar   ÚIdentityÚ	drop_path©ÚselfÚdimr*   Úlayer_scale_init_valueÚ	__class__s       €Ú$/root/disk_3tb/thao_isbi/convnext.pyr   ÚBlock.__init__   s¾   ø€ Ü‰ÑÔÜ—i’i °aÀÈ3ÑOˆŒÜ˜c tÑ,ˆŒ	Ü—y’y  a¨#¡gÓ.ˆŒÜ—7’7“9ˆŒÜ—y’y  S¡¨#Ó.ˆŒà;QÐTUÓ;Uô —\’\Ð"8¼5¿:º:ÀsÓ;LÑ"LØ26ò8Ø[_ð 	Œ
à09¸B³œ )Ó,ˆ�ÄBÇKÂKÃMˆ�ó    c                 ób  • UnU R                  U5      nUR                  SSSS5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  b  U R                  U-  nUR                  SSSS5      nX R                  U5      -   nU$ ©Nr   é   r   é   ©r   Úpermuter   r!   r#   r$   r(   r*   )r,   ÚxÚinputs      r0   ÚforwardÚBlock.forward&   s�   € ØˆØ�K‰K˜‹NˆØ�I‰I�a˜˜A˜qÓ!ˆØ�I‰I�a‹LˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�L‰L˜‹OˆØ�:‰:Ñ!Ø—
‘
˜Q‘ˆAØ�I‰I�a˜˜A˜qÓ!ˆà—N‘N 1Ó%Ñ%ˆØˆr2   ©r#   r*   r   r(   r   r!   r$   ©r   r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r;   Ú__static_attributes__Ú__classcell__©r/   s   @r0   r	   r	      s   ø† ñ	÷	R÷ð r2   r	   c                   óX   ^ • \ rS rSrSrSS/ SQ/ SQSSS	4U 4S
 jjrS rS rS rSr	U =r
$ )ÚConvNeXté5   a‰  ConvNeXt
    A PyTorch impl of : `A ConvNet for the 2020s`  -
      https://arxiv.org/pdf/2201.03545.pdf

Args:
    in_chans (int): Number of input image channels. Default: 3
    num_classes (int): Number of classes for classification head. Default: 1000
    depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]
    dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]
    drop_path_rate (float): Stochastic depth rate. Default: 0.
    layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.
    head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.
r   éè  ©r   r   é	   r   ©é`   éÀ   é€  é   r   r   ç      ð?c                 óî  >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " [        R                  " XS   SSS9[        US   SSS95      nU R                  R                  U5        [        S5       H\  n	[        R
                  " [        XI   SSS9[        R                  " XI   XIS-      S	S	S95      n
U R                  R                  U
5        M^     [        R                  " 5       U l
        [        R                  " SU[        U5      5       Vs/ s H  o»R                  5       PM     nnSn[        S5       Hd  n	[        R
                  " [        X9   5       Vs/ s H  n[        XI   XÍU-      US
9PM     sn6 nU R                  R                  U5        XÓU	   -  nMf     [        R                  " US   SS9U l        [        R"                  " US   U5      U l        U R'                  U R(                  5        U R$                  R*                  R,                  R/                  U5        U R$                  R0                  R,                  R/                  U5        g s  snf s  snf )Nr   r   ©r   Ústrider   Úchannels_first©r   Údata_formatr   r6   r5   ©r-   r*   r.   éÿÿÿÿr   )r   r   r   Ú
ModuleListÚdownsample_layersÚ
Sequentialr   r   ÚappendÚrangeÚstagesr&   ÚlinspaceÚsumÚitemr	   r   r    ÚheadÚapplyÚ_init_weightsÚweightÚdataÚmul_Úbias)r,   Úin_chansÚnum_classesÚdepthsÚdimsÚdrop_path_rater.   Úhead_init_scaleÚstemÚiÚdownsample_layerr9   Údp_ratesÚcurÚjÚstager/   s                   €r0   r   ÚConvNeXt.__init__C   sþ  ø€ ô 	‰ÑÔä!#§¢£ˆÔÜ�}Š}Ü�IŠI�h Q¡°Q¸qÑAÜ�d˜1‘g 4Ð5EÑFó
ˆð 	×Ñ×%Ñ% dÔ+Ü�q–ˆAÜ!Ÿ}š}Ü˜d™g¨4Ð=MÑNÜ—I’I˜d™g t¨a©C¡y¸aÈÑJó Ðð ×"Ñ"×)Ñ)Ð*:Ö;ñ ô —m’m“oˆŒÜ$)§N¢N°1°nÄcÈ&ÃkÔ$RÓSÒ$R˜q—&‘&–(Ñ$RˆÐSØˆÜ�q–ˆAÜ—M’MäHMÈfÉiÔHXóZÚHXÀ1ô ˜D™G¨x¸a¹Ñ/@Ø'=ô?ÙHXñZðˆEð �K‰K×Ñ˜uÔ%Ø˜!‘9ÑŠCñ ô —L’L  b¡¨tÑ4ˆŒ	Ü—I’I˜d 2™h¨Ó4ˆŒ	à�
‰
�4×%Ñ%Ô&Ø�	‰	×Ñ×Ñ×"Ñ" ?Ô3Ø�	‰	�‰×Ñ× Ñ  Õ1ùò TùòZs   Ä'I-Å1I2
c                 óà   • [        U[        R                  [        R                  45      (       a?  [	        UR
                  SS9  [        R                  R                  UR                  S5        g g )Nç{®Gáz”?©Ústdr   )	Ú
isinstancer   r   r    r   rh   ÚinitÚ	constant_rk   ©r,   Úms     r0   rg   ÚConvNeXt._init_weightsh   sG   € Ü�aœ"Ÿ)™)¤R§Y¡YÐ/×0Ñ0Ü˜!Ÿ(™(¨Ò,Ü�G‰G×Ñ˜aŸf™f aÕ(ð 1r2   c                 ó¾   • [        S5       H-  nU R                  U   " U5      nU R                  U   " U5      nM/     U R                  UR	                  SS/5      5      $ )Nr   éþÿÿÿr[   )r`   r]   ra   r   Úmean©r,   r9   rs   s      r0   Úforward_featuresÚConvNeXt.forward_featuresm   sU   € Ü�q–ˆAØ×&Ñ& qÒ)¨!Ó,ˆAØ—‘˜A’˜qÓ!ŠAñ ð �y‰y˜Ÿ™  R Ó)Ó*Ð*r2   c                 óJ   • U R                  U5      nU R                  U5      nU$ ©N)rˆ   re   ©r,   r9   s     r0   r;   ÚConvNeXt.forwardt   s$   € Ø×!Ñ! !Ó$ˆØ�I‰I�a‹LˆØˆr2   )r]   re   r   ra   ©r@   rA   rB   rC   rD   r   rg   rˆ   r;   rE   rF   rG   s   @r0   rI   rI   5   s6   ø† ñð !"¨tÚ$Ò+>ÈrØ(,¸b÷#2òJ)ò
+÷ð r2   rI   c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )r   éy   a8  LayerNorm that supports two data formats: channels_last (default) or channels_first. 
The ordering of the dimensions in the inputs. channels_last corresponds to inputs with 
shape (batch_size, height, width, channels) while channels_first corresponds to inputs 
with shape (batch_size, channels, height, width).
c                 ó2  >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        [        R                  " [        R                  " U5      5      U l        X l	        X0l
        U R                  S;  a  [        eU4U l        g )N)Úchannels_lastrW   )r   r   r   r%   r&   r'   rh   Úzerosrk   r   rY   ÚNotImplementedErrorÚnormalized_shape)r,   r•   r   rY   r/   s       €r0   r   ÚLayerNorm.__init__   sm   ø€ Ü‰ÑÔÜ—l’l¤5§:¢:Ð.>Ó#?Ó@ˆŒÜ—L’L¤§¢Ð-=Ó!>Ó?ˆŒ	ØŒØ&ÔØ×ÑÐ#FÓFÜ%Ð%Ø!1Ð 4ˆÕr2   c                 óÒ  • U R                   S:X  aA  [        R                  " XR                  U R                  U R
                  U R                  5      $ U R                   S:X  a†  UR                  SSS9nX-
  R                  S5      R                  SSS9nX-
  [        R                  " X0R                  -   5      -  nU R                  S S 2S S 4   U-  U R
                  S S 2S S 4   -   nU$ g )Nr’   rW   r6   T)Úkeepdimr5   )rY   ÚFÚ
layer_normr•   rh   rk   r   r†   Úpowr&   Úsqrt)r,   r9   ÚuÚss       r0   r;   ÚLayerNorm.forward‰   sÌ   € Ø×Ñ˜Ó.Ü—<’< ×#8Ñ#8¸$¿+¹+ÀtÇyÁyÐRV×RZÑRZÓ[Ð[Ø×ÑÐ!1Ó1Ø—‘�q $�Ð'ˆAØ‘—‘˜A“×#Ñ# A¨tÐ#Ð4ˆAØ‘œ%Ÿ*š* Q¯©¡\Ó2Ñ2ˆAØ—‘šA˜t T˜MÑ*¨QÑ.°·±º1¸dÀD¸=Ñ1IÑIˆAØˆHð 2r2   )rk   rY   r   r•   rh   )r   r’   r?   rG   s   @r0   r   r   y   s   ø† ñ÷
5÷ð r2   r   zDhttps://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pthzEhttps://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pthzDhttps://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pthzEhttps://dl.fbaipublicfiles.com/convnext/convnext_large_1k_224_ema.pthzAhttps://dl.fbaipublicfiles.com/convnext/convnext_tiny_22k_224.pthzBhttps://dl.fbaipublicfiles.com/convnext/convnext_small_22k_224.pthzAhttps://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pthzBhttps://dl.fbaipublicfiles.com/convnext/convnext_large_22k_224.pthzChttps://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_224.pth)	Úconvnext_tiny_1kÚconvnext_small_1kÚconvnext_base_1kÚconvnext_large_1kÚconvnext_tiny_22kÚconvnext_small_22kÚconvnext_base_22kÚconvnext_large_22kÚconvnext_xlarge_22kc                 óÐ   • [        S
/ SQ/ SQS.UD6nU (       aL  U(       a	  [        S   O[        S   n[        R                  R	                  USSS9nUR                  US	   5        U$ )NrL   rN   ©rn   ro   r¤   r    ÚcpuT)ÚurlÚmap_locationÚ
check_hashÚmodel© ©rI   Ú
model_urlsr&   ÚhubÚload_state_dict_from_urlÚload_state_dict©Ú
pretrainedÚin_22kÚkwargsr¯   r¬   Ú
checkpoints         r0   Úconvnext_tinyr»       sc   € äÐMšLÒ/BÑMÀfÑM€EÞÞ17ŒjÐ,Ò-¼ZÐHZÑ=[ˆÜ—Y‘Y×7Ñ7¸CÈeÐ`dÐ7Ðeˆ
Ø×Ñ˜j¨Ñ1Ô2Ø€Lr2   c                 óÎ   • [        S	/ SQ/ SQS.UD6nU (       aK  U(       a	  [        S   O[        S   n[        R                  R	                  USS9nUR                  US   5        U$ )
N©r   r   é   r   rN   rª   r¥   r¡   r«   ©r¬   r­   r¯   r°   r±   r¶   s         r0   Úconvnext_smallrÀ   ©   s`   € äÐNšMÒ0CÑNÀvÑN€EÞÞ28ŒjÐ-Ò.¼jÐI\Ñ>]ˆÜ—Y‘Y×7Ñ7¸CÈeÐ7ÐTˆ
Ø×Ñ˜j¨Ñ1Ô2Ø€Lr2   c                 ó   • [        S/ SQ/ SQS.UD6nU (       at  U(       a	  [        S   O[        S   n[        R                  R	                  USS9S   n[        UR                  5       5       H  nS	U;   d  M  XV	 M     UR                  US
S9  U$ )Nr½   )é€   é   é   é   rª   r¦   r¢   r«   r¿   r¯   re   F)Ústrictr°   )rI   r²   r&   r³   r´   ÚlistÚkeysrµ   )r·   r¸   r¹   r¯   r¬   rº   Úks          r0   Úconvnext_baserÊ   ²   sŒ   € äÐPšMÒ0EÑPÈÑP€EÞÞ17ŒjÐ,Ò-¼ZÐHZÑ=[ˆÜ—Y‘Y×7Ñ7¸CÈeÐ7ÐTÐU\Ñ]ˆ
ô �j—o‘oÓ'Ö(ˆAØ˜�{Ø’Mñ )ð 	×Ñ˜j°ÐÑ7Ø€Lr2   c                 óÎ   • [        S	/ SQ/ SQS.UD6nU (       aK  U(       a	  [        S   O[        S   n[        R                  R	                  USS9nUR                  US   5        U$ )
Nr½   )rP   rQ   rR   i   rª   r§   r£   r«   r¿   r¯   r°   r±   r¶   s         r0   Úconvnext_largerÌ   Á   s`   € äÐPšMÒ0EÑPÈÑP€EÞÞ28ŒjÐ-Ò.¼jÐI\Ñ>]ˆÜ—Y‘Y×7Ñ7¸CÈeÐ7ÐTˆ
Ø×Ñ˜j¨Ñ1Ô2Ø€Lr2   c                 óÊ   • [        S	/ SQ/ SQS.UD6nU (       aI  U(       d   S5       e[        S   n[        R                  R	                  USS9nUR                  US   5        U$ )
Nr½   )rÃ   rÄ   rÅ   i   rª   zNonly ImageNet-22K pre-trained ConvNeXt-XL is available; please set in_22k=Truer¨   r«   r¿   r¯   r°   r±   r¶   s         r0   Úconvnext_xlargerÎ   Ê   sd   € äÐQšMÒ0FÑQÈ&ÑQ€EÞÞÐgÐgÓgˆvÜÐ.Ñ/ˆÜ—Y‘Y×7Ñ7¸CÈeÐ7ÐTˆ
Ø×Ñ˜j¨Ñ1Ô2Ø€Lr2   c                   óB   ^ • \ rS rSrSS\S\S\4U 4S jjjrS rSrU =r	$ )	ÚFFNéÕ   Úd_modelÚd_ffÚdropoutc                 óø   >• [         TU ]  5         [        R                  " X5      U l        [        R                  " X!5      U l        [        R                  " 5       U l        [        R                  " U5      U l	        g r‹   )
r   r   r   r    Úfc1Úfc2r"   r#   ÚDropoutÚdrop)r,   rÒ   rÓ   rÔ   r/   s       €r0   r   ÚFFN.__init__Ö   sK   ø€ Ü‰ÑÔÜ—9’9˜WÓ+ˆŒÜ—9’9˜TÓ+ˆŒÜ—7’7“9ˆŒÜ—J’J˜wÓ'ˆ�	r2   c           	      ó~   • U R                  U R                  U R                  U R                  U5      5      5      5      $ r‹   )r×   rÙ   r#   rÖ   rŒ   s     r0   r;   ÚFFN.forwardÝ   s+   € Ø�x‰x˜Ÿ	™	 $§(¡(¨4¯8©8°A«;Ó"7Ó8Ó9Ð9r2   )r#   rÙ   rÖ   r×   )çš™™™™™¹?)
r@   rA   rB   rC   ÚintÚfloatr   r;   rE   rF   rG   s   @r0   rÐ   rÐ   Õ   s,   ø† ñ( ð (¨3ð (¸÷ (ð (÷:ð :r2   rÐ   c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )ÚDecoderLayeréá   zp
One layer: (SelfAttn + FFN) then (CrossAttn + FFN), each with Pre-LN + residual.
Order: SA -> CA (DETR-style).
c                 óŠ  >• [         TU ]  5         [        R                  " U5      U l        [        R                  " U5      U l        [        R                  " XUSS9U l        [        R                  " U5      U l	        [        R                  " U5      U l
        [        XU-  US9U l        [        R                  " U5      U l        g )NT)rÔ   Úbatch_first)rÔ   )r   r   r   r   Úln_q_caÚln_memÚMultiheadAttentionÚcarØ   Údrop_caÚln_ffn2rÐ   Úffn2Ú	drop_ffn2)r,   rÒ   ÚnheadrÔ   Úffn_multr/   s        €r0   r   ÚDecoderLayer.__init__æ   sŠ   ø€ Ü‰ÑÔô Ÿš WÓ-ˆŒÜŸš WÓ-ˆŒÜ×-Ò-¨gÀgÐ[_Ñ`ˆŒÜŸ
š
 7Ó+ˆŒÜŸš WÓ-ˆŒÜ˜G¨xÑ%7ÀÑIˆŒ	ÜŸš GÓ,ˆ�r2   c                 óø   • U R                  U5      nU R                  U5      nU R                  X4USS9u  pVXR                  U5      -   nXR	                  U R                  U R                  U5      5      5      -   nU$ )NF)Úneed_weights)rå   ræ   rè   ré   rì   rë   rê   )r,   ÚqÚmemÚq2Úm2Úca_outÚ_s          r0   r;   ÚDecoderLayer.forwardú   so   € ð �\‰\˜!‹_ˆØ�[‰[˜ÓˆØ—G‘G˜B B°U�GÐ;‰	ˆØ—‘˜VÓ$Ñ$ˆØ—‘˜tŸy™y¨¯©°a«Ó9Ó:Ñ:ˆð ˆr2   )rè   ré   rì   rë   rê   ræ   rå   )rÅ   é   rÝ   r   r?   rG   s   @r0   rá   rá   á   s   ø† ñ÷-÷(ð r2   rá   c                   óŽ   ^ • \ rS rSrSr    SS\S\S\S\S\S\4U 4S	 jjjrS
\R                  S\R                  4S jr
SrU =r$ )ÚLabelQueryHeadi  z›
Label-query decoder with:
- Learnable query embeddings initialized from scratch (C, D)
- N stacks of (CA + FFN) + (SA + FFN)
- Final per-label classifier
rm   rÒ   Ún_layersrí   rÔ   Úinit_stdc                 ó¨  >• [         TU ]  5         Xl        X l        [        R
                  " [        R                  " X5      5      U l        [        R                  R                  U R                  US9  [        R                  " [        U5       Vs/ s H  n[        UUUSS9PM     sn5      U l        [        R                  " US5      U l        g s  snf )Nr|   r   )rÒ   rí   rÔ   rî   r6   )r   r   rm   rÒ   r   r%   r&   ÚemptyÚqueryr   r   r\   r`   rá   Úlayersr    Úcls)	r,   rm   rÒ   rü   rí   rÔ   rý   r÷   r/   s	           €r0   r   ÚLabelQueryHead.__init__  s²   ø€ ô 	‰ÑÔà&ÔØŒô
 —\’\¤%§+¢+¨kÓ"CÓDˆŒ
Ü
�‰×Ñ˜dŸj™j¨hÐÑ7ô
 —m’mô ˜8”_ó%
ò %�ô ØØØØô	ñ %ñ%
ó ˆŒô —9’9˜W aÓ(ˆ�ùò%
s   ÂCró   Úreturnc                 óö   • UR                  S5      nU R                  R                  S5      R                  USS5      nU R                   H  nU" X15      nM     U R                  U5      R                  S5      nU$ )z=
mem: (B, S, D) encoder memory tokens
returns: logits (B, C)
r   r[   )Úsizer   Ú	unsqueezeÚexpandr  r  Úsqueeze)r,   ró   ÚBrò   ÚlayerÚlogitss         r0   r;   ÚLabelQueryHead.forward>  sn   € ð
 �H‰H�Q‹Kˆð �J‰J× Ñ  Ó#×*Ñ*¨1¨b°"Ó5ˆð —[”[ˆEÙ�a“ŠAñ !ð —‘˜!“×$Ñ$ RÓ(ˆØˆr2   )r  rÒ   r  rm   r   )r   rù   rÝ   r{   )r@   rA   rB   rC   rD   rÞ   rß   r   r&   ÚTensorr;   rE   rF   rG   s   @r0   rû   rû     s{   ø† ñð ØØØñ$)àð$)ð ð$)ð ð	$)ð
 ð$)ð ð$)ð ÷$)ð $)ðL˜5Ÿ<™<ð ¨E¯L©L÷ ò r2   rû   )ÚPositionalEncoding2DÚSummer)Ú	rearrangegñhãˆµøä>c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )r	   iT  z)ConvNeXt Block (stride=1, preserves HxW).c                 óü  >• [         TU ]  5         [        R                  " XSSUS9U l        [        USS9U l        [        R                  " USU-  5      U l        [        R                  " 5       U l
        [        R                  " SU-  U5      U l        US:”  a,  [        R                  " U[        R                  " U5      -  SS	9OS U l        US
:”  a  [!        U5      U l        g [        R"                  " 5       U l        g r   r   r+   s       €r0   r   r1   V  s¾   ø€ Ü‰ÑÔÜ—i’i °aÀÈ3ÑOˆŒÜ˜c tÑ,ˆŒ	Ü—y’y  a¨#¡gÓ.ˆŒÜ—7’7“9ˆŒÜ—y’y  S¡¨#Ó.ˆŒà9OÐRSÓ9Sô —\’\Ð"8¼5¿:º:ÀsÓ;LÑ"LØ04ò6ØY]ð 	Œ
à09¸B³œ )Ó,ˆ�ÄBÇKÂKÃMˆ�r2   c                 ób  • UnU R                  U5      nUR                  SSSS5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  b  U R                  U-  nUR                  SSSS5      nX R                  U5      -   nU$ r4   r7   )r,   r9   Úresiduals      r0   r;   r<   a  s�   € ØˆØ�K‰K˜‹NˆØ�I‰I�a˜˜A˜qÓ!ˆØ�I‰I�a‹LˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�L‰L˜‹OˆØ�:‰:Ñ!Ø—
‘
˜Q‘ˆAØ�I‰I�a˜˜A˜qÓ!ˆØ—~‘~ aÓ(Ñ(ˆØˆr2   r=   r>   r?   rG   s   @r0   r	   r	   T  s   ø† Ù4÷	R÷ð r2   c                   óX   ^ • \ rS rSrSrSS/ SQ/ SQSSS	4U 4S
 jjrS rS rS rSr	U =r
$ )Ú	ConvNeXt2ip  z¸
ConvNeXt2 variant with total spatial reduction /64 (instead of /32).

Change vs current ConvNeXt:
  - Add 1 extra downsample layer (stride=2)
  - Add 1 extra stage of Blocks after it
r   rK   )r   r   r¾   r   r   ©rÂ   rÃ   rÄ   rÅ   rÅ   r   r   rS   c                 ó,  >• [         TU ]  5         [        U5      [        U5      s=:X  a  S:X  d   S5       e   S5       e[        R                  " 5       U l        [        R                  " [        R                  " XS   SSS9[        US   SSS95      nU R
                  R                  U5        [        S	5       H\  n	[        R                  " [        XI   SSS9[        R                  " XI   XIS
-      SSS95      n
U R
                  R                  U
5        M^     [        R                  " [        US	   SSS9[        R                  " US	   US   SSS95      nU R
                  R                  U5        [        R                  " 5       U l        [        R                  " SU[        U5      5       Vs/ s H  oÌR                  5       PM     nnSn[        S5       Hd  n	[        R                  " [        X9   5       Vs/ s H  n[!        XI   XÞU-      US9PM     sn6 nU R                  R                  U5        XãU	   -  nMf     [#        UUS   SSSSS9U l        ['        [)        US   5      5      U l        U R-                  U R.                  5        g s  snf s  snf )Né   z1For /64 variant, provide 5-stage depths and dims.r   r   rU   r   rW   rX   r   r6   r5   rZ   r[   rR   éd   Úgelu)rm   Úinitial_num_featuresÚdecoder_embeddingÚnum_of_groupsÚ
num_layersÚ
activation)r   r   Úlenr   r\   r]   r^   r   r   r_   r`   ra   r&   rb   rc   rd   r	   r   re   r  r  Úpos_encodingrf   rg   )r,   rl   rm   rn   ro   rp   r.   rq   rr   rs   rt   Úextra_downsampler9   ru   rv   rw   rx   r/   s                    €r0   r   ÚConvNeXt2.__init__y  s^  ø€ ô 	‰ÑÔä�6‹{œc $›iÕ,¨1Ó,ÐaÐ.aÓaÑ,ÐaÐ.aÓaÐ,ô "$§¢£ˆÔÜ�}Š}Ü�IŠI�h Q¡°Q¸qÑAÜ�d˜1‘g 4Ð5EÑFó
ˆð 	×Ñ×%Ñ% dÔ+ô �q–ˆAÜ!Ÿ}š}Ü˜$™' tÐ9IÑJÜ—	’	˜$™' 4¨A©¡;¸AÀaÑHó Ðð ×"Ñ"×)Ñ)Ð*:Ö;ñ ô Ÿ=š=Ü�d˜1‘g 4Ð5EÑFÜ�IŠI�d˜1‘g˜t A™w°A¸aÑ@ó
Ðð 	×Ñ×%Ñ%Ð&6Ô7ô —m’m“oˆŒÜ&+§n¢n°Q¸ÌÈFËÔ&TÓUÒ&T —F‘F–HÑ&TˆÐUØˆÜ�q–ˆAÜ—M’Mô # 6¡9Ô-óò .˜ô Ø ™GØ"*°©7Ñ"3Ø/Eôñ
 .ñð	ˆEð �K‰K×Ñ˜uÔ%Ø˜!‘9ÑŠCñ ô Ø#Ø!% b¡Ø!ØØØñ
ˆŒ	ô #Ô#7¸¸R¹Ó#AÓBˆÔð 	�
‰
�4×%Ñ%Õ&ùò; Vùòs   Æ.JÇ8J
c                 ó�  • [        U[        R                  [        R                  45      (       aN  [	        UR
                  SS9  [        USS 5      b+  [        R                  R                  UR                  S5        g g [        U[        R                  5      (       aU  [        R                  R                  UR
                  S5        [        R                  R                  UR                  S5        g [        U[        R                  5      (       a¶  [	        UR                  SS9  UR                  b*  [        R                  R                  UR                  S5        [	        UR                  R
                  SS9  UR                  R                  b5  [        R                  R                  UR                  R                  S5        g g g )Nr{   r|   rk   r   rS   )r~   r   r   r    r   rh   Úgetattrr   r€   rk   r   rç   Úin_proj_weightÚin_proj_biasÚout_projr�   s     r0   rg   ÚConvNeXt2._init_weightsÄ  s$  € Ü�aœ"Ÿ)™)¤R§Y¡YÐ/×0Ñ0Ü˜!Ÿ(™(¨Ò-Ü�q˜& $Ó'Ñ3Ü—‘×!Ñ! !§&¡&¨!Õ,ð 4ä˜œ2Ÿ<™<×(Ñ(Ü�G‰G×Ñ˜aŸh™h¨Ô,Ü�G‰G×Ñ˜aŸf™f aÕ(Ü˜œ2×0Ñ0×1Ñ1Ü˜!×*Ñ*°Ò5Ø�~‰~Ñ)Ü—‘×!Ñ! !§.¡.°!Ô4Ü˜!Ÿ*™*×+Ñ+°Ò6Ø�z‰z�‰Ñ*Ü—‘×!Ñ! !§*¡*§/¡/°1Õ5ð +ð 2r2   c                 ó~   • [        S5       H-  nU R                  U   " U5      nU R                  U   " U5      nM/     U$ )Nr  )r`   r]   ra   r‡   s      r0   rˆ   ÚConvNeXt2.forward_featuresÔ  s>   € ä�q–ˆAØ×&Ñ& qÒ)¨!Ó,ˆAØ—‘˜A’˜qÓ!ŠAñ ð ˆr2   c                 ól   • U R                  U5      nU R                  U5      nU R                  U5      nU$ r‹   )rˆ   r#  re   rŒ   s     r0   r;   ÚConvNeXt2.forwardÛ  s4   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØ�I‰I�a‹LˆØˆr2   )r]   re   r#  ra   rŽ   rG   s   @r0   r  r  p  s:   ø† ñð ØÚÚ(ØØ#Ø÷F'òV6ò ÷ð r2   r  Ú__main__)r   r   r¾   r   r5   r  é   )rn   ro   rm   )FF)%r&   Útorch.nnr   Útorch.nn.functionalÚ
functionalr™   Útimm.models.layersr   r   Útimm.models.registryr   Ú
ml_decoderr   r   ÚModuler	   rI   r   r²   r»   rÀ   rÊ   rÌ   rÎ   rÐ   rá   rû   Ú$positional_encodings.torch_encodingsr  r  Úeinopsr  ÚNORM_EPSr  r@   r¯   Úprintr°   r2   r0   Ú<module>r=     sg  ðó Ý ß Ð ß 6Ý /ß +ô#ˆB�I‰Iô #ôJBˆr�y‰yô BôH�—	‘	ô ð8 _Ø`Ø^Ø`Ø\Ø^Ø\Ø^Ø`ñ
€
ð óó ðð óó ðð óó ðð óó ðð óó ðô	:ˆ"�)‰)ô 	:ô,�2—9‘9ô ,ô^=�R—Y‘Yô =÷@ NÝ Ø€ôˆB�I‰Iô ô8o�—	‘	ô oðf ˆzÓÙÒ-Ú6Ø"$ñ&€Eñ 
ˆ%…Lð r2   