o
    _{jxZ  ã                	   @   sæ   d dl Z d dlmZ d dlmZmZ d dlmZmZ d dl	m
Z
 ddlmZ ddlmZ d	d
lmZ z
d dlmZ dZW n eefyI   dZY nw G dd„ deƒZdeiZdd„ Ze ¡ G dd„ dejƒƒZerqe ¡ eƒ dS dS )é    N)Ú
ConvModuleÚkaiming_init)Ú_load_checkpointÚload_checkpoint)Ú	print_logé   )Úget_root_loggeré   )Ú	BACKBONESé   )ÚResNet3dTFc                       sv   e Zd ZdZdddddœ‡ fdd„
Zdddd	dd
deƒ ddddfdd„Zdd„ Zdd„ Zdd„ Z	d‡ fdd„	Z
‡  ZS )ÚResNet3dPathwaya  A pathway of Slowfast based on ResNet3d.

    Args:
        *args (arguments): Arguments same as :class:``ResNet3d``.
        lateral (bool): Determines whether to enable the lateral connection
            from another pathway. Default: False.
        speed_ratio (int): Speed ratio indicating the ratio between time
            dimension of the fast and slow pathway, corresponding to the
            ``alpha`` in the paper. Default: 8.
        channel_ratio (int): Reduce the channel number of fast pathway
            by ``channel_ratio``, corresponding to ``beta`` in the paper.
            Default: 8.
        fusion_kernel (int): The kernel size of lateral fusion.
            Default: 5.
        **kwargs (keyword arguments): Keywords arguments for ResNet3d.
    Fé   é   )ÚlateralÚspeed_ratioÚchannel_ratioÚfusion_kernelc          
         sB  || _ || _|| _|| _tƒ j|i |¤Ž | j| _| j rCt| j| j | jd | j |ddf| jddf|d d ddfd| j	d d d�	| _
g | _tt| jƒƒD ]Q}| jd|  }|| jj | _|rž|| jd kržd|d › d�}	t| |	t| j| j | jd | j |ddf| jddf|d d ddfd| j	d d d�	ƒ | j |	¡ qMd S )Nr	   r   r   F)Úkernel_sizeÚstrideÚpaddingÚbiasÚconv_cfgÚnorm_cfgÚact_cfgÚlayerZ_lateral)r   r   r   r   ÚsuperÚ__init__Úbase_channelsÚinplanesr   r   Úconv1_lateralÚlateral_connectionsÚrangeÚlenÚstage_blocksÚblockÚ	expansionÚ
num_stagesÚsetattrÚappend)
Úselfr   r   r   r   ÚargsÚkwargsÚiÚplanesÚlateral_name©Ú	__class__© úA/root/disk_3tb/HAC/mmaction/models/backbones/resnet3d_slowfast.pyr   $   sR   

ô

÷þ€ízResNet3dPathway.__init__r   ÚpytorchÚ3x1x1r   Nc                 C   sP  t |	tƒs|	n|	f| }	t |tƒs|n|f| }t|	ƒ|kr$t|ƒ|ks&J ‚| jr1|d | j }nd}|dks@|| ||j krUt|| ||j d|||fd||dd�}nd}g }| ||| |||||||	d dk|
|d dk|||||d�¡ ||j }td|ƒD ]}| |||dd|||	| dk|
|| dk|||||d�¡ qƒt	j
|Ž S )aÞ  Build residual layer for Slowfast.

        Args:
            block (nn.Module): Residual module to be built.
            inplanes (int): Number of channels for the input
                feature in each block.
            planes (int): Number of channels for the output
                feature in each block.
            blocks (int): Number of residual blocks.
            spatial_stride (int | Sequence[int]): Spatial strides
                in residual and conv layers. Default: 1.
            temporal_stride (int | Sequence[int]): Temporal strides in
                residual and conv layers. Default: 1.
            dilation (int): Spacing between kernel elements. Default: 1.
            style (str): ``pytorch`` or ``caffe``. If set to ``pytorch``,
                the stride-two layer is the 3x3 conv layer,
                otherwise the stride-two layer is the first 1x1 conv layer.
                Default: ``pytorch``.
            inflate (int | Sequence[int]): Determine whether to inflate
                for each block. Default: 1.
            inflate_style (str): ``3x1x1`` or ``1x1x1``. which determines
                the kernel sizes and padding strides for conv1 and
                conv2 in each block. Default: ``3x1x1``.
            non_local (int | Sequence[int]): Determine whether to apply
                non-local module in the corresponding block of each stages.
                Default: 0.
            non_local_cfg (dict): Config for non-local module.
                Default: ``dict()``.
            conv_cfg (dict | None): Config for conv layers. Default: None.
            norm_cfg (dict | None): Config for norm layers. Default: None.
            act_cfg (dict | None): Config for activate layers. Default: None.
            with_cp (bool): Use checkpoint or not. Using checkpoint will save
                some memory while slowing down the training speed.
                Default: False.

        Returns:
            nn.Module: A residual layer for the given config.
        r	   r   r   FN)r   r   r   r   r   r   )	ÚstyleÚinflateÚinflate_styleÚ	non_localÚnon_local_cfgr   r   r   Úwith_cp)Ú
isinstanceÚintr#   r   r   r&   r   r)   r"   ÚnnÚ
Sequential)r*   r%   r   r.   ÚblocksÚspatial_strideÚtemporal_strideÚdilationr6   r7   r8   r9   r:   r   r   r   r;   Zlateral_inplanesÚ
downsampleÚlayersr-   r2   r2   r3   Úmake_res_layerV   sˆ   7ÿÿÿÿø


ñÿ


òÿ
zResNet3dPathway.make_res_layerc           	      C   sþ   t | jƒ}d|v r|d }g }|  ¡ D ]S\}}d|v rqt|tƒrfd|v r.|d }|d }n|}| dd¡}|d |vrE| d	|› �¡ n	|  |j|||¡ |d |vr]| d	|› �¡ q|  	|j
|||¡ qt| ¡ ƒt|ƒ }|r}| d
|› �¡ dS dS )a  Inflate the resnet2d parameters to resnet3d pathway.

        The differences between resnet3d and resnet2d mainly lie in an extra
        axis of conv kernel. To utilize the pretrained parameters in 2d model,
        the weight of conv2d models should be inflated to fit in the shapes of
        the 3d counterpart. For pathway the ``lateral_connection`` part should
        not be inflated from 2d weights.

        Args:
            logger (logging.Logger): The logger used to print
                debugging infomation.
        Ú
state_dictr   rD   z.0z.1ÚconvÚbnú.weightz(Module not exist in the state_dict_r2d: z6These parameters in the 2d checkpoint are not loaded: N)r   Ú
pretrainedÚnamed_modulesr<   r   ÚreplaceÚwarningÚ_inflate_conv_paramsrH   Ú_inflate_bn_paramsrI   ÚsetÚkeysÚinfo)	r*   ÚloggerÚstate_dict_r2dÚinflated_param_namesÚnameÚmoduleÚoriginal_conv_nameÚoriginal_bn_nameÚremaining_namesr2   r2   r3   Úinflate_weightsÌ   sP   



ÿ
þ
ÿ
þ€ÿÿÿÿzResNet3dPathway.inflate_weightsc                 C   sú   |d }|| }|j }|jjj }|d }	|d |d krH|d |d  }
|}|dd… |
f |dd…  }tj|t |¡ |¡ |j¡fdd�}|j 	d¡ 
|j¡|	 }|jj |¡ | |¡ t|dƒdur{|d }|jj || ¡ | |¡ dS dS )aÀ  Inflate a conv module from 2d to 3d.

        The differences of conv modules betweene 2d and 3d in Pathway
        mainly lie in the inplanes due to lateral connections. To fit the
        shapes of the lateral connection counterpart, it will expand
        parameters by concatting conv2d parameters and extra zero paddings.

        Args:
            conv3d (nn.Module): The destination conv3d module.
            state_dict_2d (OrderedDict): The state dict of pretrained 2d model.
            module_name_2d (str): The name of corresponding conv module in the
                2d model.
            inflated_param_names (list[str]): List of parameters that have been
                inflated.
        rJ   r	   r   N©Údimr   z.bias)ÚshapeÚweightÚdataÚtorchÚcatÚzerosÚtype_asÚtoÚdeviceÚ	unsqueezeÚ	expand_asÚcopy_r)   Úgetattrr   )r*   Úconv3dÚstate_dict_2dÚmodule_name_2drV   Úweight_2d_nameÚconv2d_weightZ	old_shapeÚ	new_shapeÚkernel_tZnew_channelsÚ	pad_shapeÚ
new_weightÚbias_2d_namer2   r2   r3   rO     s:   
ÿÿüÿÿ
ýz$ResNet3dPathway._inflate_conv_paramsc                 C   s¸   | j dkr| j ¡  | j ¡ D ]}d|_qtd| j d ƒD ]<}t| d|› �ƒ}| ¡  | ¡ D ]}d|_q/|t| jƒkrY| j	rY| j
|d  }t| |ƒ}| ¡  | ¡ D ]}d|_qSqdS )zTPrevent all the parameters from being optimized before
        `self.frozen_stages`.r   Fr   r   N)Úfrozen_stagesÚconv1ÚevalÚ
parametersÚrequires_gradr"   rk   r#   Ú
res_layersr   r!   )r*   Úparamr-   Úmr/   Úconv_lateralr2   r2   r3   Ú_freeze_stages0  s"   


€ôzResNet3dPathway._freeze_stagesc                    sV   |r|| _ tƒ  ¡  | jD ]}t| |ƒ}| ¡ D ]}t|tjtj	fƒr't
|ƒ qqdS )úPInitiate the parameters either from existing checkpoint or from
        scratch.N)rK   r   Úinit_weightsr!   rk   Úmodulesr<   r>   ÚConv3dÚConv2dr   )r*   rK   Úmodule_namer   r}   r0   r2   r3   r�   F  s   


€þþzResNet3dPathway.init_weights©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚdictrF   r\   rO   r   r�   Ú__classcell__r2   r2   r0   r3   r      s0    û7
ðv9+r   Úresnet3dc                 O   s`   t | tƒr	d| v stdƒ‚|  ¡ }| d¡}|tvr!td|› �ƒ‚t| }||i |¤|¤Ž}|S )z´Build pathway.

    Args:
        cfg (None or dict): cfg should contain:
            - type (str): identify conv layer type.

    Returns:
        nn.Module: Created pathway.
    Útypez,cfg must be a dict containing the key "type"zUnrecognized pathway type )r<   r‹   Ú	TypeErrorÚcopyÚpopÚpathway_cfgÚKeyError)Úcfgr+   r,   Úcfg_Zpathway_typeZpathway_clsZpathwayr2   r2   r3   Úbuild_pathway[  s   

r–   c                       sx   e Zd ZdZdddeddddddd	d	d
d�	eddddddd	d	d�f‡ fdd„	Zddd„Zdd„ Zdd„ Zdd„ Z	‡  Z
S )ÚResNet3dSlowFasta—  Slowfast backbone.

    This module is proposed in `SlowFast Networks for Video Recognition
    <https://arxiv.org/abs/1812.03982>`_

    Args:
        pretrained (str): The file path to a pretrained model.
        resample_rate (int): A large temporal stride ``resample_rate``
            on input frames. The actual resample rate is calculated by
            multipling the ``interval`` in ``SampleFrames`` in the
            pipeline with ``resample_rate``, equivalent to the :math:`\tau`
            in the paper, i.e. it processes only one out of
            ``resample_rate * interval`` frames. Default: 8.
        speed_ratio (int): Speed ratio indicating the ratio between time
            dimension of the fast and slow pathway, corresponding to the
            :math:`\alpha` in the paper. Default: 8.
        channel_ratio (int): Reduce the channel number of fast pathway
            by ``channel_ratio``, corresponding to :math:`\beta` in the paper.
            Default: 8.
        slow_pathway (dict): Configuration of slow branch, should contain
            necessary arguments for building the specific type of pathway
            and:
            type (str): type of backbone the pathway bases on.
            lateral (bool): determine whether to build lateral connection
            for the pathway.Default:

            .. code-block:: Python

                dict(type='ResNetPathway',
                lateral=True, depth=50, pretrained=None,
                conv1_kernel=(1, 7, 7), dilations=(1, 1, 1, 1),
                conv1_stride_t=1, pool1_stride_t=1, inflate=(0, 0, 1, 1))

        fast_pathway (dict): Configuration of fast branch, similar to
            `slow_pathway`. Default:

            .. code-block:: Python

                dict(type='ResNetPathway',
                lateral=False, depth=50, pretrained=None, base_channels=8,
                conv1_kernel=(5, 7, 7), conv1_stride_t=1, pool1_stride_t=1)
    r   r�   é2   NT)r   é   r™   )r   r   r   r   r   )r   r   r   r   )	rŽ   ÚdepthrK   r   Úconv1_kernelÚ	dilationsÚconv1_stride_tÚpool1_stride_tr7   F)r   r™   r™   )rŽ   rš   rK   r   r   r›   r�   rž   c                    sR   t ƒ  ¡  || _|| _|| _|| _|d r||d< ||d< t|ƒ| _t|ƒ| _d S )Nr   r   r   )	r   r   rK   Úresample_rater   r   r–   Ú	slow_pathÚ	fast_path)r*   rK   rŸ   r   r   Zslow_pathwayZfast_pathwayr0   r2   r3   r      s   

zResNet3dSlowFast.__init__c                 C   st   |r|| _ t| j tƒr%tƒ }d| j › �}t||d� t| | j d|d� dS | j du r6| j ¡  | j ¡  dS t	dƒ‚)r€   zload model from: )rT   T)ÚstrictrT   Nz pretrained must be a str or None)
rK   r<   Ústrr   r   r   r¡   r�   r    r�   )r*   rK   rT   Úmsgr2   r2   r3   r�   Å  s   

zResNet3dSlowFast.init_weightsc                 C   s0  t jj|dd| j ddfd�}| j |¡}| j |¡}t jj|dd| j| j  ddfd�}| j |¡}| j |¡}| jj	rL| j 
|¡}tj||fdd�}t| jjƒD ]?\}}t| j|ƒ}||ƒ}t| j|ƒ}||ƒ}|t| jjƒd kr‘| jj	r‘| jj| }	t| j|	ƒ}
|
|ƒ}tj||fdd�}qR||f}|S )úõDefines the computation performed at every call.

        Args:
            x (torch.Tensor): The input data.

        Returns:
            tuple[torch.Tensor]: The feature of the input samples extracted
                by the backbone.
        Únearestç      ð?©ÚmodeÚscale_factorr   r]   ©r>   Ú
functionalÚinterpolaterŸ   r    rw   Úmaxpoolr   r¡   r   r    rb   rc   Ú	enumerater{   rk   r#   r!   ©r*   ÚxÚx_slowÚx_fastZx_fast_lateralr-   Ú
layer_nameÚ	res_layerÚres_layer_fastr/   r~   Úoutr2   r2   r3   ÚforwardØ  sD   
ýÿýÿ€zResNet3dSlowFast.forwardc                 C   s<  t jj|dd| j ddfd�}| j |¡}| j |¡}t jj|dd| j| j  ddfd�}| j |¡}| j |¡}| jj	rL| j 
|¡}tj||fdd�}t| jjƒD ]E\}}|dkr\ n<t| j|ƒ}||ƒ}t| j|ƒ}||ƒ}|t| jjƒd kr—| jj	r—| jj| }	t| j|	ƒ}
|
|ƒ}tj||fdd�}qR||f}|S )r¥   r¦   r§   r¨   r   r]   r   r«   r°   r2   r2   r3   Úget_feature  sH   
ýÿýÿ€zResNet3dSlowFast.get_featurec                 C   sD   |d }|d }t | jdƒ}||ƒ}t | jdƒ}||ƒ}||f}|S )Nr   r   Úlayer4)rk   r    r¡   )r*   r±   r²   r³   rµ   r¶   r·   r2   r2   r3   Úget_predict5  s   zResNet3dSlowFast.get_predictr†   )r‡   rˆ   r‰   rŠ   r‹   r   r�   r¸   r¹   r»   rŒ   r2   r2   r0   r3   r—   s  s>    -÷
øñ
%-0r—   )rb   Útorch.nnr>   Úmmcv.cnnr   r   Úmmcv.runnerr   r   Ú
mmcv.utilsr   Úutilsr   Úregistryr
   r�   r   Zmmdet.modelsZMMDET_BACKBONESÚmmdet_importedÚImportErrorÚModuleNotFoundErrorr   r’   r–   Úregister_moduleÚModuler—   r2   r2   r2   r3   Ú<module>   s4    ÿ  Fÿ Pÿ