o
    _{j÷)  ã                   @   s„   d dl Z d dlmZ d dlmZ d dlmZ ddlmZ ddl	m
Z
 G dd	„ d	ejƒZG d
d„ dejƒZe ¡ G dd„ de
ƒƒZdS )é    N)Ú
NonLocal3d)Ú_ntupleé   )Ú	BACKBONESé   )ÚResNetc                       s.   e Zd ZdZeƒ f‡ fdd„	Zdd„ Z‡  ZS )ÚNL3DWrappera"  3D Non-local wrapper for ResNet50.

    Wrap ResNet layers with 3D NonLocal modules.

    Args:
        block (nn.Module): Residual blocks to be built.
        num_segments (int): Number of frame segments.
        non_local_cfg (dict): Config for non-local layers. Default: ``dict()``.
    c                    s@   t t| ƒ ¡  || _|| _t| jjjjfi | j¤Ž| _	|| _
d S ©N)Úsuperr   Ú__init__ÚblockÚnon_local_cfgr   Úconv3ÚnormÚnum_featuresÚnon_local_blockÚnum_segments)Úselfr   r   r   ©Ú	__class__© ú:/root/disk_3tb/HAC/mmaction/models/backbones/resnet_tsm.pyr      s   ÿ
zNL3DWrapper.__init__c                 C   sj   |   |¡}| ¡ \}}}}| || j | j|||¡ dd¡ ¡ }|  |¡}| dd¡ ¡  ||||¡}|S ©Nr   r   )r   ÚsizeÚviewr   Ú	transposeÚ
contiguousr   ©r   ÚxÚnÚcÚhÚwr   r   r   Úforward   s   
ÿÿ
zNL3DWrapper.forward)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Údictr   r#   Ú__classcell__r   r   r   r   r   
   s    
r   c                       s8   e Zd ZdZd
‡ fdd„	Zdd„ Zeddd	„ƒZ‡  ZS )ÚTemporalShiftao  Temporal shift module.

    This module is proposed in
    `TSM: Temporal Shift Module for Efficient Video Understanding
    <https://arxiv.org/abs/1811.08383>`_

    Args:
        net (nn.module): Module to make temporal shift.
        num_segments (int): Number of frame segments. Default: 3.
        shift_div (int): Number of divisions for shift. Default: 8.
    é   é   c                    s    t ƒ  ¡  || _|| _|| _d S r	   )r
   r   Únetr   Ú	shift_div)r   r-   r   r.   r   r   r   r   5   s   

zTemporalShift.__init__c                 C   s   | j || j| jd�}|  |¡S )z¼Defines the computation performed at every call.

        Args:
            x (torch.Tensor): The input data.

        Returns:
            torch.Tensor: The output of the module.
        )r.   )Úshiftr   r.   r-   )r   r   r   r   r   r#   ;   s   	
zTemporalShift.forwardc                 C   sf  |   ¡ \}}}}|  d|||| ¡} || }| dd…dd…d|…dd…f }| dd…dd…|d| …dd…f }	| dd…dd…d| d…dd…f }
|| }|dd…dd…dd…dd…f }|dd…dd…dd…dd…f }t ||fd¡}|	|	 }|dd…dd…dd…dd…f }|	dd…dd…dd…dd…f }	t ||	fd¡}	t ||	|
fd¡}| ||||¡S )aJ  Perform temporal shift operation on the feature.

        Args:
            x (torch.Tensor): The input feature to be shifted.
            num_segments (int): Number of frame segments.
            shift_div (int): Number of divisions for shift. Default: 3.

        Returns:
            torch.Tensor: The shifted feature.
        éÿÿÿÿNr   r   )r   r   ÚtorchÚcat)r   r   r.   r   r    r!   r"   ÚfoldÚ
left_splitZ	mid_splitÚright_splitÚzerosÚblankÚoutr   r   r   r/   G   s     $$    zTemporalShift.shift)r+   r,   )r+   )	r$   r%   r&   r'   r   r#   Ústaticmethodr/   r)   r   r   r   r   r*   (   s    r*   c                       sV   e Zd ZdZdddeƒ dddf‡ fdd„	Zd	d
„ Zdd„ Zdd„ Z‡ fdd„Z	‡  Z
S )Ú	ResNetTSMa"  ResNet backbone for TSM.

    Args:
        num_segments (int): Number of frame segments. Default: 8.
        is_shift (bool): Whether to make temporal shift in reset layers.
            Default: True.
        non_local (Sequence[int]): Determine whether to apply non-local module
            in the corresponding block of each stages. Default: (0, 0, 0, 0).
        non_local_cfg (dict): Config for non-local module. Default: ``dict()``.
        shift_div (int): Number of div for shift. Default: 8.
        shift_place (str): Places in resnet layers for shift, which is chosen
            from ['block', 'blockres'].
            If set to 'block', it will apply temporal shift to all child blocks
            in each resnet layer.
            If set to 'blockres', it will apply temporal shift to each `conv1`
            layer of all child blocks in each resnet layer.
            Default: 'blockres'.
        temporal_pool (bool): Whether to add temporal pooling. Default: False.
        **kwargs (keyword arguments, optional): Arguments for ResNet.
    r,   T)r   r   r   r   ÚblockresFc	           
         sR   t ƒ j|fi |	¤Ž || _|| _|| _|| _|| _|| _t| j	ƒ|ƒ| _
|| _d S r	   )r
   r   r   Úis_shiftr.   Úshift_placeÚtemporal_poolÚ	non_localr   Ú
num_stagesÚnon_local_stagesr   )
r   Údepthr   r<   r?   r   r.   r=   r>   Úkwargsr   r   r   r   “   s   

zResNetTSM.__init__c                    s0  ˆj rˆjˆjd ˆjd ˆjd g}nˆjgd }|d dkr$tdƒ‚ˆjdkrU‡fdd„}|ˆj|d ƒˆ_|ˆj|d	 ƒˆ_|ˆj|d ƒˆ_|ˆj|d
 ƒˆ_dS dˆjv r–d	‰ tt	ˆj 
¡ ƒƒdkrid‰ ‡ ‡fdd„}|ˆj|d ƒˆ_|ˆj|d	 ƒˆ_|ˆj|d ƒˆ_|ˆj|d
 ƒˆ_dS t‚)z$Make temporal shift for some layers.r   é   r0   r   z%num_segment_list[-1] must be positiver   c                    s<   t |  ¡ ƒ}t|ƒD ]\}}t||ˆ jd�||< q
tj|Ž S )á  Make temporal shift on some blocks.

                Args:
                    stage (nn.Module): Model layers to be shifted.
                    num_segments (int): Number of frame segments.

                Returns:
                    nn.Module: The shifted blocks.
                ©r   r.   )ÚlistÚchildrenÚ	enumerater*   r.   ÚnnÚ
Sequential©Ústager   ÚblocksÚiÚb©r   r   r   Úmake_block_temporalµ   s   
ÿ
z:ResNetTSM.make_temporal_shift.<locals>.make_block_temporalr   r+   r;   é   c                    sP   t |  ¡ ƒ}t|ƒD ]\}}|ˆ  dkr"t|jj|ˆjd�|| j_q
tj|Ž S )rE   r   rF   )	rG   rH   rI   r*   Úconv1Úconvr.   rJ   rK   rL   ©Zn_roundr   r   r   rR   Ï   s   
ý€
N)r>   r   Ú
ValueErrorr=   Úlayer1Úlayer2Úlayer3Úlayer4ÚlenrG   rH   ÚNotImplementedError)r   Znum_segment_listrR   r   rV   r   Úmake_temporal_shift§   s.   þ

zResNetTSM.make_temporal_shiftc                 C   s&   G dd„ dt jƒ}|| j| jƒ| _dS )zVMake temporal pooling between layer1 and layer2, using a 3D max
        pooling layer.c                       s(   e Zd ZdZ‡ fdd„Zdd„ Z‡  ZS )z2ResNetTSM.make_temporal_pool.<locals>.TemporalPoolzôTemporal pool module.

            Wrap layer2 in ResNet50 with a 3D max pooling layer.

            Args:
                net (nn.Module): Module to make temporal pool.
                num_segments (int): Number of frame segments.
            c                    s,   t ƒ  ¡  || _|| _tjdddd�| _d S )N)r+   r   r   )r   r   r   )r   r   r   )Úkernel_sizeÚstrideÚpadding)r
   r   r-   r   rJ   Ú	MaxPool3dÚ
max_pool3d)r   r-   r   r   r   r   r   ø   s   
ÿz;ResNetTSM.make_temporal_pool.<locals>.TemporalPool.__init__c                 S   sf   |  ¡ \}}}}| || j | j|||¡ dd¡}|  |¡}| dd¡ ¡  |d |||¡}|  |¡S r   )r   r   r   r   rc   r   r-   r   r   r   r   r#   ÿ   s   ÿÿ
 
z:ResNetTSM.make_temporal_pool.<locals>.TemporalPool.forward)r$   r%   r&   r'   r   r#   r)   r   r   r   r   ÚTemporalPoolî   s    	rd   N)rJ   ÚModulerY   r   )r   rd   r   r   r   Úmake_temporal_poolê   s   zResNetTSM.make_temporal_poolc                 C   sr   t | jƒD ]1}| j| }t|ƒdkrqd|d › �}t| |ƒ}t|ƒD ]\}}|r5t|| | j| jƒ||< q#qd S )Nr   Úlayerr   )	Úranger@   rA   ÚsumÚgetattrrI   r   r   r   )r   rO   Znon_local_stageÚ
layer_nameÚ	res_layerÚidxr?   r   r   r   Úmake_non_local  s   

þ€þøzResNetTSM.make_non_localc                    sD   t ƒ  ¡  | jr|  ¡  t| jƒdkr|  ¡  | jr |  ¡  dS dS )zPInitiate the parameters either from existing checkpoint or from
        scratch.r   N)	r
   Úinit_weightsr<   r^   r\   r   rn   r>   rf   rQ   r   r   r   ro     s   
ÿzResNetTSM.init_weights)r$   r%   r&   r'   r(   r   r^   rf   rn   ro   r)   r   r   r   r   r:   |   s    øC#r:   )r1   Útorch.nnrJ   Úmmcv.cnnr   Ztorch.nn.modules.utilsr   Úregistryr   Úresnetr   re   r   r*   Úregister_moduler:   r   r   r   r   Ú<module>   s    T