o
    _{j1@  ã                   @   sÖ   d dl Zd dlZd dlmZ d dlmZmZmZm	Z	 ddl
mZ ddlmZ G dd„ dejƒZG dd	„ d	ejƒZG d
d„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZe ¡ G dd„ dejƒƒZdS )é    N)Ú
ConvModuleÚconstant_initÚnormal_initÚxavier_inité   )Ú
build_loss)ÚNECKSc                   @   s   e Zd ZdZdd„ ZdS )ÚIdentityzIdentity mapping.c                 C   s   |S ©N© ©ÚselfÚxr   r   ú//root/disk_3tb/HAC/mmaction/models/necks/tpn.pyÚforward   s   zIdentity.forwardN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r	   
   s    r	   c                
       sD   e Zd ZdZdddddedd�d	d	d
df
‡ fdd„	Zdd„ Z‡  ZS )Ú
DownSamplea  DownSample modules.

    It uses convolution and maxpooling to downsample the input feature,
    and specifies downsample position to determine `pool-conv` or `conv-pool`.

    Args:
        in_channels (int): Channel number of input features.
        out_channels (int): Channel number of output feature.
        kernel_size (int | tuple[int]): Same as :class:`ConvModule`.
            Default: (3, 1, 1).
        stride (int | tuple[int]): Same as :class:`ConvModule`.
            Default: (1, 1, 1).
        padding (int | tuple[int]): Same as :class:`ConvModule`.
            Default: (1, 0, 0).
        groups (int): Same as :class:`ConvModule`. Default: 1.
        bias (bool | str): Same as :class:`ConvModule`. Default: False.
        conv_cfg (dict | None): Same as :class:`ConvModule`.
            Default: dict(type='Conv3d').
        norm_cfg (dict | None): Same as :class:`ConvModule`. Default: None.
        act_cfg (dict | None): Same as :class:`ConvModule`. Default: None.
        downsample_position (str): Type of downsample position. Options are
            'before' and 'after'. Default: 'after'.
        downsample_scale (int | tuple[int]): downsample scale for maxpooling.
            It will be used for kernel size and stride of maxpooling.
            Default: (1, 2, 2).
    ©é   é   r   ©r   r   r   ©r   r   r   r   FÚConv3d©ÚtypeNÚafter©r   r   r   c                    sR   t ƒ  ¡  t|||||||||	|
d�
| _|dv sJ ‚|| _tj||ddd�| _d S )N)ÚgroupsÚbiasÚconv_cfgÚnorm_cfgÚact_cfg)Úbeforer   ©r   r   r   T©Ú	ceil_mode)ÚsuperÚ__init__r   ÚconvÚdownsample_positionÚnnÚ	MaxPool3dÚpool)r   Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr    r!   r"   r#   r$   r,   Údownsample_scale©Ú	__class__r   r   r*   -   s$   
öÿzDownSample.__init__c                 C   s:   | j dkr|  |¡}|  |¡}|S |  |¡}|  |¡}|S )Nr%   )r,   r/   r+   r   r   r   r   r   K   s   



þ
zDownSample.forward)r   r   r   r   Údictr*   r   Ú__classcell__r   r   r6   r   r      s    ôr   c                       s,   e Zd ZdZ	d‡ fdd„	Zdd„ Z‡  ZS )ÚLevelFusionaN  Level Fusion module.

    This module is used to aggregate the hierarchical features dynamic in
    visual tempos and consistent in spatial semantics. The top/bottom features
    for top-down/bottom-up flow would be combined to achieve two additional
    options, namely 'Cascade Flow' or 'Parallel Flow'. While applying a
    bottom-up flow after a top-down flow will lead to the cascade flow,
    applying them simultaneously will result in the parallel flow.

    Args:
        in_channels (tuple[int]): Channel numbers of input features tuple.
        mid_channels (tuple[int]): Channel numbers of middle features tuple.
        out_channels (int): Channel numbers of output features.
        downsample_scales (tuple[int | tuple[int]]): downsample scales for
            each :class:`DownSample` module. Default: ((1, 1, 1), (1, 1, 1)).
    ©r   r   c                    sª   t ƒ  ¡  t|ƒ}t ¡ | _t|ƒD ]%}t|| || dddddtddd�tddd	�d
|| d�}| j 	|¡ qt
t|ƒ|ddddtdd�tddd�tddd	�d�	| _d S )Nr   Fr&   é    ÚBN3dT©r   Úrequires_gradÚReLU©r   Úinplacer%   )	r2   r3   r!   r4   r    r#   r$   r,   r5   r   r   r   r   ©r3   r4   r!   r"   r#   r$   )r)   r*   Úlenr-   Ú
ModuleListÚdownsamplesÚranger   r8   Úappendr   ÚsumÚfusion_conv)r   r0   Úmid_channelsr1   Údownsample_scalesÚ
num_stagesÚiÚ
downsampler6   r   r   r*   g   s:   



õ

÷zLevelFusion.__init__c                    s0   ‡ fdd„t |ƒD ƒ}t |d¡}ˆ  |¡}|S )Nc                    s   g | ]\}}ˆ j | |ƒ‘qS r   )rF   )Ú.0rN   Úfeature©r   r   r   Ú
<listcomp>‹   s    z'LevelFusion.forward.<locals>.<listcomp>r   )Ú	enumerateÚtorchÚcatrJ   )r   r   Úoutr   rR   r   r   Š   s   
zLevelFusion.forward)r;   ©r   r   r   r   r*   r   r9   r   r   r6   r   r:   U   s
    ü#r:   c                       s(   e Zd ZdZ‡ fdd„Zdd„ Z‡  ZS )ÚSpatialModulationaÐ  Spatial Semantic Modulation.

    This module is used to align spatial semantics of features in the
    multi-depth pyramid. For each but the top-level feature, a stack
    of convolutions with level-specific stride are applied to it, matching
    its spatial shape and receptive field with the top one.

    Args:
        in_channels (tuple[int]): Channel numbers of input features tuple.
        out_channels (int): Channel numbers of output features tuple.
    c           
         s¼   t ƒ  ¡  t ¡ | _|D ]O}|| }tt |¡ƒ}t ¡ }|dk r%tƒ }n0t	|ƒD ]+}d| }d|d  }	| 
t|| ||	 ddddtdd�td	d
d�tdd
d�d�	¡ q)| j 
|¡ qd S )Nr   r   ©r   r   r   r   ©r   r   r   Fr   r   r=   Tr>   r@   rA   rC   )r)   r*   r-   rE   Úspatial_modulationÚintÚnpÚlog2r	   rG   rH   r   r8   )
r   r0   r1   Úchannelr5   Zdownsample_factorÚopÚfactorZ	in_factorZ
out_factorr6   r   r   r*   Ÿ   s2   



øÿ
ìzSpatialModulation.__init__c                 C   sp   g }t |ƒD ]/\}}t| j| tjƒr)|| }| j| D ]}||ƒ}q| |¡ q| | j| || ƒ¡ q|S r
   )rT   Ú
isinstancer\   r-   rE   rH   )r   r   rW   rN   Ú_Zout_ra   r   r   r   r   ¹   s   
zSpatialModulation.forwardrX   r   r   r6   r   rY   ’   s    rY   c                       s>   e Zd ZdZdedd�f‡ fdd„	Zdd„ Zdd
d„Z‡  ZS )ÚAuxHeadaÞ  Auxiliary Head.

    This auxiliary head is appended to receive stronger supervision,
    leading to enhanced semantics.

    Args:
        in_channels (int): Channel number of input features.
        out_channels (int): Channel number of output features.
        loss_weight (float): weight of loss for the auxiliary head.
            Default: 0.5.
        loss_cls (dict): loss_cls (dict): Config for building loss.
            Default: ``dict(type='CrossEntropyLoss')``.
    ç      à?ÚCrossEntropyLossr   c                    sv   t ƒ  ¡  t||d ddddtdd�tdd	d
�d�| _t d¡| _|| _tj	dd�| _
t |d |¡| _t|ƒ| _d S )Nr   rZ   r   r[   Fr   r   r=   Tr>   ©r3   r4   r!   r"   r#   r   rf   )Úp)r)   r*   r   r8   r+   r-   ÚAdaptiveAvgPool3dÚavg_poolÚloss_weightÚDropoutÚdropoutÚLinearÚfcr   Úloss_cls)r   r0   r1   rl   rq   r6   r   r   r*   Õ   s   

ùzAuxHead.__init__c                 C   sX   |   ¡ D ]%}t|tjƒrt|dd� t|tjƒrt|dd� t|tjƒr)t|dƒ qd S )Ng{®Gáz„?)ÚstdÚuniform©Údistributionr   )	Úmodulesrc   r-   ro   r   r   r   ÚBatchNorm3dr   ©r   Úmr   r   r   Úinit_weightsê   s   
€úzAuxHead.init_weightsNc                 C   s€   t ƒ }|d u r	|S |  |¡}|  |¡ d¡ d¡ d¡}|  |¡}|  |¡}|jt g ¡kr3| 	d¡}| j
|  ||¡ |d< |S )Néÿÿÿÿr   Úloss_aux)r8   r+   rk   Úsqueezern   rp   ÚshaperU   ÚSizeÚ	unsqueezerl   rq   )r   r   ÚtargetÚlossesr   r   r   r   ó   s   



zAuxHead.forwardr
   )	r   r   r   r   r8   r*   rz   r   r9   r   r   r6   r   re   Æ   s    ü	re   c                       s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )ÚTemporalModulationay  Temporal Rate Modulation.

    The module is used to equip TPN with a similar flexibility for temporal
    tempo modulation as in the input-level frame pyramid.

    Args:
        in_channels (int): Channel number of input features.
        out_channels (int): Channel number of output features.
        downsample_scale (int): Downsample scale for maxpooling. Default: 8.
    é   c                    sP   t ƒ  ¡  t||dddddtdd�d d�	| _tj|d	d	f|d	d	fd
dd�| _d S )Nr   r   r   Fr<   r   r   )r3   r4   r!   r    r"   r$   r   r&   Tr'   )r)   r*   r   r8   r+   r-   r.   r/   )r   r0   r1   r5   r6   r   r   r*     s   
ø	
þzTemporalModulation.__init__c                 C   s   |   |¡}|  |¡}|S r
   )r+   r/   r   r   r   r   r     s   

zTemporalModulation.forward)r„   rX   r   r   r6   r   rƒ     s    rƒ   c                       sB   e Zd ZdZ							d
‡ fdd„	Zdd„ Zddd	„Z‡  ZS )ÚTPNaÞ  TPN neck.

    This module is proposed in `Temporal Pyramid Network for Action Recognition
    <https://arxiv.org/pdf/2004.03548.pdf>`_

    Args:
        in_channels (tuple[int]): Channel numbers of input features tuple.
        out_channels (int): Channel number of output feature.
        spatial_modulation_cfg (dict | None): Config for spatial modulation
            layers. Required keys are `in_channels` and `out_channels`.
            Default: None.
        temporal_modulation_cfg (dict | None): Config for temporal modulation
            layers. Default: None.
        upsample_cfg (dict | None): Config for upsample layers. The keys are
            same as that in :class:``nn.Upsample``. Default: None.
        downsample_cfg (dict | None): Config for downsample layers.
            Default: None.
        level_fusion_cfg (dict | None): Config for level fusion layers.
            Required keys are 'in_channels', 'mid_channels', 'out_channels'.
            Default: None.
        aux_head_cfg (dict | None): Config for aux head layers.
            Required keys are 'out_channels'. Default: None.
        flow_type (str): Flow type to combine the features. Options are
            'cascade' and 'parallel'. Default: 'cascade'.
    NÚcascadec
                    s.  t ƒ  ¡  t|tƒsJ ‚t|tƒsJ ‚|| _|| _t|ƒ| _|d u s)t|t	ƒs)J ‚|d u s4t|t	ƒs4J ‚|d u s?t|t	ƒs?J ‚|d u sJt|t	ƒsJJ ‚|d u sUt|t	ƒsUJ ‚|d u s`t|t	ƒs`J ‚|	dvrlt
d|	› d�ƒ‚|	| _t ¡ | _t ¡ | _t ¡ | _tdi |¤Ž| _tdi |¤Ž| _t| jƒD ]F}
|d ur­|d |
 }t|d ||ƒ}| j |¡ |
| jd k rÙ|d urÆtjdi |¤Ž}| j |¡ |d urÙt||fi |¤Ž}| j |¡ q“|d }tdi |¤Ž| _t|d d	ddd
dt	dd�t	ddd�d�| _|d u�rt| jd fi |¤Ž| _nd | _|  ¡  d S )N)r†   Úparallelz<flow type in TPN should be 'cascade' or 'parallel', but got z	 instead.rL   r{   r   r1   r   i   r   Fr   r   r=   Tr>   rh   éþÿÿÿr   ) r)   r*   rc   Útupler]   r0   r1   rD   Únum_tpn_stagesr8   Ú
ValueErrorÚ	flow_typer-   rE   Útemporal_modulation_opsÚupsample_opsÚdownsample_opsr:   Úlevel_fusion_1rY   r\   rG   rƒ   rH   ÚUpsampler   Úlevel_fusion_2r   Úpyramid_fusionre   Úaux_headrz   )r   r0   r1   Zspatial_modulation_cfgZtemporal_modulation_cfgZupsample_cfgZdownsample_cfgZlevel_fusion_cfgZaux_head_cfgrŒ   rN   r5   Útemporal_modulationÚupsamplerO   Úout_dimsr6   r   r   r*   A  s‚   



ÿ
ÿÿÿ


ÿÿ
ÿ
ÿ€
ø

zTPN.__init__c                 C   sX   |   ¡ D ]}t|tjƒrt|dd� t|tjƒrt|dƒ q| jd ur*| j ¡  d S d S )Nrs   rt   r   )	rv   rc   r-   r   r   rw   r   r”   rz   rx   r   r   r   rz   ‘  s   
€
ÿzTPN.init_weightsc                 C   sD  t ƒ }| jd ur|  |d |¡}|  |¡}g }t| jƒD ]\}}| ||| ƒ¡ qdd„ |D ƒ}t| jƒdkrYt| j	d ddƒD ]}||d  | j|d  || ƒ ||d < qA|  
|¡}	| jdkrjdd„ |D ƒ}t| jƒdkrŽt| j	d ƒD ]}||d  | j| || ƒ ||d < qx|  |¡}
|  t |	|
gd¡¡}||fS )	Nrˆ   c                 S   ó   g | ]}|  ¡ ‘qS r   ©Úclone©rP   rW   r   r   r   rS   «  ó    zTPN.forward.<locals>.<listcomp>r   r   r{   r‡   c                 S   r˜   r   r™   r›   r   r   r   rS   µ  rœ   )r8   r”   r\   rT   r�   rH   rD   rŽ   rG   rŠ   r�   rŒ   r�   r’   r“   rU   rV   )r   r   r�   r|   Zspatial_modulation_outsZtemporal_modulation_outsrN   r•   ÚoutsZtop_down_outsZbotton_up_outsr   r   r   r   ›  s0   


ÿ,

(
ÿzTPN.forward)NNNNNNr†   r
   )r   r   r   r   r*   rz   r   r9   r   r   r6   r   r…   %  s    ÷P
r…   )Únumpyr^   rU   Útorch.nnr-   Úmmcv.cnnr   r   r   r   Úbuilderr   Úregistryr   ÚModuler	   r   r:   rY   re   rƒ   Úregister_moduler…   r   r   r   r   Ú<module>   s    D=4="