o
    _{j7  ã                	   @   s  d dl Z d dlZd dlmZ d dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ z
d dlmZ dZW n eefyG   d	ZY nw G d
d„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZere ¡ eƒ dS dS )é    N)Ú
ConvModuleÚconstant_initÚkaiming_init)Úload_checkpoint)Ú
_BatchNorm)ÚLFB)Úget_root_logger)ÚSHARED_HEADSTFc                       sB   e Zd ZdZ							d‡ fdd„	Zddd	„Zd
d„ Z‡  ZS )ÚNonLocalLayeraó  Non-local layer used in `FBONonLocal` is a variation of the vanilla non-
    local block.

    Args:
        st_feat_channels (int): Channels of short-term features.
        lt_feat_channels (int): Channels of long-term features.
        latent_channels (int): Channels of latent features.
        use_scale (bool): Whether to scale pairwise_weight by
            `1/sqrt(latent_channels)`. Default: True.
        pre_activate (bool): Whether to use the activation function before
            upsampling. Default: False.
        conv_cfg (Dict | None): The config dict for convolution layers. If
            not specified, it will use `nn.Conv2d` for convolution layers.
            Default: None.
        norm_cfg (Dict | None): he config dict for normalization layers.
            Default: None.
        dropout_ratio (float, optional): Probability of dropout layer.
            Default: 0.2.
        zero_init_out_conv (bool): Whether to use zero initialization for
            out_conv. Default: False.
    TNçš™™™™™É?Fc                    s  t ƒ  ¡  |	d u rtdd�}	|| _|| _|| _|| _|| _|| _|| _	|| _
|| _|| _t| j| jd|	|
d d�| _t| j| jd|	|
d d�| _t| j| jd|	|
d d�| _|r`t ||ddg¡| _n
t ||ddg¡| _t ¡ | _t| j| jd|	|
d d�| _| jdkrŠt | j¡| _d S d S )NÚConv3d)Útypeé   )Úkernel_sizeÚconv_cfgÚnorm_cfgÚact_cfgr   )ÚsuperÚ__init__ÚdictÚst_feat_channelsÚlt_feat_channelsÚlatent_channelsÚnum_st_featÚnum_lt_featÚ	use_scaleÚpre_activateÚpre_activate_with_lnÚdropout_ratioÚzero_init_out_convr   Úst_feat_convÚlt_feat_convÚglobal_convÚnnÚ	LayerNormÚlnÚReLUÚreluÚout_convÚDropoutÚdropout)Úselfr   r   r   r   r   r   r   r   r   r   r   r   ©Ú	__class__© ú4/root/disk_3tb/HAC/mmaction/models/heads/fbo_head.pyr   *   sh   

úúú
ú
ÿzNonLocalLayer.__init__c                 C   sš   t |tƒrtƒ }| d|› �¡ t| |d|d� dS |du rI|  ¡ D ]}t |tjƒr/t|ƒ q"t |t	ƒr9t
|dƒ q"| jrGt
| jddd� dS dS tdƒ‚)	zPInitiate the parameters either from existing checkpoint or from
        scratch.zload model from: F©ÚstrictÚloggerNr   r   )Úbiasú pretrained must be a str or None)Ú
isinstanceÚstrr   Úinfor   Úmodulesr#   r   r   r   r   r   r(   Ú	TypeError)r+   Ú
pretrainedr2   Úmr.   r.   r/   Úinit_weightso   s   



€ÿzNonLocalLayer.init_weightsc                 C   s  |  d¡| j}}| j| j}}|  |¡}| |||¡}|  |¡}| |||¡}|  |¡}	|	 |||¡}	t 	| 
ddd¡|¡}
| jrH|
|d  }
|
jdd�}t 	|	| 
ddd¡¡ |||dd¡}| jrp| jrk|  |¡}|  |¡}|  |¡}| js}|  |¡}| jdkr‡|  |¡}|S )Nr   é   r   g      à?éÿÿÿÿ©Údim)Úsizer   r   r   r    Úviewr!   r"   ÚtorchÚmatmulÚpermuter   Úsoftmaxr   r   r%   r'   r(   r   r*   )r+   Úst_featÚlt_featÚnÚcr   r   ÚthetaÚphiÚgZ	theta_phiÚpÚoutr.   r.   r/   Úforward�   s.   


$





zNonLocalLayer.forward)TTTNNr   F©N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r<   rP   Ú__classcell__r.   r.   r,   r/   r
      s    ô
Er
   c                       s>   e Zd ZdZ					d‡ fdd„	Zdd	d
„Zdd„ Z‡  ZS )ÚFBONonLocalaÉ  Non local feature bank operator.

    Args:
        st_feat_channels (int): Channels of short-term features.
        lt_feat_channels (int): Channels of long-term features.
        latent_channels (int): Channles of latent features.
        num_st_feat (int): Number of short-term roi features.
        num_lt_feat (int): Number of long-term roi features.
        num_non_local_layers (int): Number of non-local layers, which is
            at least 1. Default: 2.
        st_feat_dropout_ratio (float): Probability of dropout layer for
            short-term features. Default: 0.2.
        lt_feat_dropout_ratio (float): Probability of dropout layer for
            long-term features. Default: 0.2.
        pre_activate (bool): Whether to use the activation function before
            upsampling in non local layers. Default: True.
        zero_init_out_conv (bool): Whether to use zero initialization for
            out_conv in NonLocalLayer. Default: False.
    r=   r   TFc                    s  t ƒ  ¡  |dksJ dƒ‚|| _|| _|| _|| _|| _|| _|| _|| _	|	| _
|
| _tj||dd�| _tj||dd�| _| jdkrIt | j¡| _| j	dkrUt | j	¡| _| j
s]t ¡ | _g | _t| jƒD ] }d|d › �}|  |t|||||| j
| jd�¡ | j |¡ qed S )Nr   z'At least one non_local_layer is needed.)r   r   Znon_local_layer_)r   r   )r   r   r   r   r   r   r   Únum_non_local_layersÚst_feat_dropout_ratioÚlt_feat_dropout_ratior   r   r#   r   r    r!   r)   Úst_feat_dropoutÚlt_feat_dropoutr&   r'   Únon_local_layersÚrangeÚ
add_moduler
   Úappend)r+   r   r   r   r   r   rY   rZ   r[   r   r   ÚidxÚ
layer_namer,   r.   r/   r   À   sV   

ÿÿÿ


ùþ
ôzFBONonLocal.__init__Nc                 C   sn   t |tƒrtƒ }t| |d|d� d S |d u r3t| jƒ t| jƒ | jD ]}t| |ƒ}|j	|d� q#d S t
dƒ‚)NFr0   ©r:   r4   )r5   r6   r   r   r   r    r!   r^   Úgetattrr<   r9   )r+   r:   r2   rc   Únon_local_layerr.   r.   r/   r<   ö   s   




þzFBONonLocal.init_weightsc                 C   s€   |   |¡}| jdkr|  |¡}|  |¡}| jdkr|  |¡}| jD ]}|}t| |ƒ}|||ƒ}|| }| js;|  	|¡}|}q!|S )Nr   )
r    rZ   r\   r!   r[   r]   r^   re   r   r'   )r+   rG   rH   rc   Úidentityrf   Znl_outr.   r.   r/   rP     s   









zFBONonLocal.forward)r=   r   r   TFrQ   rR   r.   r.   r,   r/   rX   «   s    ö
6rX   c                       ó2   e Zd ZdZ‡ fdd„Zd	dd„Zdd„ Z‡  ZS )
ÚFBOAvgzAvg pool feature bank operator.c                    ó   t ƒ  ¡  t d¡| _d S ©N©r   NN)r   r   r#   ÚAdaptiveAvgPool3dÚavg_pool©r+   r,   r.   r/   r     ó   
zFBOAvg.__init__Nc                 C   ó   d S rQ   r.   ©r+   r:   r.   r.   r/   r<   "  ó   zFBOAvg.init_weightsc                 C   ó   |   |¡}|S rQ   )rn   ©r+   rG   rH   rO   r.   r.   r/   rP   &  ó   
zFBOAvg.forwardrQ   rR   r.   r.   r,   r/   ri     ó
    
ri   c                       rh   )
ÚFBOMaxzMax pool feature bank operator.c                    rj   rk   )r   r   r#   ÚAdaptiveMaxPool3dÚmax_poolro   r,   r.   r/   r   .  rp   zFBOMax.__init__Nc                 C   rq   rQ   r.   rr   r.   r.   r/   r<   2  rs   zFBOMax.init_weightsc                 C   rt   rQ   )rz   ru   r.   r.   r/   rP   6  rv   zFBOMax.forwardrQ   rR   r.   r.   r,   r/   rx   +  rw   rx   c                       sL   e Zd ZdZeeedœZ		d‡ fdd„	Zddd	„Z	d
d„ Z
dd„ Z‡  ZS )ÚFBOHeada¥  Feature Bank Operator Head.

    Add feature bank operator for the spatiotemporal detection model to fuse
    short-term features and long-term features.

    Args:
        lfb_cfg (Dict): The config dict for LFB which is used to sample
            long-term features.
        fbo_cfg (Dict): The config dict for feature bank operator (FBO). The
            type of fbo is also in the config dict and supported fbo type is
            `fbo_dict`.
        temporal_pool_type (str): The temporal pool type. Choices are 'avg' or
            'max'. Default: 'avg'.
        spatial_pool_type (str): The spatial pool type. Choices are 'avg' or
            'max'. Default: 'max'.
    )Ú	non_localÚavgÚmaxr}   r~   c                    sÈ   t ƒ  ¡  | dd¡}|tjv sJ ‚|dv sJ ‚|dv sJ ‚t |¡| _t |¡| _t	di | j¤Ž| _
| j| di | j¤Ž| _|dkrJt d¡| _nt d¡| _|dkr\t d¡| _d S t d¡| _d S )Nr   r|   )r~   r}   r}   rl   )Nr   r   r.   )r   r   Úpopr{   Úfbo_dictÚcopyÚdeepcopyÚlfb_cfgÚfbo_cfgr   ÚlfbÚfbor#   rm   Útemporal_poolry   Úspatial_pool)r+   rƒ   r„   Útemporal_pool_typeÚspatial_pool_typeZfbo_typer,   r.   r/   r   O  s   
zFBOHead.__init__Nc                 C   s   | j j|d� dS )z¡Initialize the weights in the module.

        Args:
            pretrained (str, optional): Path to pre-trained weights.
                Default: None.
        rd   N)r†   r<   rr   r.   r.   r/   r<   j  s   zFBOHead.init_weightsc                 C   sr   |dd…df   tj¡}g }|D ]}| | j|| d   ¡ ¡ qtj|dd�}| ddd¡ ¡ }| 	d¡ 	d¡S )z/Sample long-term features for each ROI feature.Nr   Úimg_keyr?   r=   r   r>   )
r   rC   Úint64ra   r…   ÚtoÚstackrE   Ú
contiguousÚ	unsqueeze)r+   ÚroisÚ	img_metasÚindsZlt_feat_listÚindrH   r.   r.   r/   Ú
sample_lfbs  s   zFBOHead.sample_lfbc           	      C   sN   |   |¡}|  |¡}|}|  ||¡ |j¡}|  ||¡}tj||gdd�}|S )Nr   r?   )r‡   rˆ   r•   r�   Údevicer†   rC   Úcat)	r+   Úxr‘   r’   rG   rg   rH   Zfbo_featrO   r.   r.   r/   rP   ~  s   

zFBOHead.forward)r}   r~   rQ   )rS   rT   rU   rV   rX   ri   rx   r€   r   r<   r•   rP   rW   r.   r.   r,   r/   r{   ;  s    ü
	r{   )r�   rC   Útorch.nnr#   Úmmcv.cnnr   r   r   Úmmcv.runnerr   Z
mmcv.utilsr   Zmmaction.models.commonr   Úmmaction.utilsr   Zmmdet.models.builderr	   ZMMDET_SHARED_HEADSÚmmdet_importedÚImportErrorÚModuleNotFoundErrorÚModuler
   rX   ri   rx   r{   Úregister_moduler.   r.   r.   r/   Ú<module>   s.    ÿ pRÿ