o
    _{j’E  ã                   @   s€   d dl Z d dlZd dlZd dlmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ e ¡ G d
d„ deƒƒZdS )é    Né   )Útemporal_iopÚtemporal_ioué   )Ú
build_loss)Ú
LOCALIZERSé   )ÚBaseLocalizer)Úpost_processingc                       s˜   e Zd ZdZdedd�dddf‡ fdd	„	Zd
d„ Zd$dd„Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Z			d%dd„Zedd„ ƒZd d!„ Zd"d#„ Z‡  ZS )&ÚBMNaè  Boundary Matching Network for temporal action proposal generation.

    Please refer `BMN: Boundary-Matching Network for Temporal Action Proposal
    Generation <https://arxiv.org/abs/1907.09702>`_.
    Code Reference https://github.com/JJBOY/BMN-Boundary-Matching-Network

    Args:
        temporal_dim (int): Total frames selected for each video.
        boundary_ratio (float): Ratio for determining video boundaries.
        num_samples (int): Number of samples for each proposal.
        num_samples_per_bin (int): Number of bin samples for each sample.
        feat_dim (int): Feature dimension.
        soft_nms_alpha (float): Soft NMS alpha.
        soft_nms_low_threshold (float): Soft NMS low threshold.
        soft_nms_high_threshold (float): Soft NMS high threshold.
        post_process_top_k (int): Top k proposals in post process.
        feature_extraction_interval (int):
            Interval used in feature extraction. Default: 16.
        loss_cls (dict): Config for building loss.
            Default: ``dict(type='BMNLoss')``.
        hidden_dim_1d (int): Hidden dim for 1d conv. Default: 256.
        hidden_dim_2d (int): Hidden dim for 2d conv. Default: 128.
        hidden_dim_3d (int): Hidden dim for 3d conv. Default: 512.
    é   ÚBMNLoss)Útypeé   é€   i   c                    s2  t t| ƒ ¡  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _t|ƒ| _|| _|| _|| _|  ¡  t tj| j| jdddd�tjdd�tj| j| jdddd�tjdd�¡| _t tj| j| jdddd�tjdd�tj| jddd�t ¡ ¡| _t tj| j| jdddd�tjdd�tj| jddd�t ¡ ¡| _t tj| j| jddd�tjdd�¡| _t tj| j| j| jddfd�tjdd�¡| _t tj| j| jdd�tjdd�tj| j| jddd�tjdd�tj| j| jddd�tjdd�tj| jd	dd�t ¡ ¡| _|   d
d¡\| _!| _"|  #¡ | _$|  %¡ | _&d S )Nr   r   é   )Úkernel_sizeÚpaddingÚgroupsT)Úinplace)r   )r   r   r   g      à¿g      ø?)'Úsuperr	   Ú__init__ÚtscaleÚboundary_ratioÚnum_samplesÚnum_samples_per_binÚfeat_dimÚsoft_nms_alphaÚsoft_nms_low_thresholdÚsoft_nms_high_thresholdÚpost_process_top_kÚfeature_extraction_intervalr   Úloss_clsÚhidden_dim_1dÚhidden_dim_2dÚhidden_dim_3dÚ_get_interp1d_maskÚnnÚ
SequentialÚConv1dÚReLUÚx_1d_bÚSigmoidÚx_1d_sÚx_1d_eÚx_1d_pÚConv3dÚx_3d_pÚConv2dÚx_2d_pÚ_temporal_anchorsÚanchors_tminsÚanchors_tmaxsÚ
_match_mapÚ	match_mapÚ_get_bm_maskÚbm_mask)ÚselfZtemporal_dimr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   ©Ú	__class__© ú4/root/disk_3tb/HAC/mmaction/models/localizers/bmn.pyr   )   sÂ   
û
û
ôû
ùû
ù
ü
û
ý
ü
ü
ü
óÿ
zBMN.__init__c                 C   s’   d| j  }g }t| j ƒD ]%}g }|| }td| j d ƒD ]}|||  }| ||g¡ q| |¡ qt |¡}t |g d¢¡}t |ddg¡}|S )zGenerate match map.ç      ð?r   )r   r   r   éÿÿÿÿr   )r   ÚrangeÚappendÚnpÚarrayÚ	transposeÚreshape)r;   Útemporal_gapr8   ÚidxZmatch_windowÚtminÚjdxÚtmaxr>   r>   r?   r7   ‹   s   

zBMN._match_mapç        r@   c                 C   sN   d| j  }g }g }t| j ƒD ]}| |||  ¡ | |||  ¡ q||fS )a„  Generate temporal anchors.

        Args:
            tmin_offset (int): Offset for the minimum value of temporal anchor.
                Default: 0.
            tmax_offset (int): Offset for the maximun value of temporal anchor.
                Default: 1.

        Returns:
            tuple[Sequence[float]]: The minimum and maximum values of temporal
                anchors.
        r@   )r   rB   rC   )r;   Ztmin_offsetZtmax_offsetrH   r5   r6   Úir>   r>   r?   r4   ›   s   
zBMN._temporal_anchorsc                 C   sb   |   |¡}|  |¡ d¡}|  |¡ d¡}|  |¡}|  |¡}|  |¡ d¡}|  |¡}|||fS )z»Define the computation performed at every call.

        Args:
            x (torch.Tensor): The input data.

        Returns:
            torch.Tensor: The output of the module.
        r   r   )r+   r-   Úsqueezer.   r/   Ú_boundary_matching_layerr1   r3   )r;   ÚxZbase_featureÚstartÚendÚconfidence_mapr>   r>   r?   Ú_forward±   s   





zBMN._forwardc                 C   s6   |  ¡ }t || j¡ |d |d | j| j| j¡}|S )zGenerate matching layer.r   r   )ÚsizeÚtorchÚmatmulÚsample_maskrG   r   r   )r;   rQ   Ú
input_sizeÚoutr>   r>   r?   rP   Ì   s   ÿýÿzBMN._boundary_matching_layerc                 C   sB  |   |¡\}}}|d  ¡  ¡ }|d  ¡  ¡ }|d d  ¡  ¡ }|d d  ¡  ¡ }	t|ƒ}
t|ƒ}t t|ƒ¡}d|d< t t|ƒ¡}d|d< td| jd ƒD ]L}|| ||d  krm|| ||d  krmd||< n|| d|
 kryd||< || ||d  kr’|| ||d  kr’d||< qR|| d| kržd||< qRg }t| jƒD ]U}t| jƒD ]M}|}|| d }|| jk rú|| dkrú|| dkrú|| j }|| j }|| }|| }|||f }|	||f }|| | | }| 	|||||||g¡ q­q¦t 
|¡}t|d ƒ}t||| j| j| j| j| jƒ}t|d |d�g}|S )z<Define the computation performed at every call when testing.r   r   rA   g      à?Ú
video_name)r\   Úproposal_list)rU   ÚcpuÚnumpyÚmaxrD   ÚzerosÚlenrB   r   rC   ÚstackÚdictr
   r   r   r   r    r!   )r;   Úraw_featureÚ
video_metarT   rR   rS   Ústart_scoresÚ
end_scoresZcls_confidenceZreg_confidenceÚ	max_startÚmax_endÚ
start_binsÚend_binsrI   Únew_proposalsrK   Ústart_indexÚ	end_indexrJ   rL   Ú
tmin_scoreÚ
tmax_scoreÚ	cls_scoreZ	reg_scoreÚscoreÚ
video_infor]   Úoutputr>   r>   r?   Úforward_testÖ   s„   ÿ
ÿ
€ÿÿ


þ€ô
ûþÿzBMN.forward_testc           
      C   sB   |   |¡\}}}|  ||||||| j |j¡¡}t|d d�}	|	S )z=Define the computation performed at every call when training.r   )Úloss)rU   r"   r:   ÚtoÚdevicerd   )
r;   re   Úlabel_confidenceÚlabel_startÚ	label_endrT   rR   rS   rw   Z	loss_dictr>   r>   r?   Úforward_train  s   þzBMN.forward_trainc                 C   sô  g }g }g }|D ]Ý}g }|D ]<\}}t |tjƒr| ¡ }t |tjƒr&| ¡ }t| jdd…df | jdd…df ||ƒ}	t |	| j| jg¡}	| 	|	¡ qt 
|¡ tj¡}tj|dd�}|dd…df }
|dd…df }dd| j  }tj|
|d  |
|d  fdd�}tj||d  ||d  fdd�}g }g }t| j| jƒD ]6\}}| 	t t|||dd…df |dd…df ƒ¡¡ | 	t t|||dd…df |dd…df ƒ¡¡ qŸ| 	|¡ | 	|¡ | 	|¡ qt |¡}t |¡}t |¡}|||fS )zGenerate training labels.Nr   r   ©Úaxisr   r@   r   )Ú
isinstancerW   ÚTensorr_   r   r8   rD   rG   r   rC   rE   ÚastypeÚfloat32r`   rc   Úzipr5   r6   r   )r;   Úgt_bboxZmatch_score_confidence_listZmatch_score_start_listZmatch_score_end_listZevery_gt_bboxZ
gt_iou_maprR   rS   Zcurrent_gt_iou_mapÚgt_tminsÚgt_tmaxsZ
gt_len_padZgt_start_bboxsZgt_end_bboxsZmatch_score_startZmatch_score_endZanchor_tminZanchor_tmaxr>   r>   r?   Úgenerate_labels  sv   þ
ÿÿÿÿÿÿÿÿÿÿÿ




ÿzBMN.generate_labelsNTc           	      C   sT   |r$|   |¡\}}}|j}| |¡}| |¡}| |¡}|  ||||¡S |  ||¡S )z/Define the computation performed at every call.)rˆ   ry   rx   r}   rv   )	r;   re   r…   rf   Úreturn_lossrz   r{   r|   ry   r>   r>   r?   ÚforwardT  s   ÿ


ÿzBMN.forwardc                    s"  t |ˆ ƒ}||| d  ‰ ‡ ‡fdd„t|| ƒD ƒ}g }t|ƒD ]e}||| |d | … }	t |¡}
|	D ]D}t |¡}t |¡\}}dt|ƒ  krS|d kran n|
t|ƒ  d| 7  < dt|ƒ  kro|d kr{n q7|
t|ƒ  |7  < q7d| |
 }
| |
¡ q"tj	|dd�}|S )z2Generate sample mask for a boundary-matching pair.r@   c                    s   g | ]}ˆˆ |  ‘qS r>   r>   )Ú.0rN   ©Zplen_sampleÚseg_tminr>   r?   Ú
<listcomp>l  s    
ÿÿz.BMN._get_interp1d_bin_mask.<locals>.<listcomp>r   r   r~   )
ÚfloatrB   rD   ra   ÚmathÚceilÚmodfÚintrC   rc   )r�   Zseg_tmaxr   r   r   ÚplenZtotal_samplesÚp_maskrI   Zbin_samplesZ
bin_vectorÚsampleZsample_upperZsample_decimalZsample_downr>   rŒ   r?   Ú_get_interp1d_bin_maskf  s.   
þÿ

  €zBMN._get_interp1d_bin_maskc              	   C   sú   g }t | jƒD ]V}g }t | jƒD ]@}|| | jk rB|}|| }t|| ƒd }||| j  }||| j  }	|  ||	| j| j| j¡}
n	t | j| jg¡}
| 	|
¡ qtj
|dd�}| 	|¡ qtj
|dd�}| tj¡}tjt |¡ | jd¡dd�| _dS )	z=Generate sample mask for each point in Boundary-Matching Map.r   r   r~   r   rA   F)Úrequires_gradN)rB   r   r�   r   r—   r   r   rD   ra   rC   rc   r‚   rƒ   r'   Ú	ParameterrW   ÚtensorÚviewrY   )r;   Zmask_matrn   Zmask_mat_vectorZduration_indexZp_tminZp_tmaxZ
center_lenZsample_tminZsample_tmaxr•   r>   r>   r?   r&   �  s.   þÿzBMN._get_interp1d_maskc                 C   sL   g }t | jƒD ]}dg| j|  dg|  }| |¡ qtj|tjd�}|S )z Generate Boundary-Matching Mask.r   r   )Údtype)rB   r   rC   rW   rš   r�   )r;   r:   rI   Zmask_vectorr>   r>   r?   r9   š  s   zBMN._get_bm_mask)rM   r@   )NNT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rd   r   r7   r4   rU   rP   rv   r}   rˆ   rŠ   Ústaticmethodr—   r&   r9   Ú__classcell__r>   r>   r<   r?   r      s.    #òb

>
8
ü
r   )r�   r_   rD   rW   Útorch.nnr'   Úlocalizationr   r   Úbuilderr   Úregistryr   Úbaser	   Úutilsr
   Úregister_moduler   r>   r>   r>   r?   Ú<module>   s    