o
    _{j>  ã                   @   sž   d dl Zd dlZd dlmZ d dlm  mZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ e ¡ G d
d„ deƒƒZe ¡ G dd„ deƒƒZdS )é    Né   )Útemporal_iopé   )Ú
build_loss)Ú
LOCALIZERSé   )ÚBaseLocalizer)Úpost_processingc                       sn   e Zd ZdZedd�dddddf‡ fdd	„	Zddd„Zdd„ Zdd„ Zdd„ Z	dd„ Z
			ddd„Z‡  ZS )ÚTEMaœ  Temporal Evaluation Model for Boundary Sensetive Network.

    Please refer `BSN: Boundary Sensitive Network for Temporal Action
    Proposal Generation <http://arxiv.org/abs/1806.02964>`_.

    Code reference
    https://github.com/wzmsltw/BSN-boundary-sensitive-network

    Args:
        tem_feat_dim (int): Feature dimension.
        tem_hidden_dim (int): Hidden layer dimension.
        tem_match_threshold (float): Temporal evaluation match threshold.
        loss_cls (dict): Config for building loss.
            Default: ``dict(type='BinaryLogisticRegressionLoss')``.
        loss_weight (float): Weight term for action_loss. Default: 2.
        output_dim (int): Output dimension. Default: 3.
        conv1_ratio (float): Ratio of conv1 layer output. Default: 1.0.
        conv2_ratio (float): Ratio of conv2 layer output. Default: 1.0.
        conv3_ratio (float): Ratio of conv3 layer output. Default: 0.01.
    ÚBinaryLogisticRegressionLoss)Útyper   r   r   g{®Gáz„?c                    sº   t t| ƒ ¡  || _|| _|| _|| _|| _|| _t	|ƒ| _
|| _|	| _|
| _|| _tj| j| jddddd�| _tj| j| jddddd�| _tj| j| jdddd�| _|  ¡ \| _| _d S )Nr   r   )Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingÚgroupsr   )r   r   r   r   r   )Úsuperr   Ú__init__Útemporal_dimÚboundary_ratioÚfeat_dimZc_hiddenÚmatch_thresholdÚ
output_dimr   Úloss_clsÚloss_weightÚconv1_ratioÚconv2_ratioÚconv3_ratioÚnnÚConv1dÚconv1Úconv2Úconv3Ú_temporal_anchorsÚanchors_tminsÚanchors_tmaxs)Úselfr   r   Ztem_feat_dimZtem_hidden_dimZtem_match_thresholdr   r   r   r   r   r   ©Ú	__class__© ú4/root/disk_3tb/HAC/mmaction/models/localizers/bsn.pyr   $   sH   
úúûzTEM.__init__ç        ç      ð?c                 C   sN   d| j  }g }g }t| j ƒD ]}| |||  ¡ | |||  ¡ q||fS )a„  Generate temporal anchors.

        Args:
            tmin_offset (int): Offset for the minimum value of temporal anchor.
                Default: 0.
            tmax_offset (int): Offset for the maximun value of temporal anchor.
                Default: 1.

        Returns:
            tuple[Sequence[float]]: The minimum and maximum values of temporal
                anchors.
        r-   )r   ÚrangeÚappend)r'   Útmin_offsetÚtmax_offsetÚtemporal_gapr%   r&   Úir*   r*   r+   r$   T   s   
zTEM._temporal_anchorsc                 C   sF   t  | j|  |¡ ¡}t  | j|  |¡ ¡}t | j|  	|¡ ¡}|S ©z»Define the computation performed at every call.

        Args:
            x (torch.Tensor): The input data.

        Returns:
            torch.Tensor: The output of the module.
        )
ÚFÚrelur   r!   r   r"   ÚtorchÚsigmoidr   r#   ©r'   Úxr*   r*   r+   Ú_forwardj   s   	zTEM._forwardc                 C   s’   |   |¡}|dd…ddd…f }|dd…ddd…f }|dd…ddd…f }|  ||| j¡}	|  ||| j¡}
|  ||| j¡}|	| j |
|dœ}|S )ú=Define the computation performed at every call when training.Nr   r   r   )Úloss_actionZ
loss_startZloss_end)r;   r   r   r   )r'   Úraw_featureÚlabel_actionÚlabel_startÚ	label_endÚ
tem_outputÚscore_actionZscore_startZ	score_endr=   Zloss_start_smallZloss_end_smallÚ	loss_dictr*   r*   r+   Úforward_trainx   s$   
ÿÿÿýzTEM.forward_trainc                 C   sÊ   |   |¡ ¡  ¡ }|dd…ddd…f }|dd…ddd…f }|dd…ddd…f }dd„ |D ƒ}g }t|ƒD ]+\}	}
||	 d }||	 }||	 }||	 }tj|||| j| jfdd�}| ||f¡ q7|S )	ú<Define the computation performed at every call when testing.Nr   r   r   c                 S   s   g | ]}t |ƒ‘qS r*   )Údict)Ú.0r:   r*   r*   r+   Ú
<listcomp>”   s    z$TEM.forward_test.<locals>.<listcomp>Ú
video_name©Úaxis)	r;   ÚcpuÚnumpyÚ	enumerateÚnpÚstackr%   r&   r/   )r'   r>   Ú
video_metarB   Zbatch_actionZbatch_startZ	batch_endZvideo_meta_listZvideo_resultsÚ	batch_idxÚ_rJ   Zvideo_actionZvideo_startZ	video_endZvideo_resultr*   r*   r+   Úforward_test�   s$   
ÿþzTEM.forward_testc                 C   s’  g }g }g }|D ]¬}|dd…df   ¡  ¡ }|dd…df   ¡  ¡ }|| }t d| j | j| ¡}	tj||	d  ||	d  fdd�}
tj||	d  ||	d  fdd�}g }g }g }t| j| j	ƒD ]C\}}| 
t t||||ƒ¡¡ | 
t t|||
dd…df |
dd…df ƒ¡¡ | 
t t|||dd…df |dd…df ƒ¡¡ qa| 
|¡ | 
|¡ | 
|¡ qt |¡}t |¡}t |¡}|||fS )zGenerate training labels.Nr   r   r-   r   rK   )rM   rN   rP   Úmaximumr   r   rQ   Úzipr%   r&   r/   Úmaxr   r7   ÚTensor)r'   Úgt_bboxZmatch_score_action_listÚmatch_score_start_listÚmatch_score_end_listÚevery_gt_bboxÚgt_tminsÚgt_tmaxsZgt_lensÚ
gt_len_padÚgt_start_bboxsÚgt_end_bboxsZmatch_score_actionÚmatch_score_startÚmatch_score_endÚanchor_tminÚanchor_tmaxr*   r*   r+   Úgenerate_labels£   sl   ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ




ÿzTEM.generate_labelsNTc           	      C   sT   |r$|   |¡\}}}|j}| |¡}| |¡}| |¡}|  ||||¡S |  ||¡S ©z/Define the computation performed at every call.)rg   ÚdeviceÚtorE   rU   )	r'   r>   rZ   rR   Úreturn_lossr?   r@   rA   ri   r*   r*   r+   ÚforwardÑ   s   ÿ



ÿzTEM.forward)r,   r-   )NNT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rG   r   r$   r;   rE   rU   rg   rl   Ú__classcell__r*   r*   r(   r+   r
      s$    õ
00ür
   c                       sZ   e Zd ZdZ				d‡ fdd„	Zdd„ Zd	d
„ Zdd„ Z							ddd„Z‡  Z	S )ÚPEMaÚ  Proposals Evaluation Model for Boundary Sensetive Network.

    Please refer `BSN: Boundary Sensitive Network for Temporal Action
    Proposal Generation <http://arxiv.org/abs/1806.02964>`_.

    Code reference
    https://github.com/wzmsltw/BSN-boundary-sensitive-network

    Args:
        pem_feat_dim (int): Feature dimension.
        pem_hidden_dim (int): Hidden layer dimension.
        pem_u_ratio_m (float): Ratio for medium score proprosals to balance
            data.
        pem_u_ratio_l (float): Ratio for low score proprosals to balance data.
        pem_high_temporal_iou_threshold (float): High IoU threshold.
        pem_low_temporal_iou_threshold (float): Low IoU threshold.
        soft_nms_alpha (float): Soft NMS alpha.
        soft_nms_low_threshold (float): Soft NMS low threshold.
        soft_nms_high_threshold (float): Soft NMS high threshold.
        post_process_top_k (int): Top k proposals in post process.
        feature_extraction_interval (int):
            Interval used in feature extraction. Default: 16.
        fc1_ratio (float): Ratio for fc1 layer output. Default: 0.1.
        fc2_ratio (float): Ratio for fc2 layer output. Default: 0.1.
        output_dim (int): Output dimension. Default: 1.
    é   çš™™™™™¹?r   c                    s’   t t| ƒ ¡  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _|| _|| _|| _tj| j| jdd�| _tj| j| jdd�| _d S )NT)Úin_featuresÚout_featuresÚbias)r   r   r   r   Ú
hidden_dimÚ	u_ratio_mÚ	u_ratio_lÚpem_high_temporal_iou_thresholdÚpem_low_temporal_iou_thresholdÚsoft_nms_alphaÚsoft_nms_low_thresholdÚsoft_nms_high_thresholdÚpost_process_top_kÚfeature_extraction_intervalÚ	fc1_ratioÚ	fc2_ratior   r   ÚLinearÚfc1Úfc2)r'   Zpem_feat_dimZpem_hidden_dimZpem_u_ratio_mZpem_u_ratio_lr{   r|   r}   r~   r   r€   r�   r‚   rƒ   r   r(   r*   r+   r     s.   
ÿýzPEM.__init__c                 C   s>   t  t|ƒ¡}t | j|  |¡ ¡}t  | j|  	|¡ ¡}|S r4   )
r7   ÚcatÚlistr5   r6   r‚   r…   r8   rƒ   r†   r9   r*   r*   r+   r;   (  s   	zPEM._forwardc                 C   sv  |   |¡}t t|ƒ¡}|j}| |¡}| d¡}|| jk ¡ }|| jk|| j	k@  ¡ }|| j	k ¡ }t 
|¡}	t 
|¡}
t 
|¡}| j|	 |
 }t |t dg¡ |¡¡d }tj| ¡ d |d�}|| }|d| k ¡ }| j|	 | }t |t dg¡ |¡¡d }tj| ¡ d |d�}|| }|d| k ¡ }|| | }t ||¡}t 
|| ¡t 
|¡ }t|d�}|S )r<   éÿÿÿÿr-   r   )ri   )Útemporal_iou_loss)r;   r7   r‡   rˆ   ri   rj   Úviewr{   Úfloatr|   Úsumry   ÚminrY   ÚrandÚsizerz   r5   Úsmooth_l1_lossrG   )r'   Úbsp_featureÚreference_temporal_iouÚ
pem_outputri   Zanchors_temporal_iouZu_hmaskZu_mmaskZu_lmaskZnum_hZnum_mZnum_lZr_mZu_smmaskZr_lZu_slmaskZtemporal_iou_weightsrŠ   rD   r*   r*   r+   rE   6  s\   


ÿÿÿýÿÿ


ÿÿÿþ
zPEM.forward_trainc                 C   s  |   |¡ d¡ ¡  ¡  dd¡}| d¡ ¡  ¡  dd¡}| d¡ ¡  ¡  dd¡}| d¡ ¡  ¡  dd¡}| d¡ ¡  ¡  dd¡}t || | ¡ dd¡}tj||||||fdd�}	|	 dd¡}	t|d ƒ}
t	|	|
| j
| j| j| j| jƒ}t|
d |d�g}|S )rF   r‰   r   rK   é   r   rJ   )rJ   Úproposal_list)r;   r‹   rM   rN   ÚreshaperP   ÚarrayÚconcatenaterG   r	   r}   r~   r   r€   r�   )r'   r’   ÚtminÚtmaxÚ
tmin_scoreÚ
tmax_scorerR   r”   ÚscoreÚresultÚ
video_infor–   Úoutputr*   r*   r+   rU   a  s4   ÿÿûþÿzPEM.forward_testNTc	           	      C   s$   |r|   ||¡S |  ||||||¡S rh   )rE   rU   )	r'   r’   r“   rš   r›   rœ   r�   rR   rk   r*   r*   r+   rl   }  s
   
ÿzPEM.forward)rs   rt   rt   r   )NNNNNNT)
rm   rn   ro   rp   r   r;   rE   rU   rl   rq   r*   r*   r(   r+   rr   ä   s$    &ò'+ørr   )rN   rP   r7   Útorch.nnr   Útorch.nn.functionalÚ
functionalr5   Úlocalizationr   Úbuilderr   Úregistryr   Úbaser   Úutilsr	   Úregister_moduler
   rr   r*   r*   r*   r+   Ú<module>   s     W