o
    _{j­%  ã                   @   s,   d dl Z d dlm  mZ G dd„ dƒZdS )é    Nc                   @   s>   e Zd ZdZddd„Zdd„ Zddd	„Zd
d„ Zddd„ZdS )ÚGradCAMaf  GradCAM class helps create visualization results.

    Visualization results are blended by heatmaps and input images.
    This class is modified from
    https://github.com/facebookresearch/SlowFast/blob/master/slowfast/visualization/gradcam_utils.py # noqa
    For more information about GradCAM, please visit:
    https://arxiv.org/pdf/1610.02391.pdf
    Úviridisc                 C   s¦   ddl m}m} t||ƒrd| _nt||ƒrd| _ntdƒ‚|| _| j ¡  d| _d| _	ddl
m} | |¡| _t |jjd ¡| _t |jjd	 ¡| _|  |¡ dS )
a3  Create GradCAM class with recognizer, target layername & colormap.

        Args:
            model (nn.Module): the recognizer model to be used.
            target_layer_name (str): name of convolutional layer to
                be used to get gradients and feature maps from for creating
                localization maps.
            colormap (Optional[str]): matplotlib colormap used to create
                heatmap. Default: 'viridis'. For more information, please visit
                https://matplotlib.org/3.3.0/tutorials/colors/colormaps.html
        é   )ÚRecognizer2DÚRecognizer3DTFz7GradCAM utils only support Recognizer2D & Recognizer3D.Nr   ÚmeanÚstd)Zmodels.recognizersr   r   Ú
isinstanceÚis_recognizer2dÚ
ValueErrorÚmodelÚevalÚtarget_gradientsÚtarget_activationsZmatplotlib.pyplotZpyplotZget_cmapÚcolormapÚtorchÚtensorÚcfgZimg_norm_cfgÚ	data_meanÚdata_stdÚ_register_hooks)Úselfr   Ztarget_layer_namer   r   r   Zplt© r   ú2/root/disk_3tb/HAC/mmaction/utils/gradcam_utils.pyÚ__init__   s"   

ÿ
zGradCAM.__init__c                    sX   ‡ fdd„}‡ fdd„}|  d¡}ˆ j}|D ]}|j| }q|}| |¡ | |¡ dS )z¶Register forward and backward hook to a layer, given layer_name, to
        obtain gradients and activations.

        Args:
            layer_name (str): name of the layer.
        c                    s   |d   ¡ ˆ _d S )Nr   )Údetachr   )ÚmoduleÚ
grad_inputÚgrad_output©r   r   r   Úget_gradients7   ó   z.GradCAM._register_hooks.<locals>.get_gradientsc                    s   |  ¡  ¡ ˆ _d S )N)Úcloner   r   )r   ÚinputÚoutputr   r   r   Úget_activations:   r!   z0GradCAM._register_hooks.<locals>.get_activationsú/N)Úsplitr   Ú_modulesÚregister_forward_hookÚregister_backward_hook)r   Z
layer_namer    r%   Zlayer_lsZprev_moduleÚlayerZtarget_layerr   r   r   r   /   s   

zGradCAM._register_hooksç#B’¡œÇ;c                 C   sP  |d   ¡ |d< | jdddi|¤Ž}|r+|d }|jdkr"| d¡}tj|d|d�}n	tj|dd�d	 }| j ¡  t |¡}| 	¡  | j
rQ|d  ¡ \}}}	}
}n|d  ¡ \}}}	}}
}|| }| j}| j}| j
rw| ¡ \}}}	}	|| }n| ¡ \}	}}}	}	| d	d
ddd¡}| d	d
ddd¡}tj| |||d¡dd�}| |||dd¡}| |||gt| ¡ dd… ƒ ¡}tj|| d
dd�}t |¡}| d	d
ddd¡}tj|||
|fddd�}tj| |d¡ddd�d	 tj| |d¡ddd�d	 }}tj||ddddfd�}tj||ddddfd�}|| || |  }|j}|jdd�|fS )aj  Calculate localization map for all inputs with Grad-CAM.

        Args:
            inputs (dict): model inputs, generated by test pipeline,
                at least including two keys, ``imgs`` and ``label``.
            use_labels (bool): Whether to use given labels to generate
                localization map. Labels are in ``inputs['label']``.
            delta (float): used in localization map normalization,
                must be small enough. Please make sure
                `localization_map_max - localization_map_min >> delta`
        Returns:
            tuple[torch.Tensor, torch.Tensor]: (localization_map, preds)
                localization_map (torch.Tensor): the localization map for
                    input imgs.
                preds (torch.Tensor): Model predictions for `inputs` with
                    shape (batch_size, num_classes).
        ÚimgsZgradcamTÚlabelé   éÿÿÿÿ)ÚdimÚindex)r1   r   r   é   é   éþÿÿÿN)r1   ÚkeepdimÚ	trilinearF)ÚsizeÚmodeÚalign_corners)Úshaper   )r"   r   ÚndimÚ	unsqueezer   ÚgatherÚmaxÚ	zero_gradÚsumÚbackwardr
   r8   r   r   Úpermuter   ÚviewÚlistÚFÚreluÚinterpolateÚminÚreshapeÚdataÚsqueeze)r   ÚinputsÚ
use_labelsÚdeltaÚpredsÚlabelsÚscoreÚbÚtÚ_ÚhÚwÚb1Úb2Ú	gradientsZactivationsZb_tgÚcÚtgÚweightsÚlocalization_mapZlocalization_map_minZlocalization_map_maxr   r   r   Ú_calculate_localization_mapF   sj   




ÿ
ÿ
üþÿÿ
ÿz#GradCAM._calculate_localization_mapc                 C   sÒ   |  ¡ }|  | ¡  ¡ ¡}|dd…dd…dd…dd…dd…f }t |¡}| jr2| ddddd¡}n| dgt	| 
¡ dd… ƒ ¡}| ddddd¡}|  ¡ }|| j9 }|| j7 }|d }|| d| |  }|S )	aI  Blend heatmaps and model input images and get visulization results.

        Args:
            localization_map (torch.Tensor): localization map for all inputs,
                generated with Grad-CAM
            input_imgs (torch.Tensor): model inputs, normed images.
            alpha (float): transparency level of the heatmap,
                in the range [0, 1].
        Returns:
            torch.Tensor: blending results for localization map and input
                images, with shape [B, T, H, W, 3] and pixel values in
                RGB order within range [0, 1].
        Nr3   r   r/   r4   r   r0   g     ào@)Úcpur   r   Únumpyr   Ú
from_numpyr
   rC   rD   rE   r8   r   r   )r   r^   Z
input_imgsÚalphaZheatmapZcurr_inpÚblended_imgsr   r   r   Ú_alpha_blending™   s   &
 

zGradCAM._alpha_blendingFç      à?c                 C   s,   | j ||d�\}}|  ||d |¡}||fS )a¼  Visualize the localization maps on their corresponding inputs as
        heatmap, using Grad-CAM.

        Generate visualization results for **ALL CROPS**.
        For example, for I3D model, if `clip_len=32, num_clips=10` and
        use `ThreeCrop` in test pipeline, then for every model inputs,
        there are 960(32*10*3) images generated.

        Args:
            inputs (dict): model inputs, generated by test pipeline,
                at least including two keys, ``imgs`` and ``label``.
            use_labels (bool): Whether to use given labels to generate
                localization map. Labels are in ``inputs['label']``.
            alpha (float): transparency level of the heatmap,
                in the range [0, 1].
        Returns:
            blended_imgs (torch.Tensor): Visualization results, blended by
                localization maps and model inputs.
            preds (torch.Tensor): Model predictions for inputs.
        )rN   r-   )r_   re   )r   rM   rN   rc   r^   rP   rd   r   r   r   Ú__call__Ä   s   
ÿÿzGradCAM.__call__N)r   )r,   )Frf   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r_   re   rg   r   r   r   r   r      s    
	 
S+r   )r   Útorch.nn.functionalÚnnÚ
functionalrF   r   r   r   r   r   Ú<module>   s    