o
    _{jì  ã                   @   s²   d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d
d„ Ze ¡ ddd„ƒZG dd„ deƒZdS )é    N)ÚMMDistributedDataParallel)ÚHook)Ú	print_log)Ú	GroupNorm)Ú
_BatchNorm)Ú_InstanceNorm)ÚDataParallelÚDistributedDataParallel)Ú
DataLoaderc                 C   s   t ttf}t| |ƒrdS dS )ap  Check if a module is a parallel module.

    The following 3 modules (and their subclasses) are regarded as parallel
    modules: DataParallel, DistributedDataParallel,
    MMDistributedDataParallel (the deprecated version).

    Args:
        module (nn.Module): The module to be checked.
    Returns:
        bool: True if the input module is a parallel module.
    TF)r   r	   r   Ú
isinstance)ÚmoduleZ	parallels© r   ú//root/disk_3tb/HAC/mmaction/utils/precise_bn.pyÚis_parallel_module   s   ÿ
r   éÈ   c              	   C   s  |   ¡  t|ƒ|ksJ dt|ƒ› d|› �ƒ‚t| ƒr | }| j} n| }dd„ |  ¡ D ƒ}t|ƒdkr;td|tjd� dS tt|ƒ› d	�|d
� |  ¡ D ]}|jr^t	|t
tfƒr^td|tjd� qJdd„ |D ƒ}|D ]}d|_qhdd„ |D ƒ}	dd„ |D ƒ}
d}t t|ƒ¡}t|ƒD ]\\}}t ¡ � |di |¤ddi¤Ž W d  ƒ n1 s§w   Y  | ¡  t|ƒD ]&\}}|	|  |j|	|  |d  7  < |
|  |j|
|  |d  7  < q´|d |kråd} nq‰|sïJ d|› �ƒ‚t|ƒD ]\}}|	| |_|
| |_|| |_qódS )a  Recompute and update the batch norm stats to make them more precise.

    During
    training both BN stats and the weight are changing after every iteration,
    so the running average can not precisely reflect the actual stats of the
    current model.
    In this function, the BN stats are recomputed with fixed weights, to make
    the running average more precise. Specifically, it computes the true
    average of per-batch mean/variance instead of the running average.

    Args:
        model (nn.Module): The model whose bn stats will be recomputed.
        data_loader (iterator): The DataLoader iterator.
        num_iters (int): number of iterations to compute the stats.
        logger (:obj:`logging.Logger` | None): Logger for logging.
            Default: None.
    zlength of dataloader z' must be greater than iteration number c                 S   s    g | ]}|j rt|tƒr|‘qS r   )Útrainingr   r   )Ú.0Úmr   r   r   Ú
<listcomp>G   s    ÿÿÿz#update_bn_stats.<locals>.<listcomp>r   zNo BN found in model)ÚloggerÚlevelNz	 BN found©r   z*IN/GN stats will be updated like training.c                 S   s   g | ]}|j ‘qS r   )Úmomentum©r   Úbnr   r   r   r   ]   s    g      ð?c                 S   ó   g | ]}t  |j¡‘qS r   )ÚtorchÚ
zeros_likeÚrunning_meanr   r   r   r   r   b   ó    c                 S   r   r   )r   r   Úrunning_varr   r   r   r   r   c   r   FÚreturn_lossé   Tz$Dataloader stopped before iteration r   )ÚtrainÚlenr   r   Úmodulesr   ÚloggingÚWARNINGr   r   r   r   r   ÚmmcvÚProgressBarÚ	enumerater   Úno_gradÚupdater   r    )ÚmodelÚdata_loaderÚ	num_itersr   Zparallel_moduleZ	bn_layersr   Zmomentum_actualr   r   r    Zfinish_before_loaderÚprog_barÚindÚdataÚir   r   r   Úupdate_bn_stats'   sh   ÿÿÿý€

ÿ"$þÿ

ür4   c                   @   s"   e Zd ZdZd	dd„Zdd„ ZdS )
ÚPreciseBNHooka
  Precise BN hook.

    Attributes:
        dataloader (DataLoader): A PyTorch dataloader.
        num_iters (int): Number of iterations to update the bn stats.
            Default: 200.
        interval (int): Perform precise bn interval (by epochs). Default: 1.
    r   r"   c                 C   s2   t |tƒstdt|ƒ› �ƒ‚|| _|| _|| _d S )Nz1dataloader must be a pytorch DataLoader, but got )r   r
   Ú	TypeErrorÚtypeÚ
dataloaderÚintervalr/   )Úselfr8   r/   r9   r   r   r   Ú__init__ˆ   s   
ÿ
zPreciseBNHook.__init__c                 C   sh   |   || j¡r2t d¡ td| j› d�|jd� t|j| j	| j|jd� td|jd� t d¡ d S d S )Ng       @zRunning Precise BN for z iterationsr   zBN stats updated)
Úevery_n_epochsr9   ÚtimeÚsleepr   r/   r   r4   r-   r8   )r:   Úrunnerr   r   r   Úafter_train_epoch�   s   
þüózPreciseBNHook.after_train_epochN)r   r"   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r;   r@   r   r   r   r   r5   ~   s    
	r5   )r   N)r&   r=   r(   r   Úmmcv.parallelr   Úmmcv.runnerr   Ú
mmcv.utilsr   Útorch.nnr   Útorch.nn.modules.batchnormr   Útorch.nn.modules.instancenormr   Ztorch.nn.parallelr   r	   Útorch.utils.datar
   r   r+   r4   r5   r   r   r   r   Ú<module>   s    V