o
    �Ð$jkD  ã                   @  s’  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	m
Z
mZ ddlZddlZddlm  mZ ddlmZ ddlmZ ejg d¢ejd	� d
dd
d
¡Zejg d¢ejd	� d
dd
d
¡Zejddgejd	� d
dd
d
¡Zejddgejd	� d
dd
d
¡ZdLdd„ZdMdd„Z		dNdOd!d"„Z dPd$d%„Z!dQd-d.„Z"dRd0d1„Z#dSd3d4„Z$dTd6d7„Z%dUd:d;„Z&dVd>d?„Z'	@dWdXdHdI„Z(G dJdK„ dKeƒZ)dS )YaÏ  
Fast HAC dataloader for offline-decoded tensors.

This loader intentionally reproduces the expensive parts of HACDOMAIN/MMAction only where needed:
  - re-sample temporal frame indices every __getitem__ for train
  - apply train/val spatial transforms at runtime
  - normalize and FormatShape to NCTHW
  - derive the audio crop from the same sampled frame indices, then apply train-time audio augmentation

Expected offline .npz keys from preprocess_offline_rewrite.py:
  video:         uint8 RGB frames after Resize(short_side=256), shape (F,H,W,3)
  flow_x/flow_y: uint8 optical-flow streams after Resize(short_side=256), shape (F,H,W)
  audio_samples: float32 mono samples
  samplerate:    int
  label:         int
é    )ÚannotationsN)ÚPath)ÚIterableÚOptional)Úsignal)ÚDataset)g33333ë^@gR¸…ë]@gR¸…ëáY@©Zdtypeé   é   )gÃõ(\�2M@g�Âõ(\�L@g     °L@g      `@é   ÚreturnÚintc                 C  s    t | tjƒrt|  ¡ ƒS t| ƒS ©N)Ú
isinstanceÚnpZndarrayr   Úitem)Úvalue© r   ú%/root/disk_3tb/HAC/dataloader_fast.pyÚ	_safe_int(   s   r   Úframesútorch.TensorÚsizeútuple[int, int]c              	   C  s<   zt j| |dddd�W S  ty   t j| |ddd� Y S w )NZbilinearFT)r   ÚmodeÚalign_cornersZ	antialias)r   r   r   )ÚFZinterpolateÚ	TypeError)r   r   r   r   r   Ú_resize_nchw.   s
   ÿr   ©g{®Gáz´?ç      ð?©g      è?gUUUUUUõ?ÚheightÚwidthÚscaleútuple[float, float]Úratioútuple[int, int, int, int]c                 C  sp  | | }t  |d ¡t  |d ¡f}tdƒD ]a}|t |d |d ¡ }t  t |d |d ¡¡}ttt  || ¡ƒƒ}	ttt  || ¡ƒƒ}
d|	  k rQ|krwn qd|
  k r]| krwn qt 	d| |
 ¡}t 	d||	 ¡}|||
|	f  S q||  }||d k r�|}	tt|	|d  ƒƒ}
n||d kr¢| }
tt|
|d  ƒƒ}	n|}	| }
| |
 d }||	 d }|||
|	fS )z<Torchvision-compatible RandomResizedCrop parameter sampling.r   r	   é
   r   )
ÚmathÚlogÚrangeÚrandomÚuniformZexpr   ÚroundZsqrtÚrandint)r"   r#   r$   r&   ZareaZ	log_ratioÚ_Ztarget_areaZaspect_ratioÚcrop_wÚcrop_hÚtopÚleftZin_ratior   r   r   Ú_random_resized_crop_params5   s0   0€r5   Ú	crop_sizec                 C  s^   | |k s||k rt d|› d| › d|› �ƒ‚tt| | d ƒƒ}tt|| d ƒƒ}||||fS )NzCenterCrop(z#) requires H,W >= crop_size, got H=z, W=ç       @)Ú
ValueErrorr   r.   )r"   r#   r6   r3   r4   r   r   r   Ú_center_crop_params[   s
   r9   Únum_available_framesÚclip_lenÚframe_intervalÚ	num_clipsÚ	test_modeÚboolú
np.ndarrayc                 C  sl  t dt| ƒƒ}|| }|r9||d kr/|| d t|ƒ }t |¡| }||d   tj¡}	nctj|ftjd�}	nY|| d | }|dkrWt |¡| }|tjj	||d� }	n;|t ||ƒkrnt 
tjj	|| d |d�¡}	n$|dkr‰|| d | }
t t |¡|
 ¡ tj¡}	n	tj|ftjd�}	|	dd…df tj|tjd�ddd…f |  }t ||¡}| tj¡S )a  
    MMAction-style SampleFrames approximation for the configs used here.

    Important: original HACDOMAIN passes total_frames=end_frame-start_frame = frame_num-1.
    To stay close to that behavior, caller should pass max(1, actual_frame_count - 1).
    r	   r7   r   r   ©r   r    N)Úmaxr   Úfloatr   ZarangeÚastypeZint64Úzerosr,   r/   ÚsortZaroundÚmod)r:   r;   r<   r=   r>   Ztotal_framesZori_clip_lenZavg_intervalZbase_offsetsZclip_offsetsr&   Ú
frame_indsr   r   r   Ú_sample_framesc   s*    0rI   Úvideo_npc                 C  s”   | j dkrtj| d ddd�} | j dkrtd| j› �ƒ‚| jd dkr+| dd d…f } | jd dkr:tj| ddd�} t t | ¡¡ d	ddd
¡ 	¡ S )Nr
   ).Néÿÿÿÿ©Zaxisé   z#Expected RGB video THWC, got shape=.r	   r   r   )
Úndimr   Zrepeatr8   ÚshapeÚtorchÚ
from_numpyÚascontiguousarrayÚpermuterC   )rJ   r   r   r   Ú_to_rgb_tchwŒ   s   

 rT   Úflow_npc                 C  s@   | j dkr	| d } | j dkrtd| j› �ƒ‚t t | ¡¡ ¡ S )NrM   ©.r   r
   z"Expected flow THW/THWC, got shape=)rN   r8   rO   rP   rQ   r   rR   rC   )rU   r   r   r   Ú_to_gray_thw˜   s
   

rW   rH   c                 C  s   |  d¡}| | }t|ƒS )NrK   )ÚreshaperT   )rJ   rH   Ú	flat_indsr   r   r   r   Ú_gather_rgb    s   
rZ   Ú	flow_x_npÚ	flow_y_npc                 C  s4   |  d¡}t| | ƒ}t|| ƒ}tj||gdd�S )NrK   r	   )Údim)rX   rW   rP   Ústack)r[   r\   rH   rY   ÚxÚyr   r   r   Ú_gather_flow¦   s   
ra   ÚtrainÚis_flowc                 C  s¦  | j dkrtdt| jƒ› �ƒ‚| j\}}}}|rFt||ƒ\}}	}
}d}t ¡ dk }| dd…dd…|||
 …|	|	| …f } t| ||fd�} n&|rJdnd}t|||ƒ\}}	}
}d}| dd…dd…|||
 …|	|	| …f } |r“tj	| d	d
�} |r“d| dd…ddd…dd…f  | dd…ddd…dd…f< |r§|dkr td|› �ƒ‚| t
 t } n|dkr²td|› �ƒ‚| t t } |j\}}|  |||||¡} |  ddddd¡ ¡ } | S )zJApply MMAction train/val spatial transform and return (num_clips,C,T,H,W).rM   z(Expected frames as N*T,C,H,W, got shape=éà   g      à?NrA   é   F)rK   )Zdimsg     ào@r   r   zExpected 2-channel flow, got C=r
   zExpected 3-channel RGB, got C=r	   )rN   r8   ÚtuplerO   r5   r,   r   r9   rP   ZflipÚ	FLOW_MEANÚFLOW_STDÚRGB_MEANÚRGB_STDÚviewrS   Ú
contiguous)r   rH   rb   rc   r0   Zchannelsr"   r#   r3   r4   r2   r1   Zout_sizeZdo_flipr=   r;   r   r   r   Ú_apply_spatial_and_format­   s8   
((8
rm   FÚsamplesÚ
start_timerC   Úend_timeÚdurationÚ
samplerateÚtrainingc                 C  sä  | j dkr| jdd�} |dkrtt| ƒt|ƒ dƒ}tt || t| ƒ ¡ƒ}tt || t| ƒ ¡ƒ}tdt|t| ƒƒƒ}t|t|t| ƒƒƒ}| ||… }|dd… }	t|	ƒdkrbtj	dtj
d�}	t|	ƒdk rxt |	d	¡dd… }	t|	ƒdk shtj|	tj
d�}	d
|	|	d
k< d|	|	dk < tj|	|ddd�\}
}
}t |d ¡}|t |¡ t |¡d  }|rêd}|tj dd|j¡ }|jd |krêtd| |jd | ƒ}|dkrÜtj |d¡d nd}d|||| …dd…f< |jtj
dd�S )zRSame audio logic as dataloader_HAC.get_spectrogram_piece, but from cached samples.r	   rL   r   g�íµ ÷Æ°>Né q )rt   r   r(   r    g      ð¿i   ia  )ZnpersegZnoverlapgH¯¼šò×z>g•Ö&è.>é	   çš™™™™™©¿çš™™™™™©?re   )r	   F)Úcopy)rN   ÚmeanrB   ÚlenrC   r   r   r.   ÚminrE   Úfloat32ZtileZasarrayr   Úspectrogramr*   Zstdr,   r-   rO   ZchoicerD   )rn   ro   rp   rq   rr   rs   ÚstartÚendZpieceZ	resamplesr0   r}   ÚintervalÚhighZ
start_maskr   r   r   Ú"get_spectrogram_piece_from_samplesÚ   s:   
	ÿr‚   c                   @  s~   e Zd Z											d0d1dd„Zd2dd„Zd3dd„Zd3d d!„Zd4d#d$„Zd4d%d&„Zd5d*d+„Z	d6d,d-„Z
d7d.d/„ZdS )8ÚFastHACÚtestT©ZhumanÚrgbNú/path/to/HAC/ÚsplitÚstrÚsourcer?   ÚdomainúIterable[str]ÚmodalityÚ	use_videoÚuse_flowÚ	use_audioÚdatapathÚ
tensor_dirúOptional[str]c              	   C  s¦  t |
ƒ| _|d urt |ƒn| jd | _g | _g | _g | _|| _|	| _|| _|dk| _	| j	r/dnd| _
|| _|| _|| _|D ]“}| jd d|› d|› d� }|jdd	��+}t |¡D ]}|s^qY| j |d
 ¡ | j t|d ƒ¡ | j |¡ qYW d   ƒ n1 s�w   Y  |dkrÐ|sÐ| jd d|› d� }|jdd	��+}t |¡D ]}|s¨q£| j |d
 ¡ | j t|d ƒ¡ | j |¡ q£W d   ƒ n1 sËw   Y  q=d S )NZprocessed_tensorsrb   ÚvalZ
HAC_SplitsZHAC_Z_only_z.csvÚ )Únewliner   r	   r„   ZHAC_train_only_)r   Ú	base_pathr’   Ú
video_listÚ
label_listÚprefix_listrŽ   r�   r�   rb   Úpipeline_modeÚcfgÚcfg_flowr�   ÚopenÚcsvÚreaderÚappendr   )Úselfrˆ   rŠ   r‹   r�   rœ   r�   rŽ   r�   r�   r‘   r’   ZdomZ
split_fileÚfZrowZ
train_filer   r   r   Ú__init__
  sJ   

ûÿûÿ€ôzFastHAC.__init__Úindexr   r   r   c                 C  s*   | j | j› d| j| › d| j| › d� S )Nr0   z.npz)r’   r›   rš   r˜   )r¢   r¥   r   r   r   Ú_tensor_path<  s   *zFastHAC._tensor_pathÚ
num_framesr@   c                 C  ó    t td|d ƒddd| j d�S )Nr	   é    r   ©r:   r;   r<   r=   r>   ©rI   rB   rb   ©r¢   r§   r   r   r   Ú_sample_video_indices?  ó   ûzFastHAC._sample_video_indicesc                 C  r¨   )Nr	   é   rM   rª   r«   r¬   r   r   r   Ú_sample_flow_indicesI  r®   zFastHAC._sample_flow_indicesú*tuple[dict[str, torch.Tensor], np.ndarray]c                 C  sR   d|j v r	|d n|d }|  |jd ¡}t||ƒ}t||| jdd�}d|i|fS )NZvideoÚclipr   F©rb   rc   Úimgs)Úfilesr­   rO   rZ   rm   rb   )r¢   ÚnpzrJ   rH   r   r´   r   r   r   Ú_load_videoS  s
   
zFastHAC._load_videoc           	      C  sN  d|j v r'd|j v r'|d }|d }|  t|jd |jd ƒ¡}t|||ƒ}nq|d }|jdkr;|jd dv r;|d }|  |jd d	krK|jd d	 n|jd ¡}|| d¡ }|jdkra|d }|jd d	 dkrp|d d… }t t	 
|¡¡ ¡  dd	|jd
 |jd ¡}|d d …d |jd |jd  …f }t||| jdd�}d|i|fS )NZflow_xZflow_yr   ÚflowrM   rK   )r	   r
   rV   r   éþÿÿÿTr³   r´   )rµ   r°   r{   rO   ra   rN   rX   rP   rQ   r   rR   rC   rk   rm   rb   )	r¢   r¶   r[   r\   rH   r   rU   Úflatr´   r   r   r   Ú
_load_flow[  s$   ,
,$zFastHAC._load_flowÚprimary_frame_indsúOptional[np.ndarray]r   c                 C  s2  d|j v rRd|j v rR|d }t|d ƒ}|dkr t|ƒt|ƒ nd}|d ur<| d¡}t|d ƒd }t|d ƒd }nd}|}t|||||| jd�}	t |	¡ ¡ S t |d  	¡ ¡ ¡ }	| jr—|	t 
|	¡ d	d
¡ }	d}
|	jd |
kr—td|
 |	jd |
 ƒ}|dkr‰t |¡nd}d|	|||
 …d d …f< |	S )NZaudio_samplesrr   r   g        rK   g      8@)rn   ro   rp   rq   rr   rs   r}   rv   rw   ru   re   )rµ   r   rz   rC   rX   r‚   rb   rP   rQ   rx   Z
empty_likeZuniform_rO   r{   r,   Z	randrange)r¢   r¶   r¼   rn   rr   rq   rº   ro   rp   Úspecr€   r�   r~   r   r   r   Ú_load_audios  s8   
úzFastHAC._load_audioc                 C  sà   |   |¡}| ¡ std|› d�ƒ‚d }tj|dd��H}| jr(|  |¡\}}|}nd}| jr;|  |¡\}}|d u r:|}nd}| j	rG|  
||¡}	nd}	d|jv rTt|d ƒnt| j| ƒ}
W d   ƒ n1 sew   Y  |||	|
fS )NzMissing preprocessed tensor: zO. Run preprocess_offline_rewrite.py first, or check split/source/domain naming.F)Zallow_pickler   Úlabel)r¦   ÚexistsÚFileNotFoundErrorr   ÚloadrŽ   r·   r�   r»   r�   r¿   rµ   r   r   r™   )r¢   r¥   Ztensor_pathr¼   r¶   r²   Zvideo_frame_indsr¸   Zflow_frame_indsr}   rÀ   r   r   r   Ú__getitem__–  s.   

ÿ€&ízFastHAC.__getitem__c                 C  s
   t | jƒS r   )rz   r˜   )r¢   r   r   r   Ú__len__¶  s   
zFastHAC.__len__)r„   Tr…   r†   NNTTTr‡   N)rˆ   r‰   rŠ   r?   r‹   rŒ   r�   r‰   rŽ   r?   r�   r?   r�   r?   r‘   r‰   r’   r“   )r¥   r   r   r   )r§   r   r   r@   )r   r±   )r¼   r½   r   r   )r¥   r   ©r   r   )Ú__name__Ú
__module__Ú__qualname__r¤   r¦   r­   r°   r·   r»   r¿   rÄ   rÅ   r   r   r   r   rƒ   	  s*    ô
2







# rƒ   rÆ   )r   r   r   r   r   r   )r   r!   )
r"   r   r#   r   r$   r%   r&   r%   r   r'   )r"   r   r#   r   r6   r   r   r'   )r:   r   r;   r   r<   r   r=   r   r>   r?   r   r@   )rJ   r@   r   r   )rU   r@   r   r   )rJ   r@   rH   r@   r   r   )r[   r@   r\   r@   rH   r@   r   r   )
r   r   rH   r@   rb   r?   rc   r?   r   r   )F)rn   r@   ro   rC   rp   rC   rq   rC   rr   r   rs   r?   r   r@   )*Ú__doc__Z
__future__r   rŸ   r)   Úosr,   Zpathlibr   Útypingr   r   Únumpyr   rP   Útorch.nn.functionalÚnnÚ
functionalr   Zscipyr   Ztorch.utils.datar   Útensorr|   rk   ri   rj   rg   rh   r   r   r5   r9   rI   rT   rW   rZ   ra   rm   r‚   rƒ   r   r   r   r   Ú<module>   s@        


ü
&

)



3ú/