o
    °”.e–Y  ã                   @   sÐ   d dl mZ d dl Z d dlZd dlmZ d dlmZ d dl	Z	G dd„ dej
ƒZG dd„ dej
ƒZG d	d
„ d
ej
ƒZG dd„ dej
ƒZG dd„ dej
ƒZG dd„ dej
ƒZG dd„ dej
ƒZG dd„ dej
ƒZdS )é    )ÚnnN)Ú
functional)Úpartialc                       sD   e Zd ZdZddededef‡ fdd„Zedd	„ ƒZd
d„ Z	‡  Z
S )ÚScaledEmbeddingzQScale up learning rate for the embedding, otherwise, it can move too slowly.
    ç      $@Únum_embeddingsÚembedding_dimÚscalec                    s4   t ƒ  ¡  t ||¡| _| jj j|  _|| _d S ©N)ÚsuperÚ__init__r   Ú	EmbeddingÚ	embeddingÚweightÚdatar	   )Úselfr   r   r	   ©Ú	__class__© úZ/Users/qiuzelin/Desktop/project_20/ICASSP2024/task1_match_mismatch/models/convrnn_torch.pyr   
   s   

zScaledEmbedding.__init__c                 C   s   | j j| j S r
   )r   r   r	   ©r   r   r   r   r      s   zScaledEmbedding.weightc                 C   s   |   |¡| j S r
   )r   r	   ©r   Úxr   r   r   Úforward   s   zScaledEmbedding.forward)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatr   Úpropertyr   r   Ú__classcell__r   r   r   r   r      s    
r   c                	       sD   e Zd ZdZddedededef‡ fdd„Zd	d
„ Zdd„ Z‡  Z	S )ÚSubjectLayerszPer subject linear layer.FÚin_channelsÚout_channelsÚ
n_subjectsÚinit_idc                    sf   t ƒ  ¡  t t |||¡¡| _|r%||ksJ ‚t |¡d  | jjd d …< | j jd|d  9  _d S )Né   g      à?)	r   r   r   Ú	ParameterÚtorchÚrandnÚweightsÚeyer   )r   r#   r$   r%   r&   r   r   r   r      s   
zSubjectLayers.__init__c                 C   s>   | j j\}}}| j  d| ddd¡ d||¡¡}t d||¡S )Nr   éÿÿÿÿr'   zbct,bcd->bdt)r+   ÚshapeÚgatherÚviewÚexpandr)   Úeinsum)r   r   ZsubjectsÚ_ÚCÚDr+   r   r   r   r   "   s   "zSubjectLayers.forwardc                 C   s&   | j j\}}}d|› d|› d|› d�S )NzSubjectLayers(z, ú))r+   r.   )r   ÚSr4   r5   r   r   r   Ú__repr__'   s   zSubjectLayers.__repr__)F)
r   r   r   r   r   Úboolr   r   r8   r!   r   r   r   r   r"      s
     r"   c                       s8   e Zd ZdZddededef‡ fdd„Zd	d
„ Z‡  ZS )Ú
LayerScalez¢Layer scale from [Touvron et al 2021] (https://arxiv.org/pdf/2103.17239.pdf).
    This rescales diagonaly residual outputs close to 0 initially, then learnt.
    çš™™™™™¹?ç      @ÚchannelsÚinitÚboostc                    s>   t ƒ  ¡  t tj|dd�¡| _|| | jjd d …< || _d S )NT)Úrequires_grad)	r   r   r   r(   r)   Úzerosr	   r   r?   )r   r=   r>   r?   r   r   r   r   0   s   

zLayerScale.__init__c                 C   s   | j | jd d …d f  | S r
   )r?   r	   r   r   r   r   r   6   s   zLayerScale.forward)r;   r<   )	r   r   r   r   r   r   r   r   r!   r   r   r   r   r:   ,   s    r:   c                +       s¬   e Zd Z							d#d	eje d
ededeje dededededededededeje dededededededej	ddf*‡ fdd„Z
d ej	dej	fd!d"„Z‡  ZS )$ÚConvSequenceé   r'   Né   ç        Fr   Tr=   ÚkernelÚdilation_growthÚdilation_periodÚstrideÚdropoutÚ	leakinessÚgroupsÚdecodeÚ
batch_normÚdropout_inputÚskipr	   ÚrewriteÚactivation_on_lastÚ	post_skipÚgluÚglu_contextÚglu_gluÚ
activationÚreturnc                    sf  t ƒ  ¡  d}t|ƒ}|| _t ¡ | _t ¡ | _|d u r"ttj	|ƒ}|	s'tj
ntj}tt|d d… |dd … ƒƒD ]÷\}\}}g }|t|ƒd k}|dkre|red|  k rZdk s]J ‚ J ‚| t |¡¡ |dkrs|d dkssJ dƒ‚|r}|| dkr}d}|d | }| ||||||||dkr’|ndd�¡ ||9 }|sŸ|sÊ|
rª| tj|d�¡ | |ƒ ¡ |rº| t |¡¡ |rÊ|t 
||d¡t 	|¡g7 }||krê|rê|d urÜ| t||ƒ¡ |rê| |||d|dd	�¡ | j tj|Ž ¡ |�r*|d | dk�r*|�rd| n|}|�rtjdd
�n|ƒ }| j t tj
||dd|  |d�|¡¡ q9| j d ¡ q9d S )Nr'   r-   rD   r   z.Supports only odd kernel with dilation for now)ÚdilationrL   )Únum_featuresF)rL   Úbias©Údim)Úpadding)r   r   ÚtuplerP   r   Ú
ModuleListÚsequenceÚglusr   Ú	LeakyReLUÚConv1dÚConvTranspose1dÚ	enumerateÚzipÚlenÚappendÚDropoutÚBatchNorm1dr:   Ú
SequentialÚGLU)r   r=   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   r	   rQ   rR   rS   rT   rU   rV   rW   rY   ÚConvÚkZchinZchoutÚlayersÚis_lastÚpadÚchÚactr   r   r   r   <   s`   


*ÿÿÿÖzConvSequence.__init__r   c                 C   sX   t | jƒD ]$\}}|}||ƒ}| jr|j|jkr|| }| j| }|d ur)||ƒ}q|S r
   )rf   ra   rP   r.   rb   )r   r   Ú
module_idxÚmoduleZold_xrT   r   r   r   r   y   s   
€zConvSequence.forward)rC   r'   NrD   rE   rE   r'   FFr   FNFTFr   r   TN)r   r   r   ÚtpÚSequencer   ÚOptionalr   r9   ÚAnyr   r   r!   r   r   r   r   rB   :   s\    úÿÿþþþýýýüüüûûûúúúú=rB   c                       s(   e Zd ZdZ‡ fdd„Zdd„ Z‡  ZS )ÚLSTMznA wrapper for nn.LSTM that outputs the same amount
    of features if bidirectional or not bidirectional.
    c                    sD   t ƒ  ¡  tj|||||d�| _d | _|r t d| |¡| _d S d S )N)Ú
input_sizeÚhidden_sizeÚ
num_layersrJ   ÚbidirectionalrD   )r   r   r   r{   ÚlstmÚlinearÚLinear)r   r|   r}   r~   rJ   r   r   r   r   r   Š   s   
üÿzLSTM.__init__c                 C   s&   |   |¡\}}| jr|  |¡}||fS r
   )r€   r�   )r   r   Úhr   r   r   r   •   s   
zLSTM.forward)r   r   r   r   r   r   r!   r   r   r   r   r{   …   s    r{   c                       s4   e Zd Zd
dededef‡ fdd„Zdd	„ Z‡  ZS )Ú	Attentioné2   rC   r=   ÚradiusÚheadsc                    sæ   t ƒ  ¡  || dksJ ‚t ||d¡| _t ||d¡| _t ||d¡| _t |d d || ¡| _| jj	j
}| d¡t dt|ƒd ¡ ¡  dd¡ ¡  |d d …< || _|| _t |¡| _t ||d¡| _t t |gd¡¡| _d S )Nr   r'   rD   r-   r;   )r   r   r   rd   ÚcontentÚqueryÚkeyr   r   r   r   Úcumsumr)   Úarangerh   r   r0   Úsqrtr‡   r†   rk   ÚbnÚfcr(   Úfullr	   )r   r=   r†   r‡   r   r   r   r   r   �   s   

6zAttention.__init__c                    sn  ‡ fdd„}|ˆ   |¡ƒ}|ˆ  |¡ƒ}|ˆ  |¡ƒ}|j\}}}}	t d||¡}
tj|	|jd�}|d d …d f |d d d …f  }ˆ jj	 
dˆ j| ˆ j ˆ j¡ dd¡ d|¡ ¡}| |	|	d¡}|
dt d	||¡ 7 }
t | ¡ ˆ jk|
t td
ƒ ¡ |¡¡}
tj|
dd�}t d||¡}|dt d||¡ 7 }| |d|	¡}t ˆ  ˆ  |¡¡¡ˆ j ddd¡ }|S )Nc                    s   |   | jd ˆ jd| jd ¡S )Nr   r-   rD   )r0   r.   r‡   )Úyr   r   r   Ú_split¯   s   z!Attention.forward.<locals>._splitzbhct,bhcs->bhts)Údevicer   r-   r'   g333333Ó?zbhct,tsc->bhtsÚinfr\   zbhts,bhcs->bhctzbhts,tsc->bhct)rˆ   r‰   rŠ   r.   r)   r2   rŒ   r“   r   r   r/   r†   Úclamp_r0   r1   ÚwhereÚabsÚtensorr   ÚtoÚsoftmaxÚreshapeÚFÚrelurŽ   r�   r	   )r   r   r’   rˆ   r‰   rŠ   Ú
batch_sizer3   r]   ÚlengthÚdotsÚstepsÚrelativeZembsr+   Úoutr   r   r   r   ®   s,    (ÿ"ÿ&zAttention.forward)r…   rC   )r   r   r   r   r   r   r!   r   r   r   r   r„   œ   s    r„   c                5       sê   e Zd Zdddddddddddddddddddd	gddd
fdejeef dedejeef dededededededed	edededededededededededed ej	e d!ed"ed#ef4‡ fd$d%„Z
d&d'„ Zd(d)„ Zd*d+„ Z‡  ZS ),ÚConvRNNrD   FrC   ç      ð?r   rE   éÈ   é@   r€   Úinputr#   r$   ÚhiddenÚdepthÚ
linear_outÚcomplex_outÚconcatenateÚkernel_sizerI   ÚgrowthÚ	flip_lstmÚbidirectional_lstmÚ	attentionr‡   Úconv_dropoutÚlstm_dropoutrO   rN   Úrelu_leakinessr%   Úsubject_dimÚembedding_locationÚembedding_scaleÚsubject_layersÚsubject_layers_dimc               	      ó¼  t ƒ  ¡  t| ¡ ƒtˆ ¡ ƒkr$tdt| ¡ ƒ› dtˆ ¡ ƒ› d�ƒ‚|| _|| _|| _|	| _|| _	d | _
|rVd|v s>J ‚|d }ˆd |dœ| }t|||ƒ| _
||d< d | _|rnt|||ƒ| _d|v rn|d  |7  < |r€dt| ¡ ƒi}dtˆ ¡ ƒi‰i ‰|D ]‰|ˆ gˆˆ< ˆˆ  ‡ ‡‡fdd	„t|ƒD ƒ7  < q„t‡fd
d„|D ƒƒ‰ˆ}d|v r¶||7 }t||	||||d�‰t ‡fdd„ˆ ¡ D ƒ¡| _d | _d | _|rät|ˆ|||d�| _|| _t ¡ | _t|ƒD ]}| j tˆ|d�¡ qí‡ ‡fdd	„t|d ƒD ƒ}d | _|�r|�rJ ‚t |d |d¡| _n3|�r@t  t |d d|d  d¡t !¡ t d|d  |d¡¡| _ndˆd< ||d< |dk�sQJ dƒ‚t"|fddiˆ¤Ž| _#d S )Nú%Channels and hidden keys must match (ú and r6   Úmeg©r©   r¨   r¨   Úconcatc                    ó$   g | ]}t tˆˆ ˆ |  ƒƒ‘qS r   ©r   Úround©Ú.0ro   ©r¯   r©   Únamer   r   Ú
<listcomp>  ó   $ z$ConvRNN.__init__.<locals>.<listcomp>c                 3   ó   � | ]	}ˆ | d  V  qdS ©r-   Nr   ©rÅ   Ún©Úsizesr   r   Ú	<genexpr>  ó   € z#ConvRNN.__init__.<locals>.<genexpr>r€   ©rF   rI   rK   rJ   rO   rN   c                    ó"   i | ]\}}|t |fi ˆ ¤Ž“qS r   ©rB   ©rÅ   rÇ   r=   ©Úparamsr   r   Ú
<dictcomp>  ó    ÿz$ConvRNN.__init__.<locals>.<dictcomp>©r|   r}   rJ   r~   r   ©r‡   c                    ó    g | ]}t tˆˆ |  ƒƒ‘qS r   rÂ   rÄ   ©r¯   Úlstm_hiddenr   r   rÈ   3  ó     r'   r-   rD   FrR   r   ú#if no linear out, depth must be > 0rM   T©$r   r   ÚsetÚkeysÚ
ValueErrorÚ_concatenaterª   r®   rI   r·   r¹   r"   Zsubject_embeddingr   ÚsumÚvaluesÚrangeÚdictr   Ú
ModuleDictÚitemsÚencodersr€   r�   r{   Ú
_flip_lstmr`   Ú
attentionsri   r„   Úfinalrd   rl   ÚReLUrB   Údecoder) r   r#   r$   r©   rª   r«   r¬   r­   r®   rI   r¯   r€   r°   r±   r²   r‡   r³   r´   rO   rN   rµ   r%   r¶   r·   r¸   r¹   rº   Zmeg_dimr]   Ú
lstm_inputr3   Údecoder_sizesr   ©r¯   r©   rÞ   rÇ   r×   rÏ   r   r   Ï   óŽ   
#
ÿ

ÿ(þ
ÿû

ýzConvRNN.__init__c                 C   óT   t | jƒD ]}t || j ¡d }t|dƒ}qt | jƒD ]	}|d | j }qt|ƒS ©a8  
        Return the nearest valid length to use with the model so that
        there is no time steps left over in a convolutions, e.g. for all
        layers, (size of the input - kernel_size) % stride = 0.

        If the input has a valid length, the output
        will have exactly the same length.
        r'   ©rè   rª   ÚmathÚceilrI   Úmaxr   ©r   rŸ   Úidxr   r   r   Úvalid_lengthC  ó   	zConvRNN.valid_lengthc                 C   ó,   |  d¡}|  |¡}|| }t |d|f¡S ©Nr-   r   ©Úsizerþ   rœ   rr   ©r   r   rŸ   rþ   Údeltar   r   r   rr   S  ó   

zConvRNN.padc                 C   sÎ   |j d }| jd |  |¡ƒ}|}| jd ur=| ddd¡}| jr&| dg¡}|  |¡\}}| jr6| dg¡}| ddd¡}| jD ]}|||ƒ }q@|  |¡}| j	d urX|  	|¡}|d d …d d …d |…f }|S )Nr-   r¾   rD   r   r'   )
r.   rì   rr   r€   Úpermuterí   Úfliprî   rñ   rï   )r   ÚinputsÚbatchrŸ   r   r3   r²   r£   r   r   r   r   Y  s$   





zConvRNN.forward)r   r   r   rw   ÚDictÚstrr   r9   r   ÚListr   rþ   rr   r   r!   r   r   r   r   r¤   Î   s     ßþýüúùø	÷õôóñðïíìêéèçæäãâá à!ßtr¤   c                5       sä   e Zd Zddidddiddddddddddddd	d	d	dd	d
ddgdddfdedededededededededededededededededededed ed!ed"eje	 d#ed$ed%e	f4‡ fd&d'„Z
d(d)„ Zd*d+„ Zd,d-„ Z‡  ZS ).ÚConvRNN_torch_myÚeegr§   é   rD   FrC   r¥   r   rE   r¦   r€   r¨   r#   r$   r©   rª   r«   r¬   r­   r®   rI   r¯   r°   r±   r²   r‡   r³   r´   rO   rN   rµ   r%   r¶   r·   r¸   r¹   rº   c               	      r»   )Nr¼   r½   r6   r  r¿   r¨   rÀ   c                    rÁ   r   rÂ   rÄ   rÆ   r   r   rÈ   Ô  rÉ   z-ConvRNN_torch_my.__init__.<locals>.<listcomp>c                 3   rÊ   rË   r   rÌ   rÎ   r   r   rÐ   Ö  rÑ   z,ConvRNN_torch_my.__init__.<locals>.<genexpr>r€   rÒ   c                    rÓ   r   rÔ   rÕ   rÖ   r   r   rØ   à  rÙ   z-ConvRNN_torch_my.__init__.<locals>.<dictcomp>rÚ   rÛ   c                    rÜ   r   rÂ   rÄ   rÝ   r   r   rÈ   ô  rß   r'   r-   rD   FrR   r   rà   rM   Trá   ) r   r#   r$   r©   rª   r«   r¬   r­   r®   rI   r¯   r€   r°   r±   r²   r‡   r³   r´   rO   rN   rµ   r%   r¶   r·   r¸   r¹   rº   Zeeg_dimr]   rò   r3   ró   r   rô   r   r   �  rõ   zConvRNN_torch_my.__init__c                 C   rö   r÷   rø   rü   r   r   r   rþ     rÿ   zConvRNN_torch_my.valid_lengthc                 C   r   r  r  r  r   r   r   rr     r  zConvRNN_torch_my.padc           
         s(  |j d }‡ fdd„| ¡ D ƒ}i }| ¡ D ]\}}ˆ j| ˆ  |¡ƒ||< qdd„ t| ¡ ƒD ƒ}ˆ j}| tj|dd�}ˆ jd urj| dd	d¡}ˆ j	rS| 
d	g¡}ˆ  |¡\}}ˆ j	rc| 
d	g¡}| ddd	¡}ˆ jD ]}|||ƒ }qmˆ  |¡}ˆ jd ur…ˆ  |¡}|d d …d d …d |…f }	|	S )
Nr-   c                    s   i | ]
\}}|ˆ   |¡“qS r   )rr   )rÅ   rÇ   Úinput_r   r   r   rØ   *  s    z,ConvRNN_torch_my.forward.<locals>.<dictcomp>c                 S   s   g | ]}|d  ‘qS )r'   r   )rÅ   r   r   r   r   rÈ   0  s    z,ConvRNN_torch_my.forward.<locals>.<listcomp>r'   r\   rD   r   )r.   rë   rì   rr   Úsortedr)   Úcatr€   r  rí   r  rî   rñ   rï   )
r   r	  r
  r   rŸ   ÚencodedrÇ   r3   r²   r£   r   r   r   r     s0   





zConvRNN_torch_my.forward)r   r   r   ré   r   r9   r   rw   r  r  r   rþ   rr   r   r!   r   r   r   r   r  �  s¦    ßþýüúùø	÷õôóñðïíìêéèçæäãâá à!ßtr  )r)   r   Útypingrw   Útorch.nnr   rœ   Ú	functoolsr   rù   ÚModuler   r"   r:   rB   r{   r„   r¤   r  r   r   r   r   Ú<module>   s    K2 B