Ë
    Ï—¬j* ã                   óš  — d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
Z
 ddlZddlZddlmZ ddlZddlmZ ddlmc mZ ddlmZ ddlmZmZ ej8                  Zd	„ Zej>                  Z ejB                  Z" e# e$e%ejL                  jO                  d
«      dd «      «      dk\  rd„ Z(nd„ Z(e)ejT                  fdejV                  fd„Z, e,e)«      Z-	 	 ddl.m/Z0 ddl1m2Z2 dZ3e-ji                  d«       	 ddl6Z6ddl7m8Z8 dZ9e-ji                  d«       d„ Z:d„ Z;d„ Z<d„ Z=d„ Z?dGd„Z@d „ ZA	 dHd!„ZB	 	 	 	 dId"„ZC G d#„ d$ejˆ                  «      ZE	 dJd%„ZF	 d&„ ZGd'ej�                  fd(„ZId'ej�                  dd)fd*„ZJd+d,d'd'ej�                  dd-dd-dd.d-d-dfd/„ZKd0„ ZLd1d2gd3d4gg d5¢d6œZMd7„ ZN	 dKd8„ZOdKd9„ZP G d:„ d;ejˆ                  «      ZQ G d<„ d=eQ«      ZR G d>„ d?eR«      ZS G d@„ dAeS«      ZTeSeSeTeTeTdBœZU G dC„ dDejˆ                  «      ZV G dE„ dFejˆ                  «      ZWy#  e-jk                  d«       dZ3Y �Œ9xY w# e>$ r dZ9e3se-jk                  d«       Y �Œ/w xY w)LzAStandalone version of Structured State Space sequence model (S4).é    )Údefaultdict)ÚOptionalÚMappingÚTupleÚUnionN)Úpartial)Úspecial)Úrank_zero_only)Ú	rearrangeÚrepeatc                 óP   — t        j                  | | j                  «       gd¬«      S )Néÿÿÿÿ©Údim)ÚtorchÚcatÚconj©Úxs    úc/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/SuperHugeAAD/superhuge/model/ssmamba/s4.pyÚ<lambda>r      s   € ”%—)‘)˜Q §¡£˜M¨rÕ2ó    Ú.é   )é   é
   c                 ó>   — | j                  «       j                  «       S ©N)r   Úresolve_conjr   s    r   r   r      s   € ˜aŸf™f›h×3Ñ3Ô5r   c                 ó"   — | j                  «       S r   )r   r   s    r   r   r      s
   € ˜aŸf™fœhr   Úreturnc                 ó    — t        j                  | «      }|j                  |«       dD ]"  }t        ||t	        t        ||«      «      «       Œ$ |S )z-Initializes multi-GPU-friendly python logger.)ÚdebugÚinfoÚwarningÚerrorÚ	exceptionÚfatalÚcritical)ÚloggingÚ	getLoggerÚsetLevelÚsetattrr
   Úgetattr)ÚnameÚlevelÚloggers      r   Ú
get_loggerr2      sM   € ô ×Ñ˜tÓ$€FØ
‡O�O�EÔó YˆÜ�˜œ~¬g°f¸eÓ.DÓEÕFð Yð €Mr   )Úcauchy_mult)Úlog_vandermonde_cudaTzTCUDA extension for structured kernels (Cauchy and Vandermonde multiplication) found.a  CUDA extension for structured kernels (Cauchy and Vandermonde multiplication) not found. Install by going to extensions/kernels/ and running `python setup.py install`, for improved speed and memory efficiency. Note that the kernel changed for state-spaces 4.0 and must be recompiled.F)ÚGenredzPykeops installation found.c            
      óú   — t        | D �cg c]  }t        |j                  «      ‘Œ c}«      }| D �cg c]9  }|j                  d|t        |j                  «      z
  z  |j                  z   «      ‘Œ; } }| S c c}w c c}w )N©r   )ÚmaxÚlenÚshapeÚview)ÚtensorsÚtensorÚmax_dims      r   Ú_broadcast_dimsr?   A   sn   € Ü±wÓ?±w¨V”s˜6Ÿ<™<Õ(°wÑ?Ó@ˆÙ\cÓdÑ\cÐRX�6—;‘;˜t W¬S°·±Ó->Ñ%>Ñ?ÀÇÁÑLÕMÐ\cˆÐdØˆùò @ùÚds
   ŠA3±>A8c                 óÒ   — d}d}t        d|› d|› d�g d¢dd¬	«      }t        | ||«      \  } }}t        | «      } t        |«      }t        |«      }d
 || ||d¬«      z  }t        |«      S )Nz-z * ComplexReal(v) - Real2Complex(Sum(v * w))zComplexMult(z-w, z-Conj(w))zComplexDivide(ú, Ú))ú	v = Vj(2)z	z = Vi(2)z	w = Vj(2)ÚSumr   ©Úreduction_opÚaxisr   ÚGPU©Úbackend)r5   r?   Ú_c2rÚ_r2c)ÚvÚzÚwÚexpr_numÚ
expr_denomr3   Úrs          r   Úcauchy_keopsrS   F   s„   € ØBˆØ2ˆ
äØ˜X˜J b¨¨°AÐ6òð
 Øô	
ˆô " ! Q¨Ó*‰ˆˆ1ˆaÜ�‹GˆÜ�‹GˆÜ�‹Gˆà‰k˜!˜Q ¨5Ô1Ñ1ˆÜ�A‹wˆr   c                 ó  — d}t        |g d¢dd¬«      }t        j                  |«      j                  |«      }t	        | ||«      \  } }}t        | «      } t        |«      }t        |«      } || ||d¬«      }dt        |«      j                  z  S )	Nz-ComplexMult(v, ComplexExp(ComplexMult(x, l))))rC   z	x = Vj(2)z	l = Vi(2)rD   r   rE   rH   rI   r   )r5   r   ÚarangeÚtor?   rK   rL   Úreal)rM   r   ÚLÚexprÚvandermonde_multÚlrR   s          r   Úlog_vandermonde_keopsr\   ]   s�   € Ø>ˆÜ!Øòð
 Øô	
Ðô �L‰L˜‹O×Ñ˜qÓ!ˆÜ! ! Q¨Ó*‰ˆˆ1ˆaÜ�‹GˆÜ�‹GˆÜ�‹Gˆá˜Q  1¨eÔ4ˆØ”�a“—‘‰~Ðr   c                 ó  — d}t        |g d¢dd¬«      }t        j                  |«      j                  |«      }t	        | |||«      \  } }}}t        | «      } t        |«      }t        |«      }t        |«      } || |||d¬«      }t        |«      S )z¢
        u: ... H L
        v: ... H N
        x: ... H N
        Returns: ... H N

        V = Vandermonde(a, L) : (H N L)
        contract_L(V * u * v)
        z=ComplexMult(ComplexMult(v, u), ComplexExp(ComplexMult(x, l))))z	u = Vj(2)z	v = Vi(2)z	x = Vi(2)z	l = Vj(2)rD   r   rE   rH   rI   )r5   r   rU   rV   r?   rK   rL   )ÚurM   r   rX   rY   rZ   r[   rR   s           r   Úlog_vandermonde_transpose_keopsr_   s   s–   € ð OˆÜ!Øòð Øô

Ðô �L‰L˜‹O×Ñ˜qÓ!ˆÜ$ Q¨¨1¨aÓ0‰
ˆˆ1ˆa�Ü�‹GˆÜ�‹GˆÜ�‹GˆÜ�‹Gˆá˜Q  1 a°Ô7ˆÜ�A‹wˆr   z‘Falling back on slow Cauchy and Vandermonde kernel. Install at least one of pykeops or the CUDA extension for better speed and memory efficiency.c                 óÆ   — t        | «      } t        |«      }| j                  d«      |j                  d«      |j                  d«      z
  z  }t        j                  |d¬«      S )zX
    v: (..., N)
    z: (..., L)
    w: (..., N)
    returns: (..., L) \sum v/(z-w)
    r   éþÿÿÿr   )Ú_conjÚ	unsqueezer   Úsum)rM   rN   rO   Úcauchy_matrixs       r   Úcauchy_naiverf   œ   sN   € ô 	ˆa‹€AÜˆa‹€AØ—K‘K “O q§{¡{°2£¸¿¹ÀR»Ñ'HÑI€MÜ�9‰9�]¨Ô+Ð+r   c                 óÌ   — t        j                  |j                  d«      t        j                  |«      j	                  |«      z  «      }t        d| |«      }d|j                  z  S )zF
    v: (..., N)
    x: (..., N)
    returns: (..., L) \sum v x^l
    r   z... n, ... n l -> ... lr   )r   Úexprc   rU   rV   ÚcontractrW   )rM   r   rX   r   Úvandermonde_matrixÚvandermonde_prods         r   Úlog_vandermonde_naiverl   ¨   sV   € ô Ÿ™ 1§;¡;¨r£?´U·\±\À!³_×5GÑ5GÈÓ5JÑ#JÓKÐÜÐ 9¸1Ð>PÓQÐØÐ×"Ñ"Ñ"Ð"r   c                 óð   — t        j                  |j                  d«      t        j                  |«      j	                  |«      z  «      }t        d| j	                  |«      |j	                  |«      |«      }|S )Nr   z... l, ... n, ... n l -> ... n)r   rh   rc   rU   rV   ri   )r^   rM   r   rX   rj   rk   s         r   Úlog_vandermonde_transpose_naivern   ²   s[   € ÜŸ™ 1§;¡;¨r£?´U·\±\À!³_×5GÑ5GÈÓ5JÑ#JÓKÐÜÐ @À!Ç$Á$ÀqÃ'È1Ï4É4ÐPQË7ÐTfÓgÐØÐr   c                 óø  — | dv rt        j                  «       S | dk(  rt        j                  «       S | dk(  rt        j                  «       S | dk(  rt        j                  «       S | dk(  rt        j
                  «       S | dv rt        j                  «       S | dk(  rt        j                  |¬«      S | d	k(  rt        j                  «       S | d
k(  rt        j                  «       S t        dj                  | «      «      ‚)N)NÚidÚidentityÚlinearÚtanhÚreluÚgeluÚelu)ÚswishÚsiluÚglur   ÚsigmoidÚsoftplusz)hidden activation '{}' is not implemented)ÚnnÚIdentityÚTanhÚReLUÚGELUÚELUÚSiLUÚGLUÚSigmoidÚSoftplusÚNotImplementedErrorÚformat)Ú
activationr   s     r   Ú
Activationr‰   »   sÑ   € ØÐ9Ñ9Ü�{‰{‹}ÐØ	�vÒ	Ü�w‰w‹yÐØ	�vÒ	Ü�w‰w‹yÐØ	�vÒ	Ü�w‰w‹yÐØ	�uÒ	Ü�v‰v‹xˆØ	Ð(Ñ	(Ü�w‰w‹yÐØ	�uÒ	Ü�v‰v˜#ŒÐØ	�yÒ	 Ü�z‰z‹|ÐØ	�zÒ	!Ü�{‰{‹}Ðä!Ð"M×"TÑ"TÐU_Ó"`ÓaÐar   c                 óæ   — |rt        t        j                  d¬«      nt        j                  }|�
|dk(  r|dz  } || |fd|i|¤Ž}|r)|�'t	        ||rdnd¬«      }t        j
                  ||«      }|S )	zXReturns a linear nn.Module with control over axes order, initialization, and activation.r   )Úkernel_sizery   r   Úbiasra   r   r   )r   r|   ÚConv1dÚLinearr‰   Ú
Sequential)	Úd_inputÚd_outputrŒ   Ú
transposedrˆ   ÚactivateÚkwargsÚ
linear_clsrr   s	            r   ÚLinearActivationr–   Ñ   su   € ñ 7A”œŸ™°Õ2ÄbÇiÁi€JØÐ *°Ò"5°xÀ1±}°xÙ˜ Ñ?°Ð?¸Ñ?€Fá�JÐ*Ü 
±j±ÀbÔIˆ
Ü—‘˜v zÓ2ˆØ€Mr   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )Ú	DropoutNdÚpc                 ó  •— t         ‰| �  «        |dk  s|dk\  rt        dj                  |«      «      ‚|| _        || _        || _        t        j                  j                  j                  d| j                  z
  ¬«      | _	        y)zQ
        tie: tie dropout mask across sequence lengths (Dropout1d/2d/3d)
        r   r   z3dropout probability has to be in [0, 1), but got {})ÚprobsN)ÚsuperÚ__init__Ú
ValueErrorr‡   r™   Útier’   r   ÚdistributionsÚbinomialÚBinomial)Úselfr™   rŸ   r’   Ú	__class__s       €r   r�   zDropoutNd.__init__å   ss   ø€ ô 	‰ÑÔØˆqŠ5�A˜’FÜÐU×\Ñ\Ð]^Ó_Ó`Ð`ØˆŒØˆŒØ$ˆŒÜ×+Ñ+×4Ñ4×=Ñ=ÀAÀdÇfÁfÁHÐ=ÓMˆ�r   c                 ó„  — | j                   r³| j                  st        |d«      }| j                  r"|j                  dd d|j
                  dz
  z  z   n|j                  }t        j                  |d|j                  iŽd| j                  z
  k  }||z  dd| j                  z
  z  z  }| j                  st        |d«      }|S |S )	zX: (batch, dim, lengths...).zb ... d -> b d ...Nr   r7   Údeviceç      ð?r   úb d ... -> b ... d)
Útrainingr’   r   rŸ   r:   Úndimr   Úrandr¦   r™   )r£   ÚXÚ
mask_shapeÚmasks       r   ÚforwardzDropoutNd.forwardñ   s£   € à�=Š=Ø—?’?¬	°!Ð5IÓ(J AØ:>¿(º(˜Ÿ™  !˜ t¨Q¯V©V°A©X¡Ò6ÈÏÉˆJÜ—:‘:˜zÐ;°!·(±(Ñ;¸bÀÇÁ¹iÑGˆDØ�D‘˜C  4§6¡6¡™NÑ+ˆAØ—?’?¬	°!Ð5IÓ(J AØˆHØˆr   )ç      à?TT)Ú__name__Ú
__module__Ú__qualname__Úfloatr�   r¯   Ú__classcell__©r¤   s   @r   r˜   r˜   ä   s   ø„ ñ
N˜%õ 
Nö	r   r˜   c                 ó€  — t        j                  |j                  d   «      j                  |«      }|g}d}	 | dz  dk(  r|d   |z  }| dz  } | dk(  rn/|dz  }|€|d   |d   z  g}n|j	                  |d   |d   z  «       ŒJ|€|S |j                  d«      |z
  }|j                  «       |d|d…f   z  }|dd|…f   }|dd|…f   |z   |dd|…f<   |j                  d«      dkD  rIt        |dd¬«      }|dddd…f   |j                  «       |dddd…f   z  z   }|j                  d«      dkD  rŒI||j                  d«      fS )	zSCompute A^L and the scan sum_i A^i v_i.

    A: (..., N, N)
    v: (..., N, L)
    r   r   r   r   N.z... (z l) -> ... z l)rN   )	r   Úeyer:   rV   ÚappendÚsizeÚpopr   Úsqueeze)rX   ÚArM   ÚIÚpowersr[   ÚkÚv_s           r   ÚpowerrÂ   þ   sw  € ô 	�	‰	�!—'‘'˜"‘+Ó×!Ñ! !Ó$€AàˆS€FØ	€AØ
Øˆq‰5�AŠ:˜6 "™:¨™>�qØ	ˆa‰ˆØ�Š6�5Ø	ˆQ‰ˆØˆ9Ø˜R‘j 6¨"¡:Ñ-Ð.‰Fà�M‰M˜& ™* v¨b¡zÑ1Ô2ð ð 	€y˜�(ð 	
�‰ˆr‹
�Q‰€AØ	�‰‹˜˜#˜q™r˜'™
Ñ	"€BØ	ˆ#ˆr�ˆrˆ'‰
€AØ�3˜˜˜�7‘˜b‘€A€cˆ2ˆAˆ2€g�Jð �&‰&�‹*�qŠ.Ü�aÐ/°1Ô5ˆØˆc�1’aˆi‰L˜6Ÿ:™:›<¨!¨C°²A¨I©,Ñ6Ñ6ˆð �&‰&�‹*�q‹.ð ˆa�i‰i˜‹mÐÐr   c                 ó  — | dk(  r—t        j                  |t         j                  ¬«      }d|z  dz   dz  }t        j                  ||«      \  }}|dd…df   t        j                  ||k  d||z
  z  d«      z  |ddd…f   z  }|dd…df   }| }|dz  }|dz  }||fS | dk(  rút        j                  |t         j                  ¬«      }	t        j                  |	|	«      \  }
}d|	z  dz   }t        j                  ||
k\  |d	«      t        j
                  |	«      z
   }t        j                  t        j
                  d|	z  dz   «      «      }||z  t         j                  j                  |«      z  }t        j
                  |«      dd…df   }|j                  «       }||fS | d
v rÝt        j                  |dz  «      }t        j                  t        j                  |dz  «      |gd¬«      j                  d«      dd }t         j                  t        j
                  |d«       t        j
                  |d«      z   z  }t        j                  |«      }d|d	dd…<   d|d	<   ||dd…df   |ddd…f   z  z
  }|dd…df   }||fS t        ‚)a*  A, B transition matrices for different measures.

    measure: the type of measure
      legt - Legendre (translated)
      legs - Legendre (scaled)
      glagt - generalized Laguerre (translated)
      lagt, tlagt - previous versions of (tilted) Laguerre with slightly different normalization
    Úlegt©Údtyper   r   r°   Ng      ð¿Úlegsr   ©ÚfourierÚfoutr   )rG   çÍ;fž ö?)ÚnprU   Úfloat64ÚmeshgridÚwhereÚdiagÚsqrtÚlinalgÚinvÚcopyÚstackÚzerosÚreshapeÚpir†   )ÚmeasureÚNÚmeasure_argsÚQÚRÚjÚir½   ÚBÚqÚcolÚrowrR   ÚMÚTÚfreqsÚds                    r   Ú
transitionrè   .  sY  € ð �&ÒÜ�I‰I�aœrŸz™zÔ*ˆØˆq‰S�1‰W˜‰OˆÜ�{‰{˜1˜aÓ ‰ˆˆ1ØŠa�ˆg‰JœŸ™ ! a¡%¨#°°1±©°qÓ9Ñ9¸A¸dÂA¸g¹JÑFˆØŠa�ˆg‰JˆØˆBˆð 	
ˆS‰ˆØ	ˆS‰ˆð2 ˆaˆ4€Kð/ 
�FÒ	Ü�I‰I�aœrŸz™zÔ*ˆÜ—;‘;˜q !Ó$‰ˆˆSØ�‰E�A‰IˆÜ�h‰h�s˜c‘z 1 aÓ(¬2¯7©7°1«:Ñ5Ð6ˆÜ�G‰G”B—G‘G˜A ™E A™IÓ&Ó'ˆØ�‰E”B—I‘I—M‘M !Ó$Ñ$ˆÜ�G‰G�A‹J’q˜$�wÑˆØ�F‰F‹Hˆð ˆaˆ4€Kð 
Ð'Ñ	'Ü—	‘	˜!˜Q™$“ˆÜ�H‰H”b—h‘h˜q !™t“n eÐ,°2Ô6×>Ñ>¸rÓBÀ1À2ÐFˆÜ�E‰E”B—G‘G˜A˜q“M�>¤B§G¡G¨A¨r£NÑ2Ñ3ˆÜ�H‰H�Q‹KˆØˆˆ!ˆ$ˆQˆ$‰Øˆˆ!‰ð �’!�T�'‘
˜Q˜t¢Q˜w™ZÑ'Ñ'ˆØŠa�ˆg‰Jˆð ˆaˆ4€Kô "Ð!r   r   c                 óº  — | dk(  rD|dk\  sJ ‚t        j                  dt        j                  ||¬«      z   «      j                  d«      }nÉ| dk(  r†|dk\  sJ ‚t        j                  ddt        j                  ||¬«      z  z   «      }|j	                  «       }d|dd	d…<   |j	                  «       }d|dd	d…<   t        j
                  ||gd¬
«      }|dz  }n>| dv r4t        j                  |«      }d|dd	d…<   d|d<   |j                  d«      }nt        ‚|j                  d«      }||kD  r2t        j                  |t        j                  ||z
  ||¬«      gd¬
«      }|S )z3Return low-rank matrix L such that A + L is normal.rÇ   r   r°   rÅ   r   rÄ   r   ç        Nr   çÍ;fž æ?rÈ   rË   )
r   rÑ   rU   rc   ÚclonerÕ   rÖ   r†   rº   r   )rÙ   rÚ   ÚrankrÆ   ÚPÚP0ÚP1rç   s           r   Úrank_correctionrñ   ]  sI  € ð �&ÒØ�qŠyÐˆyÜ�J‰J�rœ%Ÿ,™, q°Ô6Ñ6Ó7×AÑAÀ!ÓD‰Ø	�FÒ	Ø�qŠyÐˆyÜ�J‰J�q˜œ5Ÿ<™<¨°Ô7Ñ7Ñ7Ó8ˆØ�W‰W‹YˆØˆˆ1ˆ4ˆaˆ4‰Ø�W‰W‹YˆØˆˆ1ˆ4ˆaˆ4‰Ü�K‰K˜˜R˜ aÔ(ˆØ	ˆY‰‰Ø	Ð'Ñ	'Ü�K‰K˜‹NˆØˆˆ!ˆ$ˆQˆ$‰Øˆˆ!‰Ø�K‰K˜‹N‰Ü#Ð
#à	�‰ˆq‹	€AØˆa‚xÜ�I‰I�qœ%Ÿ+™+ d¨1¡f¨a°uÔ=Ð>ÀAÔFˆØ€Hr   ç       @c                 óð  — |t         j                  k(  s|t         j                  k(  sJ ‚|t         j                  k(  rt         j                  nt         j                  }t        | |«      \  }}t        j                  ||¬«      }t        j                  ||¬«      dd…df   }t        | |||¬«      }	|t        j                  |	j                  d«      |	j                  d«      z  d¬«      z   }
|
|
j                  dd«      z   }t        j                  ||d	   t        j                  |«      z  z
  d
z  «      |z  x}dkD  rt        d|«       t        j                  t        j                  |
«      dd¬«      }|r|
j                  t         j                  «      }
t         j                   j#                  |
dz  «      \  }}|r"|j                  |«      |j                  |«      }}|d|z  z   }t        j$                  |j&                  «      \  }}||   }|dd…|f   }|dd…d|d
z  …f   }|d|d
z   }|d   j)                  «       dkD  sJ d«       ‚|d   j)                  «       dk  rd|dd…df<   d|d<   d|d<   |t        j*                  |«      z  |j-                  «       j                  dd«      z  }t        j                  d
|j.                  z  |
z
  d
z  «      |z  x}dkD  rt        d|«       |j-                  «       j                  dd«      }t1        d||j                  |«      «      }t1        d||	j                  |«      «      }	|�3|j.                  dt        j2                  |j&                  | |¬«      z  z   }||	||fS )aö  Constructs NPLR form of HiPPO matrices.

    Returns w, p, q, V, B such that
    (w - p q^*, B) is unitarily equivalent to the original HiPPO A, B by the matrix V
    i.e. A = V[w - p q^*]V^*, B = V B

    measure: Name of HiPPO method.
    N: Size of recurrent A matrix (also known as `d_state` elsewhere).
    dtype: Single or double precision.
    diagonalize_precision: Calculate diagonalization in double precision.
    B_clip: Clip values of B, can help with stability. None for no clipping.
    rÅ   Nr   )rí   rÆ   ra   r   éýÿÿÿr   )r   r   r   gñhãˆµøä>z(WARNING: HiPPO matrix not skew symmetricT)Úkeepdimy       €      ð¿ù              ð?ç-Cëâ6?z4Only 1 zero eigenvalue allowed in diagonal part of Arê   rë   )r   r   y        Í;fž æ?)r   r   zDWarning: Diagonalization of A matrix not numerically precise - errorz
ij, j -> izij, ...j -> ...i)Úminr8   )r   r´   ÚdoubleÚcfloatÚcdoublerè   Ú	as_tensorrñ   rd   rc   Ú	transposer¸   ÚprintÚmeanÚdiagonalrV   rÒ   ÚeighÚsortÚimagÚabsÚ
diag_embedr   rW   ri   Úclamp)rÙ   rÚ   rí   rÆ   Údiagonalize_precisionÚB_clipÚcdtyper½   rà   rî   ÚAPÚ_AÚerrÚW_reÚW_imÚVÚWÚ_ÚidxÚW_sortedÚV_sortedÚ_APÚV_invs                          r   Únplrr  x  s  € ð ”E—K‘KÒ 5¬E¯L©LÒ#8Ð8Ð8Ø"¤e§k¡kÒ1ŒU�\Š\´u·}±}€Fä�g˜qÓ!�D€A€qÜ�‰˜ Ô'€AÜ�‰˜ Ô'ª¨1¨Ñ-€Aä˜ ¨°UÔ;€AØ	
ŒU�Y‰Y�q—{‘{ 2“ q§{¡{°2£Ñ6¸BÔ?Ñ	?€Bð 
ˆb�l‰l˜2˜rÓ"Ñ	"€BÜ�y‰y˜"˜r #™w¤u§y¡y°£|Ñ3Ñ3°aÑ7Ó8¸1Ñ<Ð<ˆÀÒDÜÐ8¸#Ô>ô
 �:‰:”e—n‘n RÓ(¨"°dÔ;€Dñ  2§5¡5¬¯©Ó#6˜bä�l‰l×Ñ  3¡Ó'�G€Dˆ!Ù¨¯©°«¸¿¹¸f» A˜dØˆr�D‰yÑ€Aô �Z‰Z˜Ÿ™Ó�F€A€sØ�‰v€HØ’�C�‰y€Hð 	’�E�Q˜‘T�E�Ñ€AØ��!�Q‘$ˆ€AØˆR‰5�9‰9‹;˜ÒÐUÐUÓUÐØˆ�u‡y�yƒ{�TÒØˆŠ!ˆRˆ%‰Øˆˆ%‰Øˆˆ%‰à
Œe×Ñ˜qÓ!Ñ
! A§F¡F£H×$6Ñ$6°r¸2Ó$>Ñ
>€CÜ—	‘	˜1˜SŸX™X™: b™=¨1Ñ,Ó-¨aÑ/Ð	/ˆ°4Ò7ÜÐTÐVYÔZð �F‰F‹H×Ñ˜r 2Ó&€Eô 	�˜u a§d¡d¨1£gÓ.€AäÐ# U¨A¯D©D°«GÓ4€AàÐØ�F‰F�RœŸ™ A§F¡F°°¸VÔDÑDÑDˆð
 ˆa��Aˆ:Ðr   Úhippoé@   r§   Úconstantc                 óJ
  — |t         j                  k(  s|t         j                  k(  sJ ‚|t         j                  k(  rt         j                  nt         j                  }t        j
                  t        j                  «      }|rt        j                  ||dz  «      }ndt        j                  ||dz  «      z  }||z  }|r |dz  t        j                  ||dz  «      z  }n$t        t        j                  |dz  «      d|¬«      }| dv r-t        j                  t        j                  ||dz  «      «      }nÒ| dk(  rOd|z  }|r t        j                  ||dz  «      |z  dz  }n¦dt        t        j                  |dz  «      d|¬«      z   }n~| d	v r||z  }nt| d
v rd|z  |z  |dd|z  z   z  dz
  z  }nX| dv rd|z  |z  |d|z   z  dz
  z  }n?| dv rd|z  dd|z  z   dz  z  }n)| dv rt        d|«      \  }}}}|j                   }nt         ‚||z  }| d|z  z
  }t        j"                  |j$                  dk  «      r"t        j"                  |j                  dk  «      sJ ‚|	rt&        j)                  d«       | dv rWt&        j+                  d|
›�«       t        d|d¬«      \  }}}}t        |d|¬«      j-                  «       j/                  «       }�n�|
dk(  rt        j                  ||dz  |¬«      }�nk|
dk(  rt        j                  ||dz  |¬«      }�nI|
dk(  rFt        j                  ||dz  d|¬«      }|dd…dd…dfxx   dz  cc<   |j1                  ||dz  «      }nþ|
dk(  rwt        j
                  t        j                  d|z  |z  «      |¬«      }|t        j                  d|dz  «      z  }t        |d|¬«      j-                  «       j/                  «       }n‚|
dk(  rwt        j
                  t        j                  d |z  |z  «      |¬«      }|t        j                  d|dz  «      z  }t        |d|¬«      j-                  «       j/                  «       }nt         ‚||z  }|r?| |z  }dt        j2                  t        j4                  |«      dz  dd!¬"«      z  }||dz  z  }|
dv r+t        d#|¬«      j-                  «       j/                  «       }n!t        j                  |||dz  |¬«      }||z  }t        j6                  ||¬«      dd…d|dz  …f   }t        |d$|¬«      }||||fS )%a  Directly construct a DPLR matrix.

    Args:
    - init: (str) ['rand', 'lin', inv', 'real', 'hippo'] Choices for initialization of A.
          Most of these affect the imaginary part of A, except for 'real'.
    - real_random: (bool) Initialize A.real in -U[0, 1]. Otherwise, initialize to -1/2.
    - real_scale: (float) Scaling factor of real part of A.
    - imag_random: (bool) Initialize A.imag randomly.
    - imag_scale: (bool) Scaling factor of imaginary part of A.
    - B_init: (str) ['constant' | 'random' | 'alternating' | 'unit-cw' | 'unit-ccw' | 'hippo']
          Choices for initialization of B.
    - B_scale: (float) Scaling factor for B
    - P_scale: (float) Scaling factor for P
    - normalize: (bool) Apply an automatic normalization factor on B
    r   r°   zn -> h n)Úh)Úrandomr«   rW   r   r   )rr   Úlin)ÚinverserÓ   )Úinverse2Úinv2)Ú	quadraticÚquad)rÇ   r  rÇ   rö   r÷   rê   z`'B_random' is deprecated in favor of B_init='random' and will be deprecated in a future version.z8Initializing with S4D-LegS and ignoring argument B_init=rò   )r  r  rÅ   r  Úalternatingé   Nr   zunit-cwù       €       Àzunit-ccwy               @T)r   rõ   zr n -> r h nzn m -> h n m)r   r´   rù   rú   rû   r=   ÚmathrØ   r«   Úonesr   rU   rh   Úrandnr  r  r†   ÚallrW   Úlogr%   r$   rì   Ú
contiguousr;   rd   r  r¸   )ÚinitrÚ   rí   ÚHrÆ   Úreal_randomÚ
real_scaleÚimag_randomÚ
imag_scaleÚB_randomÚB_initÚB_scaleÚP_scaleÚ	normalizerØ   Ú	real_partÚ	imag_partr½   r  rî   rà   rN   ÚnormÚzetar  s                            r   Údplrr<  Ç  sÉ  € ð: ”E—K‘KÒ 5¬E¯L©LÒ#8Ð8Ð8Ø!¤U§[¡[Ò0ŒE�LŠL´e·m±m€Eä	�‰”d—g‘gÓ	€Bñ Ü—J‘J˜q ! Q¡$Ó'‰	àœŸ™ A q¨!¡tÓ,Ñ,ˆ	Ø˜YÑ&€Iñ Ø�q‘Dœ5Ÿ:™: a¨¨A©Ó.Ñ.‰	äœ5Ÿ<™<¨¨1©Ó-¨z¸QÔ?ˆ	àÐ!Ñ!Ü—I‘IœeŸk™k¨!¨Q°©TÓ2Ó3‰	Ø	�ŠØ˜	‘Mˆ	ÙÜŸ
™
 1 a¨¡dÓ+¨aÑ/°Ñ2‰Ið œF¤5§<¡<°°1±Ó#5°zÀQÔGÑG‰IØ	Ð"Ñ	"Ø˜‘N‰	Ø	Ð#Ñ	#Ø�b‘D˜1‘H  1 Q y¡[¡=Ñ 1°!Ñ 3Ñ4‰	Ø	Ð%Ñ	%Ø�b‘D˜1‘H  1 Y¡;¡°Ñ 1Ñ2‰	Ø	Ð&Ñ	&Ø�b‘D˜A˜a 	™k™M¨AÑ-Ñ-‰	Ø	Ð"Ñ	"Ü˜& !“_‰
ˆˆ1ˆa�Ø—V‘V�G‰	Ü#Ð
#Ø˜YÑ&€Ið 
ˆ
�R˜)‘^Ñ#€AÜ�9‰9�Q—V‘V˜d‘]Ô#¬¯	©	°!·&±&¸C±-Ô(@Ð@Ð@ñ Ü�‰ÐvÔwØÐ Ñ Ü�‰ÐLÀVÀIÐNÔOô ˜& !¨CÔ0‰
ˆˆ1ˆa�Ü�1�j AÔ&×,Ñ,Ó.×9Ñ9Ó;ŠØ	�:Ò	Ü�J‰J�q˜!˜Q™$ eÔ,ŠØ	�8Ò	Ü�K‰K˜˜1˜a™4 uÔ-ŠØ	�=Ò	 Ü�J‰J�q˜!˜Q™$ ¨Ô/ˆØ	Š!ŠQ�ˆ'‹
�bÑ‹
Ø�F‰F�1�a˜‘d‹O‰Ø	�9Ò	Ü�L‰LœŸ™ 3¨¡8¨a¡<Ó0¸Ô>ˆØ”—‘˜a  a¡Ó(Ñ(ˆÜ�1�j AÔ&×,Ñ,Ó.×9Ñ9Ó;‰Ø	�:Ò	Ü�L‰LœŸ™ 2¨¡7¨Q¡;Ó/°uÔ=ˆØ”—‘˜a  a¡Ó(Ñ(ˆÜ�1�j AÔ&×,Ñ,Ó.×9Ñ9Ó;‰Ü#Ð
#Øˆ�L€Añ Øˆr�!‰tˆØ”—‘œ5Ÿ9™9 T›?¨AÑ-°2¸tÔDÑDˆØ��b‘‰Lˆð Ð"Ñ"ä�1�n¨Ô*×0Ñ0Ó2×=Ñ=Ó?‰ä�K‰K˜˜a  A¡¨UÔ3ˆØ�‰Kˆô 	�	‰	�!˜5Ô!¢! U a¨¡d U (Ñ+€AÜˆq�. AÔ&€Aàˆa��Aˆ:Ðr   c                 ó¦  — | j                  d«      s| j                  d«      r_| j                  d«      rd|d<   | dd j                  d«      }|d   d	k(  sJ ‚t        |«      d
kD  r|d
   |d<   t        d|||dœ|¤Ž\  }}}}	nKt	        | ||fi |¤Ž\  }}}}	t        |d|¬«      }t        |d|¬«      }t        |d|¬«      }t        |	d|¬«      }	||||	fS )z–Dispatcher to create single SSM initialization

    N: state size
    R: rank (for DPLR parameterization)
    H: number of independent SSM copies
    rÐ   r<  rê   r6  r%  NÚ-r   Ú r   r-  )rÚ   rí   r.  zn -> s n©Úszr n -> r s nzn m -> s n m© )Ú
startswithÚsplitr9   r<  r  r   )
r-  rÚ   rÝ   r.  Ússm_argsÚargsr½   rî   rà   r  s
             r   ÚssmrG  D  sí   € ð ‡��vÔ $§/¡/°&Ô"9Ø�?‰?˜6Ô"Ø"%ˆH�YÑØ�A�Bˆx�~‰~˜cÓ"ˆØ�A‰w˜"Š}Ðˆ}Üˆt‹9�qŠ=Ø# A™wˆH�VÑÜÐ7˜A A¨Ñ7¨hÑ7‰
ˆˆ1ˆa‘ä˜$  1Ñ1¨Ñ1‰
ˆˆ1ˆa�Ü�1�j AÔ&ˆÜ�1�n¨Ô*ˆÜ�1�j AÔ&ˆÜ�1�n¨Ô*ˆØˆa��Aˆ:Ðr   rÇ   rÉ   údiag-invúdiag-lin)rÇ   rÉ   rH  rI  )r  rÐ   r*  c                 óÎ  — t        | t        «      r| t        v r	t        |    n| g} |t        | «      z  dk(  sJ |› dt        | «      › d�«       ‚t	        | D �cg c]  }t        ||||t        | «      z  fi |¤Ž‘Œ c}Ž \  }}}}	t        j                  |d¬«      }t        j                  |d¬«      }t        j                  |d¬«      }t        j                  |	d¬«      }	||||	fS c c}w )Nr   z6 independent trainable SSM copies must be multiple of z different initsr   r   )Ú
isinstanceÚstrÚcombinationsr9   ÚziprG  r   r   )
ÚinitsrÚ   rÝ   ÚSrE  r-  r½   rî   rà   r  s
             r   ÚcombinationrQ  b  så   € Ü�%œÔØ',´Ñ'<”˜UÒ#À5À'ˆàŒs�5‹z‰>˜QÒÐx 1 #Ð%[Ô\_Ð`eÓ\fÐ[gÐgwÐ xÓxÐÜÙCHÓ	IÁ5¸4Œ#ˆd�A�q˜!œs 5›z™/Ñ
6¨XÓ
6À5Ñ	Ið�J€A€qˆ!ˆQô 	�	‰	�!˜Ô€AÜ�	‰	�!˜Ô€AÜ�	‰	�!˜Ô€AÜ�	‰	�!˜Ô€AØˆa��Aˆ:Ðùò 
Js   Á"C"c                 ó   — t        j                  | d¬«      } |dk(  r| S |dk(  rt        j                  | «      S |dk(  r| S |dk(  rt        j                  | «      S |dk(  r+t        j                  t        j                  | «      dz
  «      S t
        ‚)	z4Initialize a (positive) parameter under a transform.r÷   )rø   Únonerh   rt   rz   r{   r   )r   r  r+  Úlogitrh   r†   )ÚparamÚ	transforms     r   Úinv_transformrW  s  s…   € ä�K‰K˜ 4Ô(€EØ�FÒØˆØ	�eÒ	Ü�y‰y˜ÓÐØ	�fÒ	ØˆØ	�iÒ	Ü�{‰{˜5Ó!Ð!Ø	�jÒ	 Ü�y‰yœŸ™ 5Ó)¨!Ñ+Ó,Ð,Ü#Ð
#r   c                 ó  — |dk(  r| }|S |dk(  rt        j                  | «      }|S |dk(  rt        j                  | «      dz   }|S |dk(  rt        j                  | «      }|S |dk(  rt        j
                  | «      }|S t        ‚)z-Get a (positive) parameter under a transform.rS  rh   rt   r÷   rz   r{   )r   rh   ÚFrt   rz   r{   r†   )rU  rV  r™   s      r   Úparam_transformrZ  ‚  sœ   € à�FÒØˆð €Hð 
�eÒ	Ü�I‰I�eÓˆð €Hð 
�fÒ	ä�F‰F�5‹M˜$Ñˆð €Hð 
�iÒ	Ü�I‰I�eÓˆð €Hð 
�jÒ	 Ü�J‰J�uÓˆà€Hô $Ð
#r   c                   óà   ‡ — e Zd ZdZ	 	 	 	 	 	 ddededee   deeee   f   deeee   f   de	fˆ fd„Z
dd	„Zdd
„Zd„ Zd„ Zd„ Z ej"                  «       d„ «       Zed„ «       Zed„ «       Zˆ xZS )ÚKernelaÆ  Interface for modules that produce convolution kernels.

    A main distinction between these and normal Modules is that the forward pass
    does not take inputs. It is a mapping from parameters to a tensor that can
    be used in other modules, in particular as a convolution kernel.

    Because of the unusual parameterization, these kernels may often want special
    hyperparameter settings on their parameters. The `register` method provides
    an easy interface for controlling this, and is intended to be used with an
    optimizer hook that can be found in train.py or example.py.

    This class also defines an interface for interacting with kernels *statefully*,
    in particular for state space models (SSMs). This interface handles the setting
    when a model can be converted from a "CNN" into an "RNN".
    _setup_step()
    step()
    default_state()
    forward_state()

    See ConvKernel for the simplest instantiation of this interface.
    Úd_modelÚchannelsÚl_maxÚlrÚwdÚverbosec                 óÜ  •‡ — t         ‰‰ �  «        |dkD  sJ ‚|x‰ _        ‰ _        |x‰ _        ‰ _        |‰ _        |‰ _        |‰ _        |‰ _	        ‰ j                  r1t        |«      dkD  r#t        j                  t        ‰ «      › d|› �«       ‰ j                  �t        ‰ j                  t        «      rt!        ˆ fd„«      ‰ _        n6t!        d„ «      ‰ _        ‰ j"                  j%                  ‰ j                  «       ‰ j                  �t        ‰ j                  t        «      rt!        ˆ fd„«      ‰ _        yt!        d„ «      ‰ _        ‰ j&                  j%                  ‰ j                  «       y)aµ  General interface.

        d_model (H): Model dimension, or number of independent convolution kernels created.
        channels (C): Extra dimension in the returned output (see .forward()).
            - One interpretation is that it expands the input dimension giving it C separate "heads" per feature.
              That is convolving by this kernel maps shape (B L D) -> (B L C D)
            - This is also used to implement a particular form of bidirectionality in an efficient way.
            - In general for making a more powerful model, instead of increasing C
              it is recommended to set channels=1 and adjust H to control parameters instead.
        l_max (L): Maximum kernel length (optional). If unspecified, most Kernel instantiations
            will return kernels of arbitrary length as passed into .forward().
        lr: Optional dictionary specifying special hyperparameters for .register().
            Passing in a number (e.g. 0.001) sets attributes of SSM parameters (A, B, dt).
            A custom optimizer hook is needed to configure the optimizer to set the learning rates appropriately for these parameters.
        wd: Same as lr, but for weight decay.
        r   z extra kwargs: Nc                  ó   •— ‰ j                   S r   )r`  ©r£   s   €r   r   z!Kernel.__init__.<locals>.<lambda>Ø  ó	   ø€ ¨t¯wªwr   c                   ó   — y r   rB  rB  r   r   r   z!Kernel.__init__.<locals>.<lambda>Ú  ó   € ¨tr   c                  ó   •— ‰ j                   S r   )ra  re  s   €r   r   z!Kernel.__init__.<locals>.<lambda>à  rf  r   c                   ó   — y r   rB  rB  r   r   r   z!Kernel.__init__.<locals>.<lambda>â  rh  r   )rœ   r�   r.  r]  rX   r_  r^  r`  ra  rb  r9   r+  r$   ÚtyperK  r´   r   Úlr_dictÚupdateÚwd_dict)	r£   r]  r^  r_  r`  ra  rb  r”   r¤   s	   `       €r   r�   zKernel.__init__©  s  ù€ ô4 	‰ÑÔØ˜Š{Ðˆ{Ø 'Ð'ˆŒ�”Ø#Ð#ˆŒ�”Ø ˆŒØˆŒØˆŒØˆŒð
 �<Š<œC ›K¨!šOÜ�H‰Hœ˜T›
�| ?°6°(Ð;Ô<ð �7‰7ˆ?œj¨¯©´%Ô8Ü&£Ó7ˆD�Lä&¡|Ó4ˆDŒLØ�L‰L×Ñ §¡Ô(ð �7‰7ˆ?œj¨¯©´%Ô8Ü&£Ó7ˆD�Lä&¡|Ó4ˆDŒLØ�L‰L×Ñ §¡Õ(r   c                 ó   — t         ‚)a  General interface to generate a global convolution kernel.

        state: Initial state for recurrent updates.
            E.g. for SSMs, this should have shape (B, H, N) (batch, d_model, d_state).
        rate: Relative sampling rate.
        L: Target kernel length.

        Returns:
          - (C, H, L) (channels, d_model, l_kernel) The convolution kernel.
          - (B, H, L) (batch, d_model, l_kernel)
              Extra information for how the state affects the output of convolving by kernel.
        ©r†   )r£   ÚstateÚraterX   s       r   r¯   zKernel.forwardå  s
   € ô "Ð!r   c                 óÌ   — |dk(  r| j                  ||«       y| j                  |t        j                  |«      «       i }|�||d<   |�||d<   t	        t        | |«      d|«       y)zFRegister a tensor with a configurable learning rate and 0 weight decayrê   Nr`  Úweight_decayÚ_optim)Úregister_bufferÚregister_parameterr|   Ú	Parameterr-   r.   )r£   r/   r=   r`  ra  Úoptims         r   ÚregisterzKernel.registerô  sg   € ð �Š9Ø× Ñ   vÕ.à×#Ñ# D¬"¯,©,°vÓ*>Ô?àˆEØˆ~¨R˜u T™{Øˆ~°r˜u ^Ñ4Ü”G˜D $Ó'¨°5Õ9r   c                 ó   — t         ‚)zCConvert a model into a recurrent mode for autoregressive inference.rp  ©r£   r”   s     r   Ú_setup_stepzKernel._setup_step  ó   € ä!Ð!r   c                 ó   — t         ‚)zAStep the model for one timestep with input x and recurrent state.rp  )r£   r   rq  r”   s       r   ÚstepzKernel.step  r~  r   c                 ó   — t         ‚)zReturn a default initial state.rp  ©r£   rF  r”   s      r   Údefault_statezKernel.default_state	  r~  r   c                 ó   — t         ‚)zeForward the state through a sequence, i.e. computes the state after passing chunk through the kernel.rp  )r£   r^   rq  s      r   Úforward_statezKernel.forward_state  ó
   € ô "Ð!r   c                 ó   — t         ‚)zÀImplement this for interfaces that want to interact with a stateful layer (i.e. SSMs).

        Currently the only codepath that might use this is the StateDecoder, which is not used.
        rp  re  s    r   Úd_statezKernel.d_state  s
   € ô "Ð!r   c                 ó   — t         ‚)zKSame as d_state, only needed for niche codepaths involving recurrent state.rp  re  s    r   Ústate_to_tensorzKernel.state_to_tensor  r†  r   )r   r   NNrê   T©Nr§   N)Nrê   )r±   r²   r³   Ú__doc__Úintr   r   r´   r   Úboolr�   r¯   rz  r}  r€  rƒ  r   Úno_gradr…  Úpropertyrˆ  rŠ  rµ   r¶   s   @r   r\  r\  ’  sÛ   ø„ ñð0 ØØ#Ø.2Ø.1Øñ:)àð:)ð ð:)ð ˜‰}ð	:)ð
 �%˜ 'Ñ*Ð*Ñ+ð:)ð �%˜ 'Ñ*Ð*Ñ+ð:)ð õ:)óx"ó:ò"ò"ò"ð €U‡]�]ƒ_ñ"ó ð"ð ñ"ó ð"ð ñ"ó ô"r   r\  c            "       ó$  ‡ — e Zd ZdZd„ Zd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddededee   de	e
ee   f   d	e	e
ee   f   d
ededede
de
dedededee   dee   dee   f ˆ fd„Z ej                   «       d„ «       Zd„ Zed„ «       Zed„ «       Zˆ xZS )Ú	SSMKernelaü  Parent class for different SSM parameterizations.

    This class is abstract and only defines some initializations and flags that are common to all SSM variants.
    It is instantiated by subclasses SSMKernel{Dense,Real,Diag,DPLR}.

    Options:
    d_state (N): State size (dimensionality of parameters A, B, C). Generally shouldn't need to be adjusted and doens't affect speed much for most kernels (e.g. S4, S4D).
    deterministic: Use a deterministic initialization for dt, A, B, C.
        Useful for debugging as well as constructing a simple exponential decay kernel (e.g. used in S4ND image->video inflation).

    dt_min, dt_max: min and max values for the step size dt
    dt_tie: Keep dt tied across the N dimensions of the state. Although this theoretically makes more sense, models such as S5 and Mega have found slightly improvements by setting it to False.
    dt_transform: Transform function for parameterization of dt (default 'softplus', used to be 'exp')

    rank: Rank of low-rank correction for DPLR mode. Needs to be increased for init "legt".
    n_ssm: Number of independent trainable (A, B) SSMs, e.g.
        `n_ssm=1` means all A/B parameters are tied across the H different instantiations of C.
        `n_ssm=None` means all H SSMs are completely independent.
        Generally, changing this option can save parameters but doesn't affect performance or speed much.
        This parameter must divide H.
    init: Options for initialization of (A, B). For DPLR mode, recommendations are "legs", "fout", "hippo" (combination of both). For Diag mode, recommendations are "diag-inv", "diag-lin", "diag-legs", and "diag" (combination of diag-inv and diag-lin).
    init_args: Extra arguments passed into initialization function (see dplr.py for options).
    c                 óF  — | j                   r¨| j                  sJ d«       ‚| j                  dk(  sJ d«       ‚t        j                  t        j
                  t        j                  | j                  «      t        j                  | j                  «      | j                  «      «      j                  d«      }|S | j                  r| j                  dfn| j                  | j                  dz  f}t        j                  |d| j                  iŽt        j                  | j                  «      t        j                  | j                  «      z
  z  t        j                  | j                  «      z   }| j                  dk7  r)t        t        j                  |«      | j                  «      }|S )Nz'Deterministic dt initialization is tiedrh   z9Deterministic dt transform should be 'exp' for simplicityr   r   r   rÆ   )ÚdeterministicÚdt_tieÚdt_transformr   rh   Úlinspacer'  r+  Údt_minÚdt_maxr.  rc   rÚ   r«   rÆ   rW  )r£   Úinv_dtr:   s      r   Úinit_dtzSSMKernel.init_dt8  s1  € à×ÒØ—;’;ÐIÐ IÓI�;Ø×$Ñ$¨Ò-ÐjÐ/jÓjÐ-Ü—Y‘YœuŸ~™~¬d¯h©h°t·{±{Ó.CÄTÇXÁXÈdÏkÉkÓEZÐ\`×\bÑ\bÓcÓd×nÑnÐoqÓrˆFð ˆð $(§;¢;�T—V‘V˜Q‘K°T·V±V¸T¿V¹VÀQ¹YÐ4GˆEä—Z‘Z Ð9¨d¯j©jÑ9Ü—‘˜Ÿ™Ó%¬¯©°·±Ó(=Ñ=ñä—‘˜Ÿ™Ó%ñ&ˆFð × Ñ  EÒ)Ü&¤u§y¡y°Ó'8¸$×:KÑ:KÓL�àˆr   c                 óä  — t        | j                  | j                  «      \  }}t        j                  || j
                  ¬«      }t        j                  || j
                  ¬«      dd…df   }t        |d| j                  ¬«      j                  «       j                  «       }t        |d| j                  ¬«      j                  «       j                  «       }| j                  rKt        j                  | j                  | j                  | j                  | j
                  ¬«      }d|ddd	…f<   nAt        j                  | j                  | j                  | j                  | j
                  ¬«      }|||fS )
z<Returns (dense, real) (A, B, C) parameters for init options.rÅ   Nr   zn -> v n©rM   zn m -> v n mr§   .r   )rè   r-  rÚ   r   rü   rÆ   r   Ún_ssmrì   r,  r”  rÖ   r^  r.  r)  )r£   r½   rà   ÚCs       r   Úinit_ssm_realzSSMKernel.init_ssm_realI  s  € ô ˜$Ÿ)™) T§V¡VÓ,‰ˆˆ1Ü�O‰O˜A T§Z¡ZÔ0ˆÜ�O‰O˜A T§Z¡ZÔ0²°A°Ñ6ˆÜ�1�j D§J¡JÔ/×5Ñ5Ó7×BÑBÓDˆÜ�1�n¨¯
©
Ô3×9Ñ9Ó;×FÑFÓHˆð ×ÒÜ—‘˜DŸM™M¨4¯6©6°4·6±6ÀÇÁÔLˆAØˆAˆc�2�A�2ˆgŠJä—‘˜DŸM™M¨4¯6©6°4·6±6ÀÇÁÔLˆAà�!�Qˆwˆr   c                 óT  — t        | j                  | j                  | j                  | j                  fi | j
                  ¤Ž\  }}}}| j                  rÁt        j                  | j                  | j                  | j                  | j                  ¬«      }d|dd…dd…dd…f<   t        d|j                  «       j                  dd«      |«      }t        |d| j                  |j!                  d«      z  ¬	«      j#                  «       j%                  «       }nDt        j&                  | j                  | j                  | j                  d
z  | j                  ¬«      }| j                  |j!                  d«      z  dk(  rB| j                  |j!                  d«      z  dk(  r!| j                  |j!                  d«      z  dk(  sJ ‚t        |d| j                  |j!                  d«      z  ¬	«      j#                  «       j%                  «       }t        |d| j                  |j!                  d«      z  ¬	«      j#                  «       j%                  «       }t        |d| j                  |j!                  d«      z  ¬	«      j#                  «       j%                  «       }| xj                  d
z  c_        ||||fS )z6Returns DPLR (A, P, B, C) parameters for init options.rÅ   r§   Nr   zhmn, chn -> chmr   ra   zc t n -> c (v t) nr�  r   r   út n -> (v t) núr t n -> r (v t) n)rQ  r-  rÚ   rí   rž  Ú	init_argsr”  r   rÖ   r^  r	  ri   r   rý   r   r.  rº   rì   r,  r)  )r£   r½   rî   rà   r  rŸ  s         r   Úinit_ssm_dplrzSSMKernel.init_ssm_dplr[  sý  € ä  §¡¨D¯F©F°D·I±I¸t¿z¹zÑ\ÈTÏ^É^Ñ\‰
ˆˆ1ˆa�ð ×ÒÜ—‘˜DŸM™M¨4¯:©:°t·v±vÀTÇ[Á[ÔQˆAØˆAŠa’�B�Q�Bˆh‰KÜÐ*¨A¯F©F«H×,>Ñ,>¸rÀ2Ó,FÈÓJˆAÜ�qÐ.°$·&±&¸A¿F¹FÀ2»JÑ2FÔG×MÑMÓO×ZÑZÓ\‰Aä—‘˜DŸM™M¨4¯6©6°4·6±6¸1±9ÀDÇKÁKÔPˆAð �z‰z˜AŸF™F 2›JÑ&¨!Ò+Ø—J‘J §¡¨£Ñ+¨qÒ0Ø—J‘J §¡¨£Ñ+¨qÒ0ð	1ð 1ô �1Ð&¨$¯*©*¸¿¹¸r»
Ñ*BÔC×IÑIÓK×VÑVÓXˆÜ�1Ð*¨d¯j©j¸A¿F¹FÀ2»JÑ.FÔG×MÑMÓO×ZÑZÓ\ˆÜ�1Ð&¨$¯*©*¸¿¹¸r»
Ñ*BÔC×IÑIÓK×VÑVÓXˆð 	�Š�1‰�à�!�Q˜ˆzÐr   r]  r^  r_  r`  ra  rb  rˆ  r”  r˜  r™  r•  r–  rí   rž  rÙ   r-  c                 óv  •— t         ‰| �  ||||||¬«       || _        t        j                  t        j
                  c| _        | _        || _        |	| _	        |
| _
        || _        || _        || _        |�|n| j                  | _        |�"t         j#                  d«       |�J d«       ‚||}}|| _        || _        y )N)r]  r^  r_  r`  ra  rb  zTWarning: 'measure' option changed to 'init' and will be removed in a future version.z9'measure' and 'init' cannot both be passed into SSMKernel)rœ   r�   rÚ   r   r´   rú   rÆ   r	  r”  r˜  r™  r•  r–  rí   r.  rž  r+  r%   r-  r¤  )r£   r]  r^  r_  r`  ra  rb  rˆ  r”  r˜  r™  r•  r–  rí   rž  rÙ   r-  r¤  r¤   s                     €r   r�   zSSMKernel.__init__y  s¸   ø€ ô2 	‰Ñ °8À5ÈRÐTVÐ`gÐÔhØˆŒÜ"'§+¡+¬u¯|©|ÐˆŒ
�D”KØ*ˆÔàˆŒØˆŒØˆŒØ(ˆÔàˆŒ	Ø#Ð/‘U°T·V±VˆŒ
ØÐÜ�K‰KÐnÔoØ�<Ð\Ð!\Ó\�<Ø# T�'ˆDØˆŒ	Ø"ˆ�r   c                 ó`  — | j                  «       \  }}|j                  d«      |j                  d«      k7  }|rt        |«      }t        d||j	                  d«      «      }t        |j                  d«      ||«      \  }}t        d||«      }||z   }|r|dd|j                  d«      dz  …f   }|S )a¿  Forward the state through a sequence, i.e. computes the state after passing chunk through SSM

        This is a generic version of this functionality that works for SSMs.
        It is currently used by SSMKernelDense and SSMKernelDPLR.
        This is a suboptimal implementation; it is recommended to use SSMKernelDiag
        if this functionality is desired.

        state: (B, H, N)
        u: (B, H, L)

        Returns: (B, H, N)
        r   zh n, b h l -> b h n lzh m n, b h n -> b h m.Nr   )Ú_setup_staterº   rb   ri   ÚfliprÂ   )	r£   r^   rq  ÚdAÚdBr   rM   ÚALÚ
next_states	            r   r…  zSSMKernel.forward_state¥  s¯   € ð  ×"Ñ"Ó$‰ˆˆBà�z‰z˜"‹~ §¡¨£Ñ,ˆÙœ˜u›�äÐ,¨b°!·&±&¸³*Ó=ˆÜ�a—f‘f˜R“j " aÓ(‰ˆˆAÜÐ5°r¸5ÓAˆ
Ø !‘^ˆ
á˜j¨Ð.H°
·±ÀÓ0CÀqÑ0HÐ.HÐ)HÑI�ØÐr   c                 ó   — t         ‚)zRegister dA and dB to module.rp  re  s    r   r¨  zSSMKernel._setup_stateÂ  r~  r   c                 ó4   — | j                   | j                  z  S )z•d_state and state_to_tensor are used by specific decoders.

        These were used in earlier versions and should not be needed in general.
        )r.  rÚ   re  s    r   rˆ  zSSMKernel.d_stateÆ  s   € ð �v‰v˜Ÿ™‰Ðr   c                 ó   — d„ S )Nc                 ó   — t        d| «      S )Nz... h n -> ... (h n))r   ©rq  s    r   r   z+SSMKernel.state_to_tensor.<locals>.<lambda>Ð  s   € œYÐ'=¸uÔEr   rB  re  s    r   rŠ  zSSMKernel.state_to_tensorÎ  s   € áEÐEr   )r   r   NNrê   Tr  Fgü©ñÒMbP?gš™™™™™¹?Trh   r   NNrÇ   )r±   r²   r³   rŒ  r›  r   r¥  r�  r   r   r´   r   rŽ  rL  r�   r   r�  r…  r¨  r�  rˆ  rŠ  rµ   r¶   s   @r   r’  r’    si  ø„ ñò0ò"ò$ðB ØØ#Ø.2Ø.1ØàØ#àØØØ!àØ#Ø!%Ø$ñ+*#ð ð*#ð ð	*#ð
 ˜‰}ð*#ð �%˜ 'Ñ*Ð*Ñ+ð*#ð �%˜ 'Ñ*Ð*Ñ+ð*#ð ð*#ð ð*#ð ð*#ð ð*#ð ð*#ð ð*#ð  ð!*#ð$ ð%*#ð& ˜‰}ð'*#ð( ˜#‘ð)*#ð* �s‰mõ+*#ðX €U‡]�]ƒ_ñó ðò8"ð ñó ðð ñFó ôFr   r’  c                   ó„   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddededededee   dedefˆ fd	„Zd
„ Z	dd„Z
dd„Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚSSMKernelDiaga  SSM kernel using diagonal state matrix (S4D model).

    Options:
    disc: ['zoh' | 'bilinear' | 'dss'] Discretization options.
    dt_fast:  (experimental) Parameterize inv_dt under sinh function.
        (Ohno et al. "Fast Saturating Gate for Learning Long Time Scales with RNNs")
    real_transform, imag_transform: ['none' | 'exp' | 'relu' | 'sigmoid' | 'softplus']
        Parameterize the real/imag parts of the diagonal of A under this function.
    bandlimit: Mask high frequencies of the kernel (indices corresponding to
        diagonal elements with large imaginary part). Introduced in S4ND paper.
    backend: ['cuda' | 'keops' | 'naive'] Options for Vandermonde/Cauchy kernel (in order of efficiency).
    is_real : Real-valued SSM; can be interpreted as EMA.
    ÚdiscÚdt_fastÚreal_transformÚimag_transformÚ	bandlimitrJ   Úis_realc                 ó  •— |rd|v r|d   dz  |d<   t        ‰| �  di |¤Ž || _        || _        || _        || _        || _        || _        || _        | j                  «       }	| j                  «       \  }
}}}| j                  |
|||	|«       y )Nrˆ  r   rB  )rœ   r�   rµ  r¶  r·  r¸  r¹  rJ   rº  r›  r¥  Úregister_params)r£   rµ  r¶  r·  r¸  r¹  rJ   rº  r”   rš  r½   rî   rà   rŸ  r¤   s                 €r   r�   zSSMKernelDiag.__init__â  s¡   ø€ ñ �y FÑ*Ø & yÑ 1°AÑ 5ˆF�9ÑÜ‰ÑÑ"˜6Ò"ØˆŒ	ØˆŒØ,ˆÔØ,ˆÔØ"ˆŒØˆŒØˆŒð —‘“ˆØ×'Ñ'Ó)‰
ˆˆ1ˆa�ð 	×Ñ˜Q  1 f¨aÕ0r   c                 óJ  — | j                   dv sJ ‚| j                  rt        j                  |«      }| j                  |j                  d«      k(  sJ ‚| j                  |j                  d«      cxk(  r*|j                  d«      cxk(  r|j                  d«      k(  sJ ‚ J ‚| j                  |j                  d«      cxk(  r|j                  d«      k(  sJ ‚ J ‚| j                  |j                  d«      z  | _        t        j                  |j                  dk  «      r"t        j                  |j                  dk  «      sJ ‚|j                  t        j                  |j                  d| j                  | j                  f«      «      }|j                  d«      }| j                   |j                  d   k(  sJ ‚| j#                  d|| j$                  d	   | j&                  d	   «       | j(                  r±| j#                  d
|j                  | j$                  d
   d«       | j#                  d|j                  | j$                  d   | j&                  d   «       | j#                  dt+        |j                   | j,                  «      | j$                  d   | j&                  d   «       y| j#                  d
t/        t1        |«      «      | j$                  d
   d«       | j#                  dt/        |«      | j$                  d   | j&                  d   «       | j#                  dt+        |j                   | j,                  «      | j$                  d   | j&                  d   «       | j#                  dt+        |j                   | j2                  «      | j$                  d   | j&                  d   «       y)aG  Process the initialization into form of trainable parameters.

        A: (S, N) diagonal matrix
        B: (S, N)
        C: (C, H, N)
        dt: (H) timescale per feature

        Dimensions:
        N (or d_state): state size
        H (or d_model): total SSM copies
        S (or n_ssm): number of trainable copies of (A, B, dt); must divide H
        C (or channels): system is 1-dim to C-dim

        The forward pass of this Module returns a tensor of shape (C, H, L)

        Note: tensor shape N here denotes half the true state size, because of conjugate symmetry
        )ÚcudaÚkeopsÚnaiver   r   ra   r÷   rê   r   rš  ÚdtrŸ  Nrà   ÚA_realr½   ÚA_imag)rJ   r¶  r   Úasinhr.  rº   rÚ   rž  r   r*  rW   r  ÚexpandÚbroadcast_shapesr:   rc   r^  rz  rl  rn  rº  rW  r·  rK   Ú_resolve_conjr¸  )r£   r½   rà   rŸ  rš  rî   s         r   r¼  zSSMKernelDiag.register_params   sÊ  € ð& �|‰|Ð9Ñ9Ð9Ð9à�<Š<¤%§+¡+¨fÓ"5˜ð �v‰v˜Ÿ™ Q›Ò'Ð'Ð'Ø�v‰v˜Ÿ™ ›Ô? q§v¡v¨b£zÔ?°Q·V±V¸B³ZÒ?Ð?Ñ?Ð?Ð?Ø�z‰z˜QŸV™V B›ZÔ5¨1¯6©6°"«:Ò5Ð5Ñ5Ð5Ð5Ø—f‘f §¡ q£	Ñ)ˆŒô
 �y‰y˜Ÿ™ $™Ô'¬E¯I©I°a·f±fÀ±mÔ,DÐDÐDð �H‰H”U×+Ñ+¨A¯G©G°a¸¿¹ÀÇÁÐ5HÓIÓJˆØ�K‰K˜‹NˆØ�}‰} §¡¨¡
Ò*Ð*Ð*ð 	�‰�h ¨¯©°TÑ(:¸D¿L¹LÈÑ<NÔOà�<Š<Ø�M‰M˜#˜qŸv™v t§|¡|°CÑ'8¸$Ô?Ø�M‰M˜#˜qŸv™v t§|¡|°CÑ'8¸$¿,¹,ÀsÑ:KÔLØ�M‰M˜(¤M°1·6±6°'¸4×;NÑ;NÓ$OÐQU×Q]ÑQ]Ð^aÑQbÐdh×dpÑdpÐqtÑduÕvà�M‰M˜#œt¤M°!Ó$4Ó5°t·|±|ÀCÑ7HÈ$ÔOØ�M‰M˜#œt A›w¨¯©°SÑ(9¸4¿<¹<ÈÑ;LÔMØ�M‰M˜(¤M°1·6±6°'¸4×;NÑ;NÓ$OÐQU×Q]ÑQ]Ð^aÑQbÐdh×dpÑdpÐqtÑduÔvØ�M‰M˜(¤M°1·6±6°'¸4×;NÑ;NÓ$OÐQU×Q]ÑQ]Ð^aÑQbÐdh×dpÑdpÐqtÑduÕvr   c                 ó\  — | j                   r:t        | j                  | j                  «       }| j                  }| j
                  }not        | j                  | j                  «       dt        | j                  | j                  «      z  z
  }t        | j                  «      }t        | j
                  «      }| j                  r t        j                  | j                  «      }n| j                  }t        || j                  «      |z  }| j                  �`||z  |j                  j!                  «       z  dt"        j$                  z  z  }t        j&                  || j                  dz  k  dd«      }||z  }t)        |d| j(                  ¬«      }t)        |d| j(                  ¬«      }||z  }	||||fS )	z Process the internal parameters.rö   r   r°   r   r   r¢  r�  zb t n -> b (v t) n)rº  rZ  rÂ  r·  rà   rŸ  rÃ  r¸  rL   r¶  r   Úsinhrš  r–  r¹  r  r  r'  rØ   rÏ   r   )
r£   rr  r½   rà   rŸ  rš  rÁ  ræ   r®   ÚdtAs
             r   Ú_get_paramszSSMKernelDiag._get_params4  sY  € ð �<Š<Ü  §¡¨d×.AÑ.AÓBÐBˆAØ—‘ˆAØ—‘‰Aä  §¡¨d×.AÑ.AÓBÐBÀRÌ/ÐZ^×ZeÑZeÐgk×gzÑgzÓJ{ÑE{Ñ{ˆAÜ�T—V‘V“ˆAÜ�T—V‘V“ˆAà�<Š<¤%§*¡*¨T¯[©[Ó"9™Ø—{‘{ˆfÜ˜V T×%6Ñ%6Ó7¸$Ñ>ˆà�>‰>Ð%Ø˜‘I §¡§
¡
£Ñ,°´$·'±'±	Ñ:ˆEÜ—;‘;˜u t§~¡~¸Ñ':Ñ:¸A¸qÓAˆDØ�D‘ˆAô �1Ð&¨$¯+©+Ô6ˆÜ�1Ð*¨d¯k©kÔ:ˆð
 �1‰fˆà�1�a˜ˆ{Ðr   c                 óÂ  — | j                  |«      \  }}}}||z  }|�r||z  }	| j                  dk(  r|	d|dz  z   z  }	n9| j                  dk(  r*|	|z  |j                  «       z  |j                  «       dz
  z  }	t        j                  |	|gd¬«      }|dd…ddd…dd…f   |z  j                  d| j                  | j                  «      }t        rL|j                  t        j                  k(  r/|j                  j                  d	k(  r| j                  d	k(  rt        }
n!t        r| j                  d
v rt         }
nt"        }
| j                  dk(  r*|t        j                  |«      dz
  z  |z  } |
|||«      }�n�| j                  dk(  rG|d|dz  z
  j%                  «       z  |z  }d|dz  z   d|dz  z
  z  } |
||j'                  «       |«      }�n:| j                  dk(  �r|j)                  d«      t        j*                  ||j                  ¬«      z  }|j,                  dkD  }|j/                  «       r<t        j0                  «       5  |||dz
  z  z  }ddd«       |j)                  d«      z
  }|j                  «       }|dd|z  z
  z  }|j                  «       dz
  }||z  j                  «       dz
  }||z  }t3        |«      }|||z  dz   z  }||z  |z  }t5        d||«      j7                  «       }nt9        d| j                  › d�«      ‚|j                  d| j:                  | j                  |«      }|�|dd…dd…dd…dd…f   }nd}|ddd…dd…dd…f   }||fS # 1 sw Y   �ŒxY w)ú0See Kernel.forward() for argument documentation.NÚbilinearr§   r   Úzohrô   r   r   r¾  ©r¾  r¿  Údss©r¦   r   r   gH¯¼šò×z>zchn,hnl->chlzDiscretization z not supported)rË  rµ  rh   r   r   r;   r.  rÚ   Úhas_cuda_extensionrÆ   rú   r¦   rk  rJ   r4   Úhas_pykeopsr\   rl   Ú
reciprocalr+  rc   rU   rW   Úanyr�  rÇ  ri   r´   rž   r^  )r£   rX   rq  rr  rÁ  r½   rà   rŸ  rÊ  rA  Úlog_vandermondeÚKrª  rî   ÚA_gt_0ÚP_maxrP  ÚdtA_negÚnumÚdenr   Úx_conjrR   ÚK_states                           r   r¯   zSSMKernelDiag.forwardU  s&  € ð ×&Ñ& tÓ,‰ˆˆAˆq�!Ø�1‰fˆð ÐØ˜‘
ˆAØ�y‰y˜JÒ&Ø˜˜c !™e™Ñ$‘Ø—‘˜eÒ#Ø˜‘G˜cŸg™g›iÑ'¨3¯7©7«9°r©>Ñ:�Ü—	‘	˜1˜a˜& bÔ)ˆAð Šq�$šš1ˆ}Ñ Ñ!×'Ñ'¨¨D¯F©F°D·F±FÓ;ˆõ  !§'¡'¬U¯\©\Ò"9¸a¿h¹h¿m¹mÈvÒ>UÐZ^×ZfÑZfÐjpÒZpÜ2‰OÝ˜TŸ\™\Ð->Ñ>Ü3‰Oä3ˆOð �9‰9˜Òà”U—Y‘Y˜s“^ BÑ&Ñ'¨!Ñ+ˆAÙ  3¨Ó*ŠAØ�Y‰Y˜*Ò$Ø�R˜#˜a™%‘Z×+Ñ+Ó-Ñ-°Ñ2ˆAØ�s˜1‘u‘*  c¨!¡e¡Ñ,ˆBÙ  2§6¡6£8¨QÓ/ŠAØ�Y‰Y˜%Óà—‘˜bÓ!¤E§L¡L°¸1¿8¹8Ô$DÑDˆAØ—V‘V˜a‘ZˆFØ�z‰zŒ|Ü—]‘]•_Ø 6¨Q¨q©S¡>Ñ2�E÷ %à˜Ÿ™¨Ó+Ñ+�Ø—‘“ˆAà˜Q  6¡™\Ñ*ˆGØ—+‘+“- !Ñ#ˆCØ˜Q‘;×#Ñ#Ó%¨Ñ)ˆCð �a‘ˆAÜ" 1Ó%ˆFØ˜!˜F™( T™/Ñ*ˆAà�C‘˜!‘ˆAÜ˜¨¨AÓ.×4Ñ4Ó6‰AÜ °·±°¸>ÐJÓKÐKà�F‰F�2�t—}‘} d§f¡f¨aÓ0ˆàÐØ˜˜˜šQ¢¢1˜‘o‰GàˆGØˆb’!’Qšˆk‰Nˆà�'ˆzÐ÷5 %‘_ús   È?MÍMc                 ó¢  — | j                  «       \  }}}}||z  }| j                  dk(  r>t        j                  |«      | _        |t        j                  |«      dz
  z  |z  | _        nF| j                  dk(  r7d|dz  z   d|dz  z
  z  | _        |d|dz  z
  j                  «       z  |z  | _        t        | j
                  d«      | _        || _        y)ú6Set up dA, dB, dC discretized parameters for stepping.rÏ  r§   rÎ  r   ú1 h n -> h nN)	rË  rµ  r   rh   rª  r«  rÕ  r   ÚdC)r£   rÁ  r½   rà   rŸ  rÊ  s         r   r}  zSSMKernelDiag._setup_stepš  s¿   € ð ×'Ñ'Ó)‰ˆˆAˆq�!à�1‰fˆà�9‰9˜ÒÜ—i‘i “nˆDŒGØœ5Ÿ9™9 S›>¨"Ñ,Ñ-°Ñ1ˆD�GØ�Y‰Y˜*Ò$Ø˜C ™E‘z b¨3¨q©5¡jÑ1ˆDŒGØ˜2  A¡™:×1Ñ1Ó3Ñ3°bÑ8ˆDŒGÜ˜DŸG™G ^Ó4ˆŒØˆ�r   c                 óº   — t        | j                  «      }t        j                  g |¢| j                  ‘| j
                  ‘­|j                  |j                  dœŽ}|S )N©rÆ   r¦   )rL   rŸ  r   rÖ   r.  rÚ   rÆ   r¦   )r£   Úbatch_shaperŸ  rq  s       r   rƒ  zSSMKernelDiag.default_stateª  sI   € Ü�—‘‹LˆÜ—‘ÐY˜[ÐY¨$¯&©&ÐY°$·&±&ÑYÀÇÁÐPQ×PXÑPXÒYˆØˆr   c                 ó°   — t        d| j                  |«      t        d| j                  |«      z   }t        d| j                  |«      }d|j                  z  |fS )Nzh n, b h n -> b h nzh n, b h -> b h nzc h n, b h n -> b c hr   )ri   rª  r«  rã  rW   )r£   r^   rq  r­  Úys        r   r€  zSSMKernelDiag.step¯  sQ   € ÜÐ3°T·W±W¸eÓDÜÐ.°·±¸Ó;ñ<ˆ
äÐ,¨d¯g©g°zÓBˆØ�—‘‰x˜Ð#Ð#r   c                 óœ  — | j                  «        | j                  |j                  d«      z  }|j                  d«      j	                  | j                  «      j                  «       }t        r| j                  dv rt        }nt        } ||| j                  | j                  j                  «       |j                  d«      «      }||z  |z   }|S )z2Pass the state forward through an entire sequence.r   rÐ  )r}  rª  rº   r©  rV   r,  rÔ  rJ   r_   rn   r«  r+  )r£   r^   rq  r¬  Úlog_vandermonde_transposerM   r­  s          r   r…  zSSMKernelDiag.forward_stateµ  s›   € à×ÑÔØ�W‰W˜Ÿ™˜r›
Ñ"ˆØ�F‰F�2‹J�M‰M˜$Ÿ'™'Ó"×-Ñ-Ó/ˆå˜4Ÿ<™<Ð+<Ñ<Ü(GÑ%ä(GÐ%Ù% a¨¯©°$·'±'·+±+³-ÀÇÁÈÃÓLˆØ˜%‘Z !‘^ˆ
ØÐr   )rÏ  Frh   rS  Nr¾  F©r§   ©Nr§   )r±   r²   r³   rŒ  rL  rŽ  r   r´   r�   r¼  rË  r¯   r}  rƒ  r€  r…  rµ   r¶   s   @r   r´  r´  Ó  s”   ø„ ñð  ØØ#Ø$Ø%)ØØñ1àð1ð ð1ð ð	1ð
 ð1ð ˜E‘?ð1ð ð1ð õ1ò<2wóhóBCòJò ò
$ör   r´  c                   ó$  ‡ — e Zd ZdZ ej
                  «       d„ «       Zdd„Zˆ fd„Zdˆ fd„	Z	dd„Z
 ej
                  «       d„ «       Z ej
                  «       d„ «       Z ej
                  «       d	„ «       Zdd
„Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Zˆ xZS )ÚSSMKernelDPLRzaSSM kernel for diagonal + low rank (DPLR) state matrices, corresponding to the original S4 model.c                 óF  — | j                   j                  «       dk(  r'| j                  rt        j	                  d|› �«       d}n•|| j                   j                  «       kD  rw| j                  rNt        j	                  d| j                   j                  «       › dd| j                   j                  «       z  › �«       d}| j                   j                  «       }nyt        | j                  «      }| j                  «       \  }}t        ||«      }t        |«      }t        d	|j                  d
d«      |«      }|r| }||z
  }|dd| j                  …f   }| j                  j                  t        |«      «       |rd| j                   z  | _         y| j                   |z   | _         y)zƒConstruct C~ from C.

        Two modes are supported: go directly to length L if self.l_kernel is 1, or length is doubled
        r   z"S4: Initializing kernel to length FzS4: Doubling length from L = z to r   TNzh m n, c h n -> c h mr   ra   .)Úl_kernelÚitemrb  r+  r$   rL   rŸ  r¨  rÂ   rb   ri   rý   rÚ   Úcopy_rK   )	r£   rX   Údouble_lengthrŸ  rª  r  ÚdA_LÚC_Úprods	            r   Ú_setup_CzSSMKernelDPLR._setup_CÇ  sU  € ð �=‰=×ÑÓ 1Ò$Ø�|Š|œSŸX™XÐ(JÈ1È#Ð&NÔOØ!‰MØ�—‘×#Ñ#Ó%Ò%Ø�|Š|œSŸX™XÐ(EÀdÇmÁm×FXÑFXÓFZÐE[Ð[_Ð`aÐbf×boÑbo×btÑbtÓbvÑ`vÐ_wÐ&xÔyØ ˆMØ—‘×"Ñ"Ó$‰AØä�—‘‹LˆØ×!Ñ!Ó#‰ˆˆAÜ�Q˜‹|ˆä�1‹XˆÜÐ/°·±ÀÀBÓ1GÈÓLˆÙ $ ˜$Ø�$‰YˆØ��W�d—f‘f�W�ÑˆØ�‰�‰”T˜"“XÔá+8˜˜$Ÿ-™-™ˆ�¸d¿m¹mÈA¹oˆ�r   c                 óš  — |rHt        | d«      r<| j                  j                  d«      |dz  dz   k(  r| j                  | j                  fS t	        j
                  t        j                  dt        j                  z  |z  «      ||¬«      }|t	        j                  d|dz  dz   |¬«      z  }dd|z
  z  d|z   z  }|r|| _        || _        ||fS )	zßCalculate (and cache) FFT nodes.

        This also caches a version of the nodes "unprocessed" with the bilinear transform.
        This method should be called everytime the internal length self.l_kernel changes.
        Úomegar   r   r   r&  rå  r   rÒ  )
Úhasattrrù  rº   rN   r   r=   rÌ   rh   rØ   rU   )r£   rX   rÆ   r¦   Úcacherù  rN   s          r   Ú_omegazSSMKernelDPLR._omegaä  s½   € ñ ”W˜T 7Ô+°·
±
·±ÀÓ0CÀqÈ!ÁtÈAÁvÒ0MØ—:‘:˜tŸv™vÐ%Ð%ä—‘Ü�F‰F�3œŸ™‘; !Ñ$Ó%¨U¸6ô
ˆð œŸ™ a¨¨a©°!©¸FÔCÑCˆØ��U‘‰O˜q 5™yÑ)ˆñ ØˆDŒJØˆDŒFØ�aˆxˆr   c           	      ó2  •— | j                   r=t        j                  d| j                  › d| j                  › d| j
                  › d�«       t        ‰| �  |||||«       | j                  |j                  d   k(  sJ ‚| j                  |j                  d«      k(  sJ ‚| j                  |j                  d«      k(  sJ ‚| j                  dt        |«      | j                  d   | j                  d   «       | j!                  d	t#        j$                  d
«      «       y)aP  Process the initialization into form of trainable parameters.

        The SSM state matrix is represented by diag_embed(A) - PP^*
        Note that the A notation here is slightly overloaded:
        normally A refers to the full SSM state matrix (DPLR in this case)
        but here we're using it to refer to the diagonal part of the matrix.
        This is to make variable names compatible with the SSMKernelDiag class (DSS/S4D)
        and is a much simpler variable name (e.g. as opposed to Lambda).

        A: (S, N) diagonal part
        P: (R, S, N) low-rank part
        B: (S, N)
        C: (C, H, N)
        dt: (H) timescale per feature

        Dimensions:
        N (or d_state): state size
        H (or d_model): total SSM copies
        S (or n_ssm): number of trainable copies of (A, B, dt); must divide H
        R (or rank): rank of low-rank part
        C (or channels): system is 1-dim to C-dim

        The forward pass of this Module returns a tensor of shape (C, H, L)

        Note: tensor shape N here denotes half the true state size, because of conjugate symmetry
        zConstructing S4 (H, N, L) = (rA   rB   rô   r   ra   rî   r½   rð  r   N)rb  r+  r$   r.  rÚ   r_  rœ   r¼  rí   r:   rº   rž  rz  rK   rl  rn  rv  r   r=   )r£   r½   rà   rŸ  rš  rî   r¤   s         €r   r¼  zSSMKernelDPLR.register_paramsü  sä   ø€ ð: �<Š<Ü�H‰HÐ4°T·V±V°H¸B¸t¿v¹v¸hÀbÈÏÉÈÐTUÐVÔWô 	‰Ñ  1 a¨°Ô3ð �y‰y˜AŸG™G B™KÒ'Ð'Ð'Ø�v‰v˜Ÿ™ ›Ò#Ð#Ð#Ø�z‰z˜QŸV™V B›ZÒ'Ð'Ð'à�‰�cœ4 ›7 D§L¡L°Ñ$5°t·|±|ÀCÑ7HÔIð 	×Ñ˜Z¬¯©°a«Õ9r   c                 ó¸   •— t         ‰| �  |¬«      \  }}}}t        | j                  «      }t	        |d| j                  ¬«      }|j                  «       }||||||fS )N)rr  r£  r�  )rœ   rË  rL   rî   r   r   )	r£   rr  rÁ  r½   rà   rŸ  rî   rÜ   r¤   s	           €r   rË  zSSMKernelDPLR._get_params)  s[   ø€ Ü‘gÑ)¨tÐ)Ó4‰ˆˆAˆq�!Ü�—‘‹LˆÜ�1Ð*¨d¯k©kÔ:ˆØ�F‰F‹Hˆà�1�a˜˜A˜qÐ Ð r   c           	      ó˜  — | j                   j                  «       dk(  r6| j                  �*| j                  dkD  r| j                  | j                  «       |€&t	        | j                   j                  «       |z  «      }t	        ||z  «      }|| j                   j                  «       kD  r/| j                  |«       || j                   j                  «       kD  rŒ/t	        | j                   j                  «       |z  «      }| j                  |«      \  }}}}	}
}| j                  ||j                  |j                  |dk(  ¬«      \  }}|�‘|j                  d«      | j                  k(  rt        |«      n|}|t        |«      z  t        d|t        |«      t        |
«      «      z
  }||z  |dz  z   }|dd| j                  …f   }t        j                  ||gd	¬
«      }||z  }t        j                  ||
gd	¬
«      }t        j                  |	|gd	¬
«      }	|j                  d	«      |	j                  d«      z  }||z  }t         rL|j                  t        j"                  k(  r/|j                  j$                  dk(  r| j&                  dk(  rt(        }nt*        } ||||«      }| j,                  dk(  rL|dd…dd…dd…dd…f   |dd…dd…dd…dd…f   |dd…dd…dd…dd…f   z  d|dd…dd…dd…dd…f   z   z  z
  }�nü| j,                  dk(  �rÒ|d| j,                   …d| j,                   …dd…dd…f   }|d| j,                   …| j,                   d…dd…dd…f   }|| j,                   d…d| j,                   …dd…dd…f   }|| j,                   d…| j,                   d…dd…dd…f   }d|dd…dd…dd…dd…f   z   d|dd…dd…dd…dd…f   z   z  |dd…dd…dd…dd…f   |dd…dd…dd…dd…f   z  z
  }|dd…dd…dd…dd…f   d|dd…dd…dd…dd…f   z   z  |dd…dd…dd…dd…f   z  |dd…dd…dd…dd…f   d|dd…dd…dd…dd…f   z   z  |dd…dd…dd…dd…f   z  z   |dd…dd…dd…dd…f   |dd…dd…dd…dd…f   z  |dd…dd…dd…dd…f   z  z
  |dd…dd…dd…dd…f   |dd…dd…dd…dd…f   z  |dd…dd…dd…dd…f   z  z
  }||z  }||z
  }�n|d| j,                   …d| j,                   …dd…dd…f   }|d| j,                   …| j,                   d…dd…dd…f   }|| j,                   d…d| j,                   …dd…dd…f   }|| j,                   d…| j,                   d…dd…dd…f   }t/        |d«      }t        j0                  j3                  t        j4                  | j,                  |j                  ¬«      |z   «      }t/        |d«      }|t        j6                  d|||«      z
  }|dz  d|z   z  }t        j8                  j;                  ||¬«      }|dd|…f   }|�|dd…dd…dd…dd…f   }nd}|ddd…dd…dd…f   }||fS )rÍ  r   Nr§   )rÆ   r¦   rû  r   zbhm, rhm, rhn -> bhnr   .rô   r   éüÿÿÿr¾  r   za b h n -> h n a brÒ  zh n a b -> a b h nz$i j h n, j k h n, k l h n -> i l h n©Ún)rð  rñ  r_  r÷  ÚroundrË  rü  rÆ   r¦   rº   rÚ   rb   ri   r   r   rc   rÓ  rú   rk  ÚkernelÚcauchy_cudarf   rí   r   rÒ   rÓ   r¸   ÚeinsumÚfftÚirfft)r£   rq  rr  rX   Úcontinuous_LÚ
discrete_LrÁ  r½   rà   rŸ  rî   rÜ   rù  rN   rA  ÚsArM   r3   rR   Úk_fÚr00Úr01Úr10Úr11ÚdetrÀ   Úk_stateÚk_Bs                               r   r¯   zSSMKernelDPLR.forward1  sw  € ð �=‰=×ÑÓ 1Ò$¨¯©Ð)?ÀDÇJÁJÐQRÂNØ�M‰M˜$Ÿ*™*Ô%ð ˆ9Ü�d—m‘m×(Ñ(Ó*¨TÑ1Ó2ˆAô ˜T !™V“}ˆØ˜TŸ]™]×/Ñ/Ó1Ò1Ø�M‰M˜,Ô'ð ˜TŸ]™]×/Ñ/Ó1Ó1ä˜4Ÿ=™=×-Ñ-Ó/°Ñ4Ó5ˆ
à ×,Ñ,¨TÓ2ÑˆˆAˆq�!�Q˜ð —;‘;˜z°·±ÀÇÁÐRVÐX[ÑR[�;Ó]‰ˆˆqð Ðð
 !&§
¡
¨2£°$·&±&Ò 8”�e”¸eˆAà”E˜!“H‘ÜÐ1°1´e¸A³hÄÀaÃÓIñJð ð �B‘˜˜a™‘ˆAØ�#�w˜Ÿ™�w�,‘ˆAä—	‘	˜1˜a˜& bÔ)ˆAð �‰Fˆô �I‰I�q˜!�f "Ô%ˆÜ�I‰I�q˜!�f "Ô%ˆð �K‰K˜‹O˜aŸk™k¨"›oÑ-ˆØ�‰Fˆõ  !§'¡'¬U¯\©\Ò"9¸a¿h¹h¿m¹mÈvÒ>UÐZ^×ZeÑZeÐioÒZoÜ%‰Kô 'ˆKá˜˜1˜aÓ ˆð �9‰9˜Š>Ø�C�R�C˜˜"˜ša¢�NÑ# a¨¨¨¨R©S²!²Q¨Ñ&7¸!¸B¹CÀÀ"ÀÂaÊ¸NÑ:KÑ&KÈqÐSTÐUWÑUXÐZ\ÑZ]Ò_`ÒbcÐUcÑSdÑOdÑ&eÑeŠCØ�Y‰Y˜!‹^Ø�L�t—y‘y�j�L , T§Y¡Y J ,²²1Ð4Ñ5ˆCØ�L�t—y‘y�j�L 4§9¡9 *¡,²²1Ð4Ñ5ˆCØ�T—Y‘Y�J‘L , T§Y¡Y J ,²²1Ð4Ñ5ˆCØ�T—Y‘Y�J‘L 4§9¡9 *¡,²²1Ð4Ñ5ˆCØ�s˜2˜A˜2˜r ˜r¢1¢a˜<Ñ(Ñ(¨Q°°Q±R¸¹ºQÂ°\Ñ1BÑ-BÑCÀcÈ"È1È"ÈaÉbÒRSÒUVÈ,ÑFWÐZ]Ð^_Ñ^`ÐbdÐcdÐbdÒfgÒijÐ^jÑZkÑFkÑkˆCà’A�r˜�rš1ša�KÑ  A¨¨A©B°±²A²q¨LÑ(9Ñ$9Ñ:¸SÀÀ!ÀÂQÊÊ1ÀÑ=MÑMØ’a˜™šQ¢�kÑ" a¨#¨b¨q¨b°"°1°"²aº¨lÑ*;Ñ&;Ñ<¸sÀ1Á2ÂqÊ!ÊQÀ;Ñ?OÑOñPà’a˜˜!˜šQ¢�kÑ" c¨"¨1¨"¨a©b²!²Q¨,Ñ&7Ñ8¸3¸q¹rÂ1ÂaÊ¸{Ñ;KÑKñLð ’a˜™šQ¢�kÑ" c¨!©"¨b¨q¨b²!²Q¨,Ñ&7Ñ8¸3¸rÀ¸rÂ1ÂaÊ¸{Ñ;KÑKñLð ð �C‘ˆAØ˜‘'ŠCà�K�d—i‘i�Z�K  4§9¡9 * ªa²Ð2Ñ3ˆCØ�K�d—i‘i�Z�K $§)¡) ¡ªa²Ð2Ñ3ˆCØ�T—Y‘Y�J‘K  4§9¡9 * ªa²Ð2Ñ3ˆCØ�T—Y‘Y�J‘K $§)¡) ¡ªa²Ð2Ñ3ˆCÜ˜CÐ!5Ó6ˆCÜ—,‘,×"Ñ"¤5§9¡9¨T¯Y©Y¸q¿x¹xÔ#HÈ3Ñ#NÓOˆCÜ˜CÐ!5Ó6ˆCØœŸ™Ð%KÈSÐRUÐWZÓ[Ñ[ˆCð �A‰g˜˜U™Ñ#ˆô �I‰I�O‰O˜C :ˆOÓ.ˆð ˆc�2�A�2ˆg‰JˆàÐØ˜˜˜šQ¢¢1˜‘o‰GàˆGØ�’A’qš!�‰nˆà�Gˆ|Ðr   c                 ó@   — | j                  d| j                  z  «       y )Nr   )r÷  rð  re  s    r   ró  zSSMKernelDPLR.double_length™  s   € à�‰�a˜Ÿ™‘oÕ&r   c                 ó  — | j                  | j                  ¬«      d   }| j                  «        t        | j                  | j                  | j
                  | j                  «      }||z
  }t        dt        j                  |dz  «      «       y)zOCheck if A, B, C parameters and vanilla SSMKernel construction can be recovered)rX   r   z!checking DPLR Kernel constructionr   N)
r¯   r_  r}  Úkrylovrª  r«  rã  rþ   r   rd   )r£   rØ  ÚK_Údiffs       r   Ú_checkzSSMKernelDPLR._check�  sj   € ð �L‰L˜4Ÿ:™:ˆLÓ& qÑ)ˆà×ÑÔÜ�D—J‘J §¡¨¯©°$·'±'Ó:ˆà�2‰vˆÜÐ1´5·9±9¸TÀQ¹YÓ3GÕHr   c                 óú  — | j                  «       \  }}}}}}d|z  |z
  j                  «       }t        j                  | j                  |j
                  |j                  ¬«      dt        d|||«      j                  z  z   }t        ||z  d«      }		 t        j                  j                  ||	«      }t        |d«      }|||||d|z  |z   dœ| _        y#  t        j                  t        j                  j                  |j                  |	«      j                  «       j!                  «       j#                  «       |	j                  «       j!                  «       j#                  «       «      «      j                  |	«      }Y ŒÊxY w)	zIPreprocessing that allows fast linear-time (in state dimension) stepping.rò   rå  r   zr h n, h n, s h n -> h r szr h n -> h r nzh r n -> r h n)ÚDrÝ   rî   rÜ   rà   ÚEN)rË  rÕ  r   r¸   rí   rÆ   r¦   ri   rW   r   rÒ   Úsolver=   rÌ   rV   r,  ÚdetachÚcpuÚstep_params)
r£   rÁ  r½   rà   rŸ  rî   rÜ   r  rÝ   ÚQ_Ds
             r   Ú_setup_linearzSSMKernelDPLR._setup_linear«  sF  € ð !×,Ñ,Ó.ÑˆˆAˆq�!�Q˜ð �2‰X˜‰\×%Ñ%Ó'ˆÜ�Y‰Y�t—y‘y¨¯©¸¿¹ÔAÀAÄhÐOkÐmnÐpqÐstÓFu×FzÑFzÑDzÑzˆÜ˜˜!™Ð-Ó.ˆð	~Ü—‘×"Ñ" 1 cÓ*ˆAô �aÐ)Ó*ˆð ØØØØØ�r‘˜A‘ñ
ˆÕøð		~Ü—‘œRŸY™YŸ_™_¨Q¯T©T°#«Y×-AÑ-AÓ-C×-JÑ-JÓ-L×-PÑ-PÓ-RÐTW×TbÑTbÓTd×TkÑTkÓTm×TqÑTqÓTsÓtÓu×xÑxÐy|Ó}ŠAús   Â C ÃB'E:c                 ó  ‡ — t        ‰ j                  «      }|€6t        j                  ‰ j                  |j
                  |j                  ¬«      }|€At        j                  ‰ j                  ‰ j                  |j
                  |j                  ¬«      }‰ j                  j                  «       }|j                  d«      ‰ j                  k(  rˆ fd„}nR|j                  d«      d‰ j                  z  k(  sJ ‚|j                  «       D ��ci c]  \  }}|t        |«      “Œ }}}d„ }|d   }|d   }	|d   }
|d	   }|d
   }|d   }|	|z   ||||«      z
  }|d|z  |j                  d«      z  z   }|| |||
|«      z
  z  }|S c c}}w )a  
        Version of the step function that has time O(N) instead of O(N^2) per step, which takes advantage of the DPLR form and bilinear discretization.

        Unfortunately, as currently implemented it's about 2x slower because it calls several sequential operations.
        Perhaps a fused CUDA kernel implementation would be much faster.

        u: (H) Input
        state: (H, N/2) State with conjugate pairs. Optionally, the state can have last dimension N.

        Returns: same shape as state
        rå  r   c                 óx   •— t        dt        | «      t        |«      t        |«      «      dd ‰j                  …f   S )Nú r h n, r h m, ... h m -> ... h n.)ri   rb   rÚ   )r™   r   rè  r£   s      €r   r   z2SSMKernelDPLR._step_state_linear.<locals>.<lambda>Ù  sU   ø€ ¬(Ð3UÔW\Ð]^ÓW_ÔafÐghÓaiÔkpÐqrÓksÓ*tÐuxð  {BÐ{÷  |Bñ  |Bð  {Bð  vBò  +Cr   r   c                 ó   — t        d| ||«      S )Nr%  )ri   )r™   r   rè  s      r   r   z2SSMKernelDPLR._step_state_linear.<locals>.<lambda>Ý  s   € ¬(Ð3UÐWXÐZ[Ð]^Ô*_r   r  r  rÝ   rî   rÜ   rà   rò   )rL   rŸ  r   rÖ   r.  rÆ   r¦   rÚ   r   rÔ   rº   Úitemsrb   rc   )r£   r^   rq  rŸ  r   Úcontract_fnrÀ   rM   r  r  rÝ   rî   rÜ   rà   Ú	new_states   `              r   Ú_step_state_linearz SSMKernelDPLR._step_state_linearÃ  sn  ø€ ô �—‘‹Lˆàˆ9Ü—‘˜DŸF™F¨!¯'©'¸!¿(¹(ÔCˆAØˆ=Ü—K‘K §¡¨¯©°a·g±gÀaÇhÁhÔOˆEà×&Ñ&×+Ñ+Ó-ˆØ�:‰:�b‹>˜TŸV™VÒ#ó C‰Kà—:‘:˜b“> Q t§v¡v¡XÒ-Ð-Ð-Ø3>×3DÑ3DÔ3FÔGÑ3F©4¨1¨a˜1œe A›h™;Ð3FˆKÑGÙ_ˆKØ˜ÑˆØ˜ÑˆØ˜ÑˆØ˜ÑˆØ˜ÑˆØ˜Ñˆà˜‘I¡¨A¨q°%Ó 8Ñ8ˆ	Ø  a¡¨!¯+©+°b«/Ñ 9Ñ9ˆ	Ø˜¡[°°A°yÓ%AÑAÑBˆ	àÐùó Hs   ÄE<c                 ó¨  — | j                  «        t        | j                  «      }t        j                  d| j
                  z  |j                  |j                  ¬«      j                  d«      }| j                  |¬«      }t        |d«      }|j                  | j                  «      }| j                  |¬«      }t        |«      }t        |d«      }||fS )z3Construct dA and dB for discretized state equation.r   rå  ra   r²  zn h m -> h m n)r^   râ  )r"  rL   rŸ  r   r¸   rÚ   rÆ   r¦   rc   r*  r   Únew_onesr.  rb   )r£   rŸ  rq  rª  r^   r«  s         r   r¨  zSSMKernelDPLR._setup_stateë  s­   € ð 	×ÑÔÜ�—‘‹Lˆä—	‘	˜!˜DŸF™F™(¨!¯'©'¸!¿(¹(ÔC×MÑMÈbÓQˆØ×$Ñ$¨5Ð$Ó1ˆÜ�rÐ+Ó,ˆà�J‰J�t—v‘vÓˆØ×$Ñ$ qÐ$Ó)ˆÜ�2‹YˆÜ�r˜>Ó*ˆØ�2ˆvˆr   c                 ó´   — t        j                  | j                  | j                  |«      t        j                  | j                  | j
                  |«      z   }|S )zPMust be called after self.default_state() is used to construct an initial state!)r   r  Ústate_contractionrª  Úinput_contractionr«  )r£   r^   rq  r­  s       r   Ú_step_statezSSMKernelDPLR._step_stateü  sD   € ä—l‘l 4×#9Ñ#9¸4¿7¹7ÀEÓJÜ—|‘| D×$:Ñ$:¸D¿G¹GÀQÓGñHˆ
àÐr   c           	      óŒ  — | j                  «       \  | _        | _        t        t	        | j
                  «      «      }| j                  j                  «       dk(  r|}n¼t        | j                  j                  «       | j                  «      }t        j                  | j                  j                  d«      «      j                  |«      }t        j                  j                  ||j                  dd«      z
  |j!                  d«      «      j#                  d«      }|| _        || _        |dk(  r+d| j$                  dd…dd…d| j(                  …f   z  | _        y|dk(  rÚt        j                  j+                  | j                  «      \  }}t        j                  j-                  |«      }| j.                  rCt1        dt        j2                  |t        j4                  |«      z  |z  | j                  «      «       || _        t7        d	|| j                  «      | _        t7        d
|| j$                  «      | _        y|dk(  ryt9        d«      ‚)rá  r   r   ra   rr   r   Nr   zDiagonalization error:zh n m, h m -> h nzh n m, c h n -> c h mÚdensez?DPLR Kernel step mode must be {'dense' | 'linear' | 'diagonal'})r¨  rª  r«  rb   rL   rŸ  rð  rñ  rÂ   r   r¸   rº   rV   rÒ   r  rý   rc   r¼   rã  Ú
_step_moderÚ   ÚeigrÓ   rb  rþ   Údistr  ri   r†   )	r£   ÚmoderŸ  rã  rô  r¾   rX   r  r  s	            r   r}  zSSMKernelDPLR._setup_step  sÇ  € à×,Ñ,Ó.ÑˆŒ�”ô ”$�t—v‘v“,ÓˆØ�=‰=×ÑÓ 1Ò$Ø‰Bô ˜Ÿ™×+Ñ+Ó-¨t¯w©wÓ7ˆDÜ—	‘	˜$Ÿ'™'Ÿ,™, rÓ*Ó+×.Ñ.¨tÓ4ˆAä—‘×#Ñ#Ø�D—N‘N 2 rÓ*Ñ*Ø—‘˜B“ó÷ ‰g�b‹kð ð ˆŒð ˆŒØ�8Òð ˜Ÿ™¢¢1 g t§v¡v g Ñ.Ñ.ˆD�GØ�ZÒä—<‘<×#Ñ# D§G¡GÓ,‰DˆAˆqÜ—L‘L×$Ñ$ QÓ'ˆEà�|Š|ÜÐ.´·
±
¸1¼u×?OÑ?OÐPQÓ?RÑ;RÐUZÑ;ZÐ\`×\cÑ\cÓ0dÔeð ˆDŒGÜÐ2°E¸4¿7¹7ÓCˆDŒGÜÐ6¸¸4¿7¹7ÓCˆD�Gà�WŠ_ØÜ'Ð(iÓjÐjr   c                 óH  — t        | j                  «      }|j                  d«      }|j                  d«      }t        | dd«      }|dk7  r |dz  }|dk(  rd| _        nd	| _        d
| _        d| _        t        j                  g |¢|‘|‘­|j                  |j                  dœŽ}|S )Nr   ra   r3  r2  rr   r   r   zh n, ... h n -> ... h nzh m n, ... h n -> ... h mzh n, ... h -> ... h nzc h n, ... h n -> ... c hrå  )rL   rŸ  rº   r.   r.  r/  Úoutput_contractionr   rÖ   rÆ   r¦   )r£   ræ  rŸ  rÚ   r.  Ú	step_moderq  s          r   rƒ  zSSMKernelDPLR.default_state-  s¤   € Ü�—‘‹LˆØ�F‰F�2‹JˆØ�F‰F�2‹Jˆô ˜D ,°Ó8ˆ	Ø˜Ò Ø�‰FˆAà˜JÒ&Ø)B�Õ&ð *E�Ô&à%<ˆDÔ"à"=ˆÔä—‘ÐO˜[ÐO¨!ÐO¨QÑO°a·g±gÀaÇhÁhÒOˆØˆr   c                 óÜ   — | j                   dk(  r| j                  ||«      }n| j                  ||«      }t        j                  | j
                  | j                  |«      }|j                  |fS )zdMust have called self._setup_step() and created state with self.default_state() before calling this.rr   )r3  r*  r0  r   r  r8  rã  rW   )r£   r^   rq  r)  rè  s        r   r€  zSSMKernelDPLR.stepE  s_   € ð �?‰?˜hÒ&Ø×/Ñ/°°5Ó9‰Ià×(Ñ(¨¨EÓ2ˆIÜ�L‰L˜×0Ñ0°$·'±'¸9ÓEˆØ�v‰v�yÐ Ð r   c                 ó4   — t        j                  | g|¢­i |¤ŽS r   )r’  r…  r‚  s      r   r…  zSSMKernelDPLR.forward_stateO  s   € ô ×&Ñ& tÐ=¨dÒ=°fÑ=Ð=r   ©Trë  r‹  )NN)r2  )r±   r²   r³   rŒ  r   r�  r÷  rü  r¼  rË  r¯   ró  r  r"  r*  r¨  r0  r}  rƒ  r€  r…  rµ   r¶   s   @r   rî  rî  Ä  sµ   ø„ Ùkà€U‡]�]ƒ_ñNó ðNó8ô0+:õZ!ófðP €U‡]�]ƒ_ñ'ó ð'ð €U‡]�]ƒ_ñIó ðIð €U‡]�]ƒ_ñ
ó ð
ó.&òPò"ó)kòVò0!ö
>r   rî  )Ús4drÐ   Ús4r  r<  c                   ój   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d
ˆ fd„	Zdd„Zd„ Zd„ Zddœd„Ze	d	„ «       Z
ˆ xZS )ÚFFTConvaØ  Implements an FFT Convolution around a convolution kernel.

    d_model (H): Model dimension (in CNN terminology, this would be "channels").
    l_max (L): The maximum kernel length. Set l_max=None to always use a global kernel.
    channels: Can be interpreted as a number of "heads"; the SSM is a map from a 1-dim to C-dim sequence. It's not recommended to change this; instead, increase d_model for larger models.
    bidirectional: If True, convolution kernel will be two-sided.
    activation: Activation after the full convolution.
    transposed, dropout, tie_dropout: More general model options, see SequenceModule.
    mode: Which kernel algorithm to use. 'nplr' is the full S4 model; 'diag' is the simpler S4D. Other options can be found in the kernel registry.

    kernel_args: See the class .kernel.SSMKernel for the kernel constructor which accepts kernel_args. Relevant options that are worth considering and tuning include "mode", "init", "dt_min", "dt_max", "lr"
    Nc                 óî  •— t         ‰| �  «        || _        |x| _        | _        || _        || _        || _        || _        |�|j                  d«      r|dz  }t        || j                  rdnd¬«      | _        t        j                  t        j                  || j                  «      «      | _        | j
                  r|dz  }|�|�J d«       ‚||}}t"        |   } |d	| j                  | j                  |dœ|¤Ž| _        |	rt&        nt        j(                  }|dkD  r ||«      nt        j*                  «       | _        |
dkD  rt        j(                  |
«      | _        y t        j*                  «       | _        y )
Nry   r   r   r   r   z'Pass either mode or kernel but not both)r]  r_  r^  rê   rB  )rœ   r�   r]  rX   r_  Úbidirectionalr^  r’   Úswap_channelsrC  r‰   rˆ   r|   rx  r   r)  r  Úkernel_registryr  r˜   ÚDropoutr}   ÚdropÚdrop_kernel)r£   r]  r_  r^  rC  rB  rˆ   r’   ÚdropoutÚtie_dropoutrG  r6  r  Úkernel_argsÚ
kernel_clsÚ
dropout_fnr¤   s                   €r   r�   zFFTConv.__init__q  sO  ø€ ô  	‰ÑÔØˆŒØ#Ð#ˆŒ�”Ø*ˆÔØ ˆŒØ$ˆŒØ*ˆÔð Ð! j×&;Ñ&;¸EÔ&BØ˜‰MˆHÜ$ Z¸$¿/º/±QÈrÔRˆŒä—‘œeŸk™k¨(°D·L±LÓAÓBˆŒà×ÒØ˜‰MˆHð ÐØ�>ÐLÐ#LÓL�>ð
   �DˆFÜ$ VÑ,ˆ
Ù ð 
Ø—L‘LØ—*‘*Øñ
ð ñ	
ˆŒñ #.•Y´2·:±:ˆ
Ø+2°Sª=‘J˜wÔ'¼b¿k¹k»mˆŒ	Ø6AÀCÒ6Gœ2Ÿ:™: kÓ2ˆÕÌRÏ[É[Ë]ˆÕr   c                 óB  — | j                   s|j                  dd«      }|j                  d«      }| j                  €|n!t	        |t        | j                  |z  «      «      }| j                  |||¬«      \  }}| j                  rQt        |dd¬«      \  }	}
t        j                  |	d|f«      t        j                  |
j                  d«      |df«      z   }| j                  |«      }t        j                  j                  |||z   ¬	«      }t        j                  j                  |||z   ¬	«      }t!        d
||«      }t        j                  j#                  |||z   ¬	«      dd|…f   }|t!        d|| j$                  «      z   }|�5| j                  rJ d«       ‚||z   }| j                  j'                  ||«      }nd}| j(                  rt        |d«      }nt        |d«      }| j+                  |«      }| j                   s|j                  dd«      }| j-                  |«      }||fS )z<
        x: (B D L) if self.transposed else (B L D)
        r   ra   N)rX   rr  rq  z(s c) h l -> s c h lr   r@  r   r  zbhl,chl->bchl.zbhl,ch->bchlz1Bidirectional not supported with state forwardingzb c h l -> b (h c) lzb c h l -> b (c h) l)r’   rý   rº   rX   rø   r  r  rB  r   rY  Úpadr©  rG  r   r  Úrfftri   r  r  r…  rC  rF  rˆ   )r£   r   rq  rr  r”   rX   rð  rÀ   r  Úk0Úk1r  Úx_fÚy_frè  r­  s                   r   r¯   zFFTConv.forward§  sì  € ð �Š A§K¡K°°BÓ$7 Ø�F‰F�2‹Jˆð Ÿ™˜‘1¬C°´5¸¿¹À$¹Ó3GÓ,HˆØ—k‘k H°4¸u�kÓE‰
ˆˆ7ð ×ÒÜ˜qÐ"8¸AÔ>‰FˆB�Ü—‘�b˜1˜a˜&Ó!Ü—e‘e˜BŸG™G B›K¨!¨Q¨Ó0ñ1ˆAð ×Ñ˜QÓˆô �i‰i�n‰n˜Q (¨1¡*ˆnÓ-ˆÜ�i‰i�n‰n˜Q (¨1¡*ˆnÓ-ˆÜ�¨¨SÓ1ˆÜ�I‰I�O‰O˜C 8¨A¡:ˆOÓ.¨s°B°Q°B¨wÑ7ˆð ”˜¨¨D¯F©FÓ3Ñ3ˆð ÐØ×)Ò)Ð^Ð+^Ó^Ð)Ø�G‘ˆAØŸ™×2Ñ2°1°eÓ<‰JàˆJð ×ÒÜ˜!Ð3Ó4‰Aä˜!Ð3Ó4ˆAà�I‰I�a‹Lˆà�Š A§K¡K°°BÓ$7 Ø�O‰O˜AÓˆà�*ˆ}Ðr   c                 ó<   —  | j                   j                  di |¤Ž y ©NrB  )r  r}  r|  s     r   Ú
setup_stepzFFTConv.setup_stepå  s   € Øˆ�‰×ÑÑ) &Ó)r   c                 óÄ   — | j                   j                  ||«      \  }}||j                  d«      | j                  z  z   }t	        |d«      }| j                  |«      }||fS )z¯ Step one time step as a recurrent model. Intended to be used during validation.

        x: (B H)
        state: (B H N)
        Returns: output (B H), state (B H N)
        ra   zb c h -> b (c h))r  r€  rc   r  r   rˆ   )r£   r   rq  rè  r­  s        r   r€  zFFTConv.stepè  s^   € ð Ÿ™×(Ñ(¨¨EÓ2‰ˆˆ:Ø�—‘˜B“ $§&¡&Ñ(Ñ(ˆÜ�aÐ+Ó,ˆØ�O‰O˜AÓˆØ�*ˆ}Ðr   rÒ  c                ó4   —  | j                   j                  |Ž S r   )r  rƒ  ©r£   r¦   ræ  s      r   rƒ  zFFTConv.default_stateö  s   € ð )ˆt�{‰{×(Ñ(¨+Ð6Ð6r   c                 ó4   — | j                   | j                  z  S r   )r]  r^  re  s    r   r‘   zFFTConv.d_outputû  s   € à�|‰|˜dŸm™mÑ+Ð+r   )Nr   FFru   Trê   Frê   r<  Nrì  ©r±   r²   r³   rŒ  r�   r¯   rV  r€  rƒ  r�  r‘   rµ   r¶   s   @r   r@  r@  c  s`   ø„ ñð  ØØØØØØØØØØõ4[ól;ò|*òð 26ô 7ð
 ñ,ó ô,r   r@  c                   ój   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d
ˆ fd„	Zdd„Zd„ Zd„ Zddœd„Ze	d	„ «       Z
ˆ xZS )ÚS4Blocka[  General block design wrapping an inner layer. Currently only layer=FFTConv is supported, but easy to incorporate others.

    Arguments:
    - bottleneck: Reduce dimension of inner layer (e.g. used in GSS).
    - gate: Add multiplicative gating (e.g. used in GSS), which is essentially a multiplicative instead of additive residual branch.
    - gate_act: Activation function to apply on the gate residual branch.
    - mult_act: Activation function to apply after gate multiplication (e.g. GELU in GSS).
    - final_act: Activation function to apply after final linear layer. 'id' for no activation, None for no linear layer at all.

    - initializer: Initializer on final linear layer.
    - weight_norm: Weight normalization on final linear layer.
    - dropout: standard dropout argument. tie_dropout=True ties the dropout mask across the sequence length, emulating nn.Dropout1d

    - transposed: Choose backbone axis ordering of (B, L, H) (if False) or (B, H, L) (if True) [B=batch size, L=sequence length, H=model dimension]

    Other options are all experimental and should not need to be configured.
    Nc                 óÌ  •— t         ‰| �  «        || _        || _        || _        || _        |�=| j                  |z  | _        t        | j                  | j                  dd d¬«      | _        |�‹t        | j                  | j                  |z  d|d¬«      | _        | j                  j                  | j                  |z  k7  r9t        | j                  | j                  z  | j                  |z  dd d¬«      | _        t        |fd|
|dœ|¤Ž| _	        t        |«      | _        |rt!        t"        d¬«      nt$        j&                  }|
dkD  r ||
«      nt%        j(                  «       | _        |�|�J ‚t,        j/                  d«       ||}}|€t%        j(                  «       | _        y t        |�| j                  |z  n| j                  j                  | j                  d|d¬«      | _        y )NF)r’   rˆ   r“   T)r’   rH  rI  )r’   rê   zYWarning: 'postact' option changed to 'final_act' and will be removed in a future version.)rœ   r�   r]  r’   ÚgateÚ
bottleneckr–   Úinput_linearÚ
input_gateÚlayerr‘   r^  Úoutput_gater@  r‰   Úmult_activationr   r˜   r|   rE  r}   rF  r+  r%   Úoutput_linear)r£   r]  r`  r_  Úgate_actÚmult_actÚ	final_actÚpostactÚinitializerÚweight_normrH  rI  r’   Ú
layer_argsrL  r¤   s                  €r   r�   zS4Block.__init__  s¸  ø€ ô  	‰ÑÔàˆŒØ$ˆŒàˆŒ	Ø$ˆŒàÐ!ØŸ<™<¨:Ñ5ˆDŒLÜ 0Ø—‘Ø—‘Ø ØØô!ˆDÔð ÐÜ.Ø—‘Ø—‘˜tÑ#Ø Ø#ØôˆDŒOð �z‰z×"Ñ" d§l¡l°TÑ&9Ò9Ü#3Ø—L‘L §¡Ñ.Ø—L‘L 4Ñ'Ø$Ø#Ø"ô$�Ô ô ˜WÐo°ÀÐU`ÑoÐdnÑoˆŒ
ô
  *¨(Ó3ˆÔá=H”WœY°5Õ9ÌbÏjÉjˆ
Ø+2°Sª=‘J˜wÔ'¼b¿k¹k»mˆŒ	ð ÐØÐ$Ð$Ð$Ü�K‰KÐsÔtØ!(¨)�wˆIØÐÜ!#§¡£ˆDÕä!1Ø%)Ð%5�—‘˜TÒ!¸4¿:¹:×;NÑ;NØ—‘Ø Ø$Øô"ˆDÕr   c                 ó¢  — | j                   rt        |d«      }|j                  d«      }t        |t        «      r8||k7  r1t        j                  |t
        j                  |j                  ¬«      }nd}|�œt        |t
        j                  «      r3|j                  dk(  r$|j                  d«      d|j                  d«      fv sJ ‚t        j                  t        j                  ||j                  ¬«      dd…df   |dd…ddf   k  dd«      }||z  }| j                  �| j                  |«      }| j                  �| j!                  |«      } | j"                  |fi |¤Ž\  }}| j                  �| j%                  |«      }|z  }| j'                  |«      }| j)                  |«      }| j+                  |«      }| j                   rt        |d«      }||fS )	z¢
        x: (B H L) if self.transposed else (B L H)
        state: (H N) never needed unless you know what you're doing

        Returns: same shape as x
        r¨   r   rå  Nr   rÒ  r§   rê   )r’   r   rº   rK  r�  r   r=   Úlongr¦   ÚTensorrª   rÏ   rU   r_  rb  r`  ra  rc  rd  re  rF  rf  )	r£   r   Úlengthsr”   rX   r®   rM   rè  rq  s	            r   r¯   zS4Block.forwardf  s¡  € ð �?Š?¤	¨!Ð-AÓ B˜AØ�F‰F�1‹Iˆô �gœsÔ#Ø˜!Š|ÜŸ,™, w´e·j±jÈÏÉÔR‘à�ØÐÜ˜g¤u§|¡|Ô4¸¿¹ÈÒ9JÈwÏ|É|Ð\]ËÐcdÐfg×flÑflÐmnÓfoÐbpÑOpÐpÐpÜ—;‘;œuŸ|™|¨A°g·n±nÔEÂaÈÀgÑNÐQXÒYZÐ\`ÐbfÐYfÑQgÑgÐikÐmoÓpˆDØ�D‘ˆAà�9‰9Ð Ø—‘ Ó"ˆAØ�?‰?Ð&Ø×!Ñ! !Ó$ˆAà�4—:‘:˜aÑ* 6Ñ*‰ˆˆ5ð �9‰9Ð Ø× Ñ  Ó#ˆAØ�A‘ˆAØ× Ñ  Ó#ˆØ�I‰I�a‹LˆØ×Ñ˜qÓ!ˆà�?Š?¤	¨!Ð-AÓ B˜Aà�%ˆxˆr   c                 ó<   —  | j                   j                  di |¤Ž y rU  )rc  rV  r|  s     r   rV  zS4Block.setup_step�  s   € Øˆ�
‰
×ÑÑ' Ó'r   c                 óf  — | j                   �| j                  |«      }| j                  �| j                  |«      }| j                  j                  ||«      \  }}| j                   �| j                  |«      }|z  }| j                  |«      }| j                  |«      }| j                  |«      }||fS )z®Step one time step as a recurrent model. Intended to be used during validation.

        x: (B H)
        state: (B H N)
        Returns: output (B H), state (B H N)
        )
r_  rb  r`  ra  rc  r€  rd  re  rF  rf  )r£   r   rq  rM   rè  r­  s         r   r€  zS4Block.step’  s¨   € ð �9‰9Ð Ø—‘ Ó"ˆAØ�?‰?Ð&Ø×!Ñ! !Ó$ˆAØŸ
™
Ÿ™¨¨5Ó1‰ˆˆ:Ø�9‰9Ð Ø× Ñ  Ó#ˆAØ�A‘ˆAØ× Ñ  Ó#ˆØ�I‰I�a‹LˆØ×Ñ˜qÓ!ˆØ�*ˆ}Ðr   rÒ  c                ó4   —  | j                   j                  |Ž S r   )rc  rƒ  rY  s      r   rƒ  zS4Block.default_state§  s   € ð (ˆt�z‰z×'Ñ'¨Ð5Ð5r   c                 ó   — | j                   S r   )r]  re  s    r   r‘   zS4Block.d_output¬  s   € à�|‰|Ðr   )NNNNry   NNFrê   FTr   r[  r¶   s   @r   r]  r]     s`   ø„ ñð* ØØØØØØØØØØõOóf'òR(òð* 26ô 6ð
 ñó ôr   r]  r<  )Nr   )TFNFr   )rS  )XrŒ  Úcollectionsr   Útypingr   r   r   r   r*   Ú	functoolsr   r'  ÚnumpyrÌ   Úscipyr	   Ússr   Útorch.nnr|   Útorch.nn.functionalÚ
functionalrY  Úpytorch_lightning.utilitiesr
   Úeinopsr   r   r  ri   rb   Úview_as_realrK   Úview_as_complexrL   ÚtupleÚmapr�  Ú__version__rD  rÇ  r±   ÚINFOÚLoggerr2   r+  Úextensions.kernels.cauchyr3   r  Úextensions.kernels.vandermonder4   rÓ  r$   r%   ÚpykeopsÚpykeops.torchr5   rÔ  r?   rS   r\   r_   ÚImportErrorrf   rl   rn   r‰   r–   ÚModuler˜   rÂ   rè   r´   rñ   r  r<  rG  rM  rQ  rW  rZ  r\  r’  r´  rî  rD  r@  r]  rB  r   r   Ú<module>rŽ     s½  ðñ Hå #ß 2Ó 2Û Ý Û Û Ý Û Ý ß Ð Ý 6ß $ð �<‰<€á2€Ø×Ñ€Ø×Ñ€Ù‰ˆS�%×#Ñ#×)Ñ)¨#Ó.¨r°Ð2Ó	3Ó4¸Ò?Ù5�Má&€Mð  G§L¡Lñ °W·^±^ó ñ �Ó€à ð	ÝDÝCØÐØ‡H�HÐcÔdð^
ÛÝ$Ø€KØ‡H�HÐ*Ô+òò
ò.ò,òR
,ó#òð $óbð. !%ØØØó	ô&�—	‘	ô ð0  ó+ð\ ò-ð^ &'¨e¯k©kó ð6  5§;¡;ÀdÐSVó Mð` 
Øˆq�AØ
�+‰+ØØØØØØØØØó{òzð2 �iÐ Ø˜Ð$Ú6ñ€òð ó$óô K"ˆR�Y‰Yô K"ôZqF�ô qFôhn�Iô nôbU>�Mô U>ðp ØØ
ØØñ€ôZ,ˆb�i‰iô Z,ôznˆb�i‰iõ nøðY6Ø‡K�Kð 	fôð Óûðx ò 
Ø€KÙØ�‰ð `ô	
úð
ús   ÃH Ã1)H, ÈH)È,I
É	I
