ó
    þ°Žjó*  ã                  ó.  • S SK Jr  S SKrS SKrS SKJr  S SKrS SKrS SKJ	r	  S SK
J	s  Jr  SrSS jrSS jrS r " S	 S
\	R"                  5      r " S S\	R"                  5      r " S S\	R"                  5      rSS jrS r    SS jrS rS rSS jrg)é    )ÚannotationsN)ÚSequence)ÚnngH¯¼šò×z>c                ó   • [         R                  " U 5        [        R                   R                  U 5        [        R                  " U 5        [        R
                  R                  5       (       a   [        R
                  R                  U 5        g g ©N)ÚrandomÚseedÚnpÚtorchÚmanual_seedÚcudaÚis_availableÚmanual_seed_all)r	   s    ÚI/root/disk_3tb/earlywarning/30072026/suite10/suite7_shared_private_moe.pyÚseed_everythingr      sV   € Ü
‡K‚K�ÔÜ‡I�I‡N�N�4ÔÜ	×Ò�dÔÜ‡z�z×Ñ× Ñ Ü�
‰
×"Ñ" 4Õ(ð !ó    c                ó¸   • U S:X  a?  [         R                  " [         R                  R                  5       (       a  S5      $ S5      $ [         R                  " U 5      $ )NÚautor   Úcpu)r   Údevicer   r   )Únames    r   Úchoose_devicer      sA   € Øˆvƒ~Ü�|Š|¤e§j¡j×&=Ñ&=×&?Ñ&?˜FÓKÐKÀUÓKÐKÜ�<Š<˜ÓÐr   c                ó¤   • [         R                  " U [        S[        -
  5      n [         R                  " U 5      [         R                  " U * 5      -
  $ )Né   )r   ÚclampÚEPSÚlogÚlog1p)Úqs    r   Úlogit_tensorr       s5   € Ü�Š�A”s˜A¤™GÓ$€AÜ�9Š9�Q‹<œ%Ÿ+š+ q b›/Ñ)Ð)r   c                  ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚResidualExperté#   zExact Suite5 residual expert.c           	     óö  >• [         TU ]  5         [        U5      U l        [        R
                  " [        R                  " X5      [        R                  " 5       [        R                  " U5      [        R                  " US5      5      U l	        [        R                  R                  U R                  S   R                  5        [        R                  R                  U R                  S   R                  5        g )Nr   éÿÿÿÿ)ÚsuperÚ__init__ÚfloatÚ	delta_maxr   Ú
SequentialÚLinearÚGELUÚDropoutÚnetÚinitÚzeros_ÚweightÚbias)ÚselfÚdimÚhiddenr)   ÚdropoutÚ	__class__s        €r   r'   ÚResidualExpert.__init__&   s—   ø€ Ü‰ÑÔÜ˜yÓ)ˆŒÜ—=’=Ü�IŠI�cÓ"Ü�GŠG‹IÜ�JŠJ�wÓÜ�IŠI�f˜aÓ ó	
ˆŒô 	�‰�‰�t—x‘x ‘|×*Ñ*Ô+Ü
�‰�‰�t—x‘x ‘|×(Ñ(Õ)r   c                ó„   • U R                   [        R                  " U R                  U5      R	                  S5      5      -  $ )Nr%   )r)   r   Útanhr.   Úsqueeze©r3   Úhs     r   ÚforwardÚResidualExpert.forward2   s-   € Ø�~‰~¤§
¢
¨4¯8©8°A«;×+>Ñ+>¸rÓ+BÓ CÑCÐCr   )r)   r.   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r'   r>   Ú__static_attributes__Ú__classcell__©r7   s   @r   r"   r"   #   s   ø† Ù'õ
*÷Dð Dr   r"   c                  ó>   ^ • \ rS rSrSrSSU 4S jjjrSS jrSrU =r$ )	ÚFacultyAdapteré6   aÛ  Small residual bottleneck adapter.

Only the output projection is zero initialized. This guarantees A(h)=0 at
initialization while retaining a normally initialized feature extractor in
the down projection. In isolation the output projection can receive gradient
immediately. In Suite7, the residual experts also start with zero output
layers, so the first minibatch activates that shared residual path; the
adapter up projection then learns, followed by the down projection.
c                ó"  >• [         TU ]  5         [        R                  " [	        U5      [	        U5      5      U l        [        R                  " 5       U l        [        R                  " [        U5      5      U l
        [        R                  " [	        U5      [	        U5      5      U l        [        R                  R                  U R                  R                  5        [        R                  R                  U R                  R                  5        g r   )r&   r'   r   r+   ÚintÚdownr,   Ú
activationr-   r(   r6   Úupr/   r0   r1   r2   )r3   r4   Úbottleneck_dimr6   r7   s       €r   r'   ÚFacultyAdapter.__init__A   s•   ø€ Ü‰ÑÔÜ—I’Iœc #›h¬¨NÓ(;Ó<ˆŒ	ÜŸ'š'›)ˆŒÜ—z’z¤%¨£.Ó1ˆŒÜ—)’)œC Ó/´°S³Ó:ˆŒÜ
�‰�‰�t—w‘w—~‘~Ô&Ü
�‰�‰�t—w‘w—|‘|Õ$r   c           	     ó~   • U R                  U R                  U R                  U R                  U5      5      5      5      $ r   )rP   r6   rO   rN   r<   s     r   r>   ÚFacultyAdapter.forwardJ   s+   € Ø�w‰w�t—|‘| D§O¡O°D·I±I¸a³LÓ$AÓBÓCÐCr   )rO   rN   r6   rP   )ç        )r4   rM   rQ   rM   r6   r(   )r=   útorch.TensorÚreturnrV   r@   rH   s   @r   rJ   rJ   6   s   ø† ñ÷%ñ %÷Dò Dr   rJ   c                  óV   ^ • \ rS rSrSr           SU 4S jjrSS jrS rSrU =r	$ )	ÚSharedPrivateResidualMoEéN   a4  Suite5 Residual-MoE plus a faculty-private residual representation adapter.

IMPORTANT: shared Suite5-equivalent modules are instantiated first and in the
same order as Suite5. Adapters are instantiated afterwards. With the same
torch seed, this preserves initialization of embedding/encoder/experts/router.
c                óÐ  >• [         TU ]  5         [        U5      U l        [	        U5      U l        [	        U
5      U l        [        U5      U l        [	        U5      U l        [        R                  " [        U5      [        U5      5      U l        [        U5      [        U5      -   n[        R                  " [        R                  " U[        U5      5      [        R                  " 5       [        R                  " [        U5      5      [        R                   " U	5      [        R                  " [        U5      [        U5      5      [        R                  " 5       [        R                  " [        U5      5      5      U l        [        R$                  " ['        U R                  5       Vs/ s H"  n[)        [        U5      [        U5      X‰5      PM$     sn5      U l        [-        S[        U5      S-  5      n[        R                  " [        R                  " [        U5      U5      [        R                  " 5       [        R                   " U	5      [        R                  " UU R                  S-   5      5      U l        [        R$                  " ['        [        U5      5       Vs/ s H,  n[1        [        U5      U R                  [	        U5      5      PM.     sn5      U l        g s  snf s  snf )Né    é   r   )r&   r'   rM   Ú	n_expertsr(   r)   Úrouter_temperatureÚadapter_dimÚadapter_scaler   Ú	EmbeddingÚfaculty_embeddingr*   r+   r,   Ú	LayerNormr-   ÚencoderÚ
ModuleListÚranger"   ÚexpertsÚmaxÚrouterrJ   Úadapters)r3   Ú	input_dimÚn_facultiesr^   Úfaculty_dimÚencoder_hiddenÚrepresentation_dimÚexpert_hiddenr)   r6   r_   r`   Úadapter_dropoutra   ÚdinÚ_Úrouter_hiddenr7   s                    €r   r'   Ú!SharedPrivateResidualMoE.__init__V   s  ø€ ô  	‰ÑÔÜ˜Y›ˆŒÜ˜yÓ)ˆŒÜ"'Ð(:Ó";ˆÔÜ˜{Ó+ˆÔÜ" =Ó1ˆÔô "$§¢¬c°+Ó.>ÄÀKÓ@PÓ!QˆÔÜ�)‹nœs ;Ó/Ñ/ˆÜ—}’}Ü�IŠI�cœ3˜~Ó.Ó/Ü�GŠG‹IÜ�LŠLœ˜^Ó,Ó-Ü�JŠJ�wÓÜ�IŠI”c˜.Ó)¬3Ð/AÓ+BÓCÜ�GŠG‹IÜ�LŠLœÐ/Ó0Ó1ó
ˆŒô —}’}ô
 ˜tŸ~™~Ô.ó	ò /�Aô ÜÐ*Ó+¬S°Ó-?Àöñ /ñ	ó
ˆŒô ˜B¤Ð$6Ó 7¸1Ñ <Ó=ˆÜ—m’mÜ�IŠI”cÐ,Ó-¨}Ó=Ü�GŠG‹IÜ�JŠJ�wÓÜ�IŠI�m T§^¡^°aÑ%7Ó8ó	
ˆŒô Ÿšô
 œs ;Ó/Ô0ó	ò 1�Aô ÜÐ*Ó+¨T×-=Ñ-=¼uÀ_Ó?Uöñ 1ñ	ó
ˆ�ùò!ùò"s   Æ&)KÊ3K#c                óü   • [         R                  " U5      n[         R                  " U5       H9  n[        UR	                  5       5      nX$:H  nU R
                  U   " X   5      X6'   M;     XR                  U-  -   nXs4$ r   )r   Ú
zeros_likeÚuniquerM   Úitemrk   ra   )r3   r=   Úfaculty_idxÚadapter_outÚfiÚidxÚmaskÚadapteds           r   Úapply_faculty_adaptersÚ/SharedPrivateResidualMoE.apply_faculty_adapters“   st   € Ü×&Ò& qÓ)ˆô —,’,˜{Ö+ˆBÜ�b—g‘g“i“.ˆCØÑ$ˆDØ $§¡¨cÒ 2°1±7Ó ;ˆKÓñ ,ð ×(Ñ(¨;Ñ6Ñ6ˆØÐ#Ð#r   c                óX  • U R                  U5      nU R                  [        R                  " X/SS95      nU R	                  XR5      u  pg[        R
                  " U R                   Vs/ s H
  oˆ" U5      PM     snSS9n	U R                  U5      [        U R                  S5      -  n
[        R                  " U
SS9nUS S 2SS 24   U	-  R                  SS9n[        U5      nXÜ-   nUS S 2S 4   U	-   nU[        R                  " U5      UU	[        R                  " U5      UUUUUS.
$ s  snf )Nr   ©r4   ç�íµ ÷Æ°>)
Úfinal_logitÚ
final_probÚweightsÚ	residualsÚcounterfactual_probÚshared_representationÚadapter_outputÚadapted_representationÚrepresentationÚ
correction)rc   re   r   Úcatr�   Ústackrh   rj   ri   r_   ÚsoftmaxÚsumr    Úsigmoid)r3   Úxr{   r   Úer=   Ú	h_adaptedr|   Úmr‰   Úlogitsrˆ   r�   Úzqr†   Úcf_logits                   r   r>   Ú SharedPrivateResidualMoE.forwardž   s  € Ø×"Ñ" ;Ó/ˆØ�L‰LœŸš A 6¨qÑ1Ó2ˆØ!%×!<Ñ!<¸QÓ!LÑˆ	Ü—K’K°t·|²|Ó D²|°!  9¦±|Ñ DÈ!ÑLˆ	Ø—‘˜YÓ'¬#¨d×.EÑ.EÀtÓ*LÑLˆÜ—-’- ¨AÑ.ˆØša ¡˜e‘n yÑ0×5Ñ5¸!Ð5Ð<ˆ
Ü˜!‹_ˆØ‘oˆØ’a˜�g‘; Ñ*ˆà&ÜŸ-š-¨Ó4ØØ"Ü#(§=¢=°Ó#:Ø%&Ø)Ø&/à'Ø$ñ
ð 	
ùò !Es   Á(D')
r`   ra   rk   r)   re   rh   rc   r^   rj   r_   )é   é   é   é€   é@   g       @gš™™™™™¹?ç      ð?é   rU   r¢   )r=   rV   r{   rV   )
rA   rB   rC   rD   rE   r'   r�   r>   rF   rG   rH   s   @r   rY   rY   N   s@   ø† ñð ØØØØØØØØØØ÷;
ôz	$÷
ð 
r   rY   c                óV  • U R                  5       n [        R                  " U[        S[        -
  5      n[        R                  " U[        S[        -
  5      n[        R
                  " XSS9nU S S 2S 4   R                  U5      n[        R
                  " XFSS9nUS S 2S 4   U-
  n[        R                  " [        R                  " [        U 5      S4U R                  U R                  S9U/SS9[        [        U5      S5      -  n	[        R                  " U	R                  5       SS9n
X¨4$ )Nr   Únone©Ú	reduction)r   Údtyper„   r…   )r(   r   r   r   ÚFÚbinary_cross_entropyÚ	expand_asr�   ÚzerosÚlenr   r¨   ri   r’   Údetach)Úyr   rŠ   ÚtemperatureÚpÚlqÚyyÚlkÚutilityÚzÚtargets              r   Úcapability_targetsr¸   ¸   sò   € Ø	�‰‹	€AÜ�Š�A”s˜A¤™GÓ$€AÜ�ŠÐ'¬¨a´#©gÓ6€AÜ	
×	Ò	 °Ñ	7€BØ	
Š1ˆdˆ7‰×	Ñ	˜aÓ	 €BÜ	
×	Ò	 °Ñ	8€BØ’�D�‰k˜BÑ€GÜ�	Š	Ü	�Š”c˜!“f˜a�[¨¯©¸¿¹Ñ	AÀ7ÐKÐQRñ	äŒE�+Ó Ó%ñ	&€Aô �]Š]˜1Ÿ8™8›:¨1Ñ-€FØˆ?Ðr   c                ó
  • U S S 2SS 24   nUR                  SS9R                  S5      nXS S 2S 4   -  nUR                  5       nUR                  5       R                  S5      nX4S S 2S 4   -  R                  SS9U-  nU[        R                  " UR                  S5      5      -  R                  5       * nU[        R                  " UR                  S5      5      -  R                  SS9* nX„-  R                  5       U-  n	Xy-
  $ )Nr   r„   ç:Œ0âŽyE>r   )r“   Ú	clamp_minr®   r   r   )
rˆ   ÚnonnullÚmassÚrÚmdÚdenomÚmean_rÚh_meanÚh_rowÚmean_hs
             r   Ú!specialization_mutual_informationrÅ   Ç   só   € Ø’a˜™�e‰n€GØ�;‰;˜1ˆ;Ð×'Ñ'¨Ó-€DØ’q˜$�w‘-Ñ€AØ	�‰‹€BØ�F‰F‹H×Ñ˜tÓ$€EØ’Q˜�W‘+‰o×"Ñ" qÐ"Ð)¨EÑ1€FØœŸ	š	 &×"2Ñ"2°4Ó"8Ó9Ñ9×>Ñ>Ó@Ð@€FØ”%—)’)˜AŸK™K¨Ó-Ó.Ñ.×3Ñ3¸Ð3Ð:Ð:€EØ‰j×ÑÓ %Ñ'€FØ‰?Ðr   c                óh  • UR                  5       nUR                  5       n[        R                  " U S   U5      n[        XU S   U5      u  p‰U S   R	                  S5      n
[        R
                  " [        R                  " U
5      USS9n[        U
5      nU
S S 2SS 24   R                  5       U S   R                  S	5      -  R                  SS
9R                  5       nU[        U5      U-  -   [        U5      U-  -
  [        U5      U-  -   nUUUUUU	R                  5       UR                  5       S.$ )Nr†   rŠ   rˆ   rº   Ú	batchmeanr¦   r   r‰   r]   r„   )ÚtotalÚ
predictionrj   Úspecialization_miÚ	magnituderµ   Úcapability_target)r(   r©   Ú binary_cross_entropy_with_logitsr¸   r»   Úkl_divr   r   rÅ   r®   Úpowr“   Úmean)Úoutputr¯   r   Úrouter_lambdaÚspecialization_lambdaÚmagnitude_lambdaÚutility_temperatureÚpredr·   rµ   Úwrj   ÚmiÚmagrÈ   s                  r   Úresidual_moe_lossrÚ   Ô   s7  € ð 	
�‰‹	€AØ	�‰‹	€AÜ×-Ò-¨f°]Ñ.CÀQÓG€DÜ(Ø	ˆfÐ*Ñ+Ð-@ó�O€Fð 	ˆyÑ×#Ñ# DÓ)€AÜ�XŠX”e—i’i “l F°kÑB€FÜ	*¨1Ó	-€Bà	Š!ˆQ‰Rˆ%‰�‰Ó˜F ;Ñ/×3Ñ3°AÓ6Ñ6ß	�cˆa€c€j—‘“ð ð 	Ü
�Ó
 Ñ
'ñ	(ä
Ð%Ó
&¨Ñ
+ñ	,ô Ð Ó
! CÑ
'ñ	(ð 
ð ØØØØØ—>‘>Ó#Ø#Ÿ]™]›_ñð r   c           	     óz  • [         R                  " U [        5      n[         R                  " U[        5      nU[         R                  " [         R                  " USS5      5      -  R                  SS9* nU[        [        R                  " UR                  S   5      S5      -  n[        US S 2S4   R                  5       5      [        UR                  5       5      [        UR                  5       5      [        UR                  SS9R                  5       5      S.n[        UR                  S   S-
  5       H�  n[        US S 2US-   4   R                  5       5      USUS-    3'   [        US S 2U4   R                  5       5      USUS-    3'   [        US S 2U4   R                  5       5      USUS-    3'   Mƒ     U$ )	Nçê-�™—q=r   ©Úaxisr   )Úmean_null_weightÚmean_entropyÚnormalized_entropyÚmean_max_weightÚmean_weight_expert_Úmean_residual_expert_Ústd_residual_expert_)r
   Úasarrayr(   r   Úclipr“   ri   ÚmathÚshaperÐ   rg   Ústd)rˆ   r‰   r×   r¾   ÚentÚnormÚoutÚjs           r   Úrouter_diagnosticsrï   ú   so  € Ü
�
Š
�7œEÓ"€AÜ
�
Š
�9œeÓ$€AØ”—’”r—w’w˜q %¨Ó+Ó,Ñ,×1Ñ1°qÐ1Ð9Ð
9€CØ””T—X’X˜aŸg™g a™jÓ)¨5Ó1Ñ1€Dä! !¢A q D¡'§,¡,£.Ó1Ü˜cŸh™h›jÓ)Ü# D§I¡I£KÓ0Ü  §¡¨A  ×!3Ñ!3Ó!5Ó6ñ	€Cô �1—7‘7˜1‘: ‘>Ö"ˆÜ+0°²1°a¸!±e°8±×1AÑ1AÓ1CÓ+DˆÐ! ! A¡# Ð'Ñ(Ü-2°1²Q¸°T±7·<±<³>Ó-BˆÐ# A a¡C 5Ð)Ñ*Ü,1°!²A°q°D±'·+±+³-Ó,@ˆÐ" 1 Q¡3 %Ð(Ó)ñ #ð €Jr   c                óˆ  • [         R                  " U [        5      n[         R                  " U[        5      n[         R                  " U[        5      n[        U5      S:X  a]  [         R                  [         R                  [         R                  [         R                  [         R                  [         R                  S.$ [         R
                  R                  USS9n[         R
                  R                  USS9n[         R
                  R                  USS9n[         R                  " Xx-  S5      n	[         R                  " X5-  SS9U	-  n
[        [         R                  " U5      5      [        [         R                  " U5      5      [        [         R                  " U5      5      [        [         R                  " U[         R                  " US5      -  5      5      [        [         R                  " [         R                  " U
SS5      5      5      [        [         R                  " [         R                  " U5      5      5      S.$ )Nr   )Úmean_adapter_normÚmean_shared_normÚmean_adapted_normÚmean_relative_adapter_normÚmean_representation_cosineÚmean_abs_adapter_activationr   rÝ   rÜ   g      ð¿r¢   )r
   ræ   r(   r­   ÚnanÚlinalgrì   Úmaximumr“   rÐ   rç   Úabs)r‹   rŒ   r�   r=   ÚaÚhfÚanÚhnÚhfnrÀ   Úcoss              r   Úadapter_diagnosticsr    sr  € Ü
�
Š
Ð(¬%Ó0€AÜ
�
Š
�>¤5Ó)€AÜ	�ŠÐ*¬EÓ	2€BÜ
ˆ1ƒv�ƒ{ä!#§¡Ü "§¡Ü!#§¡Ü*,¯&©&Ü*,¯&©&Ü+-¯6©6ñ
ð 	
ô 
�‰�‰˜ ˆÐ	"€BÜ	�‰�‰˜ ˆÐ	"€BÜ
�)‰)�.‰.˜ !ˆ.Ð
$€CÜ�JŠJ�r‘x Ó'€EÜ
�&Š&�‘˜aÑ
  5Ñ
(€Cä"¤2§7¢7¨2£;Ó/Ü!¤"§'¢'¨"£+Ó.Ü"¤2§7¢7¨3£<Ó0Ü&+¬B¯GªG°B¼¿ºÀBÈÓ9NÑ4NÓ,OÓ&PÜ&+¬B¯GªG´B·G²G¸CÀÀsÓ4KÓ,LÓ&MÜ',¬R¯WªW´R·V²V¸A³YÓ-?Ó'@ñð r   c                ó&  • / n[        U5       GHk  u  p4U R                  U   n[        R                  " 5          UR	                  UU[        [        R                  R                  UR                  R                  5      R                  5       5      [        [        R                  R                  UR                  R                  5      R                  5       5      [        [        R                  R                  UR                  R                  5      R                  5       5      [        [        R                  R                  UR                  R                  5      R                  5       5      S.5        S S S 5        GMn     U$ ! , (       d  f       GM‚  = f)N)ÚfacultyÚadapter_indexÚdown_weight_l2Údown_bias_l2Úup_weight_l2Ú
up_bias_l2)Ú	enumeraterk   r   Úno_gradÚappendr(   rø   Úvector_normrN   r1   r   r2   rP   )ÚmodelÚfaculty_namesÚrowsr}   r   rû   s         r   Úadapter_parameter_rowsr  (  sü   € Ø€DÜ˜m×,‰ˆØ�N‰N˜2ÑˆÜ�]Š]�_Ø�K‰Kà#Ø%'Ü&+¬E¯L©L×,DÑ,DÀQÇVÁVÇ]Á]Ó,S×,WÑ,WÓ,YÓ&ZÜ$)¬%¯,©,×*BÑ*BÀ1Ç6Á6Ç;Á;Ó*O×*SÑ*SÓ*UÓ$VÜ$)¬%¯,©,×*BÑ*BÀ1Ç4Á4Ç;Á;Ó*O×*SÑ*SÓ*UÓ$VÜ"'¬¯©×(@Ñ(@ÀÇÁÇÁÓ(K×(OÑ(OÓ(QÓ"Rñô	÷ Š_ñ -ð €K÷ �_ús   ¸D9F Æ 
F	)é*   )r   )ç      Ð?)g      à?ç{®Gáz„?r  r  )r  rY   r  zSequence[str])Ú
__future__r   rè   r   Útypingr   Únumpyr
   r   r   Útorch.nn.functionalÚ
functionalr©   r   r   r   r    ÚModuler"   rJ   rY   r¸   rÅ   rÚ   rï   r  r  © r   r   Ú<module>r     s˜   ðå "ã Û Ý ã Û Ý ß Ð à
€ô)ôò*ô
D�R—Y‘Yô Dô&D�R—Y‘Yô Dô0g
˜rŸy™yô g
ôTò
ð" ØØØô#òLò$õ8r   