ó
    ³‹jÓ  ã                  óà   • S SK Jr  S SKrS SKrS SKrS SKrS SKJr  S SKJs  J	r
  SrSS jrSS jrS r " S S	\R                  5      r " S
 S\R                  5      rSS jrS rSS jrS rg)é    )ÚannotationsN)ÚnngH¯¼šò×z>c                ó   • [         R                  " U 5        [        R                   R                  U 5        [        R                  " U 5        [        R
                  R                  5       (       a   [        R
                  R                  U 5        g g )N)ÚrandomÚseedÚnpÚtorchÚmanual_seedÚcudaÚis_availableÚmanual_seed_all)r   s    ÚI/root/disk_3tb/earlywarning/30072026/suite5/cross_faculty_residual_moe.pyÚseed_everythingr      sQ   € Ü
‡K‚K�Ô”b—i‘i—n‘n TÔ*¬5×+<Ò+<¸TÔ+BÜ‡z�z×Ñ× Ñ ¤§¡×!;Ñ!;¸DÕ!AÐ ó    c                ó¸   • U S:X  a?  [         R                  " [         R                  R                  5       (       a  S5      $ S5      $ [         R                  " U 5      $ )NÚautor   Úcpu)r	   Údevicer   r   )Únames    r   Úchoose_devicer      sA   € ØˆVƒ|œ5Ÿ<š<´%·*±*×2IÑ2I×2KÑ2K¨ÓWÐWÐQVÓWÐWÜ�<Š<˜ÓÐr   c                ó¤   • [         R                  " U [        S[        -
  5      n [         R                  " U 5      [         R                  " U * 5      -
  $ )Né   )r	   ÚclampÚEPSÚlogÚlog1p)Úqs    r   Úlogit_tensorr      s5   € Ü‡k‚k�!”C˜œ#™Ó€AÜ�9Š9�Q‹<œŸš Q B›Ñ'Ð'r   c                  ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚResidualExperté   c           	     óö  >• [         TU ]  5         [        U5      U l        [        R
                  " [        R                  " X5      [        R                  " 5       [        R                  " U5      [        R                  " US5      5      U l	        [        R                  R                  U R                  S   R                  5        [        R                  R                  U R                  S   R                  5        g )Nr   éÿÿÿÿ)ÚsuperÚ__init__ÚfloatÚ	delta_maxr   Ú
SequentialÚLinearÚGELUÚDropoutÚnetÚinitÚzeros_ÚweightÚbias)ÚselfÚdimÚhiddenr'   ÚdropoutÚ	__class__s        €r   r%   ÚResidualExpert.__init__   s’   ø€ Ü‰ÑÔ¬%°	Ó*:˜4œ>Ü—’œrŸyšy¨Ó4´R·W²W³Y¼r¿zºzÈ'Ó?RÔSU×S\ÒS\Ð]cÐdeÓSfÓgˆŒÜ
�‰�‰�t—x‘x ‘|×*Ñ*Ô+¬B¯G©G¯N©N¸4¿8¹8ÀB¹<×;LÑ;LÕ,Mr   c                ó„   • U R                   [        R                  " U R                  U5      R	                  S5      5      -  $ )Nr#   )r'   r	   Útanhr,   Úsqueeze)r1   Úhs     r   ÚforwardÚResidualExpert.forward    s-   € Ø�~‰~œeŸjšj¨¯©°!«×)<Ñ)<¸RÓ)@ÓAÑAÐAr   )r'   r,   ©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r%   r;   Ú__static_attributes__Ú__classcell__©r5   s   @r   r    r       s   ø† õN÷Bð Br   r    c                  ó6   ^ • \ rS rSr  SU 4S jjrS rSrU =r$ )ÚResidualMoEé$   c                óä  >• [         TU ]  5         [        U5      U l        [	        U5      U l        [	        U
5      U l        [        R                  " [        U5      [        U5      5      U l	        [        U5      [        U5      -   n[        R                  " [        R                  " U[        U5      5      [        R                  " 5       [        R                  " [        U5      5      [        R                  " U	5      [        R                  " [        U5      [        U5      5      [        R                  " 5       [        R                  " [        U5      5      5      U l        [        R                   " [#        U R                  5       Vs/ s H"  n[%        [        U5      [        U5      X‰5      PM$     sn5      U l        [        R                  " [        R                  " [        U5      [)        S[        U5      S-  5      5      [        R                  " 5       [        R                  " U	5      [        R                  " [)        S[        U5      S-  5      U R                  S-   5      5      U l        g s  snf )Né    é   r   )r$   r%   ÚintÚ	n_expertsr&   r'   Úrouter_temperaturer   Ú	EmbeddingÚfaculty_embeddingr(   r)   r*   Ú	LayerNormr+   ÚencoderÚ
ModuleListÚranger    ÚexpertsÚmaxÚrouter)r1   Ú	input_dimÚn_facultiesrL   Úfaculty_dimÚencoder_hiddenÚrepresentation_dimÚexpert_hiddenr'   r4   rM   ÚdinÚ_r5   s                €r   r%   ÚResidualMoE.__init__%   sö  ø€ ä‰ÑÔ¬#¨i«.˜4œ>ÌÈiÓHX¸¼Ôqvð  xJó  rKÐY]ÔYpÜ!Ÿ|š|¬C°Ó,<¼SÀÓ=MÓNˆÔÜ�	‹Nœ3˜{Ó+Ñ+ˆÜ—]’]¤2§9¢9¨S´°^Ó1DÓ#EÄbÇgÂgÃiÔPR×P\ÒP\Ô]`ÐaoÓ]pÓPqÔrt×r|Òr|ð  ~Eó  sFÜ#%§9¢9¬S°Ó-@ÄÐEWÓAXÓ#YÔZ\×ZaÒZaÓZcÔdf×dpÒdpÔqtð  vHó  rIó  eJóKˆŒä—]’]Ôzð  AE÷  AOñ  AOô  {Pó  $Qò  {PÐuv¤N´3Ð7IÓ3JÌ3È}ÓK]Ð^gÖ$pñ  {Pñ  $Qó  RˆŒÜ—M’M¤"§)¢)¬CÐ0BÓ,CÄCÈÌ3ÐOaÓKbÐdeÑKeÓDfÓ"gÔhj×hoÒhoÓhqÔrt×r|Òr|ð  ~Eó  sFÜ"$§)¢)¬C°´3Ð7IÓ3JÈAÑ3MÓ,NÈtÏ~É~Ð^_ÑO_Ó"`óbˆ�ùò $Qs   Æ)I-c                ó,  • U R                  U5      nU R                  [        R                  " X/SS95      n[        R                  " U R
                   Vs/ s H
  of" U5      PM     snSS9nU R                  U5      [        U R                  S5      -  n[        R                  " USS9n	U	S S 2SS 24   U-  R                  SS9n
[        U5      nXº-   nUS S 2S 4   U-   nU[        R                  " U5      U	U[        R                  " U5      UU
S.$ s  snf )Nr   ©r2   ç�íµ ÷Æ°>)Úfinal_logitÚ
final_probÚweightsÚ	residualsÚcounterfactual_probÚrepresentationÚ
correction)rO   rQ   r	   ÚcatÚstackrT   rV   rU   rM   ÚsoftmaxÚsumr   Úsigmoid)r1   ÚxÚfaculty_idxr   Úer:   Úmrf   Úlogitsre   ri   Úzqrc   Úcf_logits                 r   r;   ÚResidualMoE.forward0   sû   € Ø
×
 Ñ
  Ó
-ˆØ
�,‰,”u—y’y ! ¨1Ñ-Ó
.ˆÜ—+’+¨T¯\ª\Ó:ª\¨˜q žt©\Ñ:¸qÑAˆ	Ø�{‰{˜1‹~œc $×"9Ñ"9¸$Ó?Ñ?ˆÜ—’˜f¨Ñ+ˆØšA˜a™b˜D‘M )Ñ+×0Ñ0°QÐ0Ð7ˆ
Ü˜‹?ˆØ‘MˆØ’A�d�F‘˜IÑ%ˆØ)´u·}²}À[Ó7QÐ\cØ%¼E¿MºMÈ(Ó<SÐefØ'ñ)ð 	)ùò ;s   ÁD)r'   rQ   rT   rO   rL   rV   rM   )é   é   é   é€   é@   g       @gš™™™™™¹?g      ð?r=   rD   s   @r   rF   rF   $   s   ø† ØloØOR÷	b÷)ð )r   rF   c                óV  • U R                  5       n [        R                  " U[        S[        -
  5      n[        R                  " U[        S[        -
  5      n[        R
                  " XSS9nU S S 2S 4   R                  U5      n[        R
                  " XFSS9nUS S 2S 4   U-
  n[        R                  " [        R                  " [        U 5      S4U R                  U R                  S9U/SS9[        [        U5      S5      -  n	[        R                  " U	R                  5       SS9n
X¨4$ )Nr   Únone©Ú	reduction)r   Údtypera   rb   )r&   r	   r   r   ÚFÚbinary_cross_entropyÚ	expand_asrj   ÚzerosÚlenr   r€   rU   rl   Údetach)Úyr   rg   ÚtemperatureÚpÚlqÚyyÚlkÚutilityÚzÚtargets              r   Úcapability_targetsr�   ?   sî   € Ø‡g�gƒi€A”%—+’+˜a¤ A¤c¡EÓ*�¬U¯[ª[Ð9LÌSÐQRÔSVÑQVÓ-W¨1Ü×Ò˜a¨FÑ3€BØŠˆ4ˆ�y×Ñ˜1Ó€BÜ×Ò˜a¨VÑ4€BØŠq�ˆv‰J�r‰M€GÜ‡i‚i”—’œc !›f Q˜Z¨q¯x©x¸a¿g¹gÑFÀwÐOÐTUÑVÔWZÔ[`ÐalÓ[mÐnrÓWsÑs€AÜ�=Š=˜Ÿ™›¨Ñ*€FØˆ>Ðr   c                ó
  • U S S 2SS 24   nUR                  SS9R                  S5      nXS S 2S 4   -  nUR                  5       nUR                  5       R                  S5      nX4S S 2S 4   -  R                  SS9U-  nU[        R                  " UR                  S5      5      -  R                  5       * nU[        R                  " UR                  S5      5      -  R                  SS9* nX„-  R                  5       U-  n	Xy-
  $ )Nr   ra   ç:Œ0âŽyE>r   )rm   Ú	clamp_minr†   r	   r   )
re   ÚnonnullÚmassÚrÚmdÚdenomÚmean_rÚh_meanÚh_rowÚmean_hs
             r   Ú!specialization_mutual_informationr�   J   só   € ð ’A�a‘b�D‰M€GØ	�‰˜ˆÐ	×	%Ñ	% dÓ	+€DØ’1�T�6‰lÑ€AØ‡{�{ƒ}€BØ
�&‰&‹(×
Ñ
˜TÓ
"€EØ’�4�‰j‰L×Ñ !ÐÐ$ UÑ*€FØ”E—I’I˜f×.Ñ.¨tÓ4Ó5Ñ5×:Ñ:Ó<Ð<€FØŒe�iŠi˜Ÿ™ DÓ)Ó*Ñ*×/Ñ/°AÐ/Ð6Ð
6€EØ‰H�>‰>Ó˜EÑ!€FØ‰=Ðr   c                ó`  • UR                  5       oR                  5       n[        R                  " U S   U5      n[        XU S   U5      u  p‰U S   R	                  S5      n
[        R
                  " [        R                  " U
5      USS9n[        U
5      nU
S S 2SS 24   R                  5       U S   R                  S	5      -  R                  SS
9R                  5       nU[        U5      U-  -   [        U5      U-  -
  [        U5      U-  -   nXçX¼XÙR                  5       UR                  5       S.$ )Nrc   rg   re   r’   Ú	batchmeanr~   r   rf   rJ   ra   )ÚtotalÚ
predictionrV   Úspecialization_miÚ	magnituder�   Úcapability_target)r&   r�   Ú binary_cross_entropy_with_logitsr�   r“   Úkl_divr	   r   r�   r†   Úpowrm   Úmean)Úoutputr‡   r   Úrouter_lambdaÚspecialization_lambdaÚmagnitude_lambdaÚutility_temperatureÚpredr�   r�   ÚwrV   ÚmiÚmagr    s                  r   Úresidual_moe_lossr²   Y   s,  € Ø‡g�gƒi€A—'‘'“)�Ü	
×	+Ò	+¨F°=Ñ,AÀ!Ó	D€DÜ% a¨&Ð1FÑ*GÐH[Ó\�N€FØˆYÑ×!Ñ! $Ó'€AÜ�8Š8”E—I’I˜a“L °+Ñ>€FÜ(¨Ó+€BØ	
Š1ˆQ‰Rˆ4‰�‰Ó	˜& Ñ-×1Ñ1°!Ó4Ñ	4×9Ñ9¸aÐ9Ð@×EÑEÓG€CØ
Œu�]Ó# FÑ*Ñ
*¬5Ð1FÓ+GÈÑ+JÑ
JÌ5ÐQaÓKbÐcfÑKfÑ
f€EØ°VÐ_b×m{Ñm{Óm}ð  SY÷  S`ñ  S`ó  Sbñ  cð  cr   c           	     óz  • [         R                  " U [        5      n[         R                  " U[        5      nU[         R                  " [         R                  " USS5      5      -  R                  SS9* nU[        [        R                  " UR                  S   5      S5      -  n[        US S 2S4   R                  5       5      [        UR                  5       5      [        UR                  5       5      [        UR                  SS9R                  5       5      S.n[        UR                  S   S-
  5       H�  n[        US S 2US-   4   R                  5       5      USUS-    3'   [        US S 2U4   R                  5       5      USUS-    3'   [        US S 2U4   R                  5       5      USUS-    3'   Mƒ     U$ )	Ngê-�™—q=r   )Úaxisr   )Úmean_null_weightÚmean_entropyÚnormalized_entropyÚmean_max_weightÚmean_weight_expert_Úmean_residual_expert_Ústd_residual_expert_)r   Úasarrayr&   r   Úcliprm   rU   ÚmathÚshaper¨   rS   Ústd)re   rf   r¯   r–   ÚentÚnormÚoutÚjs           r   Úrouter_diagnosticsrÅ   e   sl  € Ü‡j‚j�œÓ€A¤"§*¢*¨Y´uÓ"= Ø
ŒB�FŠF”2—7’7˜1˜U 1Ó%Ó&Ñ
&×	+Ñ	+°Ð	+Ð	3Ð3€CØ	ŒS”—’˜!Ÿ'™' !™*Ó% eÓ,Ñ	,€DÜ! !¢A a C¡&§+¡+£-Ó0ÄÀcÇhÁhÃjÓ@QÔglÐmq×mvÑmvÓmxÓgyÜ  §¡¨A  ×!3Ñ!3Ó!5Ó6ñ	8€Cä�1—7‘7˜1‘:˜a‘<Ö ˆÜ).¨q²°1°Q±3°©x¯}©}«Ó)?ˆÐ! ! A¡# Ð'Ñ(Ü+0°²1°Q°3±·±³Ó+?ˆÐ# A a¡C 5Ð)Ñ*Ü*/°²!°A°#±·
±
³Ó*=ˆÐ" 1 Q¡3 %Ð(Ó)ñ !ð €Jr   )é*   )r   )ç      Ð?)g      à?ç{®Gáz„?rÈ   rÇ   )Ú
__future__r   r¾   r   Únumpyr   r	   r   Útorch.nn.functionalÚ
functionalr�   r   r   r   r   ÚModuler    rF   r�   r�   r²   rÅ   © r   r   Ú<module>rÏ      sf   ðå "ß Û Û Ý ß Ð à€ôBô
ò
(ô
B�R—Y‘Yô Bô)�"—)‘)ô )ô6òô	có
r   