§
    iP„j0  ã                   óÆ   — d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	 e	j
        Z
ej        Z G d„ dej        j        j        ¦  «        Zej        	 	 	 	 dd„¦   «         ZdS )z5Legacy functions and classes related to optimization.é    )ÚloggingN)Úlamb)Úlegacy_adamwc                   ó4   ‡ — e Zd ZdZ	 	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚWarmUpzBApplies a warmup schedule on a given learning rate decay schedule.ç      ð?Nc                 óœ   •— t          t          | ¦  «                             ¦   «          || _        || _        || _        || _        || _        d S ©N)Úsuperr   Ú__init__Úinitial_learning_rateÚwarmup_stepsÚpowerÚdecay_schedule_fnÚname)Úselfr   r   r   r   r   Ú	__class__s         €ú^/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/official/nlp/optimization.pyr   zWarmUp.__init__   sK   ø€ õ 
�&�$ÑÔ× Ò Ñ"Ô"Ð"Ø!6€DÔØ$€DÔØ€D„JØ.€DÔØ€D„I€I€Ió    c                 ó¢  ‡ ‡‡— t          j        ‰ j        pd¦  «        5 }t          j        ‰t           j        ¦  «        }t          j        ‰ j        t           j        ¦  «        }||z  }‰ j        t           j                             |‰ j	        ¦  «        z  Št          j
        ||k     ˆfd„ˆ ˆfd„|¬¦  «        cd d d ¦  «         S # 1 swxY w Y   d S )Nr   c                  ó   •— ‰ S r
   © )Úwarmup_learning_rates   €r   ú<lambda>z!WarmUp.__call__.<locals>.<lambda>8   s   ø€ Ð&€ r   c                  ó.   •— ‰                       ‰¦  «        S r
   )r   )r   Ústeps   €€r   r   z!WarmUp.__call__.<locals>.<lambda>9   s   ø€ �$×(Ò(¨Ñ.Ô.€ r   )r   )ÚtfÚ
name_scoper   ÚcastÚfloat32r   r   ÚmathÚpowr   Úcond)r   r   r   Úglobal_step_floatÚwarmup_steps_floatÚwarmup_percent_doner   s   ``    @r   Ú__call__zWarmUp.__call__,   s  øøø€ Ý	Œ�t”yÐ, HÑ	-Ô	-ð °õ œ' $­¬
Ñ3Ô3ÐÝœ7 4Ô#4µb´jÑAÔAÐØ-Ð0BÑBÐà
Ô
$Ý
Œ'�+Š+Ð)¨4¬:Ñ
6Ô
6ñ7ð õ ŒWØ
Ð0Ò
0Ø
&Ð
&Ð
&Ð
&Ø
.Ð
.Ð
.Ð
.Ð
.Øð	ñ ô ðð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð s   ŸBCÃCÃCc                 óD   — | j         | j        | j        | j        | j        dœS )N©r   r   r   r   r   r)   )r   s    r   Ú
get_configzWarmUp.get_config<   s-   € à!%Ô!;Ø!Ô3ØÔ)Ø”Ø”	ðð ð r   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r'   r*   Ú__classcell__)r   s   @r   r   r      si   ø€ € € € € ØJÐJð Øðð ð ð ð ð ðð ð ð ð ð ð ð ð ð r   r   ç        ÚadamwçÍÌÌÌÌÌì?r   c           	      óf  — t           j        j                             | |||¬¦  «        }|rt	          | ||¬¦  «        }|dk    r,t          j        d¦  «         t          |d|ddg d¢¬	¦  «        }nB|d
k    r,t          j        d¦  «         t          |d|ddg d¢¬	¦  «        }nt          d|¦  «        ‚|S )z1Creates an optimizer with learning rate schedule.)r   Údecay_stepsÚend_learning_rater   )r   r   r   r1   zusing Adamw optimizerg{®Gáz„?g+‡ÙÎ÷ï?g�íµ ÷Æ°>)Ú	LayerNormÚ
layer_normÚbias)Úlearning_rateÚweight_decay_rateÚbeta_1Úbeta_2ÚepsilonÚexclude_from_weight_decayr   zusing Lamb optimizerzUnsupported optimizer type: )
Útf_kerasÚ
optimizersÚ	schedulesÚPolynomialDecayr   r   ÚinfoÚAdamWeightDecayÚLAMBÚ
ValueError)	Úinit_lrÚnum_train_stepsÚnum_warmup_stepsÚend_lrÚoptimizer_typer;   Ú
poly_powerÚlr_scheduleÚ	optimizers	            r   Úcreate_optimizerrO   F   s  € õ Ô#Ô-×=Ò=Ø#Ø!ØØð	 >ñ ô €+ð
 ð 'ÝØ%Ø%Ø%ð'ñ 'ô '€Kð
 �wÒÐÝ„LÐ(Ñ)Ô)Ð)ÝØ!ØØØØØ"EÐ"EÐ"EðGñ Gô G€I€Ið ˜ÒÐÝ„LÐ'Ñ(Ô(Ð(ÝØ!ØØØØØ"EÐ"EÐ"Eðñ ô €I€Iõ Ð3°^Ñ
DÔ
DÐDà	Ðr   )r0   r1   r2   r   )r.   Úabslr   ÚginÚ
tensorflowr   r?   Úofficial.modeling.optimizationr   r   rD   rE   r@   rA   ÚLearningRateScheduler   ÚconfigurablerO   r   r   r   ú<module>rV      sã   ðð <Ð ;à Ð Ð Ð Ð Ð Ø 
€
€
€
Ø !Ð !Ð !Ð !Ð !Ð !Ð !Ð !à /Ð /Ð /Ð /Ð /Ð /Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7àÔ.€Ø„y€ð'ð 'ð 'ð 'ð 'ˆXÔ Ô*Ô?ñ 'ô 'ð 'ðT Ôð  Ø$+ØØ #ð*ð *ð *ñ Ôð*ð *ð *r   