§
    iP„j€  ã                   ó~  — d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	m
Z
 ddl	mZ ddl	mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ej        Z ej        ddd¬¦  «          ej        ddd¬¦  «         d„ Zd„ Zedk    r3 ej        ¦   «           ej        g d¢¦  «          ej        e¦  «         dS dS )zTFM common training driver.é    )Úapp)Úflags)ÚloggingN)Údistribute_utils)Úregistry_imports)Útask_factory)Ú	train_lib)Útrain_utils)Úperformance)Úcontinuous_finetune_libÚpretrain_stepsz;The number of total training steps for the pretraining job.)ÚdefaultÚhelpÚenable_async_checkpointingTz>A boolean indicating whether to enable async checkpoint savingc                 óz  — d}|�r5d}	 t          j        d| j        j        | j        j        | j        j        | j        j        dœ| j                             ¦   «         ¤Ž}|                     ¦   «         5  t          j
        | j        |¬¦  «        }ddd¦  «         n# 1 swxY w Y   d}t          j        ||t          j        | |t          j        ¬¦  «         d}n^# t"          j        j        $ rG}|r8|j        r1|                     ¦   «          t-          j        d|j        ¦  «         d}n|d‚Y d}~nd}~ww xY w|�°3dS dS )	zDRuns experiment and tries to reconnect when encounting a preemption.TN)Údistribution_strategyÚall_reduce_algÚnum_gpusÚtpu_address)Úlogging_dir)r   ÚtaskÚmodeÚparamsÚ	model_dirr   FzaSome TPU workers had been preempted (message: %s), retarting training from the last checkpoint...© )r   Úget_distribution_strategyÚruntimer   r   r   ÚtpuÚmodel_parallelismÚscoper   Úget_taskr   r	   Úrun_experimentÚFLAGSr   r   ÚtfÚerrorsÚOpErrorÚpreemption_messageÚblock_until_worker_exitr   Úinfo)r   r   Úkeep_trainingÚpreemption_watcherr   r   Úes          úW/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/official/nlp/train.pyÚ(_run_experiment_with_preemption_recoveryr.   /   sì  € à€-Øñ !ØÐðÝ.ÔHð 0Ø &¤Ô DØœÔ6Ø”>Ô*Ø”nÔ(ð	0ð 0ð
 ŒN×,Ò,Ñ.Ô.ð0ð 0Ðð !×&Ò&Ñ(Ô(ð Ið IÝÔ$ V¤[¸iÐHÑHÔHˆðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Ið  Ðõ ÔØ 5ØÝŒzØØÝ%*Ô%EðGñ Gô Gð Gð €m€møÝŒ9Ôð 	ð 	ð 	Ø	ð Ð 2Ô Eð Ø×2Ò2Ñ4Ô4Ð4ÝŒð=àÔ1ñ	3ô 	3ð 	3ð ˆˆà�TÐð 	ˆˆˆˆøøøøð	øøøð1 	ñ !ð !ð !ð !ð !s<   ‰A)C Á2BÂC ÂBÂC Â!BÂ"5C ÃD3Ã,=D.Ä.D3c                 ó  — t          j        t          j        t          j        ¦  «         t          j        t          ¦  «        }t          j        }dt          j        v rt          j	        ||¦  «         t          j        dk    r-t          j        t          j        ||t          j        ¬¦  «         n:|j        j        rt          j        |j        j        ¦  «         t#          ||¦  «         t          j        t          j        |¦  «         d S )NÚtrainÚcontinuous_train_and_eval)r   )ÚginÚparse_config_files_and_bindingsr#   Úgin_fileÚ
gin_paramsr
   Úparse_configurationr   r   Úserialize_configr   Úrun_continuous_finetuner   r   Úmixed_precision_dtyper   Úset_mixed_precision_policyr.   Úsave_gin_config)Ú_r   r   s      r-   Úmainr=   V   sê   € ÝÔ%¥e¤nµeÔ6FÑGÔGÐGÝÔ*­5Ñ1Ô1€&ÝŒo€)Ø•”
ÐÐõ Ô  ¨Ñ3Ô3Ð3å
„ZÐ.Ò.Ð.ÝÔ3ÝŒ
�F˜IµeÔ6JðLñ Lô Lð Lð Lð „~Ô+ð 0ÝÔ,Ø
Œ.Ô
.ñ0ô 0ð 0å,¨V°YÑ?Ô?Ð?åÔ�eœj¨)Ñ4Ô4Ð4Ð4Ð4ó    Ú__main__)Ú
experimentr   r   )Ú__doc__Úabslr   r   r   r2   Ú
tensorflowr$   Útf_kerasÚofficial.commonr   r   Ú	tfm_flagsÚofficial.corer   r	   r
   Úofficial.modelingr   Úofficial.nlpr   r#   ÚDEFINE_integerÚDEFINE_boolr.   r=   Ú__name__Údefine_flagsÚmark_flags_as_requiredÚrunr   r>   r-   ú<module>rP      sÁ  ðð "Ð !à Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 
€
€
€
Ø !Ð !Ð !Ð !Ð !Ð !Ð !Ð !à ,Ð ,Ð ,Ð ,Ð ,Ð ,à ,Ð ,Ð ,Ð ,Ð ,Ð ,à .Ð .Ð .Ð .Ð .Ð .Ø &Ð &Ð &Ð &Ð &Ð &Ø #Ð #Ð #Ð #Ð #Ð #Ø %Ð %Ð %Ð %Ð %Ð %Ø )Ð )Ð )Ð )Ð )Ð )Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0àŒ€à €Ô ØØØ	FðHñ Hô Hð Hð
 €Ô Ø ØØ	IðKñ Kô Kð Kð$ð $ð $ðN5ð 5ð 5ð2 ˆzÒÐØ€)ÔÑÔÐØ€%ÔÐBÐBÐBÑCÔCÐCØ	€#„'ˆ$�-„-€-€-€-ð Ðr>   