§
    jP„j¥  ã                   óX  — d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	m
Z
 ddl	mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ej        Z ej        ddd¬¦  «         d„ Zd„ Zedk    r3 ej        ¦   «           ej        g d¢¦  «          ej        e¦  «         dS dS )z/TensorFlow Model Garden Vision training driver.é    )Úapp)Úflags)ÚloggingN)Údistribute_utils)Útask_factory)Ú	train_lib)Útrain_utils)Úperformance)Úregistry_imports)Úsummary_managerÚenable_async_checkpointingTz>A boolean indicating whether to enable async checkpoint saving)ÚdefaultÚhelpc                 ó|  — d}|�r6d}	 t          j        | j        j        | j        j        | j        j        | j        j        ¬¦  «        }|                     ¦   «         5  t          j	        | j
        |¬¦  «        }ddd¦  «         n# 1 swxY w Y   d}t          j        ||t          j        | |dt          j        | |¬¦  «        t          j        ¬¦  «         d}n^# t$          j        j        $ rG}|r8|j        r1|                     ¦   «          t/          j        d|j        ¦  «         d}n|d‚Y d}~nd}~ww xY w|�°4dS dS )	zDRuns experiment and tries to reconnect when encounting a preemption.TN)Údistribution_strategyÚall_reduce_algÚnum_gpusÚtpu_address)Úlogging_dir)ÚparamsÚ	model_dir)r   ÚtaskÚmoder   r   r   Úeval_summary_managerr   FzaSome TPU workers had been preempted (message: %s), retarting training from the last checkpoint...)r   Úget_distribution_strategyÚruntimer   r   r   ÚtpuÚscoper   Úget_taskr   r   Úrun_experimentÚFLAGSr   r   Ú maybe_build_eval_summary_managerr   ÚtfÚerrorsÚOpErrorÚpreemption_messageÚblock_until_worker_exitr   Úinfo)r   r   Úkeep_trainingÚpreemption_watcherr   r   Úes          úZ/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/official/vision/train.pyÚ(_run_experiment_with_preemption_recoveryr-   )   sê  € à€-Øñ %ØÐð#Ý.ÔHØ &¤Ô DØœÔ6Ø”>Ô*Ø”nÔ(ð	*ñ *ô *Ðð
 !×&Ò&Ñ(Ô(ð Ið IÝÔ$ V¤[¸iÐHÑHÔHˆðIð Ið Iñ Iô Ið Ið Ið Ið Ið Ið Iøøøð Ið Ið Ið Ið  Ðõ ÔØ 5ØÝŒzØØØÝ.ÔOØ yð ñ  ô  õ &+Ô%Eðñ ô ð ð €m€møÝŒ9Ôð 	ð 	ð 	Ø	ð Ð 2Ô Eð Ø×2Ò2Ñ4Ô4Ð4ÝŒð=àÔ1ñ	3ô 	3ð 	3ð ˆˆà�TÐð 	ˆˆˆˆøøøøð	øøøð9 	ñ %ð %ð %ð %ð %s=   ‰AC ÁBÁ9C ÂB	Â	C ÂB	ÂAC ÃD4Ã-=D/Ä/D4c                 óš  — t          j        t          j        t          j        ¦  «         t          j        t          ¦  «        }t          j        }dt          j        v rt          j	        ||¦  «         |j
        j        rt          j        |j
        j        ¦  «         t          ||¦  «         t          j        t          j        |¦  «         d S )NÚtrain)ÚginÚparse_config_files_and_bindingsr!   Úgin_fileÚ
gin_paramsr	   Úparse_configurationr   r   Úserialize_configr   Úmixed_precision_dtyper
   Úset_mixed_precision_policyr-   Úsave_gin_config)Ú_r   r   s      r,   Úmainr:   T   s¤   € ÝÔ%¥e¤nµeÔ6FÑGÔGÐGÝÔ*­5Ñ1Ô1€&ÝŒo€)Ø•”
ÐÐõ Ô  ¨Ñ3Ô3Ð3ð „^Ô)ð QÝÔ*¨6¬>Ô+OÑPÔPÐPå*¨6°9Ñ=Ô=Ð=ÝÔ�eœj¨)Ñ4Ô4Ð4Ð4Ð4ó    Ú__main__)Ú
experimentr   r   )Ú__doc__Úabslr   r   r   r0   Ú
tensorflowr#   Útf_kerasÚofficial.commonr   Ú	tfm_flagsÚofficial.corer   r   r	   Úofficial.modelingr
   Úofficial.visionr   Úofficial.vision.utilsr   r!   ÚDEFINE_boolr-   r:   Ú__name__Údefine_flagsÚmark_flags_as_requiredÚrun© r;   r,   ú<module>rN      s˜  ðð 6Ð 5à Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 
€
€
€
Ø !Ð !Ð !Ð !Ð !Ð !Ð !Ð !à ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø .Ð .Ð .Ð .Ð .Ð .Ø &Ð &Ð &Ð &Ð &Ð &Ø #Ð #Ð #Ð #Ð #Ð #Ø %Ð %Ð %Ð %Ð %Ð %Ø )Ð )Ð )Ð )Ð )Ð )Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1ð 	Œ€à €Ô Ø ØØ	IðKñ Kô Kð Kð(ð (ð (ðV5ð 5ð 5ð& ˆzÒÐØ€)ÔÑÔÐØ€%ÔÐBÐBÐBÑCÔCÐCØ	€#„'ˆ$�-„-€-€-€-ð Ðr;   