§
    jP„jc  ã                   ó<  — d Z ddlZddlZddlmZmZmZmZ ddlm	Z	 ddl
mZ ddl
mZ ddlZddlZddlmZ  ej        dd	¦  «        Zd
efd„Ze	j                             eej        ¦  «         dd„Zeej        gdf         Z G d„ d¦  «        Z G d„ d¦  «        ZdS )zCProvides a `Controller` class for managing the outer training loop.é    N)ÚCallableÚIterableÚOptionalÚUnion)Úlogging)Úrunner)Úutils)Ú
monitoringz/tensorflow/api/orbitzorbit api usageÚmessagec                 óL   — t          j        | ¦  «         t          | ¦  «         dS )z<Logs `message` to the `info` log, and also prints to stdout.N)r   ÚinfoÚprint)r   s    úU/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/orbit/controller.pyÚ_logr   &   s!   € å	„,ˆwÑÔÐÝˆ�.„.€.€.€.ó    é   c                 óÊ   ‡— t          j        | ¦  «        }|                     ¦   «         }t          |¦  «        dk    r|S ˆfd„|D ¦   «         }dd                     |¦  «        z   S )zHFormats `output`, either on one line, or indented across multiple lines.é   c                 ó    •— g | ]
}d ‰z  |z   ‘ŒS )ú © )Ú.0ÚlineÚindents     €r   ú
<listcomp>z"_format_output.<locals>.<listcomp>5   s"   ø€ Ð
1Ð
1Ð
1 4ˆ3�‰<˜$ÑÐ
1Ð
1Ð
1r   ú
)ÚpprintÚpformatÚ
splitlinesÚlenÚjoin)Úoutputr   Ú	formattedÚliness    `  r   Ú_format_outputr%   /   sh   ø€ åŒn˜VÑ$Ô$€)Ø
×
Ò
Ñ
 Ô
 €%Ýˆ�Z„Z�1‚_€_ØÐØ
1Ð
1Ð
1Ð
1¨5Ð
1Ñ
1Ô
1€%Ø	�—	’	˜%Ñ Ô Ñ	 Ð r   c                   óØ  — e Zd ZdZddddddddddddddœdej        deej                 deej	                 deej
        j                 d	eee                  d
eee                  deeeeegef         f                  deej        j                 dedee         dee         dee         deej                 deej                 fd„Zd,dedefd„Zd-dedeej                 fd„Z	 	 d.dededee         deej                 fd„Z	 	 	 d/dedeeeef                  d eeg ef                  deej                 fd!„Zd0d"ee         fd#„Zd$„ Zed%„ ¦   «         Z d&efd'„Z!d,d(efd)„Z"d*„ Z#d+„ Z$dS )1Ú
Controllera  Class that controls the outer loop of model training and evaluation.

  Orbit divides training and evaluation into "inner" and "outer" loops. Inner
  loops are implemented by users in the form of `AbstractTrainer` and
  `AbstractEvaluator` subclasses, and define how to run a given number of
  training or evaluation steps. The outer loop is provided by this `Controller`,
  and interleaves calls to the user-provided inner loops with additional actions
  such as saving checkpoints, running evaluations, writing summaries, as well as
  (optionally) user provided `Action`s (see below).

  There are four top-level "outer loops" provided:

    - `train`, which trains until a specified number of global steps is reached;
    - `evaluate`, for one-off model evaluation;
    - `train_and_evaluate`, for interleaved training and evaluation;
    - `evaluate_continuously`, for monitoring a given directory and running
      evaluations on new model checkpoints.

  While this class attempts to provide out-of-the-box solutions for common
  training and evaluation use cases, the internal details and method
  implementations are also intended to be simple enough to make subclassing or
  other custom outer loop implementations easy to achieve.

  Some additional customization can be achieved by supplying `train_actions` or
  `eval_actions` when constructing the `Controller`. Actions arbitrary callables
  that are applied by the `Controller` to the output of train steps (after each
  inner loop of `steps_per_loop` steps) or an evaluation. This provides a hook
  mechanism, enabling things like reporting metrics to Vizier, model exporting,
  additional logging, etc. See the `orbit.actions` package for a small handful
  of predefined actions and some utility classes that may be useful in defining
  your own.
  NF)ÚtrainerÚ	evaluatorÚstrategyÚtrain_actionsÚeval_actionsÚsteps_per_loopÚcheckpoint_managerÚenable_async_checkpointingÚsummary_intervalÚsummary_dirÚeval_summary_dirÚsummary_managerÚeval_summary_managerÚglobal_stepr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   c                ó\  — |€|€t          d¦  «        ‚|�—|€t          d¦  «        ‚t          |¦  «        s.t          |t          ¦  «        r|dk     rt          d|› d�¦  «        ‚|
�G|
dk    rt          d|
› d	�¦  «        ‚t          |¦  «        s|
|z  dk    rt          d
|
› d|› d�¦  «        ‚t          |t          j        ¦  «        st          d¦  «        ‚|| _        || _        |pt          j         	                    ¦   «         | _
        |€dnt          |¦  «        | _        |€dnt          |¦  «        | _        || _        || _        |	| _        t          j                             |	¬¦  «        | _        | j        �Od| _        |
| _        |r|| _        n0t/          j        |t          j        j        | j        ¬¦  «        | _        || _        | j        �X|p|}||k    r| j        �| j        | _        n:|r|| _        n0t/          j        |t          j        j        | j        ¬¦  «        | _        t          j        j                             | j        ¦  «         | j        �(|                      ¦   «         }|rtA          d|› �¦  «         tB           "                    ¦   «          #                    d¦  «         dS )a  Initializes a `Controller` instance.

    Note that if `checkpoint_manager` is provided and there are checkpoints in
    the associated model directory, the model will be restored from the most
    recent checkpoint during this `__init__` method.

    Args:
      global_step: An integer `tf.Variable` storing the global training step
        number. Usually this can be obtained from the `iterations` property of
        the model's optimizer (e.g. `trainer.optimizer.iterations`). In cases
        where multiple optimizers are used, or if one model "step" corresponds
        to more than one update to model parameters, users can create and
        increment their own global step variable as well. In this case it is
        recommended to create the `tf.Variable` inside the distribution strategy
        scope, with `aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA` (see
        also `orbit.utils.create_global_step()`).
      trainer: An instance of `orbit.AbstractTrainer`, which implements the
        inner training loop.
      evaluator: An instance of `orbit.AbstractEvaluator`, which implements
        evaluation.
      strategy: An instance of `tf.distribute.Strategy`. If not provided, the
        strategy will be initialized from the current in-scope strategy using
        `tf.distribute.get_strategy()`.
      train_actions: Optional `orbit.Action`s to call after each block of
        `steps_per_loop` training steps are run. These will be called with the
        output of `trainer.train`.
      eval_actions: Optional `orbit.Action`s to call after each evaluation.
        These will be called with the output of `evaluator.evaluate`.
      steps_per_loop: Optional integer to indicate the number of steps to run in
        each inner loop of training (passed as the `num_steps` parameter of
        `trainer.train`). It can be also a callable which takes the current
        global step value as input and returns the number of steps to run as
        output.
      checkpoint_manager: An instance of `tf.train.CheckpointManager`. If
        provided and there are checkpoints in the associated model directory,
        the model will be restored from the most recent checkpoint inside this
        `__init__` method. If not provided, the `Controller` will not
        automatically save to or restore from checkpoints.
      enable_async_checkpointing: Optional bool indicating whether to enable
        async checkpoint saving.
      summary_interval: Step interval for training summaries. Note that this
        argument only applies to `tf.summary` calls inside the `trainer.train`
        function. Summaries written by the `Controller` (specifically
        "steps_per_second" and output from the `trainer.train` method) will
        always be enabled unless the `summary_dir` parameter is `None`. If set,
        the value must be divisible by `steps_per_loop`.
      summary_dir: The directory to write summaries to. To use the same
        directory as for checkpointing, pass `checkpoint_manager.directory`. If
        `None`, no training summaries will be written.
      eval_summary_dir: The directory to write eval summaries to. If `None`, it
        will be set to `summary_dir`. If both `summary_dir` and
        `eval_summary_dir` are `None`, no eval summaries will be written.
      summary_manager: Instance of the summary manager. If set, the
        `summary_dir` will be ignored. Otherwise the summary manager will be
        created internally for TensorBoard summaries by default from the
        `summary_dir`.
      eval_summary_manager: Instance of the eval summary manager. If set, the
        `eval_summary_dir` will be ignored. Otherwise the eval summary manager
        will be created internally for TensorBoard summaries by default from the
        `eval_summary_dir`.

    Raises:
      ValueError: If both `trainer` and `evaluator` are `None`.
      ValueError: If `steps_per_loop` is not a positive integer or a callable.
      ValueError: If `summary_interval` is not a positive integer or is not
        divisible by `steps_per_loop`.
    Nz4`trainer` and `evaluator` should not both be `None`.z8`steps_per_loop` is required when `trainer` is provided.r   z`steps_per_loop` (z+) must be a positive integer or a callable.r   z`summary_interval` (z) must be larger than 0.z`summary interval` (z*) must be a multiple of `steps_per_loop` (z).z&`global_step` must be a `tf.Variable`.r   )Úenable_async)r5   zrestored from checkpoint: T)$Ú
ValueErrorÚcallableÚ
isinstanceÚintÚtfÚVariabler(   r)   Ú
distributeÚget_strategyr*   Útupler+   r,   r5   r.   Ú_enable_async_checkpoint_savingÚtrainÚCheckpointOptionsÚ_checkpoint_optionsÚ
step_timerr0   r3   r	   ÚSummaryManagerÚsummaryÚscalarÚ_steps_per_loopr4   ÚexperimentalÚset_stepÚrestore_checkpointr   Ú_orbit_api_gaugeÚget_cellÚset)Úselfr5   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   Úrestored_paths                   r   Ú__init__zController.__init__^   s:  € ðp €˜9Ð,ÝÐMÑNÔNÐNàÐØ	Ð	ÝØFñHô Hð 	Hå˜Ñ'Ô'ð Ý˜­Ñ-Ô-ðØ1?À!Ò1CÐ1CÝð ð ð ð ñô ð 	ð 
Ð	%Ø˜qÒ Ð ÝØOÐ%5ÐOÐOÐOñQô Qð Qå˜.Ñ)Ô)ð 	:Ð/?À.Ñ/PØ23ò04ð 04åð9Ð%5ð 9ð 9Ø&4ð9ð 9ð 9ñ:ô :ð :õ �k¥2¤;Ñ/Ô/ð AÝÐ?Ñ@Ô@Ð@à€D„LØ€D„NàÐ<¥¤× :Ò :Ñ <Ô <€D„Mà,Ð4˜˜½%ÀÑ:NÔ:N€DÔØ*Ð2˜˜½¸lÑ8KÔ8K€DÔà"€DÔØ0€DÔØ+E€DÔ(Ý!œx×9Ò9Ø/ð  :ñ  ô  €DÔð „|ÐØ€d„oØ.€dÔØ	ð JØ.ˆÔÐå$Ô3Ø�œÔ*¸Ô8Hð Jñ  Jô  JˆÔà+€dÔà„~Ð!Ø)Ð8¨[ÐØ	˜[Ò	(Ð	(¨T¬\Ð-Eð %)Ô$8ˆÔ!Ð!àð 	QØ&:ˆ$Ô
#Ð
#å&+Ô&:Ø¥¤
Ô 1¸tÔ?Oð'Qñ 'Qô 'Qˆ$Ô
#õ „JÔ×$Ò$ TÔ%5Ñ6Ô6Ð6ð ÔÐ*Ø×-Ò-Ñ/Ô/€mØ	ð ;ÝÐ9¨-Ð9Ð9Ñ:Ô:Ð:õ ×ÒÑÔ×#Ò# DÑ)Ô)Ð)Ð)Ð)r   TÚstepsÚcheckpoint_at_completionc                 ó¸  — |                       dd¬¦  «         | j                             ¦   «         }t          d|d›d|› d�¦  «         ||k     r`t	          ||z
  | j        ¦  «        }|                      |¦  «         |                      ¦   «          | j                             ¦   «         }||k     °`|r|                      d¬	¦  «         |                      ¦   «          d
S )aç  Runs training until the specified global step count has been reached.

    This method makes calls to `self.trainer.train()` until the global step
    count is equal to `steps`. It will additionally save checkpoints (if a
    `CheckpointManager` was passed to `Controller.__init__`) and summarize
    training output (if `summary_dir` is set).

    When async checkpointing is enabled, a sync is triggered at the end of this
    method to make sure any ongoing async checkpoint saving is finished before
    returning.

    Args:
      steps: The global step count to train up to.
      checkpoint_at_completion: Whether to save a checkpoint when this method
        returns (regardless of the checkpointing interval). Defaults to `True`.
    r(   rB   ©Ú
for_methodútrain | step: ú 6dz | training until step ú...F©Úcheck_intervalN)	Ú_requirer5   Únumpyr   Úminr-   Ú_train_n_stepsÚ_maybe_save_checkpointÚ_sync_on_async_checkpointing)rP   rS   rT   Úcurrent_stepÚ	num_stepss        r   rB   zController.train  s÷   € ð" 	‡M‚M�)¨€MÑ0Ô0Ð0ð Ô#×)Ò)Ñ+Ô+€LÝÐ	M˜,Ð	MÐ	MÐ	MÀ5Ð	MÐ	MÐ	MÑNÔNÐNØ
˜Ò
Ð
å�e˜lÑ*¨DÔ,?Ñ@Ô@€iØ
×Ò˜)Ñ$Ô$Ð$Ø
×!Ò!Ñ#Ô#Ð#ØÔ%×+Ò+Ñ-Ô-€lð ˜Ò
Ð
ð  ð 8Ø
×!Ò!°Ð!Ñ7Ô7Ð7à×%Ò%Ñ'Ô'Ð'Ð'Ð'r   éÿÿÿÿÚreturnc                 óæ  — |                       dd¬¦  «         |dk    rd|› d�}n|dk    rd}nt          d	|› d
�¦  «        ‚| j                             ¦   «         }t	          d|d›d|› �¦  «         t          j        ¦   «         }t          | j        t          j	        ¦  «        sJ ‚| j
                             ¦   «                              ¦   «         5  t          j        |t          j        ¬¦  «        }| j                             |¦  «        }ddd¦  «         n# 1 swxY w Y   t          j        ¦   «         |z
  }|pi }| j        D ]} ||¦  «         Œt          j                             t(          j        |¦  «        }|dk    r||z  }	|	|d<   d|	d›d�}
nd}
t	          d|d›d|
› d|d›dt-          |¦  «        › �¦  «         | j
                             |¦  «         | j
                             ¦   «          |S )a  Runs evaluation for the given number of steps.

    This method calls `self.evaluator.evaluate(steps)`, then writes the returned
    summaries (if any).

    Args:
      steps: The number of evaluation steps to run. The value `-1` is reserved
        as a special sentinel to indicate a "complete" evaluation that runs
        until the underlying dataset is exhausted. Support for this is dependent
        on the specific `evaluator` being used.

    Returns:
      The evaluation results as a dictionary mapping names to NumPy values.

    Raises:
      ValueError: If `evaluator` was not provided to `Controller.__init__`.
      ValueError: If no checkpoint is present in `checkpoint_manager.directory`.
      ValueError: If `steps` is not a positive value or -1.
    r)   ÚevaluaterV   r   zrunning z steps of evaluation...re   zrunning complete evaluation...z	`steps` (z) should be > 0, or == -1.z eval | step: rY   z | ©ÚdtypeNÚsteps_per_secondzsteps/sec: ú 6.1fÚ zeval time: z sec | output: )r]   r8   r5   r^   r   Útimer:   r)   r   ÚAbstractEvaluatorr4   Úsummary_writerÚ
as_defaultr<   Úconvert_to_tensorÚint32rh   r,   ÚnestÚmap_structurer	   Ú	get_valuer%   Úwrite_summariesÚflush)rP   rS   Ú	steps_msgrc   ÚstartÚsteps_tensorÚeval_outputÚelapsedÚactionrk   Ústeps_per_second_logs              r   rh   zController.evaluate#  s�  € ð( 	‡M‚M�+¨*€MÑ5Ô5Ð5àˆq‚y€yØ;˜UÐ;Ð;Ð;€i€iØ	�"ŠˆØ2€i€iåÐD 5ÐDÐDÐDÑEÔEÐEàÔ#×)Ò)Ñ+Ô+€LÝÐ	:˜,Ð	:Ð	:Ð	:¨yÐ	:Ð	:Ñ;Ô;Ð;åŒI‰KŒK€EÝ�d”n¥fÔ&>Ñ?Ô?Ð?Ð?Ð?Ø	Ô	"×	1Ò	1Ñ	3Ô	3×	>Ò	>Ñ	@Ô	@ð :ð :ÝÔ)¨%µr´xÐ@Ñ@Ô@€lØ”N×+Ò+¨LÑ9Ô9€kð:ð :ð :ñ :ô :ð :ð :ð :ð :ð :ð :øøøð :ð :ð :ð :õ Œi‰kŒk˜EÑ!€GàÐ# €KØÔ#ð ð ˆØ€fˆ[ÑÔÐÐÝ”'×'Ò'­¬¸ÑEÔE€Kàˆq‚y€yà ™ÐØ(8€kÐ$Ñ%ØFÐ+;ÐFÐFÐFÐFÐÐàÐåð 
2˜,Ð	/ð 
2ð 
2Ø ð
2ð 
2àÐ	-ð
2ð 
2õ # ;Ñ/Ô/ð
2ð 
2ñ 3ô 3ð 3ð
 	Ô×-Ò-¨kÑ:Ô:Ð:ØÔ×#Ò#Ñ%Ô%Ð%àÐs   Ã;DÄDÄDÚtrain_stepsÚ
eval_stepsÚeval_intervalc                 óÎ  — |                       dd¬¦  «         |                       dd¬¦  «         d}| j                             ¦   «         }|p||z
  }||k     rdt          ||z
  |¦  «        }||z   }|                      |d¬¦  «         |                      |¬¦  «        }| j                             ¦   «         }||k     °d|                      d¬	¦  «         |                      ¦   «          |S )
a¤  Runs interleaved training and evaluation.

    This method interleaves calls to `self.train()` and `self.evaluate()`,
    training the model until the global step count equals `train_steps`, and
    running an evaluation for `eval_steps` every `eval_interval` training steps.
    In addition, this method will run a final evaluation at the end of the
    training sequence.

    When async checkpointing is enabled, a sync is triggered at the end of this
    method to make sure any ongoing async checkpoint saving is finished before
    returning.

    Args:
      train_steps: The global step count to train up to.
      eval_steps: The number of steps to run during an evaluation. If -1, this
        method will evaluate over the entire evaluation dataset.
      eval_interval: The number of training steps to run between evaluations. If
        set, training will always stop every `eval_interval` steps, even if this
        results in a shorter inner loop than specified by `steps_per_loop`
        setting. If None, evaluation will only be performed after training is
        complete.

    Returns:
      The evaluation results as a dictionary mapping names to NumPy values.
    r(   Útrain_and_evaluaterV   r)   NF)rS   rT   )rS   r[   )r]   r5   r^   r_   rB   rh   ra   rb   )rP   r€   r�   r‚   r"   rc   Úintervalrd   s           r   r„   zController.train_and_evaluatea  sý   € ð> 	‡M‚M�)Ð(<€MÑ=Ô=Ð=Ø‡M‚M�+Ð*>€MÑ?Ô?Ð?à€FØÔ#×)Ò)Ñ+Ô+€LØ!ÐA k°LÑ&@€MØ
˜Ò
$Ð
$Ý�[ <Ñ/°Ñ?Ô?€hØ Ñ)€iØ
‡j‚j�y¸5€jÑAÔAÐAØ�}Š} :ˆ}Ñ.Ô.€fØÔ%×+Ò+Ñ-Ô-€lð ˜Ò
$Ð
$ð 	×Ò¨uÐÑ5Ô5Ð5Ø×%Ò%Ñ'Ô'Ð'Ø€Mr   ÚtimeoutÚ
timeout_fnc                 ód  — |                       dd¬¦  «         |                       dd¬¦  «         d}t          | j        t          j        j        ¦  «        sJ ‚t          j                             | j        j        ||¬¦  «        D ],}|                      |¦  «         |  	                    |¦  «        }Œ-|S )aÎ  Continuously monitors a directory and evaluates new checkpoints in it.

    This method continuously monitors a directory as specified by this
    Controller's CheckpointManager init arg and runs evaluation on the
    checkpoints found there.

    Args:
      steps: The number of steps to run when evaluating. If -1, this method will
        evaluate over the entire evaluation dataset.
      timeout: The maximum number of seconds to wait between checkpoints. See
        tf.train.checkpoints_iterator documentation.
      timeout_fn: Optional callable to call after a timeout. If the function
        returns True, then it means that no new checkpoints will be generated
        and the iterator will exit.

    Returns:
      The evaluation results as a dictionary mapping names to NumPy values.

    Raises:
      ValueError: If no checkpoint found in `self.checkpoint_manager.directory`.
      ValueError: If `evaluator` was not provided as a controller init arg.
    r)   Úevaluate_continuouslyrV   r.   N)r†   r‡   )
r]   r:   r.   r<   rB   ÚCheckpointManagerÚcheckpoints_iteratorÚ	directoryrL   rh   )rP   rS   r†   r‡   r"   Úcheckpoint_paths         r   r‰   z Controller.evaluate_continuously�  s¹   € ð8 	‡M‚M�+Ð*A€MÑBÔBÐBØ‡M‚MÐ&Ð3J€MÑKÔKÐKà€FÝ�dÔ-­r¬xÔ/IÑJÔJÐJÐJÐJÝœ8×8Ò8ØÔÔ)ØØð 9ñ ô ð $ð $ˆð ×Ò˜oÑ.Ô.Ð.Ø�}Š}˜UÑ#Ô#€f€fØ€Mr   r�   c                 óÆ  — |                       dd¬¦  «         t          | j        t          j        j        ¦  «        sJ ‚| j                             ¦   «         5  |�3t          d|› d�¦  «         | j        j	         
                    |¦  «         n(t          d¦  «         | j                             ¦   «         }ddd¦  «         n# 1 swxY w Y   |�t          d|› d	�¦  «         |S )
aá  Restores the model from a checkpoint.

    Args:
      checkpoint_path: An optional string specifying the checkpoint path to
        restore from. If `None`, will restore from the most recent checkpoint
        (or initialize the model using a custom `init_fn` if no checkpoints can
        be found) using `self.checkpoint_manager.restore_or_initialize()`.

    Returns:
      The path to the restored checkpoint if a restore happened, or `None` if no
      restore occurred.
    r.   rL   rV   Nzrestoring model from rZ   z"restoring or initializing model...zrestored model from ú.)r]   r:   r.   r<   rB   rŠ   r*   Úscoper   Ú
checkpointÚrestoreÚrestore_or_initialize)rP   r�   s     r   rL   zController.restore_checkpoint¹  s<  € ð 	‡M‚MÐ&Ð3G€MÑHÔHÐHå�dÔ-­r¬xÔ/IÑJÔJÐJÐJÐJØ	Œ×	Ò	Ñ	Ô	ð Jð Jà	Ð	$ÝÐ9 _Ð9Ð9Ð9Ñ:Ô:Ð:ØÔÔ*×2Ò2°?ÑCÔCÐCÐCåÐ1Ñ2Ô2Ð2ØÔ1×GÒGÑIÔIˆðJð Jð Jñ Jô Jð Jð Jð Jð Jð Jð Jøøøð Jð Jð Jð Jð Ð"Ý
Ð4 /Ð4Ð4Ð4Ñ5Ô5Ð5àÐs   ÁACÃCÃCc                 ó`   — |                       dd¬¦  «         |                      d¬¦  «         dS )zåSaves the model to a checkpoint.

    This method will save a checkpoint containing the current state of the
    model.

    Raises:
      ValueError: If no `checkpoint_manager` was provided to
        `Controller.__init__`.
    r.   Úsave_checkpointrV   Fr[   N)r]   ra   ©rP   s    r   r•   zController.save_checkpoint×  s9   € ð 	‡M‚MÐ&Ð3D€MÑEÔEÐEØ×Ò¨uÐÑ5Ô5Ð5Ð5Ð5r   c                 ó�   — t          | j        ¦  «        r,|                      | j                             ¦   «         ¦  «        S | j        S )z8Returns current steps_per_loop value in a training loop.)r9   rI   r5   r^   r–   s    r   r-   zController.steps_per_loopä  sC   € õ �Ô$Ñ%Ô%ð <Ø×!Ò! $Ô"2×"8Ò"8Ñ":Ô":Ñ;Ô;Ð;ØÔÐr   rd   c                 ó¦  ‡ — ‰ j         st          ‰ j        ¦  «        ‰ _         ‰ j                             ¦   «         }‰ j                             ¦   «                              ¦   «         5  d}‰ j        rˆ fd„}t          ‰ j	        t          j        ¦  «        sJ ‚t          j                             |¦  «        5  t          j        |t          j        ¬¦  «        }‰ j	                             |¦  «        }ddd¦  «         n# 1 swxY w Y   ddd¦  «         n# 1 swxY w Y   ||z   }‰ j                             ¦   «         |k    r=d|› d|› d|› d|› d	‰ j                             ¦   «         › d
�}t%          j        |¦  «         |pi }‰ j        D ]} ||¦  «         Œt          j                             t.          j        |¦  «        }‰ j                             ¦   «         }‰ j                              ¦   «         }	t5          d|d›d|	d›dt7          |¦  «        › �¦  «         |	|d<   ‰ j                             |¦  «         ‰ j                             ¦   «          dS )aÇ  Runs training for `num_steps` steps.

    Also prints/logs updates about training progress, and summarizes training
    output (if output is returned from `self.trainer.train()`, and if
    `self.summary_dir` is set).

    Args:
      num_steps: An integer specifying how many steps of training to run.

    Raises:
      RuntimeError: If `global_step` is not properly incremented by `num_steps`
        after calling `self.trainer.train(num_steps)`.
    Fc                  ó*   •— ‰ j         ‰ j        z  dk    S )Nr   )r5   r0   r–   s   €r   ú<lambda>z+Controller._train_n_steps.<locals>.<lambda>  s   ø€  Ô!1°DÔ4IÑ!IÈQÒ!N€ r   ri   Nz`trainer.train(z#)` did not update `global_step` by z. Old value was z, expected updated value to be z, but it was r�   rX   rY   z | steps/sec: rl   z | output: rk   )rE   Ú	StepTimerr5   r^   r3   rp   rq   r0   r:   r(   r   ÚAbstractTrainerr<   rG   Ú	record_ifrr   rs   rB   r   Úwarningr+   rt   ru   r	   rv   rk   r   r%   rw   rx   )
rP   rd   rc   Úshould_recordÚnum_steps_tensorÚtrain_outputÚexpected_stepr   r~   rk   s
   `         r   r`   zController._train_n_stepsë  s  ø€ ð Œ?ð 4Ý! $Ô"2Ñ3Ô3€d„oØÔ#×)Ò)Ñ+Ô+€Là	Ô	×	,Ò	,Ñ	.Ô	.×	9Ò	9Ñ	;Ô	;ð <ð <Ø€mØ	Ô	ð PàOÐOÐOÐOˆÝ˜œ¥fÔ&<Ñ=Ô=Ð=Ð=Ð=ÝŒ:×Ò Ñ.Ô.ð <ð <ÝÔ/°	ÅÄÐJÑJÔJÐØ”|×)Ò)Ð*:Ñ;Ô;ˆð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <ð ! 9Ñ,€MØÔ×ÒÑÔ =Ò0Ð0ðK˜Ið Kð KØðKð KØ(4ðKð Kà ðKð Kà/3Ô/?×/EÒ/EÑ/GÔ/GðKð Kð Kð õ „o�gÑÔÐàÐ% 2€LØÔ$ð ð ˆØ€fˆ\ÑÔÐÐÝ”7×(Ò(­¬¸,ÑGÔG€LàÔ#×)Ò)Ñ+Ô+€LØ”×7Ò7Ñ9Ô9ÐÝð 
3˜,Ð	/ð 
3ð 
3Ø'Ð	2ð
3ð 
3å" <Ñ0Ô0ð
3ð 
3ñ 4ô 4ð 4ð (8€LÐ#Ñ$ØÔ×(Ò(¨Ñ6Ô6Ð6ØÔ×ÒÑ Ô Ð Ð Ð s7   Á&ADÂ5;C<Ã0DÃ<D 	Ä DÄD 	ÄDÄDÄDr\   c                 óÌ   — | j         r\| j         j        rP| j                              | j                             ¦   «         || j        ¬¦  «        }|�t          d|› d�¦  «         dS dS )a|  Conditionally saves a checkpoint.

    A checkpoint is saved if a `CheckpointManager` is available, and if the
    required number of steps has elapsed since the last checkpoint was saved
    (although this condition can be disabled by setting `check_interval=False`).

    Args:
      check_interval: Whether to check if the checkpoint interval has fully
        elapsed. If `False`, a checkpoint is saved regardless of the elapsed
        steps since the most recent checkpoint, unless no `checkpoint_manager`
        was provided to `Controller.__init__`.

    Returns:
      A boolean indicating whether a checkpoint was saved.
    )Úcheckpoint_numberr\   ÚoptionsNzsaved checkpoint to r�   TF)r.   Úcheckpoint_intervalÚsaver5   r^   rD   r   )rP   r\   Ú	ckpt_paths      r   ra   z!Controller._maybe_save_checkpoint  s€   € ð  Ôð  4Ô#:Ô#Nð ØÔ)×.Ò.Ø Ô,×2Ò2Ñ4Ô4Ø'ØÔ*ð /ñ ,ô ,€ið 
Ð	ÝÐ0 IÐ0Ð0Ð0Ñ1Ô1Ð1ØˆtØˆ5r   c           	      óZ   — t          | |d¦  «        €t          d|› d|› d|› d�¦  «        ‚dS )zEUtility method to raise an error if the given `attribute` is not set.Nú`z` is not set. Pass `z+` to `Controller.__init__` before calling `z()`.)Úgetattrr8   )rP   Ú	attributerW   s      r   r]   zController._require9  sh   € åˆt�Y Ñ%Ô%Ð-ÝðDˆið Dð D¨Yð Dð DØ3=ðDð Dð DñEô Eð Eð .Ð-r   c                 ór   — | j         r/t          j        d¦  «         | j                              ¦   «          dS dS )zAForce to wait for the async checkpoint saving (if any) to finish.z Sync on async checkpoint saving.N)r.   r   r   Úsyncr–   s    r   rb   z'Controller._sync_on_async_checkpointing@  sE   € ð Ôð %Ý„lÐ5Ñ6Ô6Ð6Ø
Ô×"Ò"Ñ$Ô$Ð$Ð$Ð$ð%ð %r   ©T)re   )re   N)re   NN©N)%Ú__name__Ú
__module__Ú__qualname__Ú__doc__r<   r=   r   r   rœ   ro   r>   ÚStrategyr   ÚActionr   r;   r   rB   rŠ   ÚboolÚstrr	   ÚSummaryManagerInterfacerR   ÚOutputrh   r„   Úfloatr‰   rL   r•   Úpropertyr-   r`   ra   r]   rb   r   r   r   r'   r'   <   s:  € € € € € ðð ðJ 37Ø6:Ø37à26Ø15àCGØAEØ).à(,Ø#'à(,ØAEØFJð)a*ð a*ð a*ð ”;ða*ð ˜Ô.Ô/ð	a*ð
 ˜&Ô2Ô3ða*ð ˜œÔ/Ô0ða*ð ˜h vÔ.Ô/ða*ð ˜X fÔ-Ô.ða*ð ˜u S¨(°C°5¸#°:Ô*>Ð%>Ô?Ô@ða*ð # 2¤8Ô#=Ô>ða*ð #'ða*ð ! œða*ð  ˜C”=ð!a*ð$ ! œð%a*ð&   Ô =Ô>ð'a*ð( % UÔ%BÔCð)a*ð a*ð a*ð a*ðF (ð  (˜ð  (¸ð  (ð  (ð  (ð  (ðD<ð <˜Cð <¨°&´-Ô)@ð <ð <ð <ð <ðB Ø%)ð	-ð -àð-ð ð-ð ˜c”]ð	-ð
 �”Ôð-ð -ð -ð -ðb Ø-1Ø15ð	'ð 'àð'ð ˜˜c 5˜jÔ)Ô*ð'ð ˜8 B¨ HÔ-Ô.ð	'ð
 �”Ôð'ð 'ð 'ð 'ðRð °¸´ð ð ð ð ð<6ð 6ð 6ð ð ð  ñ „8ð ð2! cð 2!ð 2!ð 2!ð 2!ðhð °4ð ð ð ð ð4Eð Eð Eð%ð %ð %ð %ð %r   r'   c                   ó&   — e Zd ZdZd„ Zd„ Zdd„ZdS )r›   z)Utility class for measuring steps/second.c                 ó<   — || _         |                      ¦   «          d S r°   )Ústeprz   )rP   r¿   s     r   rR   zStepTimer.__init__K  s   € Ø€D„IØ‡J‚J�L„L€L€L€Lr   c                 ór   — | j                              ¦   «         | _        t          j        ¦   «         | _        d S r°   )r¿   r^   Úlast_iterationrn   Ú	last_timer–   s    r   rz   zStepTimer.startO  s'   € Øœ)Ÿ/š/Ñ+Ô+€DÔÝ”Y‘[”[€D„N€N€Nr   Tc                 ó¬   — | j                              ¦   «         | j        z
  t          j        ¦   «         | j        z
  z  }|r|                      ¦   «          |S r°   )r¿   r^   rÁ   rn   rÂ   rz   )rP   ÚrestartÚvalues      r   rk   zStepTimer.steps_per_secondS  sJ   € ØŒi�oŠoÑÔ $Ô"5Ñ5ÝŒi‰kŒk˜DœNÑ*ñ,€Eàð Ø
‡j‚j�l„l€lØ€Lr   Nr¯   )r±   r²   r³   r´   rR   rz   rk   r   r   r   r›   r›   H  sL   € € € € € Ø1Ð1ðð ð ð!ð !ð !ðð ð ð ð ð r   r›   )r   )r´   r   rn   Útypingr   r   r   r   Úabslr   Úorbitr   r	   Ú
tensorflowr<   Útf_kerasÚtensorflow.python.eagerr
   Ú	BoolGaugerM   r¸   r   Ú
ABSLLoggerÚregister_frame_to_skipÚ__file__r±   r%   rº   r¶   r'   r›   r   r   r   ú<module>rÐ      s‡  ðð JÐ Ià €€€Ø €€€à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6à Ð Ð Ð Ð Ð à Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ð !Ð !ð /Ð .Ð .Ð .Ð .Ð .ð (�:Ô'ØÐ.ñô Ð ð
�#ð ð ð ð ð Ô × )Ò )¨(°D´MÑ BÔ BÐ Bð!ð !ð !ð !ð 
�6”=�/ 4Ð'Ô	(€ðI%ð I%ð I%ð I%ð I%ñ I%ô I%ð I%ðXð ð ð ð ñ ô ð ð ð r   