Ë
    àáéjE  ã                   ó¢  — d Z ddlZddlZddlZddlZddlZddlZddlZej                  j                  ej                  j                  ej                  j                  e«      d«      «      Zej                  j                  dd«      Zer(ej#                  dd«      j%                  «       s ed«      ‚ej                  j                  edd	erdez   ndz   «      Zd
ZeD ]0  Z ej.                  ej                  j                  ee«      d¬«       Œ2  ej.                  ej                  j                  edd	«      d¬«       d„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Ze dk(  �rr ejB                  «       Z"e"jG                  dd¬«      Z$e$jK                  d«      Z&e&jO                  d«       e&jO                  dejP                  ¬«       e$jK                  d«       e$jK                  d«      Z&e&jO                  de)d ¬!«       e&jO                  d"e*d#¬!«       e&jO                  d$e*d%¬!«       e&jO                  d&e*d#d'¬(«       e&jO                  d)e)d*¬!«       e&jO                  d+e*d,¬!«       e&jO                  d-e*d.¬!«       e"jW                  «       Z,e,jZ                  dk(  r'e,j\                  dd/ d0gk(  re,j\                  d/d e,_.         eeed1œe,jZ                     e,«       yy)2aY  Resource-aware job queue for the decoder_lodo experiments (avoids overloading the 8-core CPU).

A job is one shell command run from the experiment root. The scheduler starts the next pending job only when
  * fewer than --max-jobs of its own jobs are running,
  * the 1-min load average is below --max-load,
  * the GPU has at least --min-gpu-free-gb free memory, and
  * at least --min-ram-free-gb of RAM is available (no swap: a job that does not fit is OOM-killed).
Jobs run under `nice -n 10` with OMP/MKL/OpenBLAS limited to 2 threads and SCORE_JOBS (parallel scorer workers
used by integrators.py / reliability.py / msm.py) set to --score-jobs. Pending jobs start in name order, so a
numeric prefix sets the priority.

  python runners/jobqueue.py add NAME -- COMMAND ...      queue a job (NAME e.g. 010_causal_onset)
  python runners/jobqueue.py status                       pending / running / done / failed
  python runners/jobqueue.py run [--max-jobs 2 ...]       scheduler loop (run it in tmux window 'jobq')
State: runners/jobs/{pending,running,done,failed}/NAME.json ; job output: logs/jobs/NAME.log
é    Nz..ÚJOBQUEUE_NAMESPACEÚ Ú_zDJOBQUEUE_NAMESPACE must contain only letters, digits, or underscoresÚrunnersÚjobs)ÚpendingÚrunningÚdoneÚfailedT)Úexist_okÚlogsc           	      ó�   — t        d„ t        j                  t        j                  j	                  t
        | «      «      D «       «      S )Nc              3   óJ   K  — | ]  }|j                  d «      sŒ|dd –— Œ y­w)ú.jsonNéûÿÿÿ)Úendswith)Ú.0Úfs     úrunners/jobqueue.pyÚ	<genexpr>zjobs.<locals>.<genexpr>%   s%   è ø€ Ð\Ñ"D˜QÈÏ
É
ÐSZÕH[�!�C�R”&Ñ"Dùs   ‚#™
#)ÚsortedÚosÚlistdirÚpathÚjoinÚQ)Ústates    r   r   r   $   s+   € ÜÑ\¤"§*¡*¬R¯W©W¯\©\¼!¸UÓ-CÔ"DÓ\Ó\Ð\ó    c                 ór  — t         j                  j                  t        || dz   «      t         j                  j                  t        || dz   «      }}t	        j
                  t        |«      «      }|j                  |«       t	        j                  |t        |d«      d¬«       t        j                  |«       |S )Nr   Úwé   ©Úindent)
r   r   r   r   ÚjsonÚloadÚopenÚupdateÚdumpÚremove)ÚnameÚaÚbÚupdÚsrcÚdstÚds          r   Úmover1   (   sx   € Ü�w‰w�|‰|œA˜q $¨¡.Ó1´2·7±7·<±<ÄÀ1ÀdÈWÁnÓ3Uˆ€CÜ�	‰	”$�s“)Ó€AØ‡H�HˆS„MÜ‡I�Iˆa”�c˜3“¨Õ*Ü‡I�Iˆc„NØ€Hr   c                  ó’   — 	 t        j                  g d¢d¬«      } t        | j                  «       d   «      dz  S # t        $ r Y yw xY w)N)z
nvidia-smiz--query-gpu=memory.freez--format=csv,noheader,nounitsT)Útextr   i   ç        )Ú
subprocessÚcheck_outputÚfloatÚsplitÚ	Exception)Úouts    r   Úgpu_free_gbr;   1   sG   € ðÜ×%Ñ%Ò&pÐw{Ô|ˆÜ�S—Y‘Y“[ ‘^Ó$ tÑ+Ð+øÜò Ùðús   ‚7: º	AÁAc                  óª   — 	 t        d«      D ]4  } | j                  d«      sŒt        | j                  «       d   «      dz  c S  	 y# t        $ r Y yw xY w)zXMemAvailable: the machine has no swap, a job that does not fit gets OOM-killed (rc 137).z/proc/meminfoÚMemAvailabler!   i   r4   )r&   Ú
startswithr7   r8   r9   )Úlns    r   Úram_free_gbr@   9   sX   € ðÜ�Ö'ˆBØ�}‰}˜^Õ,Ü˜RŸX™X›Z¨™]Ó+¨iÑ7Ò7ñ (ð
 øô ò ØØðús   ‚A ¢ A ÁA Á	AÁAc                  ó,   — t        j                  d«      S )Nz%F %T)ÚtimeÚstrftime© r   r   ÚstamprE   D   s   € Ü�=‰=˜Ó!Ð!r   c                 óÈ  ‡ — t        ˆ fd„t        D «       «      r#t        j                  d‰ j                  › d�«       dj                  ‰ j                  «      }t        j                  t        ‰ j                  |t        «       ¬«      t        t        j                  j                  t        d‰ j                  dz   «      d«      d	¬
«       t        d‰ j                  › d|› �«       y )Nc           	   3   ó¼   •K  — | ]S  }t         j                  j                  t         j                  j                  t        |‰j
                  d z   «      «      –— ŒU y­w)r   N)r   r   Úexistsr   r   r*   )r   Úsr+   s     €r   r   zcmd_add.<locals>.<genexpr>I   s9   øè ø€ Ð
PÉÀAŒ2�7‰7�>‰>œ"Ÿ'™'Ÿ,™,¤q¨!¨Q¯V©V°gÑ-=Ó>×?Éùs   ƒAAzjob z already existsÚ )r*   ÚcmdÚaddedr   r   r    r!   r"   zqueued z: )ÚanyÚSTATESÚsysÚexitr*   r   Úcommandr$   r(   ÚdictrE   r&   r   r   r   Úprint)r+   rK   s   ` r   Úcmd_addrT   H   s›   ø€ Ü
Ó
PÍÓ
PÔPÜ�‰�4˜Ÿ™�x˜Ð/Ô0Ø
�(‰(�1—9‘9Ó
€CÜ‡I�IŒd˜Ÿ™ C¬u«wÔ7¼¼b¿g¹g¿l¹lÌ1ÈiÐYZ×Y_ÑY_ÐbiÑYiÓ>jÐloÓ9pÐyzÕ{Ü	ˆG�A—F‘F�8˜2˜c˜UÐ
#Õ$r   c                 ó>  — t         D ]B  }t        |«      }t        |d›dt        |«      d›d�|sdndj	                  |dd  «      z   «       ŒD t        dt        j                  «       d	   d
›dt        j                  «       › dt        «       d
›dt        «       d
›d�	«       y )NÚ8srJ   Ú3dz  r   z, iôÿÿÿzload r   ú.1fz / z cores, GPU free z GB, RAM available ú GB)
rN   r   rS   Úlenr   r   Ú
getloadavgÚ	cpu_countr;   r@   )r+   rI   Újss      r   Ú
cmd_statusr^   P   s«   € ßˆÜ�!‹WˆÜ��2��aœ˜B› �| 2Ð&±B©"¸D¿I¹IÀbÈÈÀhÓ<OÑPÕQð ô 
ˆE”"—-‘-“/ !Ñ$ SÐ)¨¬R¯\©\«^Ð,<Ð<MÌkËmÐ\_ÐM`Ð`sÔtó  uBð  CFð  tGð  GJð  Kõ  Lr   c                 óª  ‡— i }t        t        j                  j                  t        ddt
        r	dt
        z   ndz   dz   «      dd¬«      Šˆfd	„}t        d
«      D ]  }t        |d
dd¬«       Œ  |d| j                  › d| j                  › d| j                  › d�«       	 t        |j                  «       «      D ]x  \  }}|j                  «       €Œ|j                  dk(  rdnd}t        |d
|t        «       |j                  ¬«        ||j!                  «       d›d|› d|j                  › d�«       ||= Œz t        d«      }t        j"                  «       d   t%        «       }}|�rRt'        |«      | j                  k  �r9|| j                  k  �r)|| j                  k\  �rt)        «       | j*                  k\  �r|d   }t        |dd
t        «       ¬«      }	t-        t        j.                  dddt1        | j2                  «      ¬«      }
t        t        j                  j                  t        dd|dz   «      d«      }t5        j6                  ddd d!d"|	d#   gt        |
|t4        j8                  t4        j:                  ¬$«      ||<    |d%|› d&|d'›d(|d'›d)|	d#   › �«       t=        j>                  | j@                  «       �Œt=        j>                  | j                  «       �Œ6)*Nr   Újobqueuer   r   z.logr+   r!   )Ú	bufferingc                 óv   •— t        t        «       › d| › �d¬«      ‰j                  t        «       › d| › d�«      fS )NrJ   T)ÚflushÚ
)rS   rE   Úwrite)ÚmÚlogs    €r   Ú<lambda>zcmd_run.<locals>.<lambda>Z   s9   ø€ ”Uœe›g˜Y a¨ sÐ+°4Ô8¸#¿)¹)ÄuÃwÀiÈqÐQRÐPSÐSUÐDVÓ:WÑXr   r	   r   z requeued after scheduler restart)Únotezscheduler started: max_jobs z, max_load z, min_gpu_free rY   r   r
   r   )ÚfinishedÚ
returncodeÚ6srJ   z (rc Ú))ÚstartedÚ2)ÚOMP_NUM_THREADSÚMKL_NUM_THREADSÚOPENBLAS_NUM_THREADSÚ
SCORE_JOBSr   Únicez-nÚ10Úbashz-crK   )ÚcwdÚenvÚstdoutÚstderrÚstdinzSTART  z (load rX   z, GPU free z GB): )!r&   r   r   r   ÚEXPÚ	NAMESPACEr   r1   Úmax_jobsÚmax_loadÚmin_gpu_free_gbÚlistÚitemsÚpollrk   rE   Úupperr[   r;   rZ   r@   Úmin_ram_free_gbrR   ÚenvironÚstrÚ
score_jobsr5   ÚPopenÚSTDOUTÚDEVNULLrB   ÚsleepÚsettle)r+   ÚprocsÚsayÚnÚpÚstÚpendr%   Úfreer0   rx   r:   rg   s               @r   Úcmd_runr•   W   si  ø€ Ø€EÜ
Œr�w‰w�|‰|œC ¨Í)°s¼Y²ÐY[Ñ)\Ð_eÑ)eÓfÐhkÐwxÔ
y€CÛ
X€CÜ�)Ž_ˆÜˆQ�	˜9Ð+MÖNð áÐ
& q§z¡z l°+¸a¿j¹j¸\ÈÐYZ×YjÑYjÐXkÐknÐoÔpØ
Ü˜Ÿ™›Ö'‰DˆAˆqØ�v‰v‹xÑ#Ø Ÿ|™|¨qÒ0‘V°h�Ü�Q˜	 2´³ÀAÇLÁLÕQÙ�r—x‘x“z "�o Q q c¨¨q¯|©|¨n¸AÐ>Ô?Ø˜!‘Hð (ô �I‹ˆÜ—]‘]“_ QÑ'¬«ˆdˆÚ”C˜“J §¡Ó+°°q·z±zÓ0AÀdÈa×N_ÑN_ÓF_ÔdoÓdqÐuv÷  vGñ  vGó  eGØ�Q‘ˆAÜ�Q˜	 9´e³gÔ>ˆAÜ”r—z‘z°3ÈÐbeÜ"% a§l¡lÓ"3ô5ˆCä”r—w‘w—|‘|¤C¨°¸¸V¹ÓDÀcÓJˆCÜ!×'Ñ'¨°°t¸VÀTÈ1ÈUÉ8Ð(TÔZ]ÐcfØ/2¼:×;LÑ;LÔT^×TfÑTfôhˆE�!‰Há�'˜!˜˜G D¨ :¨[¸¸c¸
À&ÈÈ5ÉÈ
ÐSÔTÜ�J‰J�q—x‘xÔ ÙÜ�
‰
�1—6‘6Ôñ) r   Ú__main__rK   )ÚdestÚrequiredÚaddr*   rQ   )ÚnargsÚstatusÚrunz
--max-jobsé   )ÚtypeÚdefaultz
--max-loadg      @z--min-gpu-free-gbg      @z--min-ram-free-gbzrstart a job only with this much available RAM (a training job takes ~4 GB; other users' processes share the 24 GB))rž   rŸ   Úhelpz--score-jobsé   z--pollg      >@z--settleg      ^@r!   z--)r™   r›   rœ   )/Ú__doc__Úargparser$   r   Úshutilr5   rO   rB   r   Úabspathr   ÚdirnameÚ__file__r|   r†   Úgetr}   ÚreplaceÚisalnumÚ
ValueErrorr   rN   rI   Úmakedirsr   r1   r;   r@   rE   rT   r^   r•   Ú__name__ÚArgumentParserÚapÚadd_subparsersÚspÚ
add_parserr‘   Úadd_argumentÚ	REMAINDERÚintr7   Ú
parse_argsr+   rK   rQ   rD   r   r   Ú<module>r·      sƒ  ðñó  Û Û 	Û Û Û 
Û à‡g�g‡o�o�b—g‘g—l‘l 2§7¡7§?¡?°8Ó#<¸dÓCÓD€Ø�J‰J�N‰NÐ/°Ó4€	Ù�i×'Ñ'¨¨RÓ0×8Ñ8Ô:Ù
Ð[Ó
\Ð\Ø‡G�G‡L�L��i ¹i¨3°ª?ÈRÑ!PÓQ€Ø	1€Û	€AØ€B‡K�K�—‘—‘˜Q Ó"¨TÖ2ð 
à €‡�ˆB�G‰G�L‰L˜˜f fÓ-¸Õ =ò]òòòò"ò%òLòð< ˆzÓØ	 ˆ×	 Ñ	 Ó	"€BØ	×	Ñ	 °Ð	Ó	5€BØ
�‰�eÓ€AØ‡N�N�6ÔØ‡N�N�9 H×$6Ñ$6€NÔ7Ø‡M�M�(ÔØ
�‰�eÓ€AØ‡N�N�< c°1€NÔ5Ø‡N�N�< e°S€NÔ9Ø‡N�NÐ&¨U¸C€NÔ@Ø‡N�NÐ&¨U¸Cð GZ€Nô [à‡N�N�>¨°Q€NÔ7Ø‡N�N�8 %°€NÔ6Ø‡N�N�: E°5€NÔ9Ø
�‰‹€AØ‡u�u�‚~˜!Ÿ)™) B Q˜-¨D¨6Ò1Ø—I‘I˜a˜b�MˆŒ	ØAˆG˜z°'Ñ:¸1¿5¹5ÑAÀ!ÕDð' r   