#!/usr/bin/env bash
# Sequential queue for M8, M10, F11, F12 (the host is shared: GPU/RAM guards, one job at a time). Log: results/queue_mt3.log
cd "$(dirname "$0")"; PY=/root/miniconda3/bin/python; LOG=results/queue_mt3.log
say() { echo "[$(date +%H:%M:%S)] $*" | tee -a $LOG; }
wait_mem() { while [ "$(awk '/MemAvailable/{print int($2/1048576)}' /proc/meminfo)" -lt "$1" ]; do sleep 30; done; }
busy() { pgrep -f "run_augment.py --res results_mt2" > /dev/null; }
say "waiting for the F12 cache"
until [ -f cache/F12.npy ]; do sleep 30; done
say "F12 cache ready; smoke test"
rm -rf results_smoke_mt3; wait_mem 4
./run.sh results_smoke_mt3 m8a_t3pre m8b_cpcpre m10_stack f12a_cwt f12b_cwt_nospec --smoke > /dev/null 2>&1
grep -E "Traceback|Error" results_smoke_mt3/run.log | head -3 | tee -a $LOG
if ! grep -q "EXIT CODE 0" results_smoke_mt3/run.log; then say "SMOKE FAILED"; tail -n 15 results_smoke_mt3/run.log >> $LOG; exit 1; fi
say "smoke ok"; tail -n 12 results_smoke_mt3/run.log | cut -c1-140 >> $LOG
say "waiting for phase 1 (results_mt2) to finish"
while busy; do sleep 60; done
say "phase 1 done; REVE embeddings"
wait_mem 5
$PY -u build_reve.py 256 2>&1 | grep --line-buffered -v -i warn | tee -a $LOG | tail -n 3
stream() {  # one job at a time inside a stream
  for v in "$@"; do
    wait_mem 4
    say "run $v"
    ./run.sh results_mt3_$v $v --save-probs probs_mt3 > /dev/null 2>&1
    say "$v: $(tail -n 1 results_mt3_$v/run.log)"
  done
}
stream m8a_t3pre m8b_cpcpre f11_reve &
sleep 120
stream m10_stack f12a_cwt f12b_cwt_nospec &
wait
say "ALL DONE"
