#!/usr/bin/env bash
# Run every experiment of a queue in order; skip finished ones; keep going after a failure.
# usage: bash run_queue.sh cpu|gpu
cd "$(dirname "$0")/.."
PY=../../.venv/bin/python
Q="runners/queues/$1.txt"
QLOG="logs/queue_$1.log"
echo "$(date '+%F %T') queue $1 started" | tee -a "$QLOG"
grep -v '^\s*#' "$Q" | grep -v '^\s*$' | while read -r dec prep; do
  exp="${dec}__${prep}"
  if [[ -f "results/$exp.json" ]]; then
    echo "$(date '+%F %T') skip $exp (already done)" | tee -a "$QLOG"
    continue
  fi
  echo "$(date '+%F %T') START $exp" | tee -a "$QLOG"
  if $PY run_experiment.py --decoder "$dec" --prep "$prep" < /dev/null >> "logs/$exp.stdout" 2>&1; then
    rm -f "results/$exp.FAILED"
    echo "$(date '+%F %T') DONE  $exp: $(grep 'MEAN over' "logs/$exp.log" | tail -1 | cut -d' ' -f3-)" | tee -a "$QLOG"
  else
    touch "results/$exp.FAILED"
    echo "$(date '+%F %T') FAILED $exp (see logs/$exp.stdout)" | tee -a "$QLOG"
  fi
  $PY summarize.py > /dev/null 2>&1 || true
done
echo "$(date '+%F %T') queue $1 finished" | tee -a "$QLOG"
