#!/usr/bin/env bash
# Step 0: best config (vlaai_ws, mat, aug + electrode-cluster dropout) with 3 more training seeds, saving the causal
# reconstructions; then the sample-level Markov switching model (msm.py, GPU) on each seed run.
cd "$(dirname "$0")/.."
PY=../../.venv/bin/python
QLOG=logs/queue_seed.log
echo "$(date '+%F %T') queue seed started" | tee -a "$QLOG"
EXPS=""
for s in 1 2 3; do
  exp="vlaai_ws__mat_aug_cluster_s${s}"; EXPS="$EXPS $exp"
  [[ -f results/$exp.json ]] && continue
  echo "$(date '+%F %T') START $exp" | tee -a "$QLOG"
  if $PY run_experiment.py --decoder vlaai_ws --prep mat --tag _aug_cluster_s${s} --aug-variant cluster --train-seed $s --save-recon < /dev/null >> logs/$exp.stdout 2>&1; then
    rm -f results/$exp.FAILED
    echo "$(date '+%F %T') DONE  $exp: $(grep 'MEAN over' logs/$exp.log | tail -1 | cut -d' ' -f3-)" | tee -a "$QLOG"
  else
    touch results/$exp.FAILED; echo "$(date '+%F %T') FAILED $exp" | tee -a "$QLOG"
  fi
  $PY summarize.py > /dev/null 2>&1 || true
done
echo "$(date '+%F %T') START msm on$EXPS" | tee -a "$QLOG"
if $PY msm.py $EXPS >> logs/msm.stdout 2>&1; then
  echo "$(date '+%F %T') DONE  msm: see logs/msm.log" | tee -a "$QLOG"
else
  echo "$(date '+%F %T') FAILED msm (see logs/msm.stdout)" | tee -a "$QLOG"
fi
echo "$(date '+%F %T') queue seed finished" | tee -a "$QLOG"
