#!/usr/bin/env bash
# SparrKULee end-to-end, unattended (tmux window 'spk'): download -> convert -> queue S1 pretraining + S2 mixing ->
# queue S1 fine-tuning as each pretrained model appears. Jobs are added only when their inputs exist, so none of them
# holds a queue slot while waiting. Log: logs/sparrkulee_chain.log
cd "$(dirname "$0")/.."
X=../external_data/SparrKULee
PY=../../.venv/bin/python
Q="$PY runners/jobqueue.py add"
BASE="--decoder vlaai_ws --prep mat_reac --aug-variant cluster --targets env,onset"
log() { echo "$(date '+%F %T') $*" | tee -a logs/sparrkulee_chain.log; }
run() {  # name tag extra-args : one LODO experiment job
  local n=$1 t=$2; shift 2
  $Q "$n" -- "$PY run_experiment.py $BASE --tag $t $* < /dev/null >> logs/vlaai_ws__mat_reac$t.stdout 2>&1 && $PY summarize.py > /dev/null"
}
log "chain started"
until [ -f $X/raw/ALL_DONE ]; do
  tmux list-windows -t decoder_lodo -F '#W' | grep -qx sparr_dl || (cd $X && python3 download_sparrkulee.py >> download.log 2>&1)
  sleep 300
done
log "download complete"
[ -f $X/cache/.done_mat_reac ] || $Q 110_convert_spk -- "cd $X && $(realpath $PY) convert_sparrkulee.py mat_reac > convert.log 2>&1"
until [ -f $X/cache/.done_mat_reac ]; do sleep 120; done
log "conversion complete: $(tail -1 $X/convert.log)"
$Q 111_pretrain_spk_s0 -- "$PY pretrain_sparrkulee.py --seed 0 > logs/pretrain_spk_s0.log 2>&1"
$Q 112_pretrain_spk_s1 -- "$PY pretrain_sparrkulee.py --seed 1 > logs/pretrain_spk_s1.log 2>&1"
run 113_lodo_xFUGSPK25_s0 _aug_cluster_onset_xFUGSPK25 --extra-train Fuglsang2020,SparrKULee --extra-frac SparrKULee=0.25
run 114_lodo_xFUGSPK25_s1 _aug_cluster_onset_xFUGSPK25_s1 --extra-train Fuglsang2020,SparrKULee --extra-frac SparrKULee=0.25 --train-seed 1
for s in 0 1; do
  until [ -f models/pretrain_spk_vlaai_ws_s$s.pt ]; do sleep 300; done
  log "pretrained model s$s ready"
  sf=$([ $s = 0 ] && echo "" || echo "_s1 --train-seed 1")
  run 11$((5 + s))_lodo_xFUG_ptSPK_s$s _aug_cluster_onset_xFUG_ptSPK$sf --extra-train Fuglsang2020 --init-from models/pretrain_spk_vlaai_ws_s$s.pt
done
log "all SparrKULee jobs queued"
