#!/usr/bin/env bash
# Progress overview: finished experiments, running experiment, last lines of the queue logs.
cd "$(dirname "$0")"
N=$(cat runners/queues/*.txt | grep -v "^\s*#" | grep -vc "^\s*$")
echo "finished: $(ls results/*.json 2>/dev/null | wc -l) / $N   failed: $(ls results/*.FAILED 2>/dev/null | wc -l)"
for q in cpu gpu ssm wu win aug skip augv w10 fft seed signal priority_p1 priority_p2 reliability; do echo "--- $q"; tail -3 logs/queue_$q.log 2>/dev/null; done
echo "--- fold files per experiment"
for d in folds/*/; do [[ -d $d ]] && echo "$(basename $d): $(ls $d | wc -l)/5"; done
echo "--- GPU"; nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader
echo "--- job queue (runners/jobqueue.py)"; ../../.venv/bin/python runners/jobqueue.py status
