#!/usr/bin/env bash
# Suite29: same root/log/results conventions as Suites 24-28.
set -Eeuo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="${AQ_ROOT:-/root/disk_3tb/earlywarning/30072026}"
PYTHON="${PYTHON:-/root/miniconda3/envs/Huy/bin/python3}"
# Excessive CPU BLAS thread counts can make even tiny padded-set batches slow.
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}"
export MKL_NUM_THREADS="${MKL_NUM_THREADS:-4}"
LOGDIR="$ROOT/log"
RESULTSDIR="$ROOT/results/suite29_enrollment_set"
mkdir -p "$LOGDIR" "$RESULTSDIR"
LOG="$LOGDIR/suite29_enrollment_set.log"
RUN_ID="$(date +%Y%m%d_%H%M%S)_$$"
RUN_LOG="$LOGDIR/suite29_enrollment_set_${RUN_ID}.log"

# Tee each line both to the stable historical file and to a per-run file.
# pipefail propagates any Python/test/preflight failure to the shell.
run_pipeline() {
  echo "=================================================================="
  echo "[START] $(date -Iseconds) Suite29 enrollment-set experiment"
  echo "[RUN] code=$HERE"
  echo "[RUN] root=$ROOT"
  echo "[RUN] python=$PYTHON"
  echo "[RUN] OMP_NUM_THREADS=$OMP_NUM_THREADS MKL_NUM_THREADS=$MKL_NUM_THREADS"
  echo "[RUN] log=$LOG"
  echo "[RUN] run_log=$RUN_LOG"
  echo "[RUN] default_results=$RESULTSDIR"
  echo "[RUN] frozen defaults: folds=1..7 faculties=all modes=self_only,set seeds=29,129,229 epochs=18 batch=64 checkpoints=on"
  printf '[RUN] user arguments:'; printf ' %q' "$@"; printf '\n'
  echo "[PREFLIGHT] Checking runtime, fold cache paths, structured cache paths, and outputs"
  "$PYTHON" -u "$HERE/check_suite29_environment.py" \
      --root "$ROOT" \
      --cache-dir "$ROOT/results/cross_faculty_residual_moe/cache" \
      --structured-dir "$ROOT/results/suite12_structured_cache" \
      --output-dir "$RESULTSDIR" \
      --folds 1,2,3,4,5,6,7 --faculties all "$@"
  echo "[TEST] Running Suite29 source-level tests"
  (cd "$HERE" && "$PYTHON" -m unittest discover -s tests -v)
  echo "[TRAIN] Launching Suite29"
  (cd "$HERE" && "$PYTHON" -u train_suite29.py \
      --cache-dir "$ROOT/results/cross_faculty_residual_moe/cache" \
      --structured-dir "$ROOT/results/suite12_structured_cache" \
      --output-dir "$RESULTSDIR" \
      --folds 1,2,3,4,5,6,7 --faculties all \
      --modes self_only,set --seeds 29,129,229 \
      --epochs 18 --patience 5 --validation-blocks 3 \
      --group-batch-size 64 --save-checkpoints "$@")
  echo "[COMPLETE] $(date -Iseconds) Suite29 successful"
}

# Capture the exit status (including any pipeline failure) in BOTH logs.
(
  set -Eeuo pipefail
  trap 'status=$?; if ((status==0)); then echo "[RUN STATUS] SUCCESS time=$(date -Iseconds)"; else echo "[RUN STATUS] FAILURE exit_code=$status time=$(date -Iseconds)"; fi' EXIT
  run_pipeline "$@"
) 2>&1 | tee -a "$LOG" "$RUN_LOG"
