#!/usr/bin/env bash
set -euo pipefail

FACULTY="${FACULTY:-khoa_cong_nghe_thong_tin}"
SPLITS_DIR="${SPLITS_DIR:?Set SPLITS_DIR to data/splits/expanding}"
OUTPUT_DIR="${OUTPUT_DIR:-./experiment_B_outputs_holdout}"
EPOCHS="${EPOCHS:-100}"
BATCH_SIZE="${BATCH_SIZE:-512}"
DEV_FOLDS="${DEV_FOLDS:-1,2,3,4,5}"
HOLDOUT_FOLDS="${HOLDOUT_FOLDS:-6,7}"
EXPERIMENTS="${EXPERIMENTS:-B0,B4,B5,B6_Q}"
SKIP_EXISTING="${SKIP_EXISTING:-1}"

IFS=',' read -r -a EXP_ARRAY <<< "$EXPERIMENTS"
mkdir -p "$OUTPUT_DIR/logs"
for exp in "${EXP_ARRAY[@]}"; do
  exp="$(echo "$exp" | xargs)"
  existing_summary=$(find "$OUTPUT_DIR/$FACULTY" -maxdepth 2 -path "*/${exp}_*/summary_holdout.json" -print -quit 2>/dev/null || true)
  if [[ "$SKIP_EXISTING" == "1" && -n "$existing_summary" ]]; then
    echo "=== SKIP $exp: $existing_summary already exists ==="
    continue
  fi
  echo "=== DIAGNOSTIC HOLDOUT $exp ==="
  PYTHONUNBUFFERED=1 python3 run_experiment_b.py \
    --faculty "$FACULTY" \
    --splits_dir "$SPLITS_DIR" \
    --output_dir "$OUTPUT_DIR" \
    --phase holdout \
    --dev_folds "$DEV_FOLDS" \
    --holdout_folds "$HOLDOUT_FOLDS" \
    --experiments "$exp" \
    --epochs "$EPOCHS" \
    --batch_size "$BATCH_SIZE" \
    > "$OUTPUT_DIR/logs/${exp}.log" 2>&1
done
python3 aggregate_results.py \
  --output_dir "$OUTPUT_DIR" \
  --faculty "$FACULTY" \
  --phase holdout \
  --experiments "$EXPERIMENTS" \
  > "$OUTPUT_DIR/experiment_B_comparison_holdout.log"
cat "$OUTPUT_DIR/experiment_B_comparison_holdout.log"
