#!/usr/bin/env bash
set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SPLITS_DIR="${1:-/root/disk_3tb/earlywarning/data/splits/expanding}"
OUTPUT_ROOT="${2:-/root/disk_3tb/earlywarning/30072026/results/suite2}"
FACULTY="${3:-khoa_cong_nghe_thong_tin}"

DEVICE="${DEVICE:-auto}"
N_ESTIMATORS="${N_ESTIMATORS:-500}"
HIST_MAX_ITER="${HIST_MAX_ITER:-250}"
SPECIALIST_MAX_ITER="${SPECIALIST_MAX_ITER:-250}"
REL_EPOCHS="${REL_EPOCHS:-40}"
WINDOW_MAX_ITER="${WINDOW_MAX_ITER:-250}"

# Four concurrent sklearn-heavy processes can otherwise each request all CPUs.
TOTAL_CPU="$(nproc 2>/dev/null || echo 16)"
DEFAULT_PER_JOB=$(( TOTAL_CPU / 4 ))
if (( DEFAULT_PER_JOB < 1 )); then DEFAULT_PER_JOB=1; fi
if (( DEFAULT_PER_JOB > 8 )); then DEFAULT_PER_JOB=8; fi
CPU_PER_JOB="${CPU_PER_JOB:-$DEFAULT_PER_JOB}"
export LOKY_MAX_CPU_COUNT="$CPU_PER_JOB"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"
export MKL_NUM_THREADS="${MKL_NUM_THREADS:-1}"
export OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-1}"
export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}"

mkdir -p "$OUTPUT_ROOT" \
  "$OUTPUT_ROOT/a_tree_specialist" \
  "$OUTPUT_ROOT/b_relational_stacker" \
  "$OUTPUT_ROOT/c_history_regime" \
  "$OUTPUT_ROOT/d_multiwindow"

COMMON=(--splits_dir "$SPLITS_DIR" --faculty "$FACULTY" --dev_folds 1,2,3,4,5 --holdout_folds 6,7 --n_estimators "$N_ESTIMATORS" --hist_max_iter "$HIST_MAX_ITER")

echo "================================================================================"
echo "SUITE2 — launching four experiments concurrently"
echo "splits   : $SPLITS_DIR"
echo "outputs  : $OUTPUT_ROOT"
echo "faculty  : $FACULTY"
echo "device   : $DEVICE (PyTorch relational parts only; sklearn tree experts use CPU)"
echo "CPU/job  : $CPU_PER_JOB | OMP=$OMP_NUM_THREADS | trees=$N_ESTIMATORS | HGB=$HIST_MAX_ITER"
echo "================================================================================"

python3 "$SCRIPT_DIR/exp_a_tree_specialist.py" "${COMMON[@]}" \
  --specialist_max_iter "$SPECIALIST_MAX_ITER" \
  --models_dir "$OUTPUT_ROOT/a_tree_specialist" \
  > "$OUTPUT_ROOT/a_tree_specialist.log" 2>&1 &
PID_A=$!

python3 "$SCRIPT_DIR/exp_b_relational_stacker.py" "${COMMON[@]}" \
  --stacker_max_iter "$SPECIALIST_MAX_ITER" --rel_epochs "$REL_EPOCHS" --device "$DEVICE" \
  --models_dir "$OUTPUT_ROOT/b_relational_stacker" \
  > "$OUTPUT_ROOT/b_relational_stacker.log" 2>&1 &
PID_B=$!

python3 "$SCRIPT_DIR/exp_c_history_regime.py" "${COMMON[@]}" \
  --specialist_max_iter "$SPECIALIST_MAX_ITER" --rel_epochs "$REL_EPOCHS" --device "$DEVICE" \
  --models_dir "$OUTPUT_ROOT/c_history_regime" \
  > "$OUTPUT_ROOT/c_history_regime.log" 2>&1 &
PID_C=$!

python3 "$SCRIPT_DIR/exp_d_multiwindow.py" "${COMMON[@]}" \
  --window_max_iter "$WINDOW_MAX_ITER" \
  --models_dir "$OUTPUT_ROOT/d_multiwindow" \
  > "$OUTPUT_ROOT/d_multiwindow.log" 2>&1 &
PID_D=$!

echo "Started PIDs:"
echo "  A strong tree specialist : $PID_A"
echo "  B relational stacker     : $PID_B"
echo "  C history regimes        : $PID_C"
echo "  D multi-window           : $PID_D"
echo

fail=0
wait_one() {
  local pid="$1" name="$2"
  if wait "$pid"; then
    echo "[DONE] $name"
  else
    code=$?
    echo "[FAIL] $name exit=$code"
    fail=1
  fi
}
wait_one "$PID_A" "A strong tree specialist"
wait_one "$PID_B" "B relational stacker"
wait_one "$PID_C" "C history regimes"
wait_one "$PID_D" "D multi-window"

echo "================================================================================"
if (( fail == 0 )); then
  echo "SUITE2 COMPLETE — all four processes exited successfully."
else
  echo "SUITE2 COMPLETE WITH FAILURES — inspect *.log files under $OUTPUT_ROOT."
fi
echo "================================================================================"
exit "$fail"
