"""Resource-aware job queue for the decoder_lodo experiments (avoids overloading the 8-core CPU).

A job is one shell command run from the experiment root. The scheduler starts the next pending job only when
  * fewer than --max-jobs of its own jobs are running,
  * the 1-min load average is below --max-load,
  * the GPU has at least --min-gpu-free-gb free memory, and
  * at least --min-ram-free-gb of RAM is available (no swap: a job that does not fit is OOM-killed).
Jobs run under `nice -n 10` with OMP/MKL/OpenBLAS limited to 2 threads and SCORE_JOBS (parallel scorer workers
used by integrators.py / reliability.py / msm.py) set to --score-jobs. Pending jobs start in name order, so a
numeric prefix sets the priority.

  python runners/jobqueue.py add NAME -- COMMAND ...      queue a job (NAME e.g. 010_causal_onset)
  python runners/jobqueue.py status                       pending / running / done / failed
  python runners/jobqueue.py run [--max-jobs 2 ...]       scheduler loop (run it in tmux window 'jobq')
State: runners/jobs/{pending,running,done,failed}/NAME.json ; job output: logs/jobs/NAME.log
"""
import argparse
import json
import os
import shutil
import subprocess
import sys
import time

EXP = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
NAMESPACE = os.environ.get("JOBQUEUE_NAMESPACE", "")
if NAMESPACE and (not NAMESPACE.replace("_", "").isalnum()):
    raise ValueError("JOBQUEUE_NAMESPACE must contain only letters, digits, or underscores")
Q = os.path.join(EXP, "runners", "jobs" + ("_" + NAMESPACE if NAMESPACE else ""))
STATES = ("pending", "running", "done", "failed")
for s in STATES:
    os.makedirs(os.path.join(Q, s), exist_ok=True)
os.makedirs(os.path.join(EXP, "logs", "jobs"), exist_ok=True)


def jobs(state):
    return sorted(f[:-5] for f in os.listdir(os.path.join(Q, state)) if f.endswith(".json"))


def move(name, a, b, **upd):
    src, dst = os.path.join(Q, a, name + ".json"), os.path.join(Q, b, name + ".json")
    d = json.load(open(src))
    d.update(upd)
    json.dump(d, open(dst, "w"), indent=1)
    os.remove(src)
    return d


def gpu_free_gb():
    try:
        out = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.free", "--format=csv,noheader,nounits"], text=True)
        return float(out.split()[0]) / 1024
    except Exception:
        return 0.0


def ram_free_gb():
    """MemAvailable: the machine has no swap, a job that does not fit gets OOM-killed (rc 137)."""
    try:
        for ln in open("/proc/meminfo"):
            if ln.startswith("MemAvailable"):
                return float(ln.split()[1]) / 1024 ** 2
    except Exception:
        pass
    return 0.0


def stamp():
    return time.strftime("%F %T")


def cmd_add(a):
    if any(os.path.exists(os.path.join(Q, s, a.name + ".json")) for s in STATES):
        sys.exit(f"job {a.name} already exists")
    cmd = " ".join(a.command)
    json.dump(dict(name=a.name, cmd=cmd, added=stamp()), open(os.path.join(Q, "pending", a.name + ".json"), "w"), indent=1)
    print(f"queued {a.name}: {cmd}")


def cmd_status(a):
    for s in STATES:
        js = jobs(s)
        print(f"{s:8s} {len(js):3d}  " + ("" if not js else ", ".join(js[-12:])))
    print(f"load {os.getloadavg()[0]:.1f} / {os.cpu_count()} cores, GPU free {gpu_free_gb():.1f} GB, RAM available {ram_free_gb():.1f} GB")


def cmd_run(a):
    procs = {}
    log = open(os.path.join(EXP, "logs", "jobqueue" + ("_" + NAMESPACE if NAMESPACE else "") + ".log"), "a", buffering=1)
    say = lambda m: (print(f"{stamp()} {m}", flush=True), log.write(f"{stamp()} {m}\n"))
    for n in jobs("running"):                      # left over from a previous scheduler: requeue
        move(n, "running", "pending", note="requeued after scheduler restart")
    say(f"scheduler started: max_jobs {a.max_jobs}, max_load {a.max_load}, min_gpu_free {a.min_gpu_free_gb} GB")
    while True:
        for n, p in list(procs.items()):
            if p.poll() is not None:
                st = "done" if p.returncode == 0 else "failed"
                move(n, "running", st, finished=stamp(), returncode=p.returncode)
                say(f"{st.upper():6s} {n} (rc {p.returncode})")
                del procs[n]
        pend = jobs("pending")
        load, free = os.getloadavg()[0], gpu_free_gb()
        if pend and len(procs) < a.max_jobs and load < a.max_load and free >= a.min_gpu_free_gb and ram_free_gb() >= a.min_ram_free_gb:
            n = pend[0]
            d = move(n, "pending", "running", started=stamp())
            env = dict(os.environ, OMP_NUM_THREADS="2", MKL_NUM_THREADS="2", OPENBLAS_NUM_THREADS="2",
                       SCORE_JOBS=str(a.score_jobs))
            out = open(os.path.join(EXP, "logs", "jobs", n + ".log"), "a")
            procs[n] = subprocess.Popen(["nice", "-n", "10", "bash", "-c", d["cmd"]], cwd=EXP, env=env,
                                        stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
            say(f"START  {n} (load {load:.1f}, GPU free {free:.1f} GB): {d['cmd']}")
            time.sleep(a.settle)                    # let the job allocate before judging resources again
            continue
        time.sleep(a.poll)


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    sp = ap.add_subparsers(dest="cmd", required=True)
    p = sp.add_parser("add")
    p.add_argument("name")
    p.add_argument("command", nargs=argparse.REMAINDER)
    sp.add_parser("status")
    p = sp.add_parser("run")
    p.add_argument("--max-jobs", type=int, default=2)
    p.add_argument("--max-load", type=float, default=7.0)
    p.add_argument("--min-gpu-free-gb", type=float, default=6.0)
    p.add_argument("--min-ram-free-gb", type=float, default=7.0, help="start a job only with this much available RAM "
                   "(a training job takes ~4 GB; other users' processes share the 24 GB)")
    p.add_argument("--score-jobs", type=int, default=3)
    p.add_argument("--poll", type=float, default=30.0)
    p.add_argument("--settle", type=float, default=120.0)
    a = ap.parse_args()
    if a.cmd == "add" and a.command[:1] == ["--"]:
        a.command = a.command[1:]
    {"add": cmd_add, "status": cmd_status, "run": cmd_run}[a.cmd](a)
