"""Download the SparrKULee derivatives needed for decoder_lodo (KU Leuven server, account required).

Credentials are NOT in this repo: curl reads them from a netrc file (default /root/.netrc_sparrkulee, chmod 600;
override with env SPARRKULEE_NETRC). Fetched:
  derivatives/preprocessed_eeg/sub-*/ses-*/*.npy      (preprocessed EEG, one file per run)
  derivatives/preprocessed_stimuli/*_-_envelope.npy   (+ .save_metadata.json)
Resumable (curl -C -); a finished file gets <file>.ok; 3 parallel streams. File list -> raw/filelist.json.
usage: python download_sparrkulee.py   (log: download.log)"""
import json
import os
import re
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor

BASE = "https://homes.esat.kuleuven.be/~spchdata/corpora/auditory_eeg_data/sparrkulee/"
NETRC = os.environ.get("SPARRKULEE_NETRC", "/root/.netrc_sparrkulee")
HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, "raw")


def log(m):
    print(f"{time.strftime('%F %T')} {m}", flush=True)


def ls(path):
    for i in range(5):
        r = subprocess.run(["curl", "-s", "--netrc-file", NETRC, "--max-time", "120", BASE + path], capture_output=True, text=True)
        if r.returncode == 0 and "<html" in r.stdout.lower():
            break
        time.sleep(30)
    out = []
    for m in re.finditer(r'<tr class="file">(.*?)</tr>', r.stdout, flags=re.S):
        row = m.group(1)
        n = re.search(r'<span class="name">([^<]+)</span>', row)
        s = re.findall(r'data-order="(-?\d+)"', row)
        if n:
            out.append((n.group(1), int(s[0]) if s else -1, "#folder" in row))
    return out


def crawl():
    fl = os.path.join(RAW, "filelist.json")
    if os.path.exists(fl):
        return json.load(open(fl))
    files = []
    st = "derivatives/preprocessed_stimuli/"
    files += [(st + n, s) for n, s, d in ls(st) if not d and (n.endswith("_-_envelope.npy") or n == ".save_metadata.json")]
    eeg = "derivatives/preprocessed_eeg/"
    subs = [n for n, _, d in ls(eeg) if d]
    for sub in subs:
        for ses, _, d in ls(eeg + sub + "/"):
            if d:
                p = eeg + sub + "/" + ses + "/"
                files += [(p + n, s) for n, s, dd in ls(p) if not dd]
        log(f"crawled {sub}: {len(files)} files so far, {sum(s for _, s in files) / 1e9:.1f} GB")
    json.dump(files, open(fl, "w"))
    return files


def get(item):
    rel, size = item
    dst = os.path.join(RAW, rel.replace("derivatives/", ""))
    if os.path.exists(dst + ".ok"):
        return 0
    os.makedirs(os.path.dirname(dst), exist_ok=True)
    for i in range(8):
        r = subprocess.run(["curl", "-s", "-f", "-C", "-", "--retry", "5", "--retry-delay", "20", "--netrc-file", NETRC,
                            "-o", dst, BASE + rel])
        if r.returncode in (0, 33) and (size < 0 or os.path.getsize(dst) == size):
            open(dst + ".ok", "w").close()
            return size
        if r.returncode == 33 or (size > 0 and os.path.exists(dst) and os.path.getsize(dst) > size):
            os.remove(dst)                                    # corrupt partial: restart
        log(f"retry {i + 1} {rel} (rc {r.returncode})")
        time.sleep(60)
    log(f"FAILED {rel}")
    return 0


if __name__ == "__main__":
    os.makedirs(RAW, exist_ok=True)
    files = crawl()
    tot = sum(max(s, 0) for _, s in files)
    log(f"{len(files)} files, {tot / 1e9:.1f} GB")
    done, t0 = 0, time.time()
    with ThreadPoolExecutor(3) as ex:
        for k, s in enumerate(ex.map(get, files)):
            done += s
            if k % 50 == 0:
                el = time.time() - t0
                log(f"{k + 1}/{len(files)} files, {done / 1e9:.1f} GB this session ({done / 1e6 / max(el, 1):.1f} MB/s)")
    n_ok = sum(os.path.exists(os.path.join(RAW, r.replace("derivatives/", "")) + ".ok") for r, _ in files)
    log(f"finished: {n_ok}/{len(files)} files ok")
    if n_ok == len(files):
        open(os.path.join(RAW, "ALL_DONE"), "w").close()
