#!/usr/bin/env bash
# Download the external datasets, 3 streams in parallel (Zenodo/Dryad throttle per connection); resumable
# (curl -C -), a finished file gets <file>.ok. Log: download.log
cd "$(dirname "$0")"
get() {  # dir file url
  local d=$1 f=$2 u=$3
  [[ -f "$d/raw/$f.ok" ]] && { echo "$(date '+%F %T') have $d/$f"; return; }
  echo "$(date '+%F %T') GET  $d/$f"
  for i in 1 2 3 4 5 6 7 8; do
    curl -sL -C - --retry 5 --retry-delay 20 -o "$d/raw/$f" "$u" && { touch "$d/raw/$f.ok"; echo "$(date '+%F %T') OK   $d/$f ($(du -h "$d/raw/$f" | cut -f1))"; return; }
    echo "$(date '+%F %T') retry $i $d/$f"; sleep 60
  done
  echo "$(date '+%F %T') FAILED $d/$f"
}
Z=https://zenodo.org/records
( get AASD "Stimuli Audio.zip" "$Z/17413336/files/Stimuli%20Audio.zip?download=1"
  get AASD "Processed EEG.zip" "$Z/17413336/files/Processed%20EEG.zip?download=1"
  get AASD "Original EEG.zip"  "$Z/17413336/files/Original%20EEG.zip?download=1" ) &
( get Broderick2018 "Cocktail Party.zip" "https://datadryad.org/api/v2/files/222584/download" ) &
( get Fuglsang2020 "ds-eeg-snhl.tar" "$Z/3618205/files/ds-eeg-snhl.tar?download=1" ) &
wait
echo "$(date '+%F %T') all downloads finished"
