crony/ia-downloader
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Rewrite using less stuff and gnu parallel insted of while loop.

Browse Source
This commit is contained in:
CronyAkatsuki 2022-12-12 20:23:57 +01:00
parent eea2093b3a
commit 60c7ea5e9c
1 changed files with 10 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ia-downloader
View File

@ -1,25 +1,18 @@
#!/bin/bash #!/bin/env bash
# script to quickly download rom sets from internet archive at max speed from its xml with the list of files # script to quickly download rom sets from internet archive at max speed from its xml with the list of files
xml_url="$1" # Save xml url to a variable
jobs_count="$1"
xml_url="$2"
# Get the url without the xml
base_url=$(dirname $xml_url) base_url=$(dirname $xml_url)
# Download command using aria2 for fastest downloads
dl="aria2c -j 16 -s 16 -x 16 -k 5M --file-allocation=none --continue=true" dl="aria2c -j 16 -s 16 -x 16 -k 5M --file-allocation=none --continue=true"
# getting roms while removing unneded files and roms like prototypes and betas # getting roms while removing unneded roms for a gaming collection, remove the second grep command if you wan't every possible rom out there.
roms=$(wget -q $xml_url -O - | grep name | cut -d \" -f 2 | sed '/.xml/d' | sed '/.sqlite/d' | sed '/(Proto)/d' | sed '/(Unl)/d' | sed '/(Beta)/d') roms=$(curl -Ls "$xml_url" | grep name | cut -d \" -f2 | grep -ivE "history/|.xml|.sqlite|.torrent|(Proto.*)|(Unl.*)|(Beta.*)|(.*Anthology.*)|(.*Virtual Console.*)|(Pre-Release)|(Kiosk Demo)|(old)|(Demo)|(Program)|(Promo)|(.*Collection.*)|\[o\]|\[b\]|\[BIOS\]")
rom_count=$(printf '%s\n' "${roms}" | wc -l)
printf '%s\n' "Downloading $rom_count roms" # Using parallel to download 6 at the same time
printf '%s\n' "$roms" | parallel -j $jobs_count $dl "$base_url/{}"
count="1"
while read rom; do
# getting the url and making it usable
rom_url=$(printf '%s\n' "${base_url}/${rom}")
printf '%s\n' "Downloading ${count}. ${rom}"
$dl "${rom_url}"
printf '%s\n' "Done, left $((${rom_count} - ${count})) roms to download"
count=$(($count + 1))
done <<< "$roms"