diff --git a/ia-downloader b/ia-downloader index 328a97a..45398a8 100755 --- a/ia-downloader +++ b/ia-downloader @@ -1,25 +1,18 @@ -#!/bin/bash +#!/bin/env bash # script to quickly download rom sets from internet archive at max speed from its xml with the list of files -xml_url="$1" +# Save xml url to a variable +jobs_count="$1" +xml_url="$2" +# Get the url without the xml base_url=$(dirname $xml_url) +# Download command using aria2 for fastest downloads dl="aria2c -j 16 -s 16 -x 16 -k 5M --file-allocation=none --continue=true" -# getting roms while removing unneded files and roms like prototypes and betas -roms=$(wget -q $xml_url -O - | grep name | cut -d \" -f 2 | sed '/.xml/d' | sed '/.sqlite/d' | sed '/(Proto)/d' | sed '/(Unl)/d' | sed '/(Beta)/d') -rom_count=$(printf '%s\n' "${roms}" | wc -l) +# getting roms while removing unneded roms for a gaming collection, remove the second grep command if you wan't every possible rom out there. +roms=$(curl -Ls "$xml_url" | grep name | cut -d \" -f2 | grep -ivE "history/|.xml|.sqlite|.torrent|(Proto.*)|(Unl.*)|(Beta.*)|(.*Anthology.*)|(.*Virtual Console.*)|(Pre-Release)|(Kiosk Demo)|(old)|(Demo)|(Program)|(Promo)|(.*Collection.*)|\[o\]|\[b\]|\[BIOS\]") -printf '%s\n' "Downloading $rom_count roms" - -count="1" - -while read rom; do - # getting the url and making it usable - rom_url=$(printf '%s\n' "${base_url}/${rom}") - printf '%s\n' "Downloading ${count}. ${rom}" - $dl "${rom_url}" - printf '%s\n' "Done, left $((${rom_count} - ${count})) roms to download" - count=$(($count + 1)) -done <<< "$roms" +# Using parallel to download 6 at the same time +printf '%s\n' "$roms" | parallel -j $jobs_count $dl "$base_url/{}"