Rewrite using less stuff and gnu parallel insted of while loop.

This commit is contained in:
CronyAkatsuki 2022-12-12 20:23:57 +01:00
parent eea2093b3a
commit 60c7ea5e9c

View File

@ -1,25 +1,18 @@
#!/bin/bash
#!/bin/env bash
# script to quickly download rom sets from internet archive at max speed from its xml with the list of files
xml_url="$1"
# Save xml url to a variable
jobs_count="$1"
xml_url="$2"
# Get the url without the xml
base_url=$(dirname $xml_url)
# Download command using aria2 for fastest downloads
dl="aria2c -j 16 -s 16 -x 16 -k 5M --file-allocation=none --continue=true"
# getting roms while removing unneded files and roms like prototypes and betas
roms=$(wget -q $xml_url -O - | grep name | cut -d \" -f 2 | sed '/.xml/d' | sed '/.sqlite/d' | sed '/(Proto)/d' | sed '/(Unl)/d' | sed '/(Beta)/d')
rom_count=$(printf '%s\n' "${roms}" | wc -l)
# getting roms while removing unneded roms for a gaming collection, remove the second grep command if you wan't every possible rom out there.
roms=$(curl -Ls "$xml_url" | grep name | cut -d \" -f2 | grep -ivE "history/|.xml|.sqlite|.torrent|(Proto.*)|(Unl.*)|(Beta.*)|(.*Anthology.*)|(.*Virtual Console.*)|(Pre-Release)|(Kiosk Demo)|(old)|(Demo)|(Program)|(Promo)|(.*Collection.*)|\[o\]|\[b\]|\[BIOS\]")
printf '%s\n' "Downloading $rom_count roms"
count="1"
while read rom; do
# getting the url and making it usable
rom_url=$(printf '%s\n' "${base_url}/${rom}")
printf '%s\n' "Downloading ${count}. ${rom}"
$dl "${rom_url}"
printf '%s\n' "Done, left $((${rom_count} - ${count})) roms to download"
count=$(($count + 1))
done <<< "$roms"
# Using parallel to download 6 at the same time
printf '%s\n' "$roms" | parallel -j $jobs_count $dl "$base_url/{}"