Rewrite using less stuff and gnu parallel insted of while loop.
This commit is contained in:
parent
eea2093b3a
commit
60c7ea5e9c
@ -1,25 +1,18 @@
|
||||
#!/bin/bash
|
||||
#!/bin/env bash
|
||||
|
||||
# script to quickly download rom sets from internet archive at max speed from its xml with the list of files
|
||||
|
||||
xml_url="$1"
|
||||
# Save xml url to a variable
|
||||
jobs_count="$1"
|
||||
xml_url="$2"
|
||||
# Get the url without the xml
|
||||
base_url=$(dirname $xml_url)
|
||||
|
||||
# Download command using aria2 for fastest downloads
|
||||
dl="aria2c -j 16 -s 16 -x 16 -k 5M --file-allocation=none --continue=true"
|
||||
|
||||
# getting roms while removing unneded files and roms like prototypes and betas
|
||||
roms=$(wget -q $xml_url -O - | grep name | cut -d \" -f 2 | sed '/.xml/d' | sed '/.sqlite/d' | sed '/(Proto)/d' | sed '/(Unl)/d' | sed '/(Beta)/d')
|
||||
rom_count=$(printf '%s\n' "${roms}" | wc -l)
|
||||
# getting roms while removing unneded roms for a gaming collection, remove the second grep command if you wan't every possible rom out there.
|
||||
roms=$(curl -Ls "$xml_url" | grep name | cut -d \" -f2 | grep -ivE "history/|.xml|.sqlite|.torrent|(Proto.*)|(Unl.*)|(Beta.*)|(.*Anthology.*)|(.*Virtual Console.*)|(Pre-Release)|(Kiosk Demo)|(old)|(Demo)|(Program)|(Promo)|(.*Collection.*)|\[o\]|\[b\]|\[BIOS\]")
|
||||
|
||||
printf '%s\n' "Downloading $rom_count roms"
|
||||
|
||||
count="1"
|
||||
|
||||
while read rom; do
|
||||
# getting the url and making it usable
|
||||
rom_url=$(printf '%s\n' "${base_url}/${rom}")
|
||||
printf '%s\n' "Downloading ${count}. ${rom}"
|
||||
$dl "${rom_url}"
|
||||
printf '%s\n' "Done, left $((${rom_count} - ${count})) roms to download"
|
||||
count=$(($count + 1))
|
||||
done <<< "$roms"
|
||||
# Using parallel to download 6 at the same time
|
||||
printf '%s\n' "$roms" | parallel -j $jobs_count $dl "$base_url/{}"
|
||||
|
Loading…
Reference in New Issue
Block a user