~gpanders/garchive

ref: refs/heads/master garchive/bin/fetch -rwxr-xr-x 678 bytes
ccadfc70 — Greg Anders Update README 6 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/bin/sh

if [ $# -lt 2 ]; then
    echo "Usage: $(basename "$0") <links.csv> <dir>" >&2
    exit 1
fi

links="$(realpath "$1")"
cd "$2" || exit 1

tmp="$(mktemp)"
awk -F'\t' '{print $2}' "$links" > "$tmp"

pids=""
while IFS= read -r url; do
    wget \
        --adjust-extension \
        --timestamping \
        --span-hosts \
        --convert-links \
        --page-requisites \
        --directory-prefix="$2" \
        --continue \
        --quiet \
        --wait=1 \
        --random-wait \
        --user-agent="" \
        --execute robots=off \
        "$url"
    pids="$! $pids"
done < "$tmp"

for pid in $pids; do
    wait "$pid"
done

rm -f "$tmp"

echo "Done!"