Sample downloader script

getem (text/plain), 2.61 KB, created by Miroslav Los on 2011-08-22 23:19:40 UTC

#!/bin/bash
# getem - Gentoo source list downloader. 
# Distributed under the GNU General Public License, version 2 or later.

# usage: getem [wishlist] [dest] [lookin]
#
# where:
#     wishlist is the list of files to download, each line containing, 
#         separated by tabs, the file name, its size and SHA256 checksum and 
#         the list of all urls to try in turn, separated by whitespace
#
#     dest is the destination directory; it will be created if necessary
#     
#     lookin is a directory to look for the file before downloading; useful if 
#     downloading on another Gentoo systems.
# 
# There are also two environment variables controlling behavior, if non-empty:
#     PREGET - download all files from their first urls at once, even if they 
#              already exist in either $dest or $lookin
#   
#     KEEP_GOING - do not exit on first unsuccessful download


WISHLIST=${1:-wishlist}
DEST=${2:-distfiles}
LOOKIN=${3:-/usr/portage/distfiles}
HASH=sha256sum

function testhash() {
    if [ ! -f "$tname" ] ; then
        echo "$fname missing..."
        return 1
    fi

    if [ -n "$size" ] ; then
        local mysize=`stat -c %s "$tname"`
        if [ "$size" = "$mysize" ] ; then
            echo -n "Size ok..."
        else
            echo "Size have $mysize want $size"
            mv "$tname" "$tname.size.bad"
            return 1
        fi
    fi

    if [ -n "$sum" ] ; then
        local mysum=`$HASH "$tname" | sed -e 's/\s.*//'`
        if [ "$sum" = "$mysum" ] ; then
            echo "Hash ok"
            return 0
        else
            echo "Hash have $mysum want $sum"
            mv "$tname" "$tname.sum.bad"
            return 1
        fi
    fi
}

function nocomments() {
    sed -e '/^\s*\(#.*\)\?$/d' "$@"
}

mkdir -p "$DEST" || exit 1

[ -n "$PREGET" ] && nocomments "$WISHLIST" |
    cut -d $'\t' -f 4 | sed -e 's/\s.*/' | sort -u | 
    (cd "$DEST" && wget -nc -i -)


nocomments "$WISHLIST" | {
declare -i wanted=0 needed=0
while IFS=$'\t' read fname size sum urls ; do
    echo "f=$fname s=$size h=$sum u=$urls"
    wanted=$((wanted + 1))
    [ -z "$fname" ] && fname=`basename "${urls/ *}"`
    tname="$DEST/$fname"
    testhash && continue
    
    if [ -f "$LOOKIN/$fname" ] ; then
        cp -v "$LOOKIN/$fname" "$DEST"
        testhash && continue
    fi

    for url in $urls ; do
        wget -nc -O "$tname" "$url"
        testhash && continue 2
    done

    [ -n "$KEEP_GOING" ] || exit 1
    needed=$((needed + 1))
done

if [ "$needed" -gt 0 ] ; then
    echo "Warning: $needed files out of $wanted have failed to download!"
    exit 1
else
    echo "Downloaded all $wanted files"
    exit 0
fi
}