From dcecd2d3e5c5a4274cf6228b5bca0a7df589822e Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Sun, 4 Nov 2018 16:17:17 -0800 Subject: [PATCH] check for keys on startup, and test for failed d/l's. Make sure that the necessary keys are present before starting. Make sure that a failed download will be properly detected and not leave behind invalid files. --- snapaid.sh | 163 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 138 insertions(+), 25 deletions(-) diff --git a/snapaid.sh b/snapaid.sh index 0b8c610..d343622 100755 --- a/snapaid.sh +++ b/snapaid.sh @@ -1,6 +1,8 @@ #!/bin/sh - STOREDIR="$HOME/.snapaid" +KEYS="0x524F0C37A0B946A3" +KEY_URLS='https://svnweb.freebsd.org/doc/head/share/pgpkeys/gjb.key?view=co' setdefaults() { GPG=$(which gpg2) @@ -27,9 +29,9 @@ fi #wget: # -N for timestamps # --backups=x for backing up - -completeurl="https://www.funkthat.com/~jmg/FreeBSD-snap/snapshot.complete.idx.xz" -currenturl="https://www.funkthat.com/~jmg/FreeBSD-snap/snapshot.idx.xz" +hostname=people.FreeBSD.org +completeurl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.complete.idx.xz" +currenturl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.idx.xz" # type release arch platform date svnrev xxx fname url mid # 1 2 3 4 5 6 7 8 9 10 @@ -55,9 +57,31 @@ gpg_first_fails() { return 1 } +# When first arg is --, just touch the file to fake a bad d/l +bad_file_dl() { + if [ x"$1" = x"--" ]; then + touch $(basename "$2") + else + $WGET_orig "$@" + fi +} + # Make sure that the storage directory is present mkstore() { mkdir "$STOREDIR" 2>/dev/null || : + if ! [ -x "$STOREDIR" -a -w "$STOREDIR" -a -d "$STOREDIR" ]; then + echo "$STOREDIR is not a writable directory." + fi +} + +check_keys() { + for i in $KEYS; do + if ! $GPG --list-keys "$i" >/dev/null 2>&1; then + return 1 + fi + done + + return 0 } # Given a message id, get the raw body and store it. @@ -72,6 +96,13 @@ get_raw() { # get the location, it's a database lookup loc=$($WGET --max-redirect=0 --method=HEAD -S -o - -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null | awk 'tolower($1) == "location:" { print $2; exit }') + # Some emails are sent to both -current and -snapshot, + # we can't handle them for now + # such as 20160529215940.GA11785@FreeBSD.org + if [ x"$loc" = x"" ]; then + return 1 + fi + # if it's relative, add https if [ x"$loc" != x"${loc#//}" ]; then # add https @@ -118,7 +149,9 @@ get_raw() { fetch() { mkstore - (cd "$STOREDIR" && $WGET -N "$1" >/dev/null 2>&1) + if ! (cd "$STOREDIR" && $WGET -N "$1" >/dev/null 2>&1); then + return 1 + fi } getvermid() { @@ -184,10 +217,52 @@ EOF echo "$hash $2" | $SHASUM -a "${algo#SHA}" -c - } +dlverify() { + fname="$8" + dlurl="$9" + vermid="${10}" + + # verify snap email + if ! get_raw "$vermid"; then + echo "Unable to fetch/verify snapshot email for: $fname" + return 1 + fi + + # fetch link + $WGET -- "$dlurl" + + if ! verifyfile "$vermid" "$fname"; then + echo 'Removing bad file.' + rm "$fname" + return 1 + fi +} + +if ! check_keys; then + echo 'Necessary keys have not been imported into key ring.' + echo "Please obtain they following keyid(s):" + echo $KEYS + echo "" + echo "The keys may be obtained from the following URLs:" + for i in $KEY_URLS; do + echo "$i" + done + echo "" + echo "and imported into GPG w/ the --import option." + echo "" + echo "For extra security, additional verification should be done, such" + echo "as manually verifying finger prints." + + exit 3 +fi + if [ x"$1" = x"verify" ]; then shift - fetch "$completeurl" + if ! fetch "$completeurl"; then + echo Failed to fetch the complete index. + exit 1 + fi for i in "$@"; do vermid=$(getvermid "$i") @@ -196,12 +271,12 @@ if [ x"$1" = x"verify" ]; then continue fi - get_raw "$vermid" - if ! verifygpg "$vermid".raw; then - echo "Unable to verify: $i" + if ! get_raw "$vermid"; then + echo "Unable to fetch snapshot email for: $i" + continue fi - verifyfile "$verurl" "$i" + verifyfile "$vermid" "$i" done elif [ x"$1" = x"find" ]; then fetch "$currenturl" @@ -235,7 +310,7 @@ BEGIN { } ' selection - read -p 'Select image, enter search term, reset, or quit: ' sel + read -p 'Select image #, enter search term, reset, or quit: ' sel if [ x"$sel" = x"reset" ]; then xzcat "$STOREDIR"/snapshot.idx.xz | sort -r -k 5 > selection; continue @@ -249,7 +324,6 @@ BEGIN { fi if [ "$sel" -ge 1 -a "$sel" -le "$cnt" ] 2>/dev/null; then - echo selected image $sel echo $(tail -n +"$sel" selection | head -n 1) > sel break else @@ -269,25 +343,62 @@ BEGIN { exit 0 fi - echo $sel - fname=$(cut -f 8 -d ' ' "$tmpdir"/sel) - dlurl=$(cut -f 9 -d ' ' "$tmpdir"/sel) - verurl=$(cut -f 10 -d ' ' "$tmpdir"/sel) + set -- ${sel} - # fetch link - $WGET -- "$dlurl" + echo selected image "$8" + + dlverify ${sel} +elif [ x"$1" = x"test" ]; then + # Setup test environment + tmpdir=$(mktemp -d -t snapaid) + + trap "rm -rf $tmpdir" 0 + + cd "$tmpdir" + + STOREDIR="$tmpdir"/snapaid + + # Make sure that the check keys function works. + echo 'Testing check_keys works...' + + # Prime the custom keyring + GPG="gpg2 --no-default-keyring --keyring pubring.gpg" + for i in $KEY_URLS; do + $WGET -O - -- "$i" 2>/dev/null | $GPG --import 2>/dev/null + done - # verify image - fetch "$verurl" - if ! verifygpg "$verurl"; then - echo "Unable to verify: $fname" + if ! check_keys; then + echo failed + exit 1 fi - if ! verifyfile "$verurl" "$fname"; then - rm "$fname" + KEYS_orig="$KEYS" + KEYS="0x1384923867573928" # bogus key + + if check_keys; then + echo failed + exit 1 fi -elif [ x"$1" = x"test" ]; then - # Run various tests + + echo passed + + # Test a bad download fails + echo 'Testing dlverify...' + WGET_orig="$WGET" + WGET=bad_file_dl + + # if dlverify is successsful, then it's a failure + if dlverify iso 13.0-CURRENT sparc64 xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz 20181026184443.GD75350@FreeBSD.org; then + echo 'failed' + exit 1 + fi + + # Make sure that a bad d/l was not left behind + if [ -e FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz ]; then + echo failed + exit 1 + fi + echo passed # Test getting the raw file echo 'Testing get_raw success...' @@ -333,6 +444,8 @@ elif [ x"$1" = x"test" ]; then echo passed setdefaults + + echo tests completed!!! else echo "Unknown verb: $1" echo "Usage:"