snapaid/snapaid.sh

#!/bin/sh -
#
# Copyright 2018, 2022 John-Mark Gurney.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	$Id$
#

STOREDIR="$HOME/.snapaid"
KEYS="78B342BA26C7B2AC681EA7BE524F0C37A0B946A3 EAF48BBA7CC77A30FEFC0DA938CECA690C6A6A6E"
KEY_URLS='https://cgit.freebsd.org/doc/plain/documentation/static/pgpkeys/gjb.key https://cgit.freebsd.org/doc/plain/documentation/static/pgpkeys/cperciva.key'

setdefaults() {
	GPG=$(which gpg2)
	WGET=$(which wget)
	SHASUM=$(which shasum)
}
setdefaults

if [ ! -x "$GPG" ]; then
	echo 'Failed to find gpg2 executable'
	exit 1
fi

if [ ! -x "$WGET" ]; then
	echo 'Failed to find wget executable'
	exit 1
fi

if [ ! -x "$SHASUM" ]; then
	echo 'Failed to find shasum executable'
	exit 1
fi

#wget:
# -N for timestamps
# --backups=x for backing up
hostname=people.FreeBSD.org
hostname=www.funkthat.com
completeurl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.complete.idx.xz"
currenturl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.idx.xz"

# type release arch platform date svnrev xxx fname url mid
# 1    2       3    4        5    6      7   8     9   10
# iso 11.1-STABLE arm-armv6 BEAGLEBONE 20180315 r330998 xxx FreeBSD-11.1-STABLE-arm-armv6-BEAGLEBONE-20180315-r330998.img.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/11.1/FreeBSD-11.1-STABLE-arm-armv6-BEAGLEBONE-20180315-r330998.img.xz 20180316000842.GA7399@FreeBSD.org

set -e

# This is used for some testing functions
copy_function() {
  declare -F "$1" > /dev/null || return 1
  local func="$(declare -f "$1")"
  eval "${2}(${func#*\(}"
}

# Test function to cause a bad input
cmd_failure() {
	exit 1
}

# First time fails, second time run real command
gpg_first_fails() {
	copy_function verifygpg_orig verifygpg
	return 1
}

# When first arg is --, just touch the file to fake a bad d/l
bad_file_dl() {
	if [ x"$1" = x"--" ]; then
		touch $(basename "$2")
	else
		$WGET_orig "$@"
	fi
}

# Make sure that the storage directory is present
mkstore() {
	mkdir "$STOREDIR" 2>/dev/null || :
	if ! [ -x "$STOREDIR" -a -w "$STOREDIR" -a -d "$STOREDIR" ]; then
		echo "$STOREDIR is not a writable directory."
	fi
}

check_keys() {
	for i in $KEYS; do
		if ! $GPG --list-keys "$i" >/dev/null 2>&1; then
			return 1
		fi
	done

	return 0
}

# Given a message id, get the raw body and store it.
get_raw() {
	mkstore

	mid="$1"

	midfile="$STOREDIR/$mid".raw

	if [ ! -e "$midfile" ]; then
		# get the location, it's a database lookup
		loc=$($WGET --max-redirect=0 --method=HEAD -S -o - -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null | awk 'tolower($1) == "location:" { print $2; exit }')

		if [ x"$loc" = x"" ]; then
			# Some emails are sent to both -current and -snapshot,
			# such as 20160529215940.GA11785@FreeBSD.org
			# try w/ some magic sed
			loc=$($WGET -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null |
			    sed -Ee '/.*(getmsg.cgi?[^"]*).*/!d;s//\/cgi\/\1/' | head -1)
		fi

		# if it's host relative, add https
		if [ x"$loc" != x"${loc#//}" ]; then
			# add https
			loc="https:$loc"
		elif [ x"$loc" != x"${loc#/[^/]}" ]; then
			# add https+host
			loc="https://docs.freebsd.org$loc"
		fi

		# get the raw part
		tmpfile="$STOREDIR/.tmp.$$.$mid".raw

		# strip out everything but message id and first signed part
		$WGET -O - "$loc"+raw 2>/dev/null | minimizeemail > "$tmpfile"

		if verifygpg "$tmpfile"; then
			mv "$tmpfile" "$STOREDIR/$mid.raw"
		else
			rm "$tmpfile"
			echo Bad signature from mail archive.
			return 1
		fi
	else
		if ! verifygpg "$midfile"; then
			rm "$midfile"
			get_raw "$mid"
			return $?
		fi
	fi
}

fetch() {
	mkstore

	if ! (cd "$STOREDIR" && $WGET -N "$1" >/dev/null 2>&1); then
		return 1
	fi
}

getvermid() {
	xzcat "$STOREDIR"/snapshot.complete.idx.xz | awk '$8 == fname {
	print $10
}' fname="$i"

}

minimizeemail() {
	awk '
	tolower($1) == "message-id:" && check == 0 {
		print
	}

	tolower($1) == "content-type:" && tolower($2) == "multipart/signed;" && check == 0 {
		getboundary = 1
		print
		next
	}

	getboundary == 1 {
		getboundary = 0
		haveboundary = 1
		print
		if (substr($2, 1, 9) == "boundary=")
			boundary=substr($2, 11, length($2) - 11)

		next
	}

	haveboundary && $1 == ("--" boundary) {
		sigbody = 1
	}

	$0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
		sigbody = 1
	}

	sigbody {
		print
	}

	$0 == "-----END PGP SIGNATURE-----" && !haveboundary {
		sigbody = 0
	}'
}

verifygpgfile() {
	local fname
	fname="$1"

	if grep "multipart/signed" "$fname" >/dev/null 2>&1; then
		tmpdir=$(mktemp -d -t snapaid)

		# Note mkfifo does not work, for some reason I got a
		# different order, they are small, so just write them to
		# disk and clean up afterward.

		awk -v FNAME="$tmpdir"/msg '
BEGIN {
	if (FNAME == "") {
		print "FNAME not specified." > "/dev/stderr"
		exit 1
	}
}

END {
	#print "exiting" > "/dev/stderr"
}

{
	#print "raw " $0 > "/dev/stderr"
}

$0 ~ "protocol=\"application/pgp-signature" {
	if (substr($2, 1, 9) == "boundary=")
		boundary=substr($2, 11, length($2) - 11)

	#print "boundary " boundary " remaining line: " $0 > "/dev/stderr"
	next
}

boundary != "" && $1 == ("--" boundary) {
	if (state == 0) {
		output = 1
		outfname = FNAME
		printf("") > outfname
		state = 1
	} else if (state == 1) {
		close(outfname)
		outfname = FNAME ".asc"
		printf("") > outfname
		state = 2
	} else if (state == 2) {
		close(outfname)
		output = 0
	}
	#print "state " state ", boundary: " boundary ", output: " output > "/dev/stderr"
	next
}


# Do not print the final line ending.  It belongs w/ the ending boundary,
# and we do not want to prepend it first time through
output {
        printf("%s%s", lineend, $0) >> outfname
        lineend = "\r\n"
}' < "$fname"

		$GPG --verify "$tmpdir"/msg.asc "$tmpdir"/msg 2>/dev/null
		exitval="$?"

		#rm -f "$tmpdir"/msg.asc "$tmpdir"/msg
		#rmdir "$tmpdir"

		return "$exitval"
	else
		$GPG --verify "$fname" 2>/dev/null
	fi
}

# takes basename of arg, which much exist in STOREDIR, and verifies
# that the signature is valid.
verifygpg() {
	local fname
	fname=$(basename "$1")
	if ! (cd "$STOREDIR" && verifygpgfile "$fname" ); then
		echo 'ERROR: PGP signature verification failed!'
		return 1
	fi
}

# Verifies the file
verifyfile() {
	local fname
	local hashinfo
	local algo hash

	fname="$STOREDIR/${1}.raw"
	hashinfo=$(awk -v FNAME="$2" '
		$0 ~ "protocol=\"application/pgp-signature" {
			if (substr($2, 1, 9) == "boundary=")
				boundary=substr($2, 11, length($2) - 11)

			#print "boundary " boundary " remaining line: " $0 > "/dev/stderr"
			next
		}

		boundary != "" && $1 == ("--" boundary) {
			if (check)
				check = 0
			else
				check = 1

			next
		}

		check && $2 == ("(" FNAME ")") {
			hashes[$1] = $4
		}

		$0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
			check = 1
		}

		$0 == "-----BEGIN PGP SIGNATURE-----" {
			check = 0
		}

		END {
			if ("SHA512" in hashes)
				algo = "SHA512"
			else if ("SHA256" in hashes)
				algo = "SHA256"
			else {
				print "unkn BADHASH"
				exit 1
			}

			print algo " " hashes[algo]
		}
		' "$fname")
	read algo hash <<-EOF
		${hashinfo}
EOF

	if [ x"$algo" == x"unkn" -o x"$algo" = x"" ]; then
		echo 'Unable to find hash for file.'
		exit 1
	fi

	echo "$hash  $2" | $SHASUM -a "${algo#SHA}" -c -
}

dlverify() {
	fname="$8"
	dlurl="$9"
	vermid="${10}"

	# verify snap email
	if ! get_raw "$vermid"; then
		echo "Unable to fetch/verify snapshot email for: $fname"
		return 1
	fi

	if ! [ -f $(basename "$dlurl") ]; then
		# fetch link
		$WGET -- "$dlurl"
	else
		echo 'Image already exists, verifying...'
	fi

	if ! verifyfile "$vermid" "$fname"; then
		echo 'Removing bad file.'
		rm "$fname"
		return 1
	fi
}

# Special wget that doesn't fetch the large file
WGET_special1() {
	if [ "$1" = "--" -a "$2" = "https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/14.0/FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz" ]; then
		return 0
	else
		$WGET_orig "$@"
	fi
}

# Allow the file to be sourced so the functions for checking PGP
# signatures can be used by addinfo.sh
if [ x"$SNAPAID_SH" = x"source" ]; then
	return 0
fi

if ! check_keys; then
	echo 'Necessary keys have not been imported into key ring.'
	echo "Please obtain they following keyid(s):"
	echo $KEYS
	echo ""
	echo "The keys may be obtained from the following URLs:"
	for i in $KEY_URLS; do
		echo "$i"
	done
	echo ""
	echo "and imported into GPG w/ the --import option.  This can be"
	echo "done via the command:"
	echo "fetch -o - $KEY_URLS | gpg --import -"
	echo ""
	echo "For extra security, additional verification should be done, such"
	echo "as manually verifying finger prints."

	exit 3
fi

if [ x"$1" = x"verify" ]; then
	shift

	if ! fetch "$completeurl"; then
		echo Failed to fetch the complete index.
		exit 1
	fi

	for i in "$@"; do
		vermid=$(getvermid "$i")
		if [ x"$vermid" = x"" ]; then
			echo "Unable to find entry for: $i"
			continue
		fi

		if ! get_raw "$vermid"; then
			echo "Unable to fetch snapshot email for: $i"
			continue
		fi

		verifyfile "$vermid" "$i"
	done
elif [ x"$1" = x"find" ]; then
	shift
	fetch "$currenturl"

	tmpdir=$(mktemp -d -t snapaid)

	trap "rm -r $tmpdir" 0

	( cd "$tmpdir";
	xzcat "$STOREDIR"/snapshot.idx.xz | sort -r -k 5 -k 2 > selection;
	while :; do
		cnt=$(wc -l < selection)
		if [ x"$1" = x"" ]; then
			# display current list
			awk '
BEGIN {
	# xzcat snapshot.complete.idx.xz | ./maxcol.awk
	# xzcat snapshot.complete.idx.xz | awk "{ print $3}" | sort -u
	# note that for powerpc-* that first part is dropped
	fmtstr = "%2s %-3s  %-15s %-14s %-18s %-8s %-11s\n"
	printf(fmtstr, "#", "TYP", "RELEASE", "ARCH", "PLATFORM/TYPE", "DATE", "REV")
	cnt = 1
}

{
	if ($3 ~ /^powerpc-/)
		$3 = substr($3, 9)
	if ($4 == "xxx")
		plt=$7
	else
		plt=$4
	printf(fmtstr, cnt, $1, $2, $3, plt, $5, $6)
	if (cnt >= 20)
		exit 0

	cnt += 1
}
			' selection
		fi

		if [ x"$1" != x"" ]; then
			sel="$1"
			shift
		else
			read -p 'Select image #, enter search term, reset, or quit: ' sel
		fi
		if [ x"$sel" = x"reset" ]; then
			xzcat "$STOREDIR"/snapshot.idx.xz | sort -r -k 5 > selection;
			continue
		elif [ x"$sel" = x"quit" ]; then
			echo "$sel" > sel
			break
		fi

		if [ "$cnt" -gt 20 ]; then
			cnt=20
		fi

		if [ "$sel" -ge 1 -a "$sel" -le "$cnt" ] 2>/dev/null; then
			echo $(tail -n +"$sel" selection | head -n 1) > sel
			break
		else
			# restrict

			if ! grep -- "$sel" selection > selection.new; then
				echo WARNING: Ignoring selection, no results.
			else
				mv selection.new selection
			fi
		fi
	done
	)

	sel=$(cat "$tmpdir"/sel)
	if [ x"$sel" = x"quit" ]; then
		exit 0
	fi

	set -- ${sel}

	echo selected image "$8"

	dlverify ${sel}
elif [ x"$1" = x"test" ]; then
	# Setup test environment
	tmpdir=$(mktemp -d -t snapaid)

	trap "rm -r $tmpdir" 0

	cd "$tmpdir"

	STOREDIR="$tmpdir"/snapaid

	# Make sure that the check keys function works.
	echo 'Testing check_keys works...'

	# Prime the custom keyring
	GPG="gpg2 --no-default-keyring --keyring pubring.gpg"
	for i in $KEY_URLS; do
		$WGET -O - -- "$i" 2>/dev/null | $GPG --import 2>/dev/null
	done

	if ! check_keys; then
		echo failed
		exit 1
	fi

	KEYS_orig="$KEYS"
	KEYS="0x1384923867573928" # bogus key

	if check_keys; then
		echo failed
		exit 1
	fi

	echo passed

	# save WGET & SHASUM
	WGET_orig="$WGET"
	SHASUM_orig="$SHASUM"

	# Testing signature that is an attachment
	echo 'Testing email with attached signature...'

	# called by dlverify
	#get_raw "20210909215942.GH1630@FreeBSD.org"
	WGET=WGET_special1
	SHASUM="echo FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz: OK; : "

	if ! dlverify iso 14.0-CURRENT amd64 xxx 20210909 xxx bootonly FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/14.0/FreeBSD-14.0-CURRENT-amd64-20210909-58a7bf124cc-249268-bootonly.iso.xz 20210909215942.GH1630@FreeBSD.org; then
		echo 'failed'
		exit 1
	fi
	echo passed
	WGET="$WGET_orig"
	SHASUM="$SHASUM_orig"

	# Test a bad download fails
	echo 'Testing dlverify...'
	WGET=bad_file_dl

	# if dlverify is successsful, then it's a failure
	if dlverify iso 13.0-CURRENT sparc64 xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz 20181026184443.GD75350@FreeBSD.org; then
		echo 'failed'
		exit 1
	fi

	# Make sure that a bad d/l was not left behind
	if [ -e FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.xz ]; then
		echo failed
		exit 1
	fi
	echo passed

	# Test getting the raw file
	echo 'Testing get_raw success...'
	mid='20160122055622.GA87581@FreeBSD.org'
	get_raw "$mid"

	# Verify resulsts
	(cd "$STOREDIR" && echo '6e53df5995b6cc423c7f2d63b6df52d5d7f70e8586c25f91433fd8a1a2466e77be6a38884bde8bedd9ff6e7deb0215a66e1c2a16e4955503c20445e649a5fb47  20160122055622.GA87581@FreeBSD.org.raw' | $SHASUM -a 512 -c)
	echo passed

	# If the file already exists, but fails verification, that
	# it will refetch and be correct
	echo 'Testing get_raw with file already present that fails verification...'
	copy_function verifygpg verifygpg_orig
	copy_function gpg_first_fails verifygpg
	get_raw "$mid"

	(cd "$STOREDIR" && echo '6e53df5995b6cc423c7f2d63b6df52d5d7f70e8586c25f91433fd8a1a2466e77be6a38884bde8bedd9ff6e7deb0215a66e1c2a16e4955503c20445e649a5fb47  20160122055622.GA87581@FreeBSD.org.raw' | $SHASUM -a 512 -c)

	echo passed

	# If the file already exists, a "broken" wget won't cause
	# a problem
	echo 'Testing get_raw with file already present...'
	WGET=cmd_failure
	get_raw "$mid"

	echo passed

	# Test failure
	echo 'Testing get_raw fails w/ bad data...'
	WGET=cmd_failure
	rm "$STOREDIR/$mid.raw"

	# it should fail
	! get_raw "$mid"

	# and the desired file should not exist
	if [ -e "$STOREDIR/$mid.raw" ]; then
		echo 'Test failed!'
		exit 1;
	fi
	echo passed

	setdefaults

	echo tests completed!!!
else
	if [ $# -gt 0 ]; then
		echo "Unknown verb: $1"
	fi
	echo "Usage:"
	echo "	$0 verify file ..."
	echo "	$0 find [ termselection ... ]"
	echo ""
	echo "The verify option will attempt to verify each file specified."
	echo ""
	echo "The find option will start up an interactive session to find"
	echo "and select the snapshot to download and verify."
fi