switch to using a local mail archive...

For the last year plus, the FreeBSD mail archives has been unreliable,
so keep our own archive...

this also makes things more simple as we don't have to handle redirection
that we did w/ the FreeBSD mail archives..

addinfo.sh

@@ -1,6 +1,6 @@
 #!/bin/sh -
 #
-# Copyright 2018, 2022 John-Mark Gurney.
+# Copyright 2018, 2022-2023 John-Mark Gurney.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -24,8 +24,6 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-#	$Id$
-#
 
 set -e
 
@@ -35,6 +33,9 @@ set -e
 # snapshot.idx, and then run "sh addinfo.sh -c YYYYMMDD" with the oldest
 # known snapshot.
 
+SNAPDIR=$(dirname $0)
+MAILDIR="$SNAPDIR"/mail
+
 usage() {
 	echo "Usage: $0 [ -m ] <file>"
 	echo "Usage: $0 -c <date>"
@@ -72,6 +73,10 @@ while :; do
 		;;
 	esac
 done
+if [ ! -d "$MAILDIR" ]; then
+	echo 'ERROR: '"$MAILDIR"' does not exist.'
+	exit 3
+fi
 if [ x"$complete" = x"1" -a x"$more" = x"1" ]; then
 	echo '-m and -c cannot be specified at the same time.'
 	usage 2
@@ -87,9 +92,11 @@ while ! mkdir "$0.running"; do
 	sleep 1;
 done
 
+trap 'rmdir "$0.running"' 0
+
 export SNAPAID_SH=source
 
-. $(dirname $0)/snapaid.sh
+. "$SNAPDIR"/snapaid.sh
 
 if [ x"$complete" = x"1" ]; then
 	if [ ! -f snapshot.complete.idx ]; then
@@ -114,7 +121,6 @@ if [ x"$complete" = x"1" ]; then
 	mv snapshot.idx.new.xz snapshot.idx.xz
 	mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
 	rm snapshot.idx snapshot.complete.idx
-	rmdir "$0.running"
 	exit 0
 fi
 
@@ -129,8 +135,26 @@ if ! verifygpgfile "$tmpfname"; then
 	exit 1
 fi
 
+mid="$(awk 'tolower($1) == "message-id:" {
+        MID=$2
+        sub(".*<", "", MID)
+        sub(">.*", "", MID)
+
+	printf "%s", MID
+
+	exit 0
+}' "$tmpfname")"
+
+midfile="$MAILDIR/$mid"
+if [ -e "$midfile" ]; then
+	echo "ERROR: $midfile already exists, failed processing $tmpfname"
+	exit 3
+fi
+
 # process file
 awk -f ./mksnapidx.awk "$tmpfname" > additional
+cp "$tmpfname" "$midfile"
+chmod og+rX "$midfile"
 rm "$tmpfname"
 
 # only check if there isn't more to come
@@ -164,5 +188,3 @@ else
 	mv snapshot.idx.new.xz snapshot.idx.xz
 	mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
 fi
-
-rmdir "$0.running"

snapaid.sh

@@ -1,6 +1,6 @@
 #!/bin/sh -
 #
-# Copyright 2018, 2022 John-Mark Gurney.
+# Copyright 2018, 2022-2023 John-Mark Gurney.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -60,6 +60,7 @@ hostname=people.FreeBSD.org
 hostname=www.funkthat.com
 completeurl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.complete.idx.xz"
 currenturl="https://${hostname}/~jmg/FreeBSD-snap/snapshot.idx.xz"
+mailarchurl="https://www.funkthat.com/~jmg/FreeBSD-snap/mail/%s"
 
 # type release arch platform date svnrev xxx fname url mid
 # 1    2       3    4        5    6      7   8     9   10
@@ -121,31 +122,13 @@ get_raw() {
 	midfile="$STOREDIR/$mid".raw
 
 	if [ ! -e "$midfile" ]; then
-		# get the location, it's a database lookup
-		loc=$($WGET --max-redirect=0 --method=HEAD -S -o - -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null | awk 'tolower($1) == "location:" { print $2; exit }')
-
-		if [ x"$loc" = x"" ]; then
-			# Some emails are sent to both -current and -snapshot,
-			# such as 20160529215940.GA11785@FreeBSD.org
-			# try w/ some magic sed
-			loc=$($WGET -O - 'https://docs.freebsd.org/cgi/mid.cgi?'"$mid" 2>/dev/null |
-			    sed -Ee '/.*(getmsg.cgi?[^"]*).*/!d;s//\/cgi\/\1/' | head -1)
-		fi
-
-		# if it's host relative, add https
-		if [ x"$loc" != x"${loc#//}" ]; then
-			# add https
-			loc="https:$loc"
-		elif [ x"$loc" != x"${loc#/[^/]}" ]; then
-			# add https+host
-			loc="https://docs.freebsd.org$loc"
-		fi
-
 		# get the raw part
 		tmpfile="$STOREDIR/.tmp.$$.$mid".raw
 
 		# strip out everything but message id and first signed part
-		$WGET -O - "$loc"+raw 2>/dev/null | minimizeemail > "$tmpfile"
+		# minimizeemail isn't necessary anymore as the archive
+		# already has done it, but keep it
+		$WGET -O - $(printf "$mailarchurl" "$mid") 2>/dev/null | minimizeemail > "$tmpfile"
 
 		if verifygpg "$tmpfile"; then
 			mv "$tmpfile" "$STOREDIR/$mid.raw"