#!/bin/sh - # # Copyright 2018, 2022-2023 John-Mark Gurney. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # set -e # # Note: It appears that sometimes files can "disappear" from the server, so, # uncompress snapshot.complete.idx.xz to both snapshot.complete.idx and # snapshot.idx, and then run "sh addinfo.sh -c YYYYMMDD" with the oldest # known snapshot. # # $ xzcat snapshot.idx.xz | awk '{ print $5 }' | sort -u # $ unxz < snapshot.complete.idx.xz > snapshot.complete.idx # $ unxz < snapshot.complete.idx.xz > snapshot.idx SNAPDIR=$(dirname $0) MAILDIR="$SNAPDIR"/mail usage() { echo "Usage: $0 [ -m ] " echo "Usage: $0 -c " echo '' echo 'Options:' echo ' -m Do not check what files are available. This is useful for' echo ' bulk import.' echo ' -c Complete a bulk import (previously using -m), and assume that' echo ' any snapshots before specified data are unfetchable.' echo '' echo 'date is specified as YYYYMMDD' if [ x"$1" != x"" ]; then exit $1 fi } args=`getopt cm $*` if [ $? -ne 0 ]; then usage 2 fi set -- $args while :; do case "$1" in -c) complete=1 shift ;; -m) more=1 shift ;; --) shift; break ;; esac done if [ ! -d "$MAILDIR" ]; then echo 'ERROR: '"$MAILDIR"' does not exist.' exit 3 fi if [ x"$complete" = x"1" -a x"$more" = x"1" ]; then echo '-m and -c cannot be specified at the same time.' usage 2 elif [ x"$complete" = x"1" -a $# -ne 1 ]; then echo 'must only specify a date with -c' usage 2 elif [ x"$complete" != x"1" -a $# -ne 1 ]; then echo 'must specify exactly one file' usage 2 fi while ! mkdir "$0.running"; do sleep 1; done trap 'rmdir "$0.running"' 0 export SNAPAID_SH=source . "$SNAPDIR"/snapaid.sh if [ x"$complete" = x"1" ]; then if [ ! -f snapshot.complete.idx ]; then echo 'snapshot.complete.idx does not exist, aborting...' rmdir "$0.running" exit 5 fi if [ ! -f snapshot.idx ]; then echo 'snapshot.idx does not exist, aborting...' rmdir "$0.running" exit 5 fi sort -u snapshot.complete.idx | xz > snapshot.complete.idx.new.xz awk '$5 == "xxx" || $5 >= "'"$1"'" { if (!system("wget --method=HEAD " $9)) print } ' snapshot.idx | sort -u | xz > snapshot.idx.new.xz chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz mv snapshot.idx.new.xz snapshot.idx.xz mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz rm snapshot.idx snapshot.complete.idx exit 0 fi # minimize file tmpfname="tmp.snapinf.asc" minimizeemail < "$1" > "$tmpfname" if ! verifygpgfile "$tmpfname"; then echo 'failed verify' rm "$tmpfname" rmdir "$0.running" exit 1 fi mid="$(awk 'tolower($1) == "message-id:" { MID=$2 sub(".*<", "", MID) sub(">.*", "", MID) printf "%s", MID exit 0 }' "$tmpfname")" midfile="$MAILDIR/$mid" if [ -e "$midfile" ]; then echo "ERROR: $midfile already exists, failed processing $tmpfname" exit 3 fi # process file awk -f ./mksnapidx.awk "$tmpfname" > additional cp "$tmpfname" "$midfile" chmod og+rX "$midfile" rm "$tmpfname" # only check if there isn't more to come if [ x"$more" = x"1" ]; then echo queuing $(wc -l additional) entries (cat snapshot.idx 2>/dev/null || xzcat snapshot.idx.xz; cat additional) > snapshot.idx.new (cat snapshot.complete.idx 2>/dev/null || xzcat snapshot.complete.idx.xz; cat additional) > snapshot.complete.idx.new else (xzcat snapshot.idx.xz; cat additional) | sort -u | awk ' { print "testing " $9 > "/dev/stderr" if (!system("wget --method=HEAD " $9)) print } ' > snapshot.idx.new xz snapshot.idx.new (xzcat snapshot.complete.idx.xz || :; cat additional) | sort -u > snapshot.complete.idx.new xz snapshot.complete.idx.new fi rm additional # install new indexes if [ x"$more" = x"1" ]; then mv snapshot.idx.new snapshot.idx mv snapshot.complete.idx.new snapshot.complete.idx else chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz mv snapshot.idx.new.xz snapshot.idx.xz mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz fi