A utility for downloading and verifying FreeBSD releases and snapshots
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

191 lines
5.0 KiB

  1. #!/bin/sh -
  2. #
  3. # Copyright 2018, 2022-2023 John-Mark Gurney.
  4. # All rights reserved.
  5. #
  6. # Redistribution and use in source and binary forms, with or without
  7. # modification, are permitted provided that the following conditions
  8. # are met:
  9. # 1. Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # 2. Redistributions in binary form must reproduce the above copyright
  12. # notice, this list of conditions and the following disclaimer in the
  13. # documentation and/or other materials provided with the distribution.
  14. #
  15. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25. # SUCH DAMAGE.
  26. #
  27. set -e
  28. #
  29. # Note: It appears that sometimes files can "disappear" from the server, so,
  30. # uncompress snapshot.complete.idx.xz to both snapshot.complete.idx and
  31. # snapshot.idx, and then run "sh addinfo.sh -c YYYYMMDD" with the oldest
  32. # known snapshot.
  33. SNAPDIR=$(dirname $0)
  34. MAILDIR="$SNAPDIR"/mail
  35. usage() {
  36. echo "Usage: $0 [ -m ] <file>"
  37. echo "Usage: $0 -c <date>"
  38. echo ''
  39. echo 'Options:'
  40. echo ' -m Do not check what files are available. This is useful for'
  41. echo ' bulk import.'
  42. echo ' -c Complete a bulk import (previously using -m), and assume that'
  43. echo ' any snapshots before specified data are unfetchable.'
  44. echo ''
  45. echo 'date is specified as YYYYMMDD'
  46. if [ x"$1" != x"" ]; then
  47. exit $1
  48. fi
  49. }
  50. args=`getopt cm $*`
  51. if [ $? -ne 0 ]; then
  52. usage 2
  53. fi
  54. set -- $args
  55. while :; do
  56. case "$1" in
  57. -c)
  58. complete=1
  59. shift
  60. ;;
  61. -m)
  62. more=1
  63. shift
  64. ;;
  65. --)
  66. shift; break
  67. ;;
  68. esac
  69. done
  70. if [ ! -d "$MAILDIR" ]; then
  71. echo 'ERROR: '"$MAILDIR"' does not exist.'
  72. exit 3
  73. fi
  74. if [ x"$complete" = x"1" -a x"$more" = x"1" ]; then
  75. echo '-m and -c cannot be specified at the same time.'
  76. usage 2
  77. elif [ x"$complete" = x"1" -a $# -ne 1 ]; then
  78. echo 'must only specify a date with -c'
  79. usage 2
  80. elif [ x"$complete" != x"1" -a $# -ne 1 ]; then
  81. echo 'must specify exactly one file'
  82. usage 2
  83. fi
  84. while ! mkdir "$0.running"; do
  85. sleep 1;
  86. done
  87. trap 'rmdir "$0.running"' 0
  88. export SNAPAID_SH=source
  89. . "$SNAPDIR"/snapaid.sh
  90. if [ x"$complete" = x"1" ]; then
  91. if [ ! -f snapshot.complete.idx ]; then
  92. echo 'snapshot.complete.idx does not exist, aborting...'
  93. rmdir "$0.running"
  94. exit 5
  95. fi
  96. if [ ! -f snapshot.idx ]; then
  97. echo 'snapshot.idx does not exist, aborting...'
  98. rmdir "$0.running"
  99. exit 5
  100. fi
  101. sort -u snapshot.complete.idx | xz > snapshot.complete.idx.new.xz
  102. awk '$5 == "xxx" || $5 >= "'"$1"'" {
  103. if (!system("wget --method=HEAD " $9))
  104. print
  105. }
  106. ' snapshot.idx | sort -u | xz > snapshot.idx.new.xz
  107. chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz
  108. mv snapshot.idx.new.xz snapshot.idx.xz
  109. mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
  110. rm snapshot.idx snapshot.complete.idx
  111. exit 0
  112. fi
  113. # minimize file
  114. tmpfname="tmp.snapinf.asc"
  115. minimizeemail < "$1" > "$tmpfname"
  116. if ! verifygpgfile "$tmpfname"; then
  117. echo 'failed verify'
  118. rm "$tmpfname"
  119. rmdir "$0.running"
  120. exit 1
  121. fi
  122. mid="$(awk 'tolower($1) == "message-id:" {
  123. MID=$2
  124. sub(".*<", "", MID)
  125. sub(">.*", "", MID)
  126. printf "%s", MID
  127. exit 0
  128. }' "$tmpfname")"
  129. midfile="$MAILDIR/$mid"
  130. if [ -e "$midfile" ]; then
  131. echo "ERROR: $midfile already exists, failed processing $tmpfname"
  132. exit 3
  133. fi
  134. # process file
  135. awk -f ./mksnapidx.awk "$tmpfname" > additional
  136. cp "$tmpfname" "$midfile"
  137. chmod og+rX "$midfile"
  138. rm "$tmpfname"
  139. # only check if there isn't more to come
  140. if [ x"$more" = x"1" ]; then
  141. echo queuing $(wc -l additional) entries
  142. (cat snapshot.idx 2>/dev/null || xzcat snapshot.idx.xz; cat additional) > snapshot.idx.new
  143. (cat snapshot.complete.idx 2>/dev/null || xzcat snapshot.complete.idx.xz; cat additional) > snapshot.complete.idx.new
  144. else
  145. (xzcat snapshot.idx.xz; cat additional) | sort -u | awk '
  146. {
  147. print "testing " $9 > "/dev/stderr"
  148. if (!system("wget --method=HEAD " $9))
  149. print
  150. }
  151. ' > snapshot.idx.new
  152. xz snapshot.idx.new
  153. (xzcat snapshot.complete.idx.xz || :; cat additional) | sort -u > snapshot.complete.idx.new
  154. xz snapshot.complete.idx.new
  155. fi
  156. rm additional
  157. # install new indexes
  158. if [ x"$more" = x"1" ]; then
  159. mv snapshot.idx.new snapshot.idx
  160. mv snapshot.complete.idx.new snapshot.complete.idx
  161. else
  162. chmod 644 snapshot.idx.new.xz snapshot.complete.idx.new.xz
  163. mv snapshot.idx.new.xz snapshot.idx.xz
  164. mv snapshot.complete.idx.new.xz snapshot.complete.idx.xz
  165. fi