A utility for downloading and verifying FreeBSD releases and snapshots
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

433 lines
13 KiB

  1. #
  2. # Copyright 2018, 2022 John-Mark Gurney.
  3. # All rights reserved.
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions
  7. # are met:
  8. # 1. Redistributions of source code must retain the above copyright
  9. # notice, this list of conditions and the following disclaimer.
  10. # 2. Redistributions in binary form must reproduce the above copyright
  11. # notice, this list of conditions and the following disclaimer in the
  12. # documentation and/or other materials provided with the distribution.
  13. #
  14. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24. # SUCH DAMAGE.
  25. #
  26. # $Id$
  27. #
  28. BEGIN {
  29. vmroot = "https://download.freebsd.org/ftp/snapshots/VM-IMAGES/"
  30. isoroot = "https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/"
  31. bciroot = "https://download.freebsd.org/ftp/snapshots/CI-IMAGES/"
  32. relvmroot = "https://download.freebsd.org/ftp/releases/VM-IMAGES/"
  33. relisoroot = "https://download.freebsd.org/ftp/releases/ISO-IMAGES/"
  34. relbciroot = "https://download.freebsd.org/ftp/releases/CI-IMAGES/"
  35. #print "begin" > "/dev/stderr"
  36. hexdigits["0"] = 0
  37. hexdigits["1"] = 1
  38. hexdigits["2"] = 2
  39. hexdigits["3"] = 3
  40. hexdigits["4"] = 4
  41. hexdigits["5"] = 5
  42. hexdigits["6"] = 6
  43. hexdigits["7"] = 7
  44. hexdigits["8"] = 8
  45. hexdigits["9"] = 9
  46. hexdigits["A"] = 10
  47. hexdigits["B"] = 11
  48. hexdigits["C"] = 12
  49. hexdigits["D"] = 13
  50. hexdigits["E"] = 14
  51. hexdigits["F"] = 15
  52. if (DO_TESTS == 1) {
  53. run_tests()
  54. exit(0)
  55. }
  56. }
  57. function run_tests() {
  58. fromhextests["3D"] = 61
  59. fromhextests["3F"] = 63
  60. for (i in fromhextests) {
  61. if (fromhex(i) != fromhextests[i]) {
  62. printf("ERROR: fromhex failed on %s, expecting %d, got %d.\n", i, fromhextests[i], fromhex(i))
  63. exit(1)
  64. }
  65. }
  66. validrevs[0] = "r369411"
  67. validrevs[1] = "c122cf32f2a"
  68. validrevs[2] = "b83b87f53e73"
  69. for (i in validrevs) {
  70. if (!isrev(validrevs[i])) {
  71. print "ERROR: valid rev not valid: " validrevs[i]
  72. exit(1)
  73. }
  74. }
  75. invalidrevs[0] = "asoijw23r"
  76. for (i in invalidrevs) {
  77. if (isrev(invalidrevs[i])) {
  78. print "ERROR: invalid rev valid: " invalidrevs[i]
  79. exit(1)
  80. }
  81. }
  82. MID = "custommid"
  83. fnametst[0] = "FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz"
  84. typeres[0] = "vm"
  85. parseres[0] = "vm 13.0-CURRENT arm64-aarch64 xxx 20200903 c122cf32f2a qcow2 FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz https://download.freebsd.org/ftp/snapshots/VM-IMAGES/13.0-CURRENT/aarch64/20200903/FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz custommid"
  86. fnametst[1] = "FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso"
  87. typeres[1] = "iso"
  88. parseres[1] = "iso 13.0-CURRENT powerpc-powerpcspe xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso custommid"
  89. fnametst[2] = "FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz"
  90. typeres[2] = "iso"
  91. parseres[2] = "iso 13.0-CURRENT arm64-aarch64 PINE64-LTS 20181026 r339752 xxx FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz custommid"
  92. fnametst[3] = "FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz"
  93. typeres[3] = "vm"
  94. parseres[3] = "vm 13.0-CURRENT i386 xxx 20181026 r339752 vmdk FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz https://download.freebsd.org/ftp/snapshots/VM-IMAGES/13.0-CURRENT/i386/20181026/FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz custommid"
  95. fnametst[4] = "FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz"
  96. typeres[4] = "iso"
  97. parseres[4] = "iso 12.1-BETA3 amd64 mini-memstick xxx unspec xxx FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.1/FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz custommid"
  98. fnametst[5] = "FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz"
  99. typeres[5] = "iso"
  100. parseres[5] = "iso 12.1-RC1 amd64 mini-memstick xxx unspec xxx FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.1/FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz custommid"
  101. fnametst[6] = "FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz"
  102. typeres[6] = "vm"
  103. parseres[6] = "vm 12.1-RELEASE arm64-aarch64 xxx xxx unspec vmdk FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz https://download.freebsd.org/ftp/releases/VM-IMAGES/12.1-RELEASE/aarch64/Latest/FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz custommid"
  104. fnametst[7] = "FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz"
  105. typeres[7] = "vm"
  106. parseres[7] = "vm 13.1-RELEASE riscv-riscv64 xxx xxx unspec vmdk FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz https://download.freebsd.org/ftp/releases/VM-IMAGES/13.1-RELEASE/riscv64/Latest/FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz custommid"
  107. fnametst[8] = "FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz"
  108. typeres[8] = "vm"
  109. parseres[8] = "vm 13.1-RELEASE amd64 BASIC-CI xxx unspec raw FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz https://download.freebsd.org/ftp/releases/CI-IMAGES/13.1-RELEASE/amd64/Latest/FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz custommid"
  110. fnametst[9] = "FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso"
  111. typeres[9] = "iso"
  112. parseres[9] = "iso 13.1-STABLE amd64 xxx 20230114 b83b87f53e73-c253383 bootonly FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.1/FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso custommid"
  113. fnametst[10] = "FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso"
  114. typeres[10] = "iso"
  115. parseres[10] = "iso 13.0-CURRENT sparc64 xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso custommid"
  116. for (tstidx in fnametst) {
  117. type = typeres[tstidx]
  118. tres = parse_fname(fnametst[tstidx])
  119. if (tres != parseres[tstidx]) {
  120. print "ERROR: invalid fname parse (idx " tstidx ") of " fnametst[tstidx]
  121. print "got:"
  122. print tres
  123. print "expected:"
  124. print parseres[tstidx]
  125. exit(1)
  126. }
  127. }
  128. }
  129. function fromhex(val) {
  130. r = 0
  131. while (val) {
  132. r = r * 16 + hexdigits[substr(val, 1, 1)]
  133. val = substr(val, 2)
  134. }
  135. return r
  136. }
  137. {
  138. #print "raw " $0 > "/dev/stderr"
  139. }
  140. !MID && tolower($1) == "message-id:" {
  141. MID=$2
  142. sub(".*<", "", MID)
  143. sub(">.*", "", MID)
  144. }
  145. !haveboundary && tolower($1) == "content-type:" && tolower($2) == "multipart/signed;" && check == 0 {
  146. getboundary = 1
  147. next
  148. }
  149. getboundary == 1 {
  150. getboundary = 0
  151. haveboundary = 1
  152. if (substr($2, 1, 9) == "boundary=")
  153. boundary=substr($2, 11, length($2) - 11)
  154. next
  155. }
  156. haveboundary && $1 == ("--" boundary) {
  157. sigbody = 1
  158. bodyheaders = 1
  159. }
  160. !haveboundary && $0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
  161. sigbody = 1
  162. }
  163. # Skip processing till we have the body
  164. !sigbody {
  165. next
  166. }
  167. bodyheaders {
  168. if (tolower($1) == "content-transfer-encoding:" && tolower($2) == "quoted-printable")
  169. quotedprintable = 1
  170. if ($0 == "") {
  171. #print "end of headers " $0 > "/dev/stderr"
  172. bodyheaders = 0
  173. }
  174. next
  175. }
  176. quotedprintable {
  177. #print "orig raw " $0 > "/dev/stderr"
  178. contline = 0
  179. if (substr($0, length($0), 1) == "=") {
  180. contline = 1
  181. $0 = substr($0, 1, length($0) - 1)
  182. }
  183. procline = $0
  184. line = ""
  185. while (1) {
  186. #print "procline " procline > "/dev/stderr"
  187. #print "line " line > "/dev/stderr"
  188. pos = index(procline, "=")
  189. #print "pos " pos > "/dev/stderr"
  190. if (pos == 0) {
  191. line = line procline
  192. break
  193. }
  194. line = line substr(procline, 1, pos - 1) sprintf("%c", fromhex(substr(procline, pos + 1, 2)))
  195. procline = substr(procline, pos + 3)
  196. }
  197. $0 = line
  198. #print " new raw " $0 > "/dev/stderr"
  199. if (contline) {
  200. #print "continuing" > "/dev/stderr"
  201. prevline = prevline $0
  202. next
  203. }
  204. $0 = prevline $0
  205. prevline = ""
  206. #print "mid " MID > "/dev/stderr"
  207. #print "proc raw " $0 > "/dev/stderr"
  208. }
  209. $0 == "== ISO CHECKSUMS ==" ||
  210. $0 == "ISO Image Checksums" {
  211. type = "iso"
  212. }
  213. $0 == "== VM IMAGE CHECKSUMS ==" ||
  214. $0 == "Virtual Machine Disk Image Checksums" {
  215. type = "vm"
  216. }
  217. function isdate(date) {
  218. m = match(date, "[0-9]+")
  219. if (m && RLENGTH == 8)
  220. return 1
  221. return 0
  222. }
  223. function isrev(s) {
  224. if (substr(s, 1, 1) == "r")
  225. return 1
  226. m = match(s, "[0-9a-f]+")
  227. if (m && RLENGTH >= 11)
  228. return 1
  229. return 0
  230. }
  231. function isrelease(relpart) {
  232. if (index(relpart, "BETA") == 1 || index(relpart, "RC") == 1 ||
  233. index(relpart, "RELEASE") == 1)
  234. return 1
  235. return 0
  236. }
  237. #FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz
  238. #FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso
  239. #FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.asc
  240. #FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
  241. #FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz
  242. #FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz
  243. #FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz
  244. #FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz
  245. #FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz
  246. #FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz
  247. #FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso
  248. /.*SHA512 (.*) = [0-9a-f]+/ {
  249. #print "raw " $0 > "/dev/stderr"
  250. # Strip parens
  251. fname = substr($2, 2, length($2) - 2)
  252. a = parse_fname(fname)
  253. print(a)
  254. }
  255. function parse_fname(fname) {
  256. split(fname, dotparts, ".")
  257. # recombine around dot in version, strips off ALL extensions (including vm type)
  258. basename = dotparts[1] "." dotparts[2]
  259. cnt = split(basename, parts, "-")
  260. # make arch part, may include additional part
  261. arch = parts[4]
  262. basearch = arch
  263. if (parts[4] == "arm" || parts[4] == "riscv" ||
  264. (parts[4] == "powerpc" && parts[5] == "powerpcspe") ||
  265. (parts[4] == "powerpc" && parts[5] == "powerpc64") ||
  266. parts[4] == "arm64") {
  267. # FreeBSD-11.3-STABLE-arm64-aarch64-20191011-r353406-memstick.img
  268. basearch = parts[5]
  269. arch = parts[4] "-" parts[5]
  270. nextidx = 6
  271. } else {
  272. # FreeBSD-11.3-STABLE-amd64-20191011-r353406.qcow2.xz
  273. nextidx = 5
  274. }
  275. # find date, may be platform first
  276. if (isrelease(parts[3])) {
  277. if (nextidx > cnt) {
  278. # FreeBSD-12.1-BETA3-i386.vhd.xz
  279. platform = "xxx"
  280. } else if (cnt == nextidx) {
  281. # FreeBSD-12.1-RC1-powerpc-powerpcspe-dvd1.iso
  282. platform = parts[nextidx]
  283. nextidx += 1
  284. } else {
  285. # FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz
  286. platform = parts[nextidx] "-" parts[nextidx + 1]
  287. nextidx += 2
  288. }
  289. date = "xxx"
  290. } else if (isdate(parts[nextidx])) {
  291. # FreeBSD-11.3-STABLE-amd64-20191011-r353406.qcow2.xz
  292. platform = "xxx"
  293. date = parts[nextidx]
  294. nextidx += 1
  295. } else {
  296. # FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
  297. platform = parts[nextidx]
  298. date = parts[nextidx + 1]
  299. if (isdate(date)) {
  300. # FreeBSD-13.0-CURRENT-arm64-aarch64-PINEBOOK-20191011-r353427.img.xz
  301. nextidx += 2
  302. } else {
  303. # FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
  304. date = parts[nextidx + 2]
  305. platform = parts[nextidx] "-" parts[nextidx + 1]
  306. nextidx += 3
  307. }
  308. }
  309. # if this part doesn't begin w/ r (for svn rev), skip it, we can't parse
  310. # others for now
  311. rev = parts[nextidx]
  312. if (isrelease(parts[3])) {
  313. # FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz
  314. rev = "unspec"
  315. } else if (!isrev(rev)) {
  316. print "invalid rev, skipping: " rev > "/dev/stderr"
  317. #print "raw " $0 > "/dev/stderr"
  318. #printf("%s %s %s %s %s %s %s %s %s %s\n", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID) > "/dev/stderr"
  319. return
  320. } else {
  321. # we have rev, but check if next part is also part of it
  322. if (parts[nextidx + 1] ~ /^[0-9]+$/) {
  323. rev = rev "-c" parts[nextidx + 1]
  324. nextidx += 1
  325. }
  326. }
  327. if (nextidx >= cnt)
  328. vers="xxx"
  329. else {
  330. vers=""
  331. sep=""
  332. for (i = nextidx + 1; i <= cnt; i++) {
  333. vers = vers sep parts[i]
  334. sep="-"
  335. }
  336. }
  337. if (isrelease(parts[3])) {
  338. if (platform == "BASIC-CI") {
  339. vers = dotparts[3]
  340. url = relbciroot parts[2] "-" parts[3] "/" basearch "/Latest/" fname
  341. } else if (type == "vm") {
  342. vers = dotparts[3]
  343. url = relvmroot parts[2] "-" parts[3] "/" basearch "/Latest/" fname
  344. } else
  345. url = relisoroot parts[2] "/" fname
  346. } else {
  347. if (platform == "BASIC-CI") {
  348. vers = dotparts[3]
  349. url = bciroot parts[2] "-" parts[3] "/" basearch "/" date "/" fname
  350. } else if (type == "vm") {
  351. vers = dotparts[3]
  352. url = vmroot parts[2] "-" parts[3] "/" basearch "/" date "/" fname
  353. } else
  354. url = isoroot parts[2] "/" fname
  355. }
  356. # double check that date is valid, if not, skip it
  357. if (date != "xxx" && !isdate(date))
  358. return
  359. res = sprintf("%s %s %s %s %s %s %s %s %s %s", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID)
  360. #printf("%s %s %s %s %s %s %s %s %s %s\n", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID) > "/dev/stderr"
  361. return res
  362. }