snapaid/mksnapidx.awk

#
# Copyright 2018, 2022 John-Mark Gurney.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	$Id$
#

BEGIN {
	vmroot = "https://download.freebsd.org/ftp/snapshots/VM-IMAGES/"
	ociroot = "https://download.freebsd.org/ftp/snapshots/OCI-IMAGES/"
	isoroot = "https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/"
	bciroot = "https://download.freebsd.org/ftp/snapshots/CI-IMAGES/"

	relvmroot = "https://download.freebsd.org/ftp/releases/VM-IMAGES/"
	relociroot = "https://download.freebsd.org/ftp/releases/OCI-IMAGES/"
	relisoroot = "https://download.freebsd.org/ftp/releases/ISO-IMAGES/"
	relbciroot = "https://download.freebsd.org/ftp/releases/CI-IMAGES/"

	#print "begin" > "/dev/stderr"
	hexdigits["0"] = 0
	hexdigits["1"] = 1
	hexdigits["2"] = 2
	hexdigits["3"] = 3
	hexdigits["4"] = 4
	hexdigits["5"] = 5
	hexdigits["6"] = 6
	hexdigits["7"] = 7
	hexdigits["8"] = 8
	hexdigits["9"] = 9
	hexdigits["A"] = 10
	hexdigits["B"] = 11
	hexdigits["C"] = 12
	hexdigits["D"] = 13
	hexdigits["E"] = 14
	hexdigits["F"] = 15

	if (DO_TESTS == 1) {
		run_tests()
		exit(0)
	}
}

function run_tests() {
	fromhextests["3D"] = 61
	fromhextests["3F"] = 63

	for (i in fromhextests) {
		if (fromhex(i) != fromhextests[i]) {
			printf("ERROR: fromhex failed on %s, expecting %d, got %d.\n", i, fromhextests[i], fromhex(i))
			exit(1)
		}
	}

	validrevs[0] = "r369411"
	validrevs[1] = "c122cf32f2a"
	validrevs[2] = "b83b87f53e73"

	for (i in validrevs) {
		if (!isrev(validrevs[i])) {
			print "ERROR: valid rev not valid: " validrevs[i]
			exit(1)
		}
	}

	invalidrevs[0] = "asoijw23r"

	for (i in invalidrevs) {
		if (isrev(invalidrevs[i])) {
			print "ERROR: invalid rev valid: " invalidrevs[i]
			exit(1)
		}
	}

	MID = "custommid"

	fnametst[0] = "FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz"
	typeres[0]  = "vm"
	parseres[0] = "vm 13.0-CURRENT arm64-aarch64 xxx 20200903 c122cf32f2a qcow2 FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz https://download.freebsd.org/ftp/snapshots/VM-IMAGES/13.0-CURRENT/aarch64/20200903/FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz custommid"

	fnametst[1] = "FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso"
	typeres[1]  = "iso"
	parseres[1] = "iso 13.0-CURRENT powerpc-powerpcspe xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso custommid"

	fnametst[2] = "FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz"
	typeres[2]  = "iso"
	parseres[2] = "iso 13.0-CURRENT arm64-aarch64 PINE64-LTS 20181026 r339752 xxx FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz custommid"

	fnametst[3] = "FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz"
	typeres[3]  = "vm"
	parseres[3] = "vm 13.0-CURRENT i386 xxx 20181026 r339752 vmdk FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz https://download.freebsd.org/ftp/snapshots/VM-IMAGES/13.0-CURRENT/i386/20181026/FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz custommid"

	fnametst[4] = "FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz"
	typeres[4]  = "iso"
	parseres[4] = "iso 12.1-BETA3 amd64 mini-memstick xxx unspec xxx FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.1/FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz custommid"

	fnametst[5] = "FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz"
	typeres[5]  = "iso"
	parseres[5] = "iso 12.1-RC1 amd64 mini-memstick xxx unspec xxx FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.1/FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz custommid"

	fnametst[6] = "FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz"
	typeres[6]  = "vm"
	parseres[6] = "vm 12.1-RELEASE arm64-aarch64 xxx xxx unspec vmdk FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz https://download.freebsd.org/ftp/releases/VM-IMAGES/12.1-RELEASE/aarch64/Latest/FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz custommid"

	fnametst[7] = "FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz"
	typeres[7]  = "vm"
	parseres[7] = "vm 13.1-RELEASE riscv-riscv64 xxx xxx unspec vmdk FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz https://download.freebsd.org/ftp/releases/VM-IMAGES/13.1-RELEASE/riscv64/Latest/FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz custommid"

	fnametst[8] = "FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz"
	typeres[8]  = "vm"
	parseres[8] = "vm 13.1-RELEASE amd64 BASIC-CI xxx unspec raw FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz https://download.freebsd.org/ftp/releases/CI-IMAGES/13.1-RELEASE/amd64/Latest/FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz custommid"

	fnametst[9] = "FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso"
	typeres[9]  = "iso"
	parseres[9] = "iso 13.1-STABLE amd64 xxx 20230114 b83b87f53e73-c253383 bootonly FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.1/FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso custommid"

	fnametst[10] = "FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso"
	typeres[10]  = "iso"
	parseres[10] = "iso 13.0-CURRENT sparc64 xxx 20181026 r339752 bootonly FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso https://download.freebsd.org/ftp/snapshots/ISO-IMAGES/13.0/FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso custommid"

	fnametst[11] = "FreeBSD-14.2-RELEASE-riscv-riscv64-container-image-static.txz"
	typeres[11]  = "oci"
	parseres[11] = "oci 14.2-RELEASE riscv-riscv64 xxx xxx unspec txz FreeBSD-14.2-RELEASE-riscv-riscv64-container-image-static.txz https://download.freebsd.org/ftp/releases/OCI-IMAGES/14.2-RELEASE/riscv64/Latest/FreeBSD-14.2-RELEASE-riscv-riscv64-container-image-static.txz custommid"
	#res = sprintf("%s %s %s %s %s %s %s %s %s %s", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID)

	fnametst[12] = "FreeBSD-15.0-CURRENT-riscv-riscv64-container-image-20241128-edfccce309a6-273911-static.txz"
	typeres[12]  = "oci"
	parseres[12] = "oci 15.0-CURRENT riscv-riscv64 xxx 20241128 edfccce309a6-c273911 txz FreeBSD-15.0-CURRENT-riscv-riscv64-container-image-20241128-edfccce309a6-273911-static.txz https://download.freebsd.org/ftp/snapshots/OCI-IMAGES/15.0-CURRENT/riscv64/20241128/FreeBSD-15.0-CURRENT-riscv-riscv64-container-image-20241128-edfccce309a6-273911-static.txz custommid"


	for (tstidx in fnametst) {
		type = typeres[tstidx]
		tres = parse_fname(fnametst[tstidx])
		if (tres != parseres[tstidx]) {
			print "ERROR: invalid fname parse (idx " tstidx ") of " fnametst[tstidx]
			print "got:"
			print tres
			print "expected:"
			print parseres[tstidx]
			exit(1)
		}
	}
}

function fromhex(val) {
	r = 0
	while (val) {
		r = r * 16 + hexdigits[substr(val, 1, 1)]
		val = substr(val, 2)
	}

	return r
}

{
	#print "raw " $0 > "/dev/stderr"
}

!MID && tolower($1) == "message-id:" {
	MID=$2
	sub(".*<", "", MID)
	sub(">.*", "", MID)
}

!haveboundary && tolower($1) == "content-type:" && tolower($2) == "multipart/signed;" && check == 0 {
	getboundary = 1
	next
}

getboundary == 1 {
	getboundary = 0
	haveboundary = 1
	if (substr($2, 1, 9) == "boundary=")
		boundary=substr($2, 11, length($2) - 11)

	next
}

haveboundary && $1 == ("--" boundary) {
	sigbody = 1
	bodyheaders = 1
}

!haveboundary && $0 == "-----BEGIN PGP SIGNED MESSAGE-----" {
	sigbody = 1
}

# Skip processing till we have the body
!sigbody {
	next
}

bodyheaders {
	if (tolower($1) == "content-transfer-encoding:" && tolower($2) == "quoted-printable")
		quotedprintable = 1

	if ($0 == "") {
		#print "end of headers " $0 > "/dev/stderr"
		bodyheaders = 0
	}

	next
}

quotedprintable {
	#print "orig raw " $0 > "/dev/stderr"

	contline = 0
	if (substr($0, length($0), 1) == "=") {
		contline = 1
		$0 = substr($0, 1, length($0) - 1)
	}

	procline = $0
	line = ""
	while (1) {
		#print "procline " procline > "/dev/stderr"
		#print "line " line > "/dev/stderr"
		pos = index(procline, "=")
		#print "pos " pos > "/dev/stderr"
		if (pos == 0) {
			line = line procline
			break
		}

		line = line substr(procline, 1, pos - 1) sprintf("%c", fromhex(substr(procline, pos + 1, 2)))
		procline = substr(procline, pos + 3)
	}

	$0 = line
	#print " new raw " $0 > "/dev/stderr"

	if (contline) {
		#print "continuing" > "/dev/stderr"
		prevline = prevline $0
		next
	}

	$0 = prevline $0

	prevline = ""

	#print "mid " MID > "/dev/stderr"
	#print "proc raw " $0 > "/dev/stderr"
}

$0 == "== ISO CHECKSUMS ==" ||
$0 == "ISO Image Checksums" {
	type = "iso"
}

$0 == "== VM IMAGE CHECKSUMS ==" ||
$0 == "Virtual Machine Disk Image Checksums" {
	type = "vm"
}

$0 == "OCI Container Checksums" {
	type = "oci"
}

function isdate(date) {
	m = match(date, "[0-9]+")
	if (m && RLENGTH == 8)
		return 1

	return 0
}

function isrev(s) {
	if (substr(s, 1, 1) == "r")
		return 1

	m = match(s, "[0-9a-f]+")
	if (m && RLENGTH >= 11)
		return 1

	return 0
}

function isrelease(relpart) {
	if (index(relpart, "BETA") == 1 || index(relpart, "RC") == 1 ||
	    index(relpart, "RELEASE") == 1)
		return 1

	return 0
}

#FreeBSD-13.0-CURRENT-arm64-aarch64-20200903-c122cf32f2a.qcow2.xz
#FreeBSD-13.0-CURRENT-powerpc-powerpcspe-20181026-r339752-bootonly.iso
#FreeBSD-13.0-CURRENT-sparc64-20181026-r339752-bootonly.iso.asc
#FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
#FreeBSD-13.0-CURRENT-i386-20181026-r339752.vmdk.xz
#FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz
#FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz
#FreeBSD-12.1-RELEASE-arm64-aarch64.vmdk.xz
#FreeBSD-13.1-RELEASE-riscv-riscv64.vmdk.xz
#FreeBSD-13.1-RELEASE-amd64-BASIC-CI.raw.xz
#FreeBSD-13.1-STABLE-amd64-20230114-b83b87f53e73-253383-bootonly.iso

/.*SHA512 (.*) = [0-9a-f]+/ {
	#print "raw " $0 > "/dev/stderr"

	# Strip parens
	fname = substr($2, 2, length($2) - 2)

	a = parse_fname(fname)

	print(a)
}

function parse_fname(fname) {
	split(fname, dotparts, ".")

	# recombine around dot in version, strips off ALL extensions (including vm type)
	basename = dotparts[1] "." dotparts[2]

	cnt = split(basename, parts, "-")

	# make arch part, may include additional part
	arch = parts[4]
	basearch = arch
	if (parts[4] == "arm" || parts[4] == "riscv" ||
	    (parts[4] == "powerpc" && parts[5] == "powerpcspe") ||
	    (parts[4] == "powerpc" && parts[5] == "powerpc64") ||
	    parts[4] == "arm64") {
		# FreeBSD-11.3-STABLE-arm64-aarch64-20191011-r353406-memstick.img
		basearch = parts[5]
		arch = parts[4] "-" parts[5]
		nextidx = 6
	} else {
		# FreeBSD-11.3-STABLE-amd64-20191011-r353406.qcow2.xz
		nextidx = 5
	}

	# find date, may be platform first
	if (isrelease(parts[3])) {
		if (nextidx > cnt) {
			# FreeBSD-12.1-BETA3-i386.vhd.xz
			platform = "xxx"
		} else if (cnt == nextidx) {
			# FreeBSD-12.1-RC1-powerpc-powerpcspe-dvd1.iso
			platform = parts[nextidx]
			nextidx += 1
		} else if (type == "oci") {
			platform = "xxx"
			nextidx += 2
		} else {
			# FreeBSD-12.1-BETA3-amd64-mini-memstick.img.xz
			platform = parts[nextidx] "-" parts[nextidx + 1]
			nextidx += 2
		}
		date = "xxx"
	} else if (isdate(parts[nextidx])) {
		# FreeBSD-11.3-STABLE-amd64-20191011-r353406.qcow2.xz
		platform = "xxx"
		date = parts[nextidx]
		nextidx += 1
	} else if (type == "oci") {
		platform = "xxx"
		nextidx += 2
		date = parts[nextidx]
		nextidx += 1
	} else {
		# FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
		platform = parts[nextidx]
		date = parts[nextidx + 1]
		if (isdate(date)) {
			# FreeBSD-13.0-CURRENT-arm64-aarch64-PINEBOOK-20191011-r353427.img.xz
			nextidx += 2
		} else {
			# FreeBSD-13.0-CURRENT-arm64-aarch64-PINE64-LTS-20181026-r339752.img.xz
			date = parts[nextidx + 2]
			platform = parts[nextidx] "-" parts[nextidx + 1]
			nextidx += 3
		}
	}

	# if this part doesn't begin w/ r (for svn rev), skip it, we can't parse
	# others for now
	rev = parts[nextidx]
	if (isrelease(parts[3])) {
		# FreeBSD-12.1-RC1-amd64-mini-memstick.img.xz
		rev = "unspec"
	} else if (!isrev(rev)) {
		print "invalid rev, skipping: " rev > "/dev/stderr"
		#print "raw " $0 > "/dev/stderr"
		#printf("%s %s %s %s %s %s %s %s %s %s\n", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID) > "/dev/stderr"
		return
	} else {
		# we have rev, but check if next part is also part of it
		if (parts[nextidx + 1] ~ /^[0-9]+$/) {
			rev = rev "-c" parts[nextidx + 1]
			nextidx += 1
		}
	}

	if (nextidx >= cnt)
		vers="xxx"
	else {
		vers=""
		sep=""
		for (i = nextidx + 1; i <= cnt; i++) {
			vers = vers sep parts[i]
			sep="-"
		}
	}
	if (isrelease(parts[3])) {
		if (platform == "BASIC-CI") {
			vers = dotparts[3]
			url = relbciroot parts[2] "-" parts[3] "/" basearch "/Latest/" fname
		} else if (type == "vm") {
			vers = dotparts[3]
			url = relvmroot parts[2] "-" parts[3] "/" basearch "/Latest/" fname
		} else if (type == "oci") {
			vers = dotparts[3]
			url = relociroot parts[2] "-" parts[3] "/" basearch "/Latest/" fname
		} else
			url = relisoroot parts[2] "/" fname
	} else {
		if (platform == "BASIC-CI") {
			vers = dotparts[3]
			url = bciroot parts[2] "-" parts[3] "/" basearch "/" date "/" fname
		} else if (type == "vm") {
			vers = dotparts[3]
			url = vmroot parts[2] "-" parts[3] "/" basearch "/" date "/" fname
		} else if (type == "oci") {
			vers = dotparts[3]
			url = ociroot parts[2] "-" parts[3] "/" basearch "/" date "/" fname
		} else
			url = isoroot parts[2] "/" fname
	}

	# double check that date is valid, if not, skip it
	if (date != "xxx" && !isdate(date))
		return

	res = sprintf("%s %s %s %s %s %s %s %s %s %s", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID)
	#printf("%s %s %s %s %s %s %s %s %s %s\n", type, parts[2] "-" parts[3], arch, platform, date, rev, vers, fname, url, MID) > "/dev/stderr"

	return res
}