release: Add support for creating ZFS-based VM images

The change extends vmimage.subr to handle a new parameter, VMFS, which
should be equal to either "ufs" or "zfs".  When it is set to ZFS, we use
makefs to create a bootable pool populated using the same dataset layout
as bsdinstall and "poudriere image" use.  The pool can be grown using
the growfs rc.d script, just as in UFS images.

This will make it easy to provide VM and cloud images with ZFS as the
root filesystem.  So far I did not do extensive testing of cloud images;
I merely verified that creation of ZFS-based AWS AMIs works and allows
me to create amd64 and arm64 EC2 instances with ZFS as the root
filesystem.

Reviewed by:	emaste, gjb
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D34426

5 files changed, 95 insertions, 23 deletions

diff --git a/release/Makefile.vm b/release/Makefile.vm
index cb44a34c4545..2ab5b4e373e3 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -8,6 +8,7 @@
 VMTARGETS=	vm-image
 VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	5120m
+VMFS?=		ufs
 SWAPSIZE?=	1g
 VMBASE?=	vm
 
@@ -86,7 +87,7 @@ cw-${_CW:tl}: emulator-portinstall
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \
 		QEMUSTATIC=${QEMUSTATIC} \
 		${.CURDIR}/scripts/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${VMFS} \
 		-i ${.OBJDIR}/${_CW:tl}.img -s ${VMSIZE} -f ${${_CW:tu}_FORMAT} \
 		-S ${WORLDDIR} -o ${.OBJDIR}/${${_CW:tu}IMAGE} -c ${${_CW:tu}CONF}
 	touch ${.TARGET}
@@ -120,7 +121,7 @@ vm-image:
 	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \
 		${.CURDIR}/scripts/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${VMFS} \
 		-i ${.OBJDIR}/${FORMAT}.img -s ${VMSIZE} -f ${FORMAT} \
 		-S ${WORLDDIR} -o ${.OBJDIR}/${VMBASE}.${FORMAT}
 . endfor
diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index d5985ceb0d25..bcf77ffd8f0f 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -41,7 +41,7 @@ usage() {
 main() {
 	local arg
 	VMCONFIG="/dev/null"
-	while getopts "C:c:d:f:i:o:s:S:" arg; do
+	while getopts "C:c:d:F:f:i:o:s:S:" arg; do
 		case "${arg}" in
 			C)
 				VMBUILDCONF="${OPTARG}"
@@ -52,6 +52,9 @@ main() {
 			d)
 				DESTDIR="${OPTARG}"
 				;;
+			F)
+				VMFS="${OPTARG}"
+				;;
 			f)
 				VMFORMAT="${OPTARG}"
 				;;
@@ -77,7 +80,8 @@ main() {
 		-z "${WORLDDIR}" -o \
 		-z "${DESTDIR}" -o \
 		-z "${VMSIZE}" -o \
-		-z "${VMIMAGE}" ];
+		-z "${VMIMAGE}" -o \
+		-z "${VMFS}" ];
 	then
 		usage || exit 0
 	fi
diff --git a/release/tools/ec2.conf b/release/tools/ec2.conf
index a61b711c24ce..34b83fa627b6 100644
--- a/release/tools/ec2.conf
+++ b/release/tools/ec2.conf
@@ -21,11 +21,11 @@ fi
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="ec2_configinit ec2_fetchkey ec2_loghostkey firstboot_freebsd_update firstboot_pkgs ntpd dev_aws_disk ec2_ephemeral_swap"
 
-# Build with a 4.9 GB UFS partition; the growfs rc.d script will expand
+# Build with a 4.9 GB partition; the growfs rc.d script will expand
 # the partition to fill the root disk after the EC2 instance is launched.
 # Note that if this is set to <N>G, we will end up with an <N+1> GB disk
-# image since VMSIZE is the size of the UFS partition, not the disk which
-# it resides within.
+# image since VMSIZE is the size of the filesystem partition, not the disk
+# which it resides within.
 export VMSIZE=5000m
 
 # No swap space; the ec2_ephemeralswap rc.d script will allocate swap
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 0d9202431822..8982e768527a 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -39,8 +39,6 @@ cleanup() {
 }
 
 vm_create_base() {
-	# Creates the UFS root filesystem for the virtual machine disk,
-	# written to the formatted disk image with mkimg(1).
 
 	mkdir -p ${DESTDIR}
 
@@ -70,8 +68,10 @@ vm_install_base() {
 
 	echo '# Custom /etc/fstab for FreeBSD VM images' \
 		> ${DESTDIR}/etc/fstab
-	echo "/dev/${ROOTLABEL}/rootfs   /       ufs     rw      1       1" \
-		>> ${DESTDIR}/etc/fstab
+	if [ "${VMFS}" != zfs ]; then
+		echo "/dev/${ROOTLABEL}/rootfs   /       ${VMFS}   rw      1       1" \
+			>> ${DESTDIR}/etc/fstab
+	fi
 	if [ -z "${NOSWAP}" ]; then
 		echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
 			>> ${DESTDIR}/etc/fstab
@@ -80,6 +80,10 @@ vm_install_base() {
 	local hostname
 	hostname="$(echo $(uname -o) | tr '[:upper:]' '[:lower:]')"
 	echo "hostname=\"${hostname}\"" >> ${DESTDIR}/etc/rc.conf
+	if [ "${VMFS}" = zfs ]; then
+		echo "zfs_enable=\"YES\"" >> ${DESTDIR}/etc/rc.conf
+		echo "zpool_reguid=\"zroot\"" >> ${DESTDIR}/etc/rc.conf
+	fi
 
 	if ! [ -z "${QEMUSTATIC}" ]; then
 		export EMULATOR=/qemu
@@ -94,6 +98,11 @@ vm_install_base() {
 
 	cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
 
+        if [ "${VMFS}" = zfs ]; then
+		echo "kern.geom.label.disk_ident.enable=0" >> ${DESTDIR}/boot/loader.conf
+		echo "zfs_load=YES" >> ${DESTDIR}/boot/loader.conf
+        fi
+
 	return 0
 }
 
@@ -169,6 +178,40 @@ vm_extra_pkg_rmcache() {
 	return 0
 }
 
+buildfs() {
+	local md tmppool
+
+	case "${VMFS}" in
+	ufs)
+		makefs ${MAKEFSARGS} -o label=rootfs -o version=2 -o softupdates=1 \
+			${VMBASE} ${DESTDIR}
+		;;
+	zfs)
+		makefs -t zfs ${MAKEFSARGS} \
+			-o poolname=zroot -o bootfs=zroot/ROOT/default -o rootpath=/ \
+			-o fs=zroot\;mountpoint=none \
+			-o fs=zroot/ROOT\;mountpoint=none \
+			-o fs=zroot/ROOT/default\;mountpoint=/ \
+			-o fs=zroot/tmp\;mountpoint=/tmp\;exec=on\;setuid=off \
+			-o fs=zroot/usr\;mountpoint=/usr\;canmount=off \
+			-o fs=zroot/usr/home \
+			-o fs=zroot/usr/ports\;setuid=off \
+			-o fs=zroot/usr/src \
+			-o fs=zroot/usr/obj \
+			-o fs=zroot/var\;mountpoint=/var\;canmount=off \
+			-o fs=zroot/var/audit\;setuid=off\;exec=off \
+			-o fs=zroot/var/log\;setuid=off\;exec=off \
+			-o fs=zroot/var/mail\;atime=on \
+			-o fs=zroot/var/tmp\;setuid=off \
+			${VMBASE} ${DESTDIR}
+		;;
+	*)
+		echo "Unexpected VMFS value '${VMFS}'"
+		exit 1
+		;;
+	esac
+}
+
 umount_loop() {
 	DIR=$1
 	i=0
@@ -188,8 +231,7 @@ umount_loop() {
 }
 
 vm_create_disk() {
-	echo "Creating image...  Please wait."
-	echo
+	local BOOTFILES BOOTPARTSOFFSET FSPARTTYPE X86GPTBOOTFILE
 
 	if [ -z "${NOSWAP}" ]; then
 		SWAPOPT="-p freebsd-swap/swapfs::${SWAPSIZE}"
@@ -199,34 +241,52 @@ vm_create_disk() {
 		BOOTPARTSOFFSET=":${VM_BOOTPARTSOFFSET}"
 	fi
 
+	case "${VMFS}" in
+	ufs)
+		FSPARTTYPE=freebsd-ufs
+		X86GPTBOOTFILE=i386/gptboot/gptboot
+		;;
+	zfs)
+		FSPARTTYPE=freebsd-zfs
+		X86GPTBOOTFILE=i386/gptzfsboot/gptzfsboot
+		;;
+	*)
+		echo "Unexpected VMFS value '${VMFS}'"
+		return 1
+		;;
+	esac
+
+	echo "Creating image...  Please wait."
+	echo
 
 	BOOTFILES="$(env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		WITH_UNIFIED_OBJDIR=yes \
 		make -C ${WORLDDIR}/stand -V .OBJDIR)"
 	BOOTFILES="$(realpath ${BOOTFILES})"
+	MAKEFSARGS="-s ${VMSIZE}"
 
 	case "${TARGET}:${TARGET_ARCH}" in
 		amd64:amd64 | i386:i386)
 			ESP=yes
 			BOOTPARTS="-b ${BOOTFILES}/i386/pmbr/pmbr \
-				   -p freebsd-boot/bootfs:=${BOOTFILES}/i386/gptboot/gptboot${BOOTPARTSOFFSET}"
-			ROOTFSPART="-p freebsd-ufs/rootfs:=${VMBASE}"
-			MAKEFSARGS="-B little"
+				   -p freebsd-boot/bootfs:=${BOOTFILES}/${X86GPTBOOTFILE}${BOOTPARTSOFFSET}"
+			ROOTFSPART="-p ${FSPARTTYPE}/rootfs:=${VMBASE}"
+			MAKEFSARGS="$MAKEFSARGS -B little"
 			;;
 		arm64:aarch64 | riscv:riscv64*)
 			ESP=yes
 			BOOTPARTS=
-			ROOTFSPART="-p freebsd-ufs/rootfs:=${VMBASE}"
-			MAKEFSARGS="-B little"
+			ROOTFSPART="-p ${FSPARTTYPE}/rootfs:=${VMBASE}"
+			MAKEFSARGS="$MAKEFSARGS -B little"
 			;;
 		powerpc:powerpc*)
 			ESP=no
 			BOOTPARTS="-p prepboot:=${BOOTFILES}/powerpc/boot1.chrp/boot1.elf -a 1"
 			ROOTFSPART="-p freebsd:=${VMBASE}"
 			if [ ${TARGET_ARCH} = powerpc64le ]; then
-				MAKEFSARGS="-B little"
+				MAKEFSARGS="$MAKEFSARGS -B little"
 			else
-				MAKEFSARGS="-B big"
+				MAKEFSARGS="$MAKEFSARGS -B big"
 			fi
 			;;
 		*)
@@ -248,8 +308,7 @@ vm_create_disk() {
 	fi
 
 	echo "Building filesystem...  Please wait."
-	makefs ${MAKEFSARGS} -o label=rootfs -o version=2 -o softupdates=1 \
-	    -s ${VMSIZE} ${VMBASE} ${DESTDIR}
+	buildfs
 
 	echo "Building final disk image...  Please wait."
 	mkimg -s ${PARTSCHEME} -f ${VMFORMAT} \
diff --git a/share/man/man7/release.7 b/share/man/man7/release.7
index be475979dd0f..855cf91697a1 100644
--- a/share/man/man7/release.7
+++ b/share/man/man7/release.7
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 21, 2022
+.Dd October 28, 2022
 .Dt RELEASE 7
 .Os
 .Sh NAME
@@ -365,6 +365,14 @@ is used, the resulting files compressed with
 .Xr xz 1
 compress to roughly the same size, regardless of the specified disk image
 size.
+.It Va VMFS
+Set to specify the file system type to use.
+Valid values are
+.Va ufs
+and
+.Va zfs .
+The default value is
+.Va ufs .
 .It Va VMFORMATS
 Set to the target virtual disk image format(s) to create.
 By default, the