release: Rework vm_extra_pre_umount

The vm_extra_pre_umount function in vmimage.subr served two purposes:
It removed /etc/resolv.conf and /qemu (if cross-building), and it
provided a function for cloudware to override in order to make cloud
specific changes to the filesystem before constructing a disk image.

This resulted in a number of bugs:
1. When cross-building, the emulator binary was left as /qemu in the
Azure, GCE, Openstack and Vagrant images.
2. The build host's resolv.conf was left as /etc/resolv.conf in the
basic-ci and basic-cloudinit images.
3. When building GCE images, a Google-specific resolv.conf file was
constructed, and then deleted before the disk image was created.

Move the bits needed for running code inside a VM staging directory
from vm_install_base into a new vm_emulation_setup routine, and move
the corresponding cleanup bits from vm_extra_pre_umount to a new
vm_emulation_cleanup routine.

Remove the /qemu and /etc/resolv.conf cleanups from the cloudware
configuration files (where they exist) since we will now be running
vm_emulation_cleanup to remove those even when vm_extra_pre_umount
has been overridden.

Override vm_emulation_cleanup in gce.conf since in that one case (and
*only* that one case) we don't want to clean up resolv.conf (since it
was constructed for the VM image rather than copied from the host).

releng/14.1 candidate.

MFC after:	1 week
Sponsored by:	https://www.patreon.com/cperciva

(cherry picked from commit 0b1c5628c74a37e2aa2aa3769c606d3e826302f8)

9 files changed, 22 insertions, 30 deletions

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index ca6a359affbd..dfd91f43140d 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -97,13 +97,14 @@ main() {
 
 	vm_create_base
 	vm_install_base
+	vm_emulation_setup
 	vm_extra_install_base
 	vm_extra_install_packages
 	vm_extra_install_ports
 	vm_extra_enable_services
 	vm_extra_pre_umount
 	vm_extra_pkg_rmcache
-	cleanup
+	vm_emulation_cleanup
 	vm_copy_base
 	vm_create_disk || return 0
 	vm_extra_create_disk
diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index b6526f21e474..9da2b19de694 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -53,7 +53,5 @@ EOF
 
 	touch ${DESTDIR}/firstboot
 
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
diff --git a/release/tools/ec2.conf b/release/tools/ec2.conf
index a233bf5981c1..ce09ab0d2367 100644
--- a/release/tools/ec2.conf
+++ b/release/tools/ec2.conf
@@ -102,10 +102,5 @@ EOF
 	# The first time the AMI boots, run "first boot" scripts.
 	touch ${DESTDIR}/firstboot
 
-	if ! [ -z "${QEMUSTATIC}" ]; then
-		rm -f ${DESTDIR}/${EMULATOR}
-	fi
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
diff --git a/release/tools/gce.conf b/release/tools/gce.conf
index aad4a74d2e39..5ad83bcded53 100644
--- a/release/tools/gce.conf
+++ b/release/tools/gce.conf
@@ -118,7 +118,15 @@ EOF
 
 	touch ${DESTDIR}/firstboot
 
-	rm -f ${DESTDIR}/etc/resolv.conf
+	return 0
+}
 
+# Do everything except deleting resolv.conf since we construct our own
+# Googlized resolv.conf file in vm_extra_install_base.
+vm_emulation_cleanup() {
+	if ! [ -z "${QEMUSTATIC}" ]; then
+		rm -f ${DESTDIR}/${EMULATOR}
+	fi
+	umount_loop ${DESTDIR}/dev
 	return 0
 }
diff --git a/release/tools/oci.conf b/release/tools/oci.conf
index c57a7683a500..a4fe54ad3031 100644
--- a/release/tools/oci.conf
+++ b/release/tools/oci.conf
@@ -90,10 +90,5 @@ EOF
 
 	touch ${DESTDIR}/firstboot
 
-	if ! [ -z "${QEMUSTATIC}" ]; then
-		rm -f ${DESTDIR}/${EMULATOR}
-	fi
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index 4c905fbae4ba..05d2d13bbb39 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -35,8 +35,6 @@ vm_extra_pre_umount() {
 	echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> \
 		${DESTDIR}/usr/local/etc/sudoers.d/cloud-init
 
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	# The console is not interactive, so we might as well boot quickly.
 	echo 'autoboot_delay="-1"' >> ${DESTDIR}/boot/loader.conf
 	echo 'beastie_disable="YES"' >> ${DESTDIR}/boot/loader.conf
diff --git a/release/tools/vagrant-virtualbox.conf b/release/tools/vagrant-virtualbox.conf
index 4dd7ca8953ad..9e0e430bbc88 100644
--- a/release/tools/vagrant-virtualbox.conf
+++ b/release/tools/vagrant-virtualbox.conf
@@ -14,5 +14,4 @@ vm_extra_pre_umount () {
 
 	# Setup the Vagrant common items
 	vagrant_common
-	rm -f ${DESTDIR}/etc/resolv.conf
 }
diff --git a/release/tools/vagrant-vmware.conf b/release/tools/vagrant-vmware.conf
index 52ff7f1aac1a..fff929829222 100644
--- a/release/tools/vagrant-vmware.conf
+++ b/release/tools/vagrant-vmware.conf
@@ -18,5 +18,4 @@ vm_extra_pre_umount () {
 
 	# Setup the Vagrant common items
 	vagrant_common
-	rm -f ${DESTDIR}/etc/resolv.conf
 }
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index eda22e061c6d..72540dad14a8 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -83,8 +83,14 @@ vm_install_base() {
 		echo "zfs_enable=\"YES\"" >> ${DESTDIR}/etc/rc.conf
 		echo "zpool_reguid=\"zroot\"" >> ${DESTDIR}/etc/rc.conf
 		echo "zpool_upgrade=\"zroot\"" >> ${DESTDIR}/etc/rc.conf
+		echo "kern.geom.label.disk_ident.enable=0" >> ${DESTDIR}/boot/loader.conf
+		echo "zfs_load=YES" >> ${DESTDIR}/boot/loader.conf
 	fi
 
+	return 0
+}
+
+vm_emulation_setup() {
 	if ! [ -z "${QEMUSTATIC}" ]; then
 		export EMULATOR=/qemu
 		cp ${QEMUSTATIC} ${DESTDIR}/${EMULATOR}
@@ -94,15 +100,8 @@ vm_install_base() {
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} ${EMULATOR} /usr/bin/newaliases
 	chroot ${DESTDIR} ${EMULATOR} /bin/sh /etc/rc.d/ldconfig forcestart
-	umount_loop ${DESTDIR}/dev
-
 	cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
 
-        if [ "${VMFS}" = zfs ]; then
-		echo "kern.geom.label.disk_ident.enable=0" >> ${DESTDIR}/boot/loader.conf
-		echo "zfs_load=YES" >> ${DESTDIR}/boot/loader.conf
-        fi
-
 	return 0
 }
 
@@ -136,15 +135,12 @@ vm_extra_install_packages() {
 	if [ -z "${VM_EXTRA_PACKAGES}" ]; then
 		return 0
 	fi
-	mkdir -p ${DESTDIR}/dev
-	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} ${EMULATOR} env ASSUME_ALWAYS_YES=yes \
 		/usr/sbin/pkg bootstrap -y
 	for p in ${VM_EXTRA_PACKAGES}; do
 		chroot ${DESTDIR} ${EMULATOR} env ASSUME_ALWAYS_YES=yes \
 			/usr/sbin/pkg install -y ${p}
 	done
-	umount_loop ${DESTDIR}/dev
 
 	return 0
 }
@@ -159,13 +155,16 @@ vm_extra_install_ports() {
 vm_extra_pre_umount() {
 	# Prototype.  When overridden, performs additional tasks within the
 	# virtual machine environment prior to unmounting the filesystem.
-	# Note: When overriding this function, removing resolv.conf in the
-	# disk image must be included.
 
+	return 0
+}
+
+vm_emulation_cleanup() {
 	if ! [ -z "${QEMUSTATIC}" ]; then
 		rm -f ${DESTDIR}/${EMULATOR}
 	fi
 	rm -f ${DESTDIR}/etc/resolv.conf
+	umount_loop ${DESTDIR}/dev
 	return 0
 }