vmm: Suspend the VM before destroying it

Otherwise we don't do anything to kick vcpu threads out of a sleep
state when destroying a VM.  For instance, suppose a guest executes hlt
on amd64 or wfi on arm64 with interrupts disabled.  Then,
bhyvectl --destroy will hang until the vcpu thread somehow comes out of
vm_handle_hlt()/vm_handle_wfi() since destroy_dev() is waiting for vCPU
threads to drain.

Note that on amd64, if hw.vmm.halt_detection is set to 1 (the default),
the guest will automatically exit in this case since it's treated as a
shutdown.  But, the above should not hang if halt_detection is set to 0.

Here, vm_suspend() wakes up vcpu threads, and a subsequent attempt to
run the vCPU will result in an error which gets propagated to userspace,
allowing destroy_dev() to proceed.

Add a new suspend code for this purpose.  Modify bhyve to exit with
status 4 ("exited due to an error") when it's received, since that's
what'll happen generally when the VM is destroyed asynchronously.

Reported by:	def
MFC after:	2 weeks
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D51761

11 files changed, 24 insertions, 6 deletions

diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index 0b3daed4f69e..e35119af8572 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -46,6 +46,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
 	VM_SUSPEND_TRIPLEFAULT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 
diff --git a/sys/arm64/include/vmm.h b/sys/arm64/include/vmm.h
index 73b5b4a09591..e839b5dd92c9 100644
--- a/sys/arm64/include/vmm.h
+++ b/sys/arm64/include/vmm.h
@@ -42,6 +42,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_RESET,
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 
diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
index 3082d2941221..1dcefa1489e9 100644
--- a/sys/arm64/vmm/vmm.c
+++ b/sys/arm64/vmm/vmm.c
@@ -1342,8 +1342,14 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 static int
 vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 {
+	struct vm *vm;
+
+	vm = vcpu->vm;
 	vcpu_lock(vcpu);
 	while (1) {
+		if (vm->suspend)
+			break;
+
 		if (vgic_has_pending_irq(vcpu->cookie))
 			break;
 
diff --git a/sys/dev/vmm/vmm_dev.c b/sys/dev/vmm/vmm_dev.c
index 9f2b009d02ec..460a508a60dc 100644
--- a/sys/dev/vmm/vmm_dev.c
+++ b/sys/dev/vmm/vmm_dev.c
@@ -901,6 +901,7 @@ vmmdev_lookup_and_destroy(const char *name, struct ucred *cred)
 	sc->cdev = NULL;
 	sx_xunlock(&vmmdev_mtx);
 
+	vm_suspend(sc->vm, VM_SUSPEND_DESTROY);
 	destroy_dev(cdev);
 	vmmdev_destroy(sc);
 
diff --git a/sys/riscv/include/vmm.h b/sys/riscv/include/vmm.h
index 1221521be368..de7119dd534a 100644
--- a/sys/riscv/include/vmm.h
+++ b/sys/riscv/include/vmm.h
@@ -49,6 +49,7 @@ enum vm_suspend_how {
 	VM_SUSPEND_RESET,
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
+	VM_SUSPEND_DESTROY,
 	VM_SUSPEND_LAST
 };
 
diff --git a/sys/riscv/vmm/vmm.c b/sys/riscv/vmm/vmm.c
index 7528ef6e4698..ec4514f70fa6 100644
--- a/sys/riscv/vmm/vmm.c
+++ b/sys/riscv/vmm/vmm.c
@@ -1036,10 +1036,14 @@ vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
 static int
 vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
 {
+	struct vm *vm;
 
+	vm = vcpu->vm;
 	vcpu_lock(vcpu);
-
 	while (1) {
+		if (vm->suspend)
+			break;
+
 		if (aplic_check_pending(vcpu->cookie))
 			break;
 
diff --git a/usr.sbin/bhyve/aarch64/vmexit.c b/usr.sbin/bhyve/aarch64/vmexit.c
index 3acad4020a3c..2457cbe76b5e 100644
--- a/usr.sbin/bhyve/aarch64/vmexit.c
+++ b/usr.sbin/bhyve/aarch64/vmexit.c
@@ -122,6 +122,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 		exit(1);
 	case VM_SUSPEND_HALT:
 		exit(2);
+	case VM_SUSPEND_DESTROY:
+		exit(4);
 	default:
 		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
 		exit(100);
diff --git a/usr.sbin/bhyve/amd64/vmexit.c b/usr.sbin/bhyve/amd64/vmexit.c
index 944f5de34645..14f89563fd0f 100644
--- a/usr.sbin/bhyve/amd64/vmexit.c
+++ b/usr.sbin/bhyve/amd64/vmexit.c
@@ -418,6 +418,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 		exit(2);
 	case VM_SUSPEND_TRIPLEFAULT:
 		exit(3);
+	case VM_SUSPEND_DESTROY:
+		exit(4);
 	default:
 		EPRINTLN("vmexit_suspend: invalid reason %d", how);
 		exit(100);
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 89c0b23961a8..c902c265da9e 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -1126,7 +1126,7 @@ powered off
 .It 2
 halted
 .It 3
-triple fault
+triple fault (amd64 only)
 .It 4
 exited due to an error
 .El
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 9ead49582a7d..bfc0b949a75d 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -561,10 +561,8 @@ fbsdrun_start_thread(void *param)
 #endif
 
 	vm_loop(vi->ctx, vi->vcpu);
-
-	/* not reached */
-	exit(1);
-	return (NULL);
+	/* We get here if the VM was destroyed asynchronously. */
+	exit(4);
 }
 
 void
diff --git a/usr.sbin/bhyve/riscv/vmexit.c b/usr.sbin/bhyve/riscv/vmexit.c
index 3bc83b3bef4e..985f8e4e9065 100644
--- a/usr.sbin/bhyve/riscv/vmexit.c
+++ b/usr.sbin/bhyve/riscv/vmexit.c
@@ -121,6 +121,8 @@ vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_run *vmrun)
 		exit(1);
 	case VM_SUSPEND_HALT:
 		exit(2);
+	case VM_SUSPEND_DESTROY:
+		exit(4);
 	default:
 		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
 		exit(100);