xen: fix dropping bitmap IPIs during resume

Current Xen resume code clears all pending bitmap IPIs on resume, which is
not correct. Instead re-inject bitmap IPI vectors on resume to all CPUs in
order to acknowledge any pending bitmap IPIs.

Sponsored by:		Citrix Systems R&D
MFC after:		2 weeks

2 files changed, 5 insertions, 25 deletions

diff --git a/sys/dev/xen/control/control.c b/sys/dev/xen/control/control.c
index f2851eea65f9..6a351d831ec8 100644
--- a/sys/dev/xen/control/control.c
+++ b/sys/dev/xen/control/control.c
@@ -128,6 +128,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/_inttypes.h>
 #include <machine/intr_machdep.h>
 
+#include <x86/apicvar.h>
+
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
@@ -256,6 +258,8 @@ xctrl_suspend()
 	gnttab_resume(NULL);
 
 #ifdef SMP
+	/* Send an IPI_BITMAP in case there are pending bitmap IPIs. */
+	lapic_ipi_vectored(IPI_BITMAP_VECTOR, APIC_IPI_DEST_ALL);
 	if (smp_started && !CPU_EMPTY(&cpu_suspend_map)) {
 		/*
 		 * Now that event channels have been initialized,
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
index c969e262b956..a8b14fda4c5c 100644
--- a/sys/x86/xen/hvm.c
+++ b/sys/x86/xen/hvm.c
@@ -58,15 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <xen/interface/vcpu.h>
 
 /*--------------------------- Forward Declarations ---------------------------*/
-#ifdef SMP
-static void xen_hvm_cpu_resume(void);
-#endif
 static void xen_hvm_cpu_init(void);
 
-/*---------------------------- Extern Declarations ---------------------------*/
-/* Variables used by mp_machdep to perform the bitmap IPI */
-extern volatile u_int cpu_ipi_pending[MAXCPU];
-
 /*-------------------------------- Local Types -------------------------------*/
 enum xen_hvm_init_type {
 	XEN_HVM_INIT_COLD,
@@ -80,7 +73,7 @@ enum xen_domain_type xen_domain_type = XEN_NATIVE;
 #ifdef SMP
 struct cpu_ops xen_hvm_cpu_ops = {
 	.cpu_init	= xen_hvm_cpu_init,
-	.cpu_resume	= xen_hvm_cpu_resume
+	.cpu_resume	= xen_hvm_cpu_init
 };
 #endif
 
@@ -107,23 +100,6 @@ int xen_disable_pv_nics = 0;
 TUNABLE_INT("hw.xen.disable_pv_disks", &xen_disable_pv_disks);
 TUNABLE_INT("hw.xen.disable_pv_nics", &xen_disable_pv_nics);
 
-#ifdef SMP
-/*---------------------- XEN diverged cpu operations -------------------------*/
-static void
-xen_hvm_cpu_resume(void)
-{
-	u_int cpuid = PCPU_GET(cpuid);
-
-	/*
-	 * Reset pending bitmap IPIs, because Xen doesn't preserve pending
-	 * event channels on migration.
-	 */
-	cpu_ipi_pending[cpuid] = 0;
-
-	/* register vcpu_info area */
-	xen_hvm_cpu_init();
-}
-#endif
 /*---------------------- XEN Hypervisor Probe and Setup ----------------------*/
 static uint32_t
 xen_hvm_cpuid_base(void)