amd64: Add cpu_stop() support to go UP after SMP

Reviewed by:	kib
Sponsored by:	Juniper Networks, Inc.
Differential Revision:	https://reviews.freebsd.org/D51622

8 files changed, 63 insertions, 5 deletions

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index e98bae9eb6c5..8691387a5a8e 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -204,6 +204,17 @@ IDTVEC(spuriousint)
 	jmp	doreti
 
 /*
+ * Executed by a CPU when it receives an IPI_OFF from another CPU.
+ * Should never return
+ */
+	INTR_HANDLER cpuoff
+	KMSAN_ENTER
+	call	cpuoff_handler
+	call	as_lapic_eoi
+	KMSAN_LEAVE
+	jmp	doreti
+
+/*
  * Executed by a CPU when it receives an IPI_SWI.
  */
 	INTR_HANDLER ipi_swi
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 00e99f9df192..96ed0a2cc3ba 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -140,6 +140,10 @@ cpu_mp_start(void)
 	setidt(IPI_STOP, pti ? IDTVEC(cpustop_pti) : IDTVEC(cpustop),
 	    SDT_SYSIGT, SEL_KPL, 0);
 
+	/* Install an inter-CPU IPI for CPU offline */
+	setidt(IPI_OFF, pti ? IDTVEC(cpuoff_pti) : IDTVEC(cpuoff),
+	    SDT_SYSIGT, SEL_KPL, 0);
+
 	/* Install an inter-CPU IPI for CPU suspend/resume */
 	setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend),
 	    SDT_SYSIGT, SEL_KPL, 0);
@@ -176,6 +180,15 @@ cpu_mp_start(void)
 #endif
 }
 
+void
+cpu_mp_stop(void)
+{
+	cpuset_t other_cpus = all_cpus;
+
+	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+	offline_cpus(other_cpus);
+}
+
 /*
  * AP CPU's call this to initialize themselves.
  */
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index bff92570ff82..28c372a2e556 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -30,6 +30,7 @@ inthand_t
 	IDTVEC(ipi_intr_bitmap_handler_pti),
 	IDTVEC(ipi_swi_pti),
 	IDTVEC(cpustop_pti),
+	IDTVEC(cpuoff_pti),
 	IDTVEC(cpususpend_pti),
 	IDTVEC(rendezvous_pti);
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 1f9577fddf9c..9f5106316018 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -242,7 +242,7 @@ generic_stop_cpus(cpuset_t map, u_int type)
 	KASSERT(
 	    type == IPI_STOP || type == IPI_STOP_HARD
 #if X86
-	    || type == IPI_SUSPEND
+	    || type == IPI_SUSPEND || type == IPI_OFF
 #endif
 	    , ("%s: invalid stop type", __func__));
 
@@ -260,7 +260,7 @@ generic_stop_cpus(cpuset_t map, u_int type)
 	 * will be lost, violating FreeBSD's assumption of reliable
 	 * IPI delivery.
 	 */
-	if (type == IPI_SUSPEND)
+	if (type == IPI_SUSPEND || type == IPI_OFF)
 		mtx_lock_spin(&smp_ipi_mtx);
 #endif
 
@@ -280,7 +280,7 @@ generic_stop_cpus(cpuset_t map, u_int type)
 #endif
 
 #if X86
-	if (type == IPI_SUSPEND)
+	if (type == IPI_SUSPEND || type == IPI_OFF)
 		cpus = &suspended_cpus;
 	else
 #endif
@@ -298,7 +298,7 @@ generic_stop_cpus(cpuset_t map, u_int type)
 	}
 
 #if X86
-	if (type == IPI_SUSPEND)
+	if (type == IPI_SUSPEND || type == IPI_OFF)
 		mtx_unlock_spin(&smp_ipi_mtx);
 #endif
 
@@ -327,6 +327,13 @@ suspend_cpus(cpuset_t map)
 
 	return (generic_stop_cpus(map, IPI_SUSPEND));
 }
+
+int
+offline_cpus(cpuset_t map)
+{
+
+	return (generic_stop_cpus(map, IPI_OFF));
+}
 #endif
 
 /*
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index 252dc9dc1cae..5b968aa69791 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -259,6 +259,7 @@ int	stop_cpus_hard(cpuset_t);
 #if defined(__amd64__) || defined(__i386__)
 int	suspend_cpus(cpuset_t);
 int	resume_cpus(cpuset_t);
+int	offline_cpus(cpuset_t);
 #endif
 
 void	smp_rendezvous_action(void);
diff --git a/sys/x86/include/apicvar.h b/sys/x86/include/apicvar.h
index c537d0ee0cdd..551f5527ac00 100644
--- a/sys/x86/include/apicvar.h
+++ b/sys/x86/include/apicvar.h
@@ -134,7 +134,8 @@
 #define	IPI_STOP	(APIC_IPI_INTS + 6)	/* Stop CPU until restarted. */
 #define	IPI_SUSPEND	(APIC_IPI_INTS + 7)	/* Suspend CPU until restarted. */
 #define	IPI_SWI		(APIC_IPI_INTS + 8)	/* Run clk_intr_event. */
-#define	IPI_DYN_FIRST	(APIC_IPI_INTS + 9)
+#define	IPI_OFF		(APIC_IPI_INTS + 9)	/* Stop CPU forever */
+#define	IPI_DYN_FIRST	(APIC_IPI_INTS + 10)
 #define	IPI_DYN_LAST	(254)			/* IPIs allocated at runtime */
 
 /*
diff --git a/sys/x86/include/x86_smp.h b/sys/x86/include/x86_smp.h
index 8b9eb2ec9b66..f5015e9d8a24 100644
--- a/sys/x86/include/x86_smp.h
+++ b/sys/x86/include/x86_smp.h
@@ -77,6 +77,7 @@ extern u_long *ipi_rendezvous_counts[MAXCPU];
 inthand_t
 	IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ 
 	IDTVEC(ipi_swi),	/* Runs delayed SWI */
+	IDTVEC(cpuoff),		/* CPU goes offline until hard reset */
 	IDTVEC(cpustop),	/* CPU stops & waits to be restarted */
 	IDTVEC(cpususpend),	/* CPU suspends & waits to be resumed */
 	IDTVEC(rendezvous);	/* handle CPU rendezvous */
@@ -93,6 +94,7 @@ void	assign_cpu_ids(void);
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
 void	cpususpend_handler(void);
+void	cpuoff_handler(void);
 void	init_secondary_tail(void);
 void	init_secondary(void);
 void	ipi_startup(int apic_id, int vector);
diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c
index c0da41a4d222..6b1715853763 100644
--- a/sys/x86/x86/mp_x86.c
+++ b/sys/x86/x86/mp_x86.c
@@ -1696,6 +1696,28 @@ cpususpend_handler(void)
 	CPU_CLR_ATOMIC(cpu, &toresume_cpus);
 }
 
+void
+cpuoff_handler(void)
+{
+	u_int cpu;
+
+	cpu = PCPU_GET(cpuid);
+
+	/* Time to go catatonic.  A reset will be required to leave. */
+	disable_intr();
+	lapic_disable();
+	CPU_SET_ATOMIC(cpu, &suspended_cpus);
+
+	/*
+	 * There technically should be no need for the `while` here, since it
+	 * cannot be interrupted (interrupts are disabled).  Be safe anyway.
+	 * Any interrupt at this point will likely be fatal, as the page tables
+	 * are likely going away shortly.
+	 */
+	while (1)
+		halt();
+}
+
 /*
  * Handle an IPI_SWI by waking delayed SWI thread.
  */