xen: add a hook to perform AP startup

AP startup on PVH follows the PV method, so we need to add a hook in
order to diverge from bare metal.

Approved by: gibbs
Sponsored by: Citrix Systems R&D

amd64/amd64/machdep.c:
 - Add hook for start_all_aps on native (using native_start_all_aps
   defined in mp_machdep).

amd64/amd64/mp_machdep.c:
 - Make some variables global because they will also be used by the
   Xen PVH AP startup code.
 - Use the start_all_aps hook to start APs.
 - Rename start_all_aps to native_start_all_aps.

amd64/include/smp.h:
 - Add declaration for native_start_all_aps.

x86/include/init.h:
 - Declare start_all_aps hook in init_ops.

x86/xen/pv.c:
 - Pick external declarations from mp_machdep.
 - Introduce Xen PV code to start APs on PVH.
 - Set start_all_aps init hook to use the Xen PVH implementation.

2 files changed, 97 insertions, 0 deletions

diff --git a/sys/x86/include/init.h b/sys/x86/include/init.h
index edc8418c98b6..e1ff5cf2133a 100644
--- a/sys/x86/include/init.h
+++ b/sys/x86/include/init.h
@@ -40,6 +40,7 @@ struct init_ops {
 	void	(*early_delay)(int);
 	void	(*parse_memmap)(caddr_t, vm_paddr_t *, int *);
 	u_int	(*mp_bootaddress)(u_int);
+	int	(*start_all_aps)(void);
 };
 
 extern struct init_ops init_ops;
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index f7d5e47c8f2f..96420e3d5209 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -34,10 +34,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/reboot.h>
 #include <sys/systm.h>
+#include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/rwlock.h>
 #include <sys/boot.h>
 #include <sys/ctype.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -50,10 +53,13 @@ __FBSDID("$FreeBSD$");
 
 #include <x86/init.h>
 #include <machine/pc/bios.h>
+#include <machine/smp.h>
 
 #include <xen/xen-os.h>
 #include <xen/hypervisor.h>
 
+#include <xen/interface/vcpu.h>
+
 #include <dev/xen/timer/timer.h>
 
 /* Native initial function */
@@ -67,6 +73,22 @@ uint64_t hammer_time_xen(start_info_t *, uint64_t);
 static caddr_t xen_pv_parse_preload_data(u_int64_t);
 static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *);
 
+#ifdef SMP
+static int xen_pv_start_all_aps(void);
+#endif
+
+/*---------------------------- Extern Declarations ---------------------------*/
+#ifdef SMP
+/* Variables used by amd64 mp_machdep to start APs */
+extern struct mtx ap_boot_mtx;
+extern void *bootstacks[];
+extern char *doublefault_stack;
+extern char *nmi_stack;
+extern void *dpcpu;
+extern int bootAP;
+extern char *bootSTK;
+#endif
+
 /*-------------------------------- Global Data -------------------------------*/
 /* Xen init_ops implementation. */
 struct init_ops xen_init_ops = {
@@ -74,6 +96,9 @@ struct init_ops xen_init_ops = {
 	.early_clock_source_init =	xen_clock_init,
 	.early_delay =			xen_delay,
 	.parse_memmap =			xen_pv_parse_memmap,
+#ifdef SMP
+	.start_all_aps =		xen_pv_start_all_aps,
+#endif
 };
 
 static struct bios_smap xen_smap[MAX_E820_ENTRIES];
@@ -151,6 +176,77 @@ hammer_time_xen(start_info_t *si, uint64_t xenstack)
 }
 
 /*-------------------------------- PV specific -------------------------------*/
+#ifdef SMP
+static bool
+start_xen_ap(int cpu)
+{
+	struct vcpu_guest_context *ctxt;
+	int ms, cpus = mp_naps;
+	const size_t stacksize = KSTACK_PAGES * PAGE_SIZE;
+
+	/* allocate and set up an idle stack data page */
+	bootstacks[cpu] =
+	    (void *)kmem_malloc(kernel_arena, stacksize, M_WAITOK | M_ZERO);
+	doublefault_stack =
+	    (char *)kmem_malloc(kernel_arena, PAGE_SIZE, M_WAITOK | M_ZERO);
+	nmi_stack =
+	    (char *)kmem_malloc(kernel_arena, PAGE_SIZE, M_WAITOK | M_ZERO);
+	dpcpu =
+	    (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, M_WAITOK | M_ZERO);
+
+	bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
+	bootAP = cpu;
+
+	ctxt = malloc(sizeof(*ctxt), M_TEMP, M_WAITOK | M_ZERO);
+	if (ctxt == NULL)
+		panic("unable to allocate memory");
+
+	ctxt->flags = VGCF_IN_KERNEL;
+	ctxt->user_regs.rip = (unsigned long) init_secondary;
+	ctxt->user_regs.rsp = (unsigned long) bootSTK;
+
+	/* Set the AP to use the same page tables */
+	ctxt->ctrlreg[3] = KPML4phys;
+
+	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
+		panic("unable to initialize AP#%d", cpu);
+
+	free(ctxt, M_TEMP);
+
+	/* Launch the vCPU */
+	if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
+		panic("unable to start AP#%d", cpu);
+
+	/* Wait up to 5 seconds for it to start. */
+	for (ms = 0; ms < 5000; ms++) {
+		if (mp_naps > cpus)
+			return (true);
+		DELAY(1000);
+	}
+
+	return (false);
+}
+
+static int
+xen_pv_start_all_aps(void)
+{
+	int cpu;
+
+	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
+
+	for (cpu = 1; cpu < mp_ncpus; cpu++) {
+
+		/* attempt to start the Application Processor */
+		if (!start_xen_ap(cpu))
+			panic("AP #%d failed to start!", cpu);
+
+		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
+	}
+
+	return (mp_naps);
+}
+#endif /* SMP */
+
 /*
  * Functions to convert the "extra" parameters passed by Xen
  * into FreeBSD boot options.