Stop linking against a direct-mapped virtual address and instead

use the PBVM. This eliminates the implied hardcoding of the
physical address at which the kernel needs to be loaded. Using the
PBVM makes it possible to load the kernel irrespective of the
physical memory organization and allows us to replicate kernel text
on NUMA machines.

While here, reduce the direct-mapped page size to the kernel's
page size so that we can support memory attributes better.

1 files changed, 124 insertions, 65 deletions

diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index aef41087034b..5804f8cd69b7 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/uuid.h>
 
 #include <machine/atomic.h>
+#include <machine/bootinfo.h>
 #include <machine/cpu.h>
 #include <machine/fpu.h>
 #include <machine/intr.h>
@@ -62,22 +63,18 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
 
+extern uint64_t bdata[];
+
 MALLOC_DEFINE(M_SMP, "SMP", "SMP related allocations");
 
 void ia64_ap_startup(void);
 
-#define	LID_SAPIC(x)		((u_int)((x) >> 16))
-#define	LID_SAPIC_ID(x)		((u_int)((x) >> 24) & 0xff)
-#define	LID_SAPIC_EID(x)	((u_int)((x) >> 16) & 0xff)
-#define	LID_SAPIC_SET(id,eid)	(((id & 0xff) << 8 | (eid & 0xff)) << 16);
-#define	LID_SAPIC_MASK		0xffff0000UL
+#define	SAPIC_ID_GET_ID(x)	((u_int)((x) >> 8) & 0xff)
+#define	SAPIC_ID_GET_EID(x)	((u_int)(x) & 0xff)
+#define	SAPIC_ID_SET(id, eid)	((u_int)(((id) & 0xff) << 8) | ((eid) & 0xff))
 
-/* Variables used by os_boot_rendez and ia64_ap_startup */
-struct pcpu *ap_pcpu;
-void *ap_stack;
-volatile int ap_delay;
-volatile int ap_awake;
-volatile int ap_spin;
+/* State used to wake and bootstrap APs. */
+struct ia64_ap_state ia64_ap_state;
 
 int ia64_ipi_ast;
 int ia64_ipi_highfp;
@@ -87,6 +84,21 @@ int ia64_ipi_rndzvs;
 int ia64_ipi_stop;
 
 static u_int
+sz2shft(uint64_t sz)
+{
+	uint64_t s;
+	u_int shft;
+
+	shft = 12;      /* Start with 4K */
+	s = 1 << shft;
+	while (s < sz) {
+		shft++;
+		s <<= 1;
+	}
+	return (shft);
+}
+
+static u_int
 ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf)
 {
 
@@ -180,16 +192,27 @@ ia64_ap_startup(void)
 {
 	uint64_t vhpt;
 
-	pcpup = ap_pcpu;
+	ia64_ap_state.as_trace = 0x100;
+
+	ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1);
+	ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (PAGE_SHIFT << 2));
+	ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (PAGE_SHIFT << 2));
+	ia64_srlz_d();
+
+	pcpup = ia64_ap_state.as_pcpu;
 	ia64_set_k4((intptr_t)pcpup);
 
+	ia64_ap_state.as_trace = 0x108;
+
 	vhpt = PCPU_GET(md.vhpt);
 	map_vhpt(vhpt);
 	ia64_set_pta(vhpt + (1 << 8) + (pmap_vhpt_log2size << 2) + 1);
 	ia64_srlz_i();
 
-	ap_awake = 1;
-	ap_delay = 0;
+	ia64_ap_state.as_trace = 0x110;
+
+	ia64_ap_state.as_awake = 1;
+	ia64_ap_state.as_delay = 0;
 
 	map_pal_code();
 	map_gateway_page();
@@ -197,14 +220,14 @@ ia64_ap_startup(void)
 	ia64_set_fpsr(IA64_FPSR_DEFAULT);
 
 	/* Wait until it's time for us to be unleashed */
-	while (ap_spin)
+	while (ia64_ap_state.as_spin)
 		cpu_spinwait();
 
 	/* Initialize curthread. */
 	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
 	PCPU_SET(curthread, PCPU_GET(idlethread));
 
-	atomic_add_int(&ap_awake, 1);
+	atomic_add_int(&ia64_ap_state.as_awake, 1);
 	while (!smp_started)
 		cpu_spinwait();
 
@@ -253,18 +276,18 @@ cpu_mp_probe(void)
 }
 
 void
-cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid)
+cpu_mp_add(u_int acpi_id, u_int id, u_int eid)
 {
 	struct pcpu *pc;
-	u_int64_t lid;
 	void *dpcpu;
-	u_int cpuid;
+	u_int cpuid, sapic_id;
 
-	lid = LID_SAPIC_SET(apicid, apiceid);
-	cpuid = ((ia64_get_lid() & LID_SAPIC_MASK) == lid) ? 0 : smp_cpus++;
+	sapic_id = SAPIC_ID_SET(id, eid);
+	cpuid = (IA64_LID_GET_SAPIC_ID(ia64_get_lid()) == sapic_id)
+	    ? 0 : smp_cpus++;
 
 	KASSERT((all_cpus & (1UL << cpuid)) == 0,
-	    ("%s: cpu%d already in CPU map", __func__, acpiid));
+	    ("%s: cpu%d already in CPU map", __func__, acpi_id));
 
 	if (cpuid != 0) {
 		pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK);
@@ -274,23 +297,26 @@ cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid)
 	} else
 		pc = pcpup;
 
-	pc->pc_acpi_id = acpiid;
-	pc->pc_md.lid = lid;
-	all_cpus |= (1UL << cpuid);
+	pc->pc_acpi_id = acpi_id;
+	pc->pc_md.lid = IA64_LID_SET_SAPIC_ID(sapic_id);
+
+	all_cpus |= (1UL << pc->pc_cpuid);
 }
 
 void
 cpu_mp_announce()
 {
 	struct pcpu *pc;
+	uint32_t sapic_id;
 	int i;
 
 	for (i = 0; i <= mp_maxid; i++) {
 		pc = pcpu_find(i);
 		if (pc != NULL) {
+			sapic_id = IA64_LID_GET_SAPIC_ID(pc->pc_md.lid);
 			printf("cpu%d: ACPI Id=%x, SAPIC Id=%x, SAPIC Eid=%x",
-			    i, pc->pc_acpi_id, LID_SAPIC_ID(pc->pc_md.lid),
-			    LID_SAPIC_EID(pc->pc_md.lid));
+			    i, pc->pc_acpi_id, SAPIC_ID_GET_ID(sapic_id),
+			    SAPIC_ID_GET_EID(sapic_id));
 			if (i == 0)
 				printf(" (BSP)\n");
 			else
@@ -302,41 +328,76 @@ cpu_mp_announce()
 void
 cpu_mp_start()
 {
+	struct ia64_sal_result result;
+	struct ia64_fdesc *fd;
 	struct pcpu *pc;
-
-	ap_spin = 1;
+	uintptr_t state;
+	u_char *stp;
+
+	state = ia64_tpa((uintptr_t)&ia64_ap_state);
+	fd = (struct ia64_fdesc *) os_boot_rendez;
+	result = ia64_sal_entry(SAL_SET_VECTORS, SAL_OS_BOOT_RENDEZ,
+	    ia64_tpa(fd->func), state, 0, 0, 0, 0);
+
+	ia64_ap_state.as_pgtbl_pte = PTE_PRESENT | PTE_MA_WB |
+	    PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW |
+	    (bootinfo->bi_pbvm_pgtbl & PTE_PPN_MASK);
+	ia64_ap_state.as_pgtbl_itir = sz2shft(bootinfo->bi_pbvm_pgtblsz) << 2;
+	ia64_ap_state.as_text_va = IA64_PBVM_BASE;
+	ia64_ap_state.as_text_pte = PTE_PRESENT | PTE_MA_WB |
+	    PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RX |
+	    (ia64_tpa(IA64_PBVM_BASE) & PTE_PPN_MASK);
+	ia64_ap_state.as_text_itir = bootinfo->bi_text_mapped << 2;
+	ia64_ap_state.as_data_va = (uintptr_t)bdata;
+	ia64_ap_state.as_data_pte = PTE_PRESENT | PTE_MA_WB |
+	    PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW |
+	    (ia64_tpa((uintptr_t)bdata) & PTE_PPN_MASK);
+	ia64_ap_state.as_data_itir = bootinfo->bi_data_mapped << 2;
+
+	/* Keep 'em spinning until we unleash them... */
+	ia64_ap_state.as_spin = 1;
 
 	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
 		pc->pc_md.current_pmap = kernel_pmap;
 		pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask;
-		if (pc->pc_cpuid > 0) {
-			ap_pcpu = pc;
-			pc->pc_md.vhpt = pmap_alloc_vhpt();
-			if (pc->pc_md.vhpt == 0) {
-				printf("SMP: WARNING: unable to allocate VHPT"
-				    " for cpu%d", pc->pc_cpuid);
-				continue;
-			}
-			ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_SMP,
-			    M_WAITOK);
-			ap_delay = 2000;
-			ap_awake = 0;
-
-			if (bootverbose)
-				printf("SMP: waking up cpu%d\n", pc->pc_cpuid);
-
-			ipi_send(pc, ia64_ipi_wakeup);
-
-			do {
-				DELAY(1000);
-			} while (--ap_delay > 0);
-			pc->pc_md.awake = ap_awake;
-
-			if (!ap_awake)
-				printf("SMP: WARNING: cpu%d did not wake up\n",
-				    pc->pc_cpuid);
-		} else
+		/* The BSP is obviously running already. */
+		if (pc->pc_cpuid == 0) {
 			pc->pc_md.awake = 1;
+			continue;
+		}
+
+		ia64_ap_state.as_pcpu = pc;
+		pc->pc_md.vhpt = pmap_alloc_vhpt();
+		if (pc->pc_md.vhpt == 0) {
+			printf("SMP: WARNING: unable to allocate VHPT"
+			    " for cpu%d", pc->pc_cpuid);
+			continue;
+		}
+
+		stp = malloc(KSTACK_PAGES * PAGE_SIZE, M_SMP, M_WAITOK);
+		ia64_ap_state.as_kstack = stp;
+		ia64_ap_state.as_kstack_top = stp + KSTACK_PAGES * PAGE_SIZE;
+
+		ia64_ap_state.as_trace = 0;
+		ia64_ap_state.as_delay = 2000;
+		ia64_ap_state.as_awake = 0;
+
+		if (bootverbose)
+			printf("SMP: waking up cpu%d\n", pc->pc_cpuid);
+
+		/* Here she goes... */
+		ipi_send(pc, ia64_ipi_wakeup);
+		do {
+			DELAY(1000);
+		} while (--ia64_ap_state.as_delay > 0);
+
+		pc->pc_md.awake = ia64_ap_state.as_awake;
+
+		if (!ia64_ap_state.as_awake) {
+			printf("SMP: WARNING: cpu%d did not wake up (code "
+			    "%#lx)\n", pc->pc_cpuid,
+			    ia64_ap_state.as_trace - state);
+		}
 	}
 }
 
@@ -372,10 +433,10 @@ cpu_mp_unleash(void *dummy)
 		}
 	}
 
-	ap_awake = 1;
-	ap_spin = 0;
+	ia64_ap_state.as_awake = 1;
+	ia64_ap_state.as_spin = 0;
 
-	while (ap_awake != smp_cpus)
+	while (ia64_ap_state.as_awake != smp_cpus)
 		cpu_spinwait();
 
 	if (smp_cpus != cpus || cpus != mp_ncpus) {
@@ -432,21 +493,19 @@ ipi_all_but_self(int ipi)
 }
 
 /*
- * Send an IPI to the specified processor. The lid parameter holds the
- * cr.lid (CR64) contents of the target processor. Only the id and eid
- * fields are used here.
+ * Send an IPI to the specified processor.
  */
 void
 ipi_send(struct pcpu *cpu, int xiv)
 {
-	u_int lid;
+	u_int sapic_id;
 
 	KASSERT(xiv != 0, ("ipi_send"));
 
-	lid = LID_SAPIC(cpu->pc_md.lid);
+	sapic_id = IA64_LID_GET_SAPIC_ID(cpu->pc_md.lid);
 
 	ia64_mf();
-	ia64_st8(&(ia64_pib->ib_ipi[lid][0]), xiv);
+	ia64_st8(&(ia64_pib->ib_ipi[sapic_id][0]), xiv);
 	ia64_mf_a();
 	CTR3(KTR_SMP, "ipi_send(%p, %d): cpuid=%d", cpu, xiv, PCPU_GET(cpuid));
 }