Move the arm64 DMAP creation to C

To simplify the creation of the direct map (DMAP) region on arm64 move
it from the pre-C code into pmap. This simplifies the DMAP creation
as we can use the notmal index macros, and should reduce the number
of pages needed to hold the level 1 tables to just those needed.

Reviewed by:	alc, dch
Tested by:	dch, kevans
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34568

2 files changed, 205 insertions, 73 deletions

diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index 534c89ac32dc..e1b0530006cb 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -38,7 +38,6 @@
 #include <machine/vmparam.h>
 
 #define	VIRT_BITS	48
-#define	DMAP_TABLES	((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
 
 	.globl	kernbase
 	.set	kernbase, KERNBASE
@@ -378,7 +377,6 @@ LEND(get_virt_delta)
  *    - The Kernel L0 table             (TTBR1)
  *    - The identity (PA = VA) L1 table
  *    - The identity (PA = VA) L0 table (TTBR0)
- *    - The DMAP L1 tables
  */
 LENTRY(create_pagetables)
 	/* Save the Link register */
@@ -476,13 +474,6 @@ common:
 	mov	x10, #1
 	bl	link_l0_pagetable
 
-	/* Link the DMAP tables */
-	ldr	x8, =DMAP_MIN_ADDRESS
-	adrp	x9, pagetable_dmap
-	add	x9, x9, :lo12:pagetable_dmap
-	mov	x10, #DMAP_TABLES
-	bl	link_l0_pagetable
-
 	/*
 	 * Build the TTBR0 maps.  As TTBR0 maps, they must specify ATTR_S1_nG.
 	 * They are only needed early on, so the VA = PA map is uncached.
@@ -771,6 +762,7 @@ END(abort)
 	 *           L0 bootstrap for user
 	 *           L0 for user
 	 */
+	.globl pagetable_l0_ttbr1
 pagetable:
 	.space	PAGE_SIZE
 pagetable_l1_ttbr1:
@@ -786,9 +778,6 @@ pagetable_l0_ttbr0_boostrap:
 pagetable_l0_ttbr0:
 	.space	PAGE_SIZE
 
-	.globl pagetable_dmap
-pagetable_dmap:
-	.space	PAGE_SIZE * DMAP_TABLES
 pagetable_end:
 
 el2_pagetable:
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 9776c36b3ef4..3a356e8b720b 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -290,8 +290,7 @@ vm_offset_t dmap_max_addr;	/* The virtual address limit of the dmap */
 CTASSERT((DMAP_MIN_ADDRESS  & ~L0_OFFSET) == DMAP_MIN_ADDRESS);
 CTASSERT((DMAP_MAX_ADDRESS  & ~L0_OFFSET) == DMAP_MAX_ADDRESS);
 
-#define	DMAP_TABLES	((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
-extern pt_entry_t pagetable_dmap[];
+extern pt_entry_t pagetable_l0_ttbr1[];
 
 #define	PHYSMAP_SIZE	(2 * (VM_PHYSSEG_MAX - 1))
 static vm_paddr_t physmap[PHYSMAP_SIZE];
@@ -808,104 +807,248 @@ pmap_early_vtophys(vm_offset_t va)
 	return (pa_page | (va & PAR_LOW_MASK));
 }
 
-static vm_offset_t
-pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot,
-    pt_entry_t **l2p, int i, vm_offset_t freemempos)
+/* State of the bootstrapped DMAP page tables */
+struct dmap_bootstrap_state {
+	vm_offset_t	va;
+	vm_paddr_t	pa;
+	pt_entry_t	*l1;
+	pt_entry_t	*l2;
+	pt_entry_t	*l3;
+	u_int		l0_slot;
+	u_int		l1_slot;
+	u_int		l2_slot;
+	vm_offset_t	freemempos;
+};
+
+static void
+pmap_bootstrap_dmap_l0_table(struct dmap_bootstrap_state *state)
+{
+	vm_paddr_t l1_pa;
+	u_int l0_slot;
+
+	/* Link the level 0 table to a level 1 table */
+	l0_slot = pmap_l0_index(state->va);
+	if (l0_slot != state->l0_slot) {
+		MPASS(state->l0_slot < l0_slot ||
+		    state->l0_slot == L0_ENTRIES);
+
+		/* Create a new L0 table entry */
+		state->l0_slot = l0_slot;
+		state->l1 = (pt_entry_t *)state->freemempos;
+		memset(state->l1, 0, PAGE_SIZE);
+		state->freemempos += PAGE_SIZE;
+
+		/* Reset lower levels */
+		state->l2 = NULL;
+		state->l3 = NULL;
+		state->l1_slot = Ln_ENTRIES;
+		state->l2_slot = Ln_ENTRIES;
+
+		l1_pa = pmap_early_vtophys((vm_offset_t)state->l1);
+		MPASS((l1_pa & Ln_TABLE_MASK) == 0);
+		MPASS(pagetable_l0_ttbr1[l0_slot] == 0);
+		pmap_store(&pagetable_l0_ttbr1[l0_slot], l1_pa |
+		    TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE);
+	}
+	KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l1_table(struct dmap_bootstrap_state *state)
 {
-	pt_entry_t *l2;
 	vm_paddr_t l2_pa;
-	u_int l1_slot, l2_slot;
-	bool first;
+	u_int l1_slot;
+
+	/* Make sure there is a valid L0 -> L1 table */
+	pmap_bootstrap_dmap_l0_table(state);
+
+	/* Link the level 1 table to a level 2 table */
+	l1_slot = pmap_l1_index(state->va);
+	if (l1_slot != state->l1_slot) {
+		MPASS(state->l1_slot < l1_slot ||
+		    state->l1_slot == Ln_ENTRIES);
+
+		/* Create a new L1 table entry */
+		state->l1_slot = l1_slot;
+		state->l2 = (pt_entry_t *)state->freemempos;
+		memset(state->l2, 0, PAGE_SIZE);
+		state->freemempos += PAGE_SIZE;
+
+		/* Reset lower levels */
+		state->l3 = NULL;
+		state->l2_slot = Ln_ENTRIES;
+
+		l2_pa = pmap_early_vtophys((vm_offset_t)state->l2);
+		MPASS((l2_pa & Ln_TABLE_MASK) == 0);
+		MPASS(state->l1[l1_slot] == 0);
+		pmap_store(&state->l1[l1_slot], l2_pa | TATTR_PXN_TABLE |
+		    L1_TABLE);
+	}
+	KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__));
+}
 
-	l2 = *l2p;
-	l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
-	if (l1_slot != *prev_l1_slot) {
-		*prev_l1_slot = l1_slot;
-		l2 = (pt_entry_t *)freemempos;
-		l2_pa = pmap_early_vtophys((vm_offset_t)l2);
-		freemempos += PAGE_SIZE;
+static void
+pmap_bootstrap_dmap_l2_table(struct dmap_bootstrap_state *state)
+{
+	vm_paddr_t l3_pa;
+	u_int l2_slot;
+
+	/* Make sure there is a valid L1 -> L2 table */
+	pmap_bootstrap_dmap_l1_table(state);
 
-		pmap_store(&pagetable_dmap[l1_slot],
-		    (l2_pa & ~Ln_TABLE_MASK) |
-		    TATTR_PXN_TABLE | L1_TABLE);
+	/* Link the level 2 table to a level 3 table */
+	l2_slot = pmap_l2_index(state->va);
+	if (l2_slot != state->l2_slot) {
+		MPASS(state->l2_slot < l2_slot ||
+		    state->l2_slot == Ln_ENTRIES);
 
-		memset(l2, 0, PAGE_SIZE);
+		/* Create a new L2 table entry */
+		state->l2_slot = l2_slot;
+		state->l3 = (pt_entry_t *)state->freemempos;
+		memset(state->l3, 0, PAGE_SIZE);
+		state->freemempos += PAGE_SIZE;
+
+		l3_pa = pmap_early_vtophys((vm_offset_t)state->l3);
+		MPASS((l3_pa & Ln_TABLE_MASK) == 0);
+		MPASS(state->l2[l2_slot] == 0);
+		pmap_store(&state->l2[l2_slot], l3_pa | TATTR_PXN_TABLE |
+		    L2_TABLE);
 	}
-	KASSERT(l2 != NULL,
-	    ("pmap_bootstrap_dmap_l2: NULL l2 map"));
-	for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1];
-	    *pa += L2_SIZE, *va += L2_SIZE) {
+	KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__));
+}
+
+static void
+pmap_bootstrap_dmap_l2_block(struct dmap_bootstrap_state *state, int i)
+{
+	u_int l2_slot;
+	bool first;
+
+	if ((physmap[i + 1] - state->pa) < L2_SIZE)
+		return;
+
+	/* Make sure there is a valid L1 table */
+	pmap_bootstrap_dmap_l1_table(state);
+
+	MPASS((state->va & L2_OFFSET) == 0);
+	for (first = true;
+	    state->va < DMAP_MAX_ADDRESS &&
+	    (physmap[i + 1] - state->pa) >= L2_SIZE;
+	    state->va += L2_SIZE, state->pa += L2_SIZE) {
 		/*
 		 * Stop if we are about to walk off the end of what the
 		 * current L1 slot can address.
 		 */
-		if (!first && (*pa & L1_OFFSET) == 0)
+		if (!first && (state->pa & L1_OFFSET) == 0)
 			break;
 
 		first = false;
-		l2_slot = pmap_l2_index(*va);
-		pmap_store(&l2[l2_slot],
-		    (*pa & ~L2_OFFSET) | ATTR_DEFAULT |
-		    ATTR_S1_XN |
-		    ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+		l2_slot = pmap_l2_index(state->va);
+		MPASS((state->pa & L2_OFFSET) == 0);
+		MPASS(state->l2[l2_slot] == 0);
+		pmap_store(&state->l2[l2_slot], state->pa | ATTR_DEFAULT |
+		    ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
 		    L2_BLOCK);
 	}
-	MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS));
-	*l2p = l2;
+	MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
+}
 
-	return (freemempos);
+static void
+pmap_bootstrap_dmap_l3_page(struct dmap_bootstrap_state *state, int i)
+{
+	u_int l3_slot;
+	bool first;
+
+	if ((physmap[i + 1] - state->pa) < L3_SIZE)
+		return;
+
+	/* Make sure there is a valid L2 table */
+	pmap_bootstrap_dmap_l2_table(state);
+
+	MPASS((state->va & L3_OFFSET) == 0);
+	for (first = true;
+	    state->va < DMAP_MAX_ADDRESS &&
+	    (physmap[i + 1] - state->pa) >= L3_SIZE;
+	    state->va += L3_SIZE, state->pa += L3_SIZE) {
+		/*
+		 * Stop if we are about to walk off the end of what the
+		 * current L2 slot can address.
+		 */
+		if (!first && (state->pa & L2_OFFSET) == 0)
+			break;
+
+		first = false;
+		l3_slot = pmap_l3_index(state->va);
+		MPASS((state->pa & L3_OFFSET) == 0);
+		MPASS(state->l3[l3_slot] == 0);
+		pmap_store(&state->l3[l3_slot], state->pa | ATTR_DEFAULT |
+		    ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+		    L3_PAGE);
+	}
+	MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
 }
 
 static vm_offset_t
 pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa,
     vm_offset_t freemempos)
 {
-	pt_entry_t *l2;
-	vm_offset_t va;
-	vm_paddr_t pa;
-	u_int l1_slot, prev_l1_slot;
+	struct dmap_bootstrap_state state;
 	int i;
 
 	dmap_phys_base = min_pa & ~L1_OFFSET;
 	dmap_phys_max = 0;
 	dmap_max_addr = 0;
-	l2 = NULL;
-	prev_l1_slot = -1;
+
+	state.l1 = state.l2 = state.l3 = NULL;
+	state.l0_slot = L0_ENTRIES;
+	state.l1_slot = Ln_ENTRIES;
+	state.l2_slot = Ln_ENTRIES;
+	state.freemempos = freemempos;
 
 	for (i = 0; i < (physmap_idx * 2); i += 2) {
-		pa = physmap[i] & ~L2_OFFSET;
-		va = pa - dmap_phys_base + DMAP_MIN_ADDRESS;
+		state.pa = physmap[i] & ~L3_OFFSET;
+		state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS;
 
-		/* Create L2 mappings at the start of the region */
-		if ((pa & L1_OFFSET) != 0) {
-			freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
-			    &prev_l1_slot, &l2, i, freemempos);
-		}
+		/* Create L3 mappings at the start of the region */
+		if ((state.pa & L2_OFFSET) != 0)
+			pmap_bootstrap_dmap_l3_page(&state, i);
+		MPASS(state.pa <= physmap[i + 1]);
 
-		for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] &&
-		    (physmap[i + 1] - pa) >= L1_SIZE;
-		    pa += L1_SIZE, va += L1_SIZE) {
-			l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
-			pmap_store(&pagetable_dmap[l1_slot],
-			    (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN |
-			    ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK);
-		}
+		/* Create L2 mappings at the start of the region */
+		if ((state.pa & L1_OFFSET) != 0)
+			pmap_bootstrap_dmap_l2_block(&state, i);
+		MPASS(state.pa <= physmap[i + 1]);
+
+		/* Create the main L1 block mappings */
+		for (; state.va < DMAP_MAX_ADDRESS &&
+		    (physmap[i + 1] - state.pa) >= L1_SIZE;
+		    state.va += L1_SIZE, state.pa += L1_SIZE) {
+			/* Make sure there is a valid L1 table */
+			pmap_bootstrap_dmap_l0_table(&state);
+			MPASS((state.pa & L1_OFFSET) == 0);
+			pmap_store(&state.l1[pmap_l1_index(state.va)],
+			    state.pa | ATTR_DEFAULT | ATTR_S1_XN |
+			    ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
+			    L1_BLOCK);
+		}
+		MPASS(state.pa <= physmap[i + 1]);
 
 		/* Create L2 mappings at the end of the region */
-		if (pa < physmap[i + 1]) {
-			freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
-			    &prev_l1_slot, &l2, i, freemempos);
-		}
+		pmap_bootstrap_dmap_l2_block(&state, i);
+		MPASS(state.pa <= physmap[i + 1]);
+
+		/* Create L3 mappings at the end of the region */
+		pmap_bootstrap_dmap_l3_page(&state, i);
+		MPASS(state.pa == physmap[i + 1]);
 
-		if (pa > dmap_phys_max) {
-			dmap_phys_max = pa;
-			dmap_max_addr = va;
+		if (state.pa > dmap_phys_max) {
+			dmap_phys_max = state.pa;
+			dmap_max_addr = state.va;
 		}
 	}
 
 	cpu_tlb_flushID();
 
-	return (freemempos);
+	return (state.freemempos);
 }
 
 static vm_offset_t