amd64 pmap: LA57 AKA 5-level paging

Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.

This patch makes pmap support both LA48 and LA57 hardware.  The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging.  It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.

To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode.  This is somewhat delicate
and done in pmap_bootstrap_la57().  AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.

I decided to not change kernel map for now.  Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing.  Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step.  Nice side effect is that minidumps are
compatible.

On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.

For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.

Tested by:	pho (LA48 hardware)
Reviewed by:	alc
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D25273

5 files changed, 83 insertions, 30 deletions

diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
index 8d866fee846a..9a550d7024fe 100644
--- a/sys/amd64/include/md_var.h
+++ b/sys/amd64/include/md_var.h
@@ -46,6 +46,8 @@ extern int	syscall_ret_l1d_flush_mode;
 extern vm_paddr_t intel_graphics_stolen_base;
 extern vm_paddr_t intel_graphics_stolen_size;
 
+extern int la57;
+
 /*
  * The file "conf/ldscript.amd64" defines the symbol "kernphys".  Its
  * value is the physical address at which the kernel is loaded.
diff --git a/sys/amd64/include/param.h b/sys/amd64/include/param.h
index ac3df693e4d4..2bd4d913a7b1 100644
--- a/sys/amd64/include/param.h
+++ b/sys/amd64/include/param.h
@@ -118,6 +118,12 @@
 #define	PML4SHIFT	39		/* LOG2(NBPML4) */
 #define	NBPML4		(1UL<<PML4SHIFT)/* bytes/page map lev4 table */
 #define	PML4MASK	(NBPML4-1)
+/* Size of the level 5 page-map level-5 table units */
+#define	NPML5EPG	(PAGE_SIZE/(sizeof (pml5_entry_t)))
+#define	NPML5EPGSHIFT	9		/* LOG2(NPML5EPG) */
+#define	PML5SHIFT	48		/* LOG2(NBPML5) */
+#define	NBPML5		(1UL<<PML5SHIFT)/* bytes/page map lev5 table */
+#define	PML5MASK	(NBPML5-1)
 
 #define	MAXPAGESIZES	3	/* maximum number of supported page sizes */
 
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index e2d7a714511b..5cdcca66d973 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -166,14 +166,22 @@
  * Pte related macros.  This is complicated by having to deal with
  * the sign extension of the 48th bit.
  */
-#define KVADDR(l4, l3, l2, l1) ( \
+#define KV4ADDR(l4, l3, l2, l1) ( \
 	((unsigned long)-1 << 47) | \
 	((unsigned long)(l4) << PML4SHIFT) | \
 	((unsigned long)(l3) << PDPSHIFT) | \
 	((unsigned long)(l2) << PDRSHIFT) | \
 	((unsigned long)(l1) << PAGE_SHIFT))
+#define KV5ADDR(l5, l4, l3, l2, l1) (		\
+	((unsigned long)-1 << 56) | \
+	((unsigned long)(l5) << PML5SHIFT) | \
+	((unsigned long)(l4) << PML4SHIFT) | \
+	((unsigned long)(l3) << PDPSHIFT) | \
+	((unsigned long)(l2) << PDRSHIFT) | \
+	((unsigned long)(l1) << PAGE_SHIFT))
 
-#define UVADDR(l4, l3, l2, l1) ( \
+#define UVADDR(l5, l4, l3, l2, l1) (	     \
+	((unsigned long)(l5) << PML5SHIFT) | \
 	((unsigned long)(l4) << PML4SHIFT) | \
 	((unsigned long)(l3) << PDPSHIFT) | \
 	((unsigned long)(l2) << PDRSHIFT) | \
@@ -187,9 +195,19 @@
  */
 #define NKPML4E		4
 
-#define	NUPML4E		(NPML4EPG/2)	/* number of userland PML4 pages */
-#define	NUPDPE		(NUPML4E*NPDPEPG)/* number of userland PDP pages */
-#define	NUPDE		(NUPDPE*NPDEPG)	/* number of userland PD entries */
+/*
+ * We use the same numbering of the page table pages for 5-level and
+ * 4-level paging structures.
+ */
+#define	NUPML5E		(NPML5EPG / 2)		/* number of userland PML5
+						   pages */
+#define	NUPML4E		(NUPML5E * NPML4EPG)	/* number of userland PML4
+						   pages */
+#define	NUPDPE		(NUPML4E * NPDPEPG)	/* number of userland PDP
+						   pages */
+#define	NUPDE		(NUPDPE * NPDEPG)	/* number of userland PD
+						   entries */
+#define	NUP4ML4E	(NPML4EPG / 2)
 
 /*
  * NDMPML4E is the maximum number of PML4 entries that will be
@@ -216,7 +234,8 @@
  * Or, in other words, KPML4I provides bits 39..47 of KERNBASE,
  * and KPDPI provides bits 30..38.)
  */
-#define	PML4PML4I	(NPML4EPG/2)	/* Index of recursive pml4 mapping */
+#define	PML4PML4I	(NPML4EPG / 2)	/* Index of recursive pml4 mapping */
+#define	PML5PML5I	(NPML5EPG / 2)	/* Index of recursive pml5 mapping */
 
 #define	KPML4BASE	(NPML4EPG-NKPML4E) /* KVM at highest addresses */
 #define	DMPML4I		rounddown(KPML4BASE-NDMPML4E, NDMPML4E) /* Below KVM */
@@ -258,25 +277,34 @@ typedef u_int64_t pd_entry_t;
 typedef u_int64_t pt_entry_t;
 typedef u_int64_t pdp_entry_t;
 typedef u_int64_t pml4_entry_t;
+typedef u_int64_t pml5_entry_t;
 
 /*
  * Address of current address space page table maps and directories.
  */
 #ifdef _KERNEL
-#define	addr_PTmap	(KVADDR(PML4PML4I, 0, 0, 0))
-#define	addr_PDmap	(KVADDR(PML4PML4I, PML4PML4I, 0, 0))
-#define	addr_PDPmap	(KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, 0))
-#define	addr_PML4map	(KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I))
-#define	addr_PML4pml4e	(addr_PML4map + (PML4PML4I * sizeof(pml4_entry_t)))
-#define	PTmap		((pt_entry_t *)(addr_PTmap))
-#define	PDmap		((pd_entry_t *)(addr_PDmap))
-#define	PDPmap		((pd_entry_t *)(addr_PDPmap))
-#define	PML4map		((pd_entry_t *)(addr_PML4map))
-#define	PML4pml4e	((pd_entry_t *)(addr_PML4pml4e))
+#define	addr_P4Tmap	(KV4ADDR(PML4PML4I, 0, 0, 0))
+#define	addr_P4Dmap	(KV4ADDR(PML4PML4I, PML4PML4I, 0, 0))
+#define	addr_P4DPmap	(KV4ADDR(PML4PML4I, PML4PML4I, PML4PML4I, 0))
+#define	addr_P4ML4map	(KV4ADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I))
+#define	addr_P4ML4pml4e	(addr_PML4map + (PML4PML4I * sizeof(pml4_entry_t)))
+#define	P4Tmap		((pt_entry_t *)(addr_P4Tmap))
+#define	P4Dmap		((pd_entry_t *)(addr_P4Dmap))
+
+#define	addr_P5Tmap	(KV5ADDR(PML5PML5I, 0, 0, 0, 0))
+#define	addr_P5Dmap	(KV5ADDR(PML5PML5I, PML5PML5I, 0, 0, 0))
+#define	addr_P5DPmap	(KV5ADDR(PML5PML5I, PML5PML5I, PML5PML5I, 0, 0))
+#define	addr_P5ML4map	(KV5ADDR(PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I, 0))
+#define	addr_P5ML5map	\
+    (KVADDR(PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I))
+#define	addr_P5ML5pml5e	(addr_P5ML5map + (PML5PML5I * sizeof(pml5_entry_t)))
+#define	P5Tmap		((pt_entry_t *)(addr_P5Tmap))
+#define	P5Dmap		((pd_entry_t *)(addr_P5Dmap))
 
 extern int nkpt;		/* Initial number of kernel page tables */
 extern u_int64_t KPDPphys;	/* physical address of kernel level 3 */
 extern u_int64_t KPML4phys;	/* physical address of kernel level 4 */
+extern u_int64_t KPML5phys;	/* physical address of kernel level 5 */
 
 /*
  * virtual address to page table entry and
@@ -333,8 +361,8 @@ struct pmap_pcids {
  */
 struct pmap {
 	struct mtx		pm_mtx;
-	pml4_entry_t		*pm_pml4;	/* KVA of level 4 page table */
-	pml4_entry_t		*pm_pml4u;	/* KVA of user l4 page table */
+	pml4_entry_t		*pm_pmltop;	/* KVA of top level page table */
+	pml4_entry_t		*pm_pmltopu;	/* KVA of user top page table */
 	uint64_t		pm_cr3;
 	uint64_t		pm_ucr3;
 	TAILQ_HEAD(,pv_chunk)	pm_pvchunk;	/* list of mappings in pmap */
@@ -447,6 +475,7 @@ bool	pmap_not_in_di(void);
 boolean_t pmap_page_is_mapped(vm_page_t m);
 void	pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
 void	pmap_pinit_pml4(vm_page_t);
+void	pmap_pinit_pml5(vm_page_t);
 bool	pmap_ps_enabled(pmap_t pmap);
 void	pmap_unmapdev(vm_offset_t, vm_size_t);
 void	pmap_invalidate_page(pmap_t, vm_offset_t);
@@ -502,6 +531,13 @@ pmap_pml4e_index(vm_offset_t va)
 	return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
 }
 
+static __inline vm_pindex_t
+pmap_pml5e_index(vm_offset_t va)
+{
+
+	return ((va >> PML5SHIFT) & ((1ul << NPML5EPGSHIFT) - 1));
+}
+
 #endif /* !LOCORE */
 
 #endif /* !_MACHINE_PMAP_H_ */
diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h
index 75f357c3a53e..e74f1626a56a 100644
--- a/sys/amd64/include/proc.h
+++ b/sys/amd64/include/proc.h
@@ -84,6 +84,8 @@ struct mdproc {
 };
 
 #define	P_MD_KPTI		0x00000001	/* Enable KPTI on exec */
+#define	P_MD_LA48		0x00000002	/* Request LA48 after exec */
+#define	P_MD_LA57		0x00000004	/* Request LA57 after exec */
 
 #define	KINFO_PROC_SIZE 1088
 #define	KINFO_PROC32_SIZE 768
diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h
index 2fe349e0beb5..64eed5760357 100644
--- a/sys/amd64/include/vmparam.h
+++ b/sys/amd64/include/vmparam.h
@@ -169,25 +169,32 @@
  * 0xffffffff80000000                        KERNBASE
  */
 
-#define	VM_MIN_KERNEL_ADDRESS	KVADDR(KPML4BASE, 0, 0, 0)
-#define	VM_MAX_KERNEL_ADDRESS	KVADDR(KPML4BASE + NKPML4E - 1, \
+#define	VM_MIN_KERNEL_ADDRESS	KV4ADDR(KPML4BASE, 0, 0, 0)
+#define	VM_MAX_KERNEL_ADDRESS	KV4ADDR(KPML4BASE + NKPML4E - 1, \
 					NPDPEPG-1, NPDEPG-1, NPTEPG-1)
 
-#define	DMAP_MIN_ADDRESS	KVADDR(DMPML4I, 0, 0, 0)
-#define	DMAP_MAX_ADDRESS	KVADDR(DMPML4I + NDMPML4E, 0, 0, 0)
+#define	DMAP_MIN_ADDRESS	KV4ADDR(DMPML4I, 0, 0, 0)
+#define	DMAP_MAX_ADDRESS	KV4ADDR(DMPML4I + NDMPML4E, 0, 0, 0)
 
-#define	LARGEMAP_MIN_ADDRESS	KVADDR(LMSPML4I, 0, 0, 0)
-#define	LARGEMAP_MAX_ADDRESS	KVADDR(LMEPML4I + 1, 0, 0, 0)
+#define	LARGEMAP_MIN_ADDRESS	KV4ADDR(LMSPML4I, 0, 0, 0)
+#define	LARGEMAP_MAX_ADDRESS	KV4ADDR(LMEPML4I + 1, 0, 0, 0)
 
-#define	KERNBASE		KVADDR(KPML4I, KPDPI, 0, 0)
+#define	KERNBASE		KV4ADDR(KPML4I, KPDPI, 0, 0)
 
-#define	UPT_MAX_ADDRESS		KVADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I)
-#define	UPT_MIN_ADDRESS		KVADDR(PML4PML4I, 0, 0, 0)
+#define	UPT_MAX_ADDRESS		KV4ADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I)
+#define	UPT_MIN_ADDRESS		KV4ADDR(PML4PML4I, 0, 0, 0)
 
-#define	VM_MAXUSER_ADDRESS	UVADDR(NUPML4E, 0, 0, 0)
+#define	VM_MAXUSER_ADDRESS_LA57	UVADDR(NUPML5E, 0, 0, 0, 0)
+#define	VM_MAXUSER_ADDRESS_LA48	UVADDR(0, NUP4ML4E, 0, 0, 0)
+#define	VM_MAXUSER_ADDRESS	VM_MAXUSER_ADDRESS_LA57
 
-#define	SHAREDPAGE		(VM_MAXUSER_ADDRESS - PAGE_SIZE)
-#define	USRSTACK		SHAREDPAGE
+#define	SHAREDPAGE_LA57		(VM_MAXUSER_ADDRESS_LA57 - PAGE_SIZE)
+#define	SHAREDPAGE_LA48		(VM_MAXUSER_ADDRESS_LA48 - PAGE_SIZE)
+#define	USRSTACK_LA57		SHAREDPAGE_LA57
+#define	USRSTACK_LA48		SHAREDPAGE_LA48
+#define	USRSTACK		USRSTACK_LA48
+#define	PS_STRINGS_LA57		(USRSTACK_LA57 - sizeof(struct ps_strings))
+#define	PS_STRINGS_LA48		(USRSTACK_LA48 - sizeof(struct ps_strings))
 
 #define	VM_MAX_ADDRESS		UPT_MAX_ADDRESS
 #define	VM_MIN_ADDRESS		(0)