arm64/vmm: Enable 16-bit VMIDs when in use by pmap

pmap_init always uses 16-bit VMIDs when supported, but we never enable
them in VTCR_EL2 (for ASIDs, locore enables them in TCR_EL1 and
pmap_init keys off whether they've been enabled, but the order in which
pmap_init and vmmops_modinit run is reversed). As a result, although the
full 16-bit value can be stored to VTTBR_EL2 and read back, the upper 8
bits are treated as 0, and so VMIDs that our VMID allocation believes
are distinct end up aliasing.

In future this interface may change such that vmm decides on the VMID
width and tells the pmap to use that, with appropriate support for
unloading and reloading vmm, but that can come as a follow-up change, as
this is a more minimal bug fix.

Reviewed by:	markj
Obtained from:	CheriBSD
Fixes:		47e073941f4e ("Import the kernel parts of bhyve/arm64")
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D55860

4 files changed, 16 insertions, 0 deletions

diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 5017b5f63c4d..556dda855c0b 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -802,6 +802,17 @@ pmap_ps_enabled(pmap_t pmap)
 }
 
 bool
+pmap_vs_enabled(void)
+{
+	/*
+	 * 8 and 16 are the only values hardware can support, but allow for the
+	 * possibility of artificially restricting the bits, e.g. for testing.
+	 */
+	KASSERT(vmids.asid_bits <= 16, ("VMID bits %d > 16", vmids.asid_bits));
+	return (vmids.asid_bits > 8);
+}
+
+bool
 pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1,
     pd_entry_t **l2, pt_entry_t **l3)
 {
diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h
index 73adf89b4182..bd77dcadc724 100644
--- a/sys/arm64/include/hypervisor.h
+++ b/sys/arm64/include/hypervisor.h
@@ -2166,6 +2166,8 @@
 #define	 VTCR_EL2_PS_44BIT	(0x4UL << VTCR_EL2_PS_SHIFT)
 #define	 VTCR_EL2_PS_48BIT	(0x5UL << VTCR_EL2_PS_SHIFT)
 #define	 VTCR_EL2_PS_52BIT	(0x6UL << VTCR_EL2_PS_SHIFT)
+#define	VTCR_EL2_VS_SHIFT	19
+#define	 VTCR_EL2_VS		(1UL << VTCR_EL2_VS_SHIFT)
 #define	VTCR_EL2_DS_SHIFT	32
 #define	VTCR_EL2_DS		(0x1UL << VTCR_EL2_DS_SHIFT)
 
diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h
index 406b6e2c5e0a..f5812893fcfa 100644
--- a/sys/arm64/include/pmap.h
+++ b/sys/arm64/include/pmap.h
@@ -156,6 +156,7 @@ void	*pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma);
 bool	pmap_page_is_mapped(vm_page_t m);
 int	pmap_pinit_stage(pmap_t, enum pmap_stage, int);
 bool	pmap_ps_enabled(pmap_t pmap);
+bool	pmap_vs_enabled(void);
 uint64_t pmap_to_ttbr0(pmap_t pmap);
 void	pmap_disable_promotion(vm_offset_t sva, vm_size_t size);
 void	pmap_map_delete(pmap_t, vm_offset_t, vm_offset_t);
diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
index 712783338214..8fe0bf62bf8b 100644
--- a/sys/arm64/vmm/vmm_arm64.c
+++ b/sys/arm64/vmm/vmm_arm64.c
@@ -391,6 +391,8 @@ vmmops_modinit(int ipinum)
 #ifdef SMP
 	el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS;
 #endif
+	if (pmap_vs_enabled())
+		el2_regs.vtcr_el2 |= VTCR_EL2_VS;
 	/*
 	 * If FEAT_LPA2 is enabled in the host then we need to enable it here
 	 * so the page tables created by pmap.c are correct. The meaning of