arm64: Add vm_page_t MTE flags

To track which pages have MTE tags. Add a flag field to md_page. We
can then use this in MD code to mark which pages have MTE tags.

Reviewed by:	andrew
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D55954

4 files changed, 44 insertions, 0 deletions

diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c
index 88f394dc72c3..ba5d9df3b01c 100644
--- a/sys/arm64/arm64/mte.c
+++ b/sys/arm64/arm64/mte.c
@@ -49,6 +49,13 @@ static u_int __read_mostly mte_version = 0;
 
 struct thread *mte_switch(struct thread *);
 
+/* Fetch the block size used by tag load and store instructions */
+static inline size_t
+mte_block_size(void)
+{
+	return (sizeof(int) << GMID_BS_SIZE(READ_SPECIALREG(GMID_EL1_REG)));
+}
+
 static void
 mte_update_sctlr(struct thread *td, uint64_t sctlr)
 {
@@ -57,6 +64,37 @@ mte_update_sctlr(struct thread *td, uint64_t sctlr)
 	td->td_md.md_sctlr |= sctlr;
 }
 
+/**
+ * Clear/sync the allocation tags for a given page. This should be done on
+ * allocation of a page to ensure a tag check fault does not occur immediately
+ * after accessing newly tagged memory.
+ */
+void
+mte_sync_tags(vm_page_t page)
+{
+	char *addr;
+	size_t block_size;
+
+	if (!MTE_HAS_TAG_CHECK)
+		return;
+
+	/* don't clear the tags on a page that's already setup for mte */
+	if ((page->md.pv_flags & PV_MTE_TAGGED) != 0)
+		return;
+
+	block_size = mte_block_size();
+	addr = PHYS_TO_DMAP(page->phys_addr);
+
+	for (size_t count = 0; count < PAGE_SIZE;
+	    count += block_size, addr += block_size)
+		asm volatile(
+		    ".arch_extension memtag	\n"
+		    "stgm xzr, [%0]		\n"
+		    ".arch_extension nomemtag" : : "r" (addr));
+
+	page->md.pv_flags |= PV_MTE_TAGGED;
+}
+
 void
 mte_fork(struct thread *new_td, struct thread *orig_td)
 {
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 0aa1ee8d5801..6914fc13b065 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -6922,6 +6922,7 @@ pmap_zero_page(vm_page_t m)
 	void *va = VM_PAGE_TO_DMAP(m);
 
 	pagezero(va);
+	m->md.pv_flags &= ~PV_MTE_TAGGED;
 }
 
 /*
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
index 8d8a80a8d02a..d36e1e56c91c 100644
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -284,6 +284,8 @@ void	mte_copy_thread(struct thread *, struct thread *);
 void	mte_thread_alloc(struct thread *);
 void	mte_thread0(struct thread *);
 
+void	mte_sync_tags(vm_page_t page);
+
 /* Functions to read the sanitised view of the special registers */
 void	update_special_regs(u_int);
 void	update_special_reg_iss(u_int, uint64_t, uint64_t);
diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h
index 69ae8e5c80b7..cf20827fa666 100644
--- a/sys/arm64/include/pmap.h
+++ b/sys/arm64/include/pmap.h
@@ -74,6 +74,9 @@ struct md_page {
 	uint8_t			pv_reserve[2];
 };
 
+/* machine page flags */
+#define PV_MTE_TAGGED	0x01	/* page is tagged with MTE */
+
 enum pmap_stage {
 	PM_INVALID,
 	PM_STAGE1,