arm64: Add MOPS implementations of copyin/copyout

Reimplement copyin() & copyout() as ifuncs.

Reviewed by:	andrew
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D54947

3 files changed, 103 insertions, 6 deletions

diff --git a/sys/arm64/arm64/copyinout.S b/sys/arm64/arm64/copyinout.S
index e41c4b5f6734..8c6fee63f410 100644
--- a/sys/arm64/arm64/copyinout.S
+++ b/sys/arm64/arm64/copyinout.S
@@ -65,9 +65,9 @@ END(copyio_fault)
 /*
  * Copies from a kernel to user address
  *
- * int copyout(const void *kaddr, void *udaddr, size_t len)
+ * int copyout_std(const void *kaddr, void *udaddr, size_t len)
  */
-ENTRY(copyout)
+ENTRY(copyout_std)
 	cbz	x2, 1f
 	check_user_access 1, 2, copyio_fault_nopcb
 
@@ -76,14 +76,37 @@ ENTRY(copyout)
 1:	mov	x0, xzr		/* return 0 */
 	ret
 
-END(copyout)
+END(copyout_std)
+
+/*
+ * Copies from a kernel to user address
+ *
+ * int copyout_mops(const void *kaddr, void *udaddr, size_t len)
+ */
+ENTRY(copyout_mops)
+	cbz	x2, 1f
+	check_user_access 1, 2, copyio_fault_nopcb
+
+	adr	x6, copyio_fault /* Get the handler address */
+	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
+
+	.inst   0x19001441      /* cpyfpwt   [x1]!, [x0]!, x2!  */
+	.inst   0x19401441      /* cpyfmwt   [x1]!, [x0]!, x2!  */
+	.inst   0x19801441      /* cpyfewt   [x1]!, [x0]!, x2!  */
+
+	SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
+
+1:	mov	x0, xzr		/* return 0 */
+	ret
+
+END(copyout_mops)
 
 /*
  * Copies from a user to kernel address
  *
- * int copyin(const void *uaddr, void *kdaddr, size_t len)
+ * int copyin_std(const void *uaddr, void *kdaddr, size_t len)
  */
-ENTRY(copyin)
+ENTRY(copyin_std)
 	cbz	x2, 1f
 	check_user_access 0, 2, copyio_fault_nopcb
 
@@ -92,7 +115,30 @@ ENTRY(copyin)
 1:	mov	x0, xzr		/* return 0 */
 	ret
 
-END(copyin)
+END(copyin_std)
+
+/*
+ * Copies from a user to kernel address
+ *
+ * int copyin_mops(const void *uaddr, void *kdaddr, size_t len)
+ */
+ENTRY(copyin_mops)
+	cbz	x2, 1f
+	check_user_access 0, 2, copyio_fault_nopcb
+
+	adr	x6, copyio_fault /* Get the handler address */
+	SET_FAULT_HANDLER(x6, x7) /* Set the handler */
+
+	.inst   0x19002441      /* cpyfprt   [x1]!, [x0]!, x2!  */
+	.inst   0x19402441      /* cpyfmrt   [x1]!, [x0]!, x2!  */
+	.inst   0x19802441      /* cpyfert   [x1]!, [x0]!, x2!  */
+
+	SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */
+
+1:	mov	x0, xzr		/* return 0 */
+	ret
+
+END(copyin_mops)
 
 /*
  * Copies a string from a user to kernel address
diff --git a/sys/arm64/arm64/copyinout_ifunc.c b/sys/arm64/arm64/copyinout_ifunc.c
new file mode 100644
index 000000000000..33d9f11fc7e6
--- /dev/null
+++ b/sys/arm64/arm64/copyinout_ifunc.c
@@ -0,0 +1,50 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 ARM Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+
+#include <machine/cpu.h>
+#include <machine/elf.h>
+#include <machine/ifunc.h>
+#include <machine/md_var.h>
+
+int copyout_std(const void *kaddr, void *udaddr, size_t len);
+int copyout_mops(const void *kaddr, void *udaddr, size_t len);
+int copyin_std(const void *uaddr, void *kdaddr, size_t len);
+int copyin_mops(const void *uaddr, void *kdaddr, size_t len);
+
+DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t))
+{
+        return ((elf_hwcap2 & HWCAP2_MOPS) != 0 ? copyout_mops : copyout_std);
+}
+
+DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t))
+{
+        return ((elf_hwcap2 & HWCAP2_MOPS) != 0 ? copyin_mops : copyin_std);
+}
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 5439cc5edde0..3b12999a6c87 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -36,6 +36,7 @@ arm64/arm64/busdma_bounce.c			standard
 arm64/arm64/busdma_machdep.c			standard
 arm64/arm64/clock.c				standard
 arm64/arm64/copyinout.S				standard
+arm64/arm64/copyinout_ifunc.c			standard
 arm64/arm64/cpu_errata.c			standard
 arm64/arm64/cpu_feat.c				standard
 arm64/arm64/cpufunc_asm.S			standard