Support LSE atomics in the arm64 casue* functions

As with atomic(9) use the ARMv8.1 Large System Extension atomic
instructions to implement the userspace compare and swap functions.

Reviewed by:	kib
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D35234

3 files changed, 105 insertions, 6 deletions

diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 4771814a9b45..885af5676159 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -56,9 +56,9 @@ fsu_fault_nopcb:
 END(fsu_fault)
 
 /*
- * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
+ * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
  */
-ENTRY(casueword32)
+ENTRY(casueword32_llsc)
 	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	mov	w5, #1
@@ -73,12 +73,32 @@ ENTRY(casueword32)
 	str	w4, [x2]		/* Store the read data */
 	mov	w0, w5			/* Result same as store status */
 	ret				/* Return */
-END(casueword32)
+END(casueword32_llsc)
 
 /*
- * int casueword(volatile u_long *, u_long, u_long *, u_long)
+ * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
  */
-ENTRY(casueword)
+ENTRY(casueword32_lse)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+	mov	w7, w1			/* Back up the compare value */
+	.arch_extension lse
+	cas	w1, w3, [x0]		/* Compare and Swap */
+	.arch_extension nolse
+	cmp	w1, w7			/* Check if successful */
+	cset	w0, ne			/* Return 0 on success, 1 on failure */
+	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)	/* Reset the fault handler */
+	str	w1, [x2]		/* Store the read data */
+	ret				/* Return */
+END(casueword32_lse)
+
+/*
+ * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long)
+ */
+ENTRY(casueword_llsc)
 	check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb
 	adr	x6, fsu_fault		/* Load the fault handler */
 	mov	w5, #1
@@ -93,7 +113,27 @@ ENTRY(casueword)
 	str	x4, [x2]		/* Store the read data */
 	mov	w0, w5			/* Result same as store status */
 	ret				/* Return */
-END(casueword)
+END(casueword_llsc)
+
+/*
+ * int casueword_lse(volatile u_long *, u_long, u_long *, u_long)
+ */
+ENTRY(casueword_lse)
+	check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb
+	adr	x6, fsu_fault		/* Load the fault handler */
+	SET_FAULT_HANDLER(x6, x4)	/* And set it */
+	ENTER_USER_ACCESS(w6, x4)
+	mov	x7, x1			/* Back up the compare value */
+	.arch_extension lse
+	cas	x1, x3, [x0]		/* Compare and Swap */
+	.arch_extension nolse
+	cmp	x1, x7			/* Check if successful */
+	cset	w0, ne			/* Return 0 on success, 1 on failure */
+	EXIT_USER_ACCESS(w6)
+	SET_FAULT_HANDLER(xzr, x6)	/* Reset the fault handler */
+	str	x1, [x2]		/* Store the read data */
+	ret				/* Return */
+END(casueword_lse)
 
 .macro fsudata insn, ret_reg, user_arg
 	adr	x7, fsu_fault		/* Load the fault handler */
diff --git a/sys/arm64/arm64/support_ifunc.c b/sys/arm64/arm64/support_ifunc.c
new file mode 100644
index 000000000000..8abf3f3c75af
--- /dev/null
+++ b/sys/arm64/arm64/support_ifunc.c
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * This software was developed by Andrew Turner under sponsorship from
+ * the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+
+#include <machine/atomic.h>
+#include <machine/ifunc.h>
+
+int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
+int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t);
+
+int casueword_llsc(volatile u_long *, u_long, u_long *, u_long);
+int casueword_lse(volatile u_long *, u_long, u_long *, u_long);
+
+DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *base, uint32_t oldval,
+    uint32_t *oldvalp, uint32_t newval))
+{
+	if (lse_supported)
+		return (casueword32_lse);
+
+	return (casueword32_llsc);
+}
+
+DEFINE_IFUNC(, int, casueword, (volatile u_long *base, u_long oldval,
+    u_long *oldvalp, u_long newval))
+{
+	if (lse_supported)
+		return (casueword_lse);
+
+	return (casueword_llsc);
+}
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 30358a5d67d1..e0f529506d3d 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -72,6 +72,7 @@ arm64/arm64/pmap.c				standard
 arm64/arm64/ptrace_machdep.c			standard
 arm64/arm64/sigtramp.S				standard
 arm64/arm64/stack_machdep.c			optional ddb | stack
+arm64/arm64/support_ifunc.c			standard
 arm64/arm64/support.S				standard
 arm64/arm64/swtch.S				standard
 arm64/arm64/sys_machdep.c			standard