aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndriy Gapon <avg@FreeBSD.org>2019-10-09 11:26:36 +0000
committerAndriy Gapon <avg@FreeBSD.org>2019-10-09 11:26:36 +0000
commitf5c4c7209be61707dd84b50f1852d994d0df661a (patch)
treeb398e0185a4030d58099fddd840c4deb820f63cd
parent3af7f97c4e196d987da10541088b539f283e91c0 (diff)
downloadsrc-f5c4c7209be61707dd84b50f1852d994d0df661a.tar.gz
src-f5c4c7209be61707dd84b50f1852d994d0df661a.zip
cleanup of illumos compatibility atomics
atomic_cas_32 is implemented using atomic_fcmpset_32 on all platforms. Ditto for atomic_cas_64 and atomic_fcmpset_64 on platforms that have it. The only exception is sparc64 that provides MD atomic_cas_32 and atomic_cas_64. This is slightly inefficient as fcmpset reports whether the operation updated the target and that information is not needed for cas. Nevertheless, there is less code to maintain and to add for new platforms. Also, the operations are done inline now as opposed to function calls before. atomic_add_64_nv is implemented using atomic_fetchadd_64 on platforms that provide it. casptr, cas32, atomic_or_8, atomic_or_8_nv are completely removed as they have no users. atomic_mtx that is used to emulate 64-bit atomics on platforms that lack them is defined only on those platforms. As a result, platform specific opensolaris_atomic.S files have lost most of their code. The only exception is i386 where the compat+contrib code provides 64-bit atomics for userland use. That code assumes availability of cmpxchg8b instruction. FreeBSD does not have that assumption for i386 userland and does not provide 64-bit atomics. Hopefully, this can and will be fixed. MFC after: 3 weeks
Notes
Notes: svn path=/head/; revision=353340
-rw-r--r--sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c41
-rw-r--r--sys/cddl/compat/opensolaris/sys/atomic.h63
-rw-r--r--sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S52
-rw-r--r--sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S34
-rw-r--r--sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S22
-rw-r--r--sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S55
-rw-r--r--sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S61
7 files changed, 38 insertions, 290 deletions
diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
index c82772242fec..57f23014ff75 100644
--- a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
+++ b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
@@ -32,6 +32,9 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/atomic.h>
+#if !defined(__LP64__) && !defined(__mips_n32) && \
+ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
+
#ifdef _KERNEL
#include <sys/kernel.h>
@@ -52,8 +55,6 @@ atomic_init(void)
}
#endif
-#if !defined(__LP64__) && !defined(__mips_n32) && \
- !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
void
atomic_add_64(volatile uint64_t *target, int64_t delta)
{
@@ -94,7 +95,6 @@ atomic_load_64(volatile uint64_t *a)
mtx_unlock(&atomic_mtx);
return (ret);
}
-#endif
uint64_t
atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
@@ -107,27 +107,6 @@ atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
return (newval);
}
-#if defined(__powerpc__) || defined(__arm__) || defined(__mips__)
-void
-atomic_or_8(volatile uint8_t *target, uint8_t value)
-{
- mtx_lock(&atomic_mtx);
- *target |= value;
- mtx_unlock(&atomic_mtx);
-}
-#endif
-
-uint8_t
-atomic_or_8_nv(volatile uint8_t *target, uint8_t value)
-{
- uint8_t newval;
-
- mtx_lock(&atomic_mtx);
- newval = (*target |= value);
- mtx_unlock(&atomic_mtx);
- return (newval);
-}
-
uint64_t
atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
{
@@ -140,19 +119,7 @@ atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
mtx_unlock(&atomic_mtx);
return (oldval);
}
-
-uint32_t
-atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
-{
- uint32_t oldval;
-
- mtx_lock(&atomic_mtx);
- oldval = *target;
- if (oldval == cmp)
- *target = newval;
- mtx_unlock(&atomic_mtx);
- return (oldval);
-}
+#endif
void
membar_producer(void)
diff --git a/sys/cddl/compat/opensolaris/sys/atomic.h b/sys/cddl/compat/opensolaris/sys/atomic.h
index 700e60251bd5..51300ef2680a 100644
--- a/sys/cddl/compat/opensolaris/sys/atomic.h
+++ b/sys/cddl/compat/opensolaris/sys/atomic.h
@@ -32,10 +32,6 @@
#include <sys/types.h>
#include <machine/atomic.h>
-#define casptr(_a, _b, _c) \
- atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b), (uintptr_t) (_c))
-#define cas32 atomic_cmpset_32
-
#if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE))
#define I386_HAVE_ATOMIC64
#endif
@@ -46,27 +42,12 @@ extern void atomic_add_64(volatile uint64_t *target, int64_t delta);
extern void atomic_dec_64(volatile uint64_t *target);
extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value);
extern uint64_t atomic_load_64(volatile uint64_t *a);
-#endif
-#ifndef __sparc64__
-extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp,
- uint32_t newval);
+extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta);
extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp,
uint64_t newval);
#endif
-extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta);
-extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value);
-extern void membar_producer(void);
-#if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \
- defined(__mips__) || defined(__aarch64__) || defined(__riscv)
-extern void atomic_or_8(volatile uint8_t *target, uint8_t value);
-#else
-static __inline void
-atomic_or_8(volatile uint8_t *target, uint8_t value)
-{
- atomic_set_8(target, value);
-}
-#endif
+extern void membar_producer(void);
static __inline uint32_t
atomic_add_32_nv(volatile uint32_t *target, int32_t delta)
@@ -81,6 +62,18 @@ atomic_add_int_nv(volatile u_int *target, int delta)
}
static __inline void
+atomic_inc_32(volatile uint32_t *target)
+{
+ atomic_add_32(target, 1);
+}
+
+static __inline uint32_t
+atomic_inc_32_nv(volatile uint32_t *target)
+{
+ return (atomic_add_32_nv(target, 1));
+}
+
+static __inline void
atomic_dec_32(volatile uint32_t *target)
{
atomic_subtract_32(target, 1);
@@ -89,8 +82,17 @@ atomic_dec_32(volatile uint32_t *target)
static __inline uint32_t
atomic_dec_32_nv(volatile uint32_t *target)
{
- return (atomic_fetchadd_32(target, -1) - 1);
+ return (atomic_add_32_nv(target, -1));
+}
+
+#ifndef __sparc64__
+static inline uint32_t
+atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
+{
+ (void)atomic_fcmpset_32(target, &cmp, newval);
+ return (cmp);
}
+#endif
#if defined(__LP64__) || defined(__mips_n32) || \
defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64)
@@ -99,19 +101,22 @@ atomic_dec_64(volatile uint64_t *target)
{
atomic_subtract_64(target, 1);
}
-#endif
-static __inline void
-atomic_inc_32(volatile uint32_t *target)
+static inline uint64_t
+atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
{
- atomic_add_32(target, 1);
+ return (atomic_fetchadd_64(target, delta) + delta);
}
-static __inline uint32_t
-atomic_inc_32_nv(volatile uint32_t *target)
+#ifndef __sparc64__
+static inline uint64_t
+atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
{
- return (atomic_add_32_nv(target, 1));
+ (void)atomic_fcmpset_64(target, &cmp, newval);
+ return (cmp);
}
+#endif
+#endif
static __inline void
atomic_inc_64(volatile uint64_t *target)
diff --git a/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S b/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
index 0321c7d74e12..9441c311e33e 100644
--- a/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
+++ b/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S
@@ -28,58 +28,6 @@
#include <machine/asm.h>
-/*
- * uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta)
- */
-ENTRY(atomic_add_64_nv)
-1: ldxr x2, [x0] /* Load *target */
- add x2, x2, x1 /* x2 = x2 + delta */
- stxr w3, x2, [x0] /* Store *target */
- cbnz w3, 1b /* Check if the store succeeded */
- mov x0, x2 /* Return the new value */
- ret
-END(atomic_add_64_nv)
-
-/*
- * uint32_t
- * atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval)
- */
-ENTRY(atomic_cas_32)
-1: ldxr w3, [x0] /* Load *target */
- cmp w3, w1 /* Does *targe == cmp? */
- b.ne 2f /* If not exit */
- stxr w4, w2, [x0] /* Store newval to *target */
- cbnz w4, 1b /* Check if the store succeeded */
-2: mov w0, w3 /* Return the old value */
- ret
-END(atomic_cas_32)
-
-/*
- * uint64_t
- * atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval)
- */
-ENTRY(atomic_cas_64)
-1: ldxr x3, [x0] /* Load *target */
- cmp x3, x1 /* Does *targe == cmp? */
- b.ne 2f /* If not exit */
- stxr w4, x2, [x0] /* Store newval to *target */
- cbnz w4, 1b /* Check if the store succeeded */
-2: mov x0, x3 /* Return the old value */
- ret
-END(atomic_cas_64)
-
-/*
- * uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value)
- */
-ENTRY(atomic_or_8_nv)
-1: ldxrb w2, [x0] /* Load *target */
- orr w2, w2, w1 /* x2 = x2 | delta */
- stxrb w3, w2, [x0] /* Store *target */
- cbnz w3, 1b /* Check if the store succeeded */
- mov w0, w2 /* Return the new value */
- ret
-END(atomic_or_8_nv)
-
ENTRY(membar_producer)
dmb ish
ret
diff --git a/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S b/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
index 6d0a1f8fbee8..69c131dde2b9 100644
--- a/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
+++ b/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S
@@ -28,40 +28,6 @@
#define _ASM
#include <sys/asm_linkage.h>
- ENTRY(atomic_add_64_nv)
- mov %rsi, %rax // %rax = delta addend
- lock
- xaddq %rsi, (%rdi) // %rsi = old value, (%rdi) = sum
- addq %rsi, %rax // new value = original value + delta
- ret
- SET_SIZE(atomic_add_64_nv)
-
- ENTRY(atomic_or_8_nv)
- movb (%rdi), %al // %al = old value
-1:
- movb %sil, %cl
- orb %al, %cl // %cl = new value
- lock
- cmpxchgb %cl, (%rdi) // try to stick it in
- jne 1b
- movzbl %cl, %eax // return new value
- ret
- SET_SIZE(atomic_or_8_nv)
-
- ENTRY(atomic_cas_32)
- movl %esi, %eax
- lock
- cmpxchgl %edx, (%rdi)
- ret
- SET_SIZE(atomic_cas_32)
-
- ENTRY(atomic_cas_64)
- movq %rsi, %rax
- lock
- cmpxchgq %rdx, (%rdi)
- ret
- SET_SIZE(atomic_cas_64)
-
ENTRY(membar_producer)
sfence
ret
diff --git a/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S b/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
index dd61186ab413..b7a844d28d7a 100644
--- a/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
+++ b/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S
@@ -89,28 +89,6 @@
SET_SIZE(atomic_add_64_nv)
SET_SIZE(atomic_add_64)
- ENTRY(atomic_or_8_nv)
- movl 4(%esp), %edx // %edx = target address
- movb (%edx), %al // %al = old value
-1:
- movl 8(%esp), %ecx // %ecx = delta
- orb %al, %cl // %cl = new value
- lock
- cmpxchgb %cl, (%edx) // try to stick it in
- jne 1b
- movzbl %cl, %eax // return new value
- ret
- SET_SIZE(atomic_or_8_nv)
-
- ENTRY(atomic_cas_32)
- movl 4(%esp), %edx
- movl 8(%esp), %eax
- movl 12(%esp), %ecx
- lock
- cmpxchgl %ecx, (%edx)
- ret
- SET_SIZE(atomic_cas_32)
-
ENTRY(atomic_cas_64)
pushl %ebx
pushl %esi
diff --git a/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S b/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
index 0ab5d19c5a68..62ec109d89ba 100644
--- a/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
+++ b/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S
@@ -27,61 +27,6 @@
#include <machine/asm.h>
-ENTRY(atomic_add_64_nv)
- 1: ldarx %r5,0,%r3
- add %r5,%r4,%r5
- stdcx. %r5,0,%r3
- bne- 1b
-
- mr %r3,%r5
- blr
-
-ENTRY(atomic_cas_32)
- 1: lwarx %r6,0,%r3
- cmplw %r6,%r4
- bne 2f
- stwcx. %r5,0,%r3
- bne- 1b
- b 3f
-
- 2: stwcx. %r6,0,%r3 /* clear reservation */
-
- 3: mr %r3,%r6
- blr
-
-ENTRY(atomic_cas_64)
- 1: ldarx %r6,0,%r3
- cmpld %r6,%r4
- bne 2f
- stdcx. %r5,0,%r3
- bne- 1b
- b 3f
-
- 2: stdcx. %r6,0,%r3 /* clear reservation */
-
- 3: mr %r3,%r6
- blr
-
-ENTRY(atomic_or_8_nv)
- li %r6,3
- andc. %r6,%r3,%r6 /* r6 = r3 & ~4 */
- addi %r7,%r6,3
- sub %r7,%r7,%r3 /* offset in r7 */
- sldi %r7,%r7,3 /* bits to shift in r7 */
-
- rlwinm %r4,%r4,0,24,31 /* mask and rotate the argument */
- slw %r4,%r4,%r7
-
- 1: lwarx %r5,0,%r6
- or %r5,%r4,%r5
- stwcx. %r5,0,%r6
- bne- 1b
-
- srw %r3,%r5,%r7
- rlwinm %r3,%r3,0,24,31 /* mask return value */
-
- blr
-
ENTRY(membar_producer)
eieio
blr
diff --git a/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S b/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
index 3a498fe8963e..406a84ec0f06 100644
--- a/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
+++ b/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S
@@ -40,67 +40,6 @@
#endif
/*
- * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
- * separated, you need to also edit the libc sparcv9 platform
- * specific mapfile and remove the NODYNSORT attribute
- * from atomic_add_64_nv.
- */
- ENTRY(atomic_add_64)
- ALTENTRY(atomic_add_64_nv)
- ALTENTRY(atomic_add_ptr)
- ALTENTRY(atomic_add_ptr_nv)
- ALTENTRY(atomic_add_long)
- ALTENTRY(atomic_add_long_nv)
-add_64:
- ldx [%o0], %o2
-1:
- add %o2, %o1, %o3
- casxa [%o0] __ASI_ATOMIC, %o2, %o3
- cmp %o2, %o3
- bne,a,pn %xcc, 1b
- mov %o3, %o2
- retl
- add %o2, %o1, %o0 ! return new value
- SET_SIZE(atomic_add_long_nv)
- SET_SIZE(atomic_add_long)
- SET_SIZE(atomic_add_ptr_nv)
- SET_SIZE(atomic_add_ptr)
- SET_SIZE(atomic_add_64_nv)
- SET_SIZE(atomic_add_64)
-
- /*
- * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
- * separated, you need to also edit the libc sparcv9 platform
- * specific mapfile and remove the NODYNSORT attribute
- * from atomic_or_8_nv.
- */
- ENTRY(atomic_or_8)
- ALTENTRY(atomic_or_8_nv)
- ALTENTRY(atomic_or_uchar)
- and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right
- xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left
- sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left
- set 0xff, %o3 ! %o3 = mask
- sll %o3, %g1, %o3 ! %o3 = shifted to bit offset
- sll %o1, %g1, %o1 ! %o1 = shifted to bit offset
- and %o1, %o3, %o1 ! %o1 = single byte value
- andn %o0, 0x3, %o0 ! %o0 = word address
- ld [%o0], %o2 ! read old value
-1:
- or %o2, %o1, %o5 ! or in the new value
- casa [%o0] __ASI_ATOMIC, %o2, %o5
- cmp %o2, %o5
- bne,a,pn %icc, 1b
- mov %o5, %o2 ! %o2 = old value
- or %o2, %o1, %o5
- and %o5, %o3, %o5
- retl
- srl %o5, %g1, %o0 ! %o0 = new value
- SET_SIZE(atomic_or_uchar)
- SET_SIZE(atomic_or_8_nv)
- SET_SIZE(atomic_or_8)
-
- /*
* Spitfires and Blackbirds have a problem with membars in the
* delay slot (SF_ERRATA_51). For safety's sake, we assume
* that the whole world needs the workaround.