1 files changed, 25 insertions, 19 deletions

diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 99df700c8836..c024de5cc8cb 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -283,6 +283,7 @@ void
 _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
 {
 	struct rwlock *rw;
+	uintptr_t tid, v;
 
 	if (SCHEDULER_STOPPED())
 		return;
@@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
 	    ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
 	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
 	    line, NULL);
-	__rw_wlock(rw, curthread, file, line);
+	tid = (uintptr_t)curthread;
+	v = RW_UNLOCKED;
+	if (!_rw_write_lock_fetch(rw, &v, tid))
+		_rw_wlock_hard(rw, v, tid, file, line);
+	else
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
+		    0, 0, file, line, LOCKSTAT_WRITER);
+
 	LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
 	WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	TD_LOCKS_INC(curthread);
@@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
 	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
 	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
 	    line);
-	__rw_wunlock(rw, curthread, file, line);
+	if (rw->rw_recurse)
+		rw->rw_recurse--;
+	else
+		_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
+
 	TD_LOCKS_DEC(curthread);
 }
 
@@ -440,7 +452,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 			 * if the lock has been unlocked and write waiters
 			 * were present.
 			 */
-			if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
+			if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v,
 			    v + RW_ONE_READER)) {
 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
 					CTR4(KTR_LOCK,
@@ -449,7 +461,6 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 					    (void *)(v + RW_ONE_READER));
 				break;
 			}
-			v = RW_READ_VALUE(rw);
 			continue;
 		}
 #ifdef KDTRACE_HOOKS
@@ -675,7 +686,7 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 		 * just drop one and return.
 		 */
 		if (RW_READERS(x) > 1) {
-			if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
+			if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
 			    x - RW_ONE_READER)) {
 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
 					CTR4(KTR_LOCK,
@@ -684,7 +695,6 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 					    (void *)(x - RW_ONE_READER));
 				break;
 			}
-			x = RW_READ_VALUE(rw);
 			continue;
 		}
 		/*
@@ -694,14 +704,13 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 		if (!(x & RW_LOCK_WAITERS)) {
 			MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
 			    RW_READERS_LOCK(1));
-			if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
+			if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
 			    RW_UNLOCKED)) {
 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
 					CTR2(KTR_LOCK, "%s: %p last succeeded",
 					    __func__, rw);
 				break;
 			}
-			x = RW_READ_VALUE(rw);
 			continue;
 		}
 		/*
@@ -769,8 +778,8 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
  * read or write lock.
  */
 void
-__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
-    int line)
+__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
+    const char *file, int line)
 {
 	struct rwlock *rw;
 	struct turnstile *ts;
@@ -779,7 +788,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 	int spintries = 0;
 	int i;
 #endif
-	uintptr_t v, x;
+	uintptr_t x;
 #ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
@@ -803,7 +812,6 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 	lock_delay_arg_init(&lda, NULL);
 #endif
 	rw = rwlock2rw(c);
-	v = RW_READ_VALUE(rw);
 
 	if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) {
 		KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
@@ -825,9 +833,8 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 #endif
 	for (;;) {
 		if (v == RW_UNLOCKED) {
-			if (_rw_write_lock(rw, tid))
+			if (_rw_write_lock_fetch(rw, &v, tid))
 				break;
-			v = RW_READ_VALUE(rw);
 			continue;
 		}
 #ifdef KDTRACE_HOOKS
@@ -1003,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 		return;
 
 	rw = rwlock2rw(c);
+	MPASS(!rw_recursed(rw));
 
-	if (rw_wlocked(rw) && rw_recursed(rw)) {
-		rw->rw_recurse--;
-		if (LOCK_LOG_TEST(&rw->lock_object, 0))
-			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
+	    LOCKSTAT_WRITER);
+	if (_rw_write_unlock(rw, tid))
 		return;
-	}
 
 	KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
 	    ("%s: neither of the waiter flags are set", __func__));