Consistently use a reader/writer flag for lockstat probes in rwlock(9) and

sx(9), rather than using the probe function name to determine whether a
given lock is a read lock or a write lock. Update lockstat(1) accordingly.

6 files changed, 49 insertions, 31 deletions

diff --git a/sys/kern/kern_lockstat.c b/sys/kern/kern_lockstat.c
index bedd108e9ccf..c5a26a333055 100644
--- a/sys/kern/kern_lockstat.c
+++ b/sys/kern/kern_lockstat.c
@@ -43,16 +43,16 @@ SDT_PROBE_DEFINE1(lockstat, , , spin__acquire, "struct mtx *");
 SDT_PROBE_DEFINE1(lockstat, , , spin__release, "struct mtx *");
 SDT_PROBE_DEFINE2(lockstat, , , spin__spin, "struct mtx *", "uint64_t");
 
-SDT_PROBE_DEFINE1(lockstat, , , rw__acquire, "struct rwlock *");
-SDT_PROBE_DEFINE1(lockstat, , , rw__release, "struct rwlock *");
+SDT_PROBE_DEFINE2(lockstat, , , rw__acquire, "struct rwlock *", "int");
+SDT_PROBE_DEFINE2(lockstat, , , rw__release, "struct rwlock *", "int");
 SDT_PROBE_DEFINE5(lockstat, , , rw__block, "struct rwlock *", "uint64_t", "int",
     "int", "int");
 SDT_PROBE_DEFINE2(lockstat, , , rw__spin, "struct rwlock *", "uint64_t");
 SDT_PROBE_DEFINE1(lockstat, , , rw__upgrade, "struct rwlock *");
 SDT_PROBE_DEFINE1(lockstat, , , rw__downgrade, "struct rwlock *");
 
-SDT_PROBE_DEFINE1(lockstat, , , sx__acquire, "struct sx *");
-SDT_PROBE_DEFINE1(lockstat, , , sx__release, "struct sx *");
+SDT_PROBE_DEFINE2(lockstat, , , sx__acquire, "struct sx *", "int");
+SDT_PROBE_DEFINE2(lockstat, , , sx__release, "struct sx *", "int");
 SDT_PROBE_DEFINE5(lockstat, , , sx__block, "struct sx *", "uint64_t", "int",
     "int", "int");
 SDT_PROBE_DEFINE2(lockstat, , , sx__spin, "struct sx *", "uint64_t");
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 4a6525701dbf..2921de4ef71e 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -301,8 +301,8 @@ __rw_try_wlock(volatile uintptr_t *c, const char *file, int line)
 		WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 		    file, line);
 		if (!rw_recursed(rw))
-			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire,
-			    rw, 0, 0, file, line);
+			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
+			    rw, 0, 0, file, line, LOCKSTAT_WRITER);
 		curthread->td_locks++;
 	}
 	return (rval);
@@ -561,8 +561,8 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
 	 * however.  turnstiles don't like owners changing between calls to
 	 * turnstile_wait() currently.
 	 */
-	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire, rw, contested,
-	    waittime, file, line);
+	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
+	    waittime, file, line, LOCKSTAT_READER);
 	LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
 	WITNESS_LOCK(&rw->lock_object, 0, file, line);
 	curthread->td_locks++;
@@ -594,8 +594,8 @@ __rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
 			LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
 			    line);
 			WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
-			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire,
-			    rw, 0, 0, file, line);
+			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire,
+			    rw, 0, 0, file, line, LOCKSTAT_READER);
 			curthread->td_locks++;
 			curthread->td_rw_rlocks++;
 			return (1);
@@ -713,7 +713,7 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
 		turnstile_chain_unlock(&rw->lock_object);
 		break;
 	}
-	LOCKSTAT_PROFILE_RELEASE_LOCK(rw__release, rw);
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER);
 	curthread->td_locks--;
 	curthread->td_rw_rlocks--;
 }
@@ -920,8 +920,8 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
 		    LOCKSTAT_READER, (state & RW_LOCK_READ) == 0,
 		    (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state));
 #endif
-	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire, rw, contested,
-	    waittime, file, line);
+	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
+	    waittime, file, line, LOCKSTAT_WRITER);
 }
 
 /*
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index ee6c01bb70fd..1ba1ab2aff53 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -288,8 +288,8 @@ sx_try_slock_(struct sx *sx, const char *file, int line)
 		if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
 			LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
 			WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
-			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire,
-			    sx, 0, 0, file, line);
+			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
+			    sx, 0, 0, file, line, LOCKSTAT_READER);
 			curthread->td_locks++;
 			return (1);
 		}
@@ -351,8 +351,8 @@ sx_try_xlock_(struct sx *sx, const char *file, int line)
 		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
 		    file, line);
 		if (!sx_recursed(sx))
-			LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire,
-			    sx, 0, 0, file, line);
+			LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire,
+			    sx, 0, 0, file, line, LOCKSTAT_WRITER);
 		curthread->td_locks++;
 	}
 
@@ -728,8 +728,8 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 		    (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 #endif
 	if (!error)
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire, sx,
-		    contested, waittime, file, line);
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+		    contested, waittime, file, line, LOCKSTAT_WRITER);
 	GIANT_RESTORE();
 	return (error);
 }
@@ -992,8 +992,8 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 		    (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
 #endif
 	if (error == 0)
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire, sx, contested,
-		    waittime, file, line);
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+		    contested, waittime, file, line, LOCKSTAT_READER);
 	GIANT_RESTORE();
 	return (error);
 }
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index e53a7e4b88c8..1cb9a21198ce 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -90,15 +90,25 @@ SDT_PROBE_DECLARE(lockstat, , , thread__spin);
 	LOCKSTAT_RECORD0(probe, lp);					\
 } while (0)
 
+#define	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(probe, lp, c, wt, f, l, a) do { \
+	lock_profile_obtain_lock_success(&(lp)->lock_object, c, wt, f, l); \
+	LOCKSTAT_RECORD1(probe, lp, a);					\
+} while (0)
+
 #define	LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp) do {			\
 	lock_profile_release_lock(&(lp)->lock_object);			\
 	LOCKSTAT_RECORD0(probe, lp);					\
 } while (0)
 
+#define	LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a) do {		\
+	lock_profile_release_lock(&(lp)->lock_object);			\
+	LOCKSTAT_RECORD1(probe, lp, a);					\
+} while (0)
+
 extern int lockstat_enabled;
 
 struct lock_object;
-extern uint64_t lockstat_nsecs(struct lock_object *);
+uint64_t lockstat_nsecs(struct lock_object *);
 
 #else /* !KDTRACE_HOOKS */
 
@@ -111,9 +121,15 @@ extern uint64_t lockstat_nsecs(struct lock_object *);
 #define	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(probe, lp, c, wt, f, l)	\
 	lock_profile_obtain_lock_success(&(lp)->lock_object, c, wt, f, l)
 
+#define	LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(probe, lp, c, wt, f, l, a) \
+	LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(probe, lp, c, wt, f, l)
+
 #define	LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)  			\
 	lock_profile_release_lock(&(lp)->lock_object)
 
+#define	LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a)  			\
+	LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
+
 #endif /* !KDTRACE_HOOKS */
 #endif /* _KERNEL */
 #endif /* _SYS_LOCKSTAT_H */
diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index 960d4af7e263..f8947c55617a 100644
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -99,8 +99,8 @@
 	if (!_rw_write_lock((rw), _tid))				\
 		_rw_wlock_hard((rw), _tid, (file), (line));		\
 	else 								\
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire, rw,	\
-		    0, 0, file, line);					\
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,	\
+		    0, 0, file, line, LOCKSTAT_WRITER);			\
 } while (0)
 
 /* Release a write lock. */
@@ -110,7 +110,8 @@
 	if ((rw)->rw_recurse)						\
 		(rw)->rw_recurse--;					\
 	else {								\
-		LOCKSTAT_PROFILE_RELEASE_LOCK(rw__release, rw);		\
+		LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,	\
+		    LOCKSTAT_WRITER);					\
 		if (!_rw_write_unlock((rw), _tid))			\
 			_rw_wunlock_hard((rw), _tid, (file), (line));	\
 	}								\
diff --git a/sys/sys/sx.h b/sys/sys/sx.h
index c01735c75f51..96a664f35e52 100644
--- a/sys/sys/sx.h
+++ b/sys/sys/sx.h
@@ -153,8 +153,8 @@ __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
 	if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid))
 		error = _sx_xlock_hard(sx, tid, opts, file, line);
 	else 
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire, sx,
-		    0, 0, file, line);
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+		    0, 0, file, line, LOCKSTAT_WRITER);
 
 	return (error);
 }
@@ -166,7 +166,8 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
 	uintptr_t tid = (uintptr_t)td;
 
 	if (sx->sx_recurse == 0)
-		LOCKSTAT_PROFILE_RELEASE_LOCK(sx__release, sx);
+		LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx,
+		    LOCKSTAT_WRITER);
 	if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
 		_sx_xunlock_hard(sx, tid, file, line);
 }
@@ -182,8 +183,8 @@ __sx_slock(struct sx *sx, int opts, const char *file, int line)
 	    !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER))
 		error = _sx_slock_hard(sx, opts, file, line);
 	else
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(sx__acquire, sx,
-		    0, 0, file, line);
+		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
+		    0, 0, file, line, LOCKSTAT_READER);
 
 	return (error);
 }
@@ -200,7 +201,7 @@ __sx_sunlock(struct sx *sx, const char *file, int line)
 {
 	uintptr_t x = sx->sx_lock;
 
-	LOCKSTAT_PROFILE_RELEASE_LOCK(sx__release, sx);
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER);
 	if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) ||
 	    !atomic_cmpset_rel_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER))
 		_sx_sunlock_hard(sx, file, line);