aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--sys/kern/kern_mutex.c20
-rw-r--r--sys/sys/lockstat.h8
-rw-r--r--sys/sys/mutex.h21
-rw-r--r--sys/sys/sdt.h3
4 files changed, 31 insertions, 21 deletions
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 8e6133d6bfde..f49d87c81999 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -265,6 +265,7 @@ void
__mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
{
struct mtx *m;
+ uintptr_t tid, v;
if (SCHEDULER_STOPPED())
return;
@@ -282,7 +283,13 @@ __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) |
LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL);
- __mtx_lock(m, curthread, opts, file, line);
+ tid = (uintptr_t)curthread;
+ v = MTX_UNOWNED;
+ if (!_mtx_obtain_lock_fetch(m, &v, tid))
+ _mtx_lock_sleep(m, v, tid, opts, file, line);
+ else
+ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,
+ m, 0, 0, file, line);
LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
line);
WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE,
@@ -310,7 +317,7 @@ __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line)
line);
mtx_assert(m, MA_OWNED);
- __mtx_unlock(m, curthread, opts, file, line);
+ __mtx_unlock_sleep(c, opts, file, line);
TD_LOCKS_DEC(curthread);
}
@@ -887,20 +894,17 @@ __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line)
{
struct mtx *m;
struct turnstile *ts;
- uintptr_t v;
if (SCHEDULER_STOPPED())
return;
m = mtxlock2mtx(c);
- v = MTX_READ_VALUE(m);
- if (v == (uintptr_t)curthread) {
+ if (!mtx_recursed(m)) {
+ LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, m);
if (_mtx_release_lock(m, (uintptr_t)curthread))
return;
- }
-
- if (mtx_recursed(m)) {
+ } else {
if (--(m->mtx_recurse) == 0)
atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
if (LOCK_LOG_TEST(&m->lock_object, opts))
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index e5503539298f..958e9503185e 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -107,6 +107,10 @@ extern int lockstat_enabled;
LOCKSTAT_RECORD1(probe, lp, a); \
} while (0)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) SDT_PROBE_ENABLED(lockstat, , , probe)
+#endif
+
struct lock_object;
uint64_t lockstat_nsecs(struct lock_object *);
@@ -130,6 +134,10 @@ uint64_t lockstat_nsecs(struct lock_object *);
#define LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a) \
LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) 0
+#endif
+
#endif /* !KDTRACE_HOOKS */
#endif /* _KERNEL */
#endif /* _SYS_LOCKSTAT_H */
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index e72d23d1cac4..e4c69b76c939 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -171,10 +171,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#define _mtx_obtain_lock(mp, tid) \
atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
-#define _mtx_obtain_lock_fetch(mp, vp, tid) ({ \
- *vp = MTX_UNOWNED; \
- atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)); \
-})
+#define _mtx_obtain_lock_fetch(mp, vp, tid) \
+ atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid))
/* Try to release mtx_lock if it is unrecursed and uncontested. */
#define _mtx_release_lock(mp, tid) \
@@ -193,13 +191,11 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
/* Lock a normal mutex. */
#define __mtx_lock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
- if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\
+ !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \
_mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\
- else \
- LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \
- mp, 0, 0, file, line); \
} while (0)
/*
@@ -211,7 +207,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#ifdef SMP
#define __mtx_lock_spin(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
spinlock_enter(); \
if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) { \
@@ -270,9 +266,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#define __mtx_unlock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
\
- if ((mp)->mtx_recurse == 0) \
- LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \
- if (!_mtx_release_lock((mp), _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\
+ !_mtx_release_lock((mp), _tid))) \
_mtx_unlock_sleep((mp), (opts), (file), (line)); \
} while (0)
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index 25423d764e3c..42598837fd72 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -160,6 +160,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype);
#define SDT_PROBE_DECLARE(prov, mod, func, name) \
extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1]
+#define SDT_PROBE_ENABLED(prov, mod, func, name) \
+ __predict_false((sdt_##prov##_##mod##_##func##_##name->id))
+
#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do { \
if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id)) \
(*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id, \