aboutsummaryrefslogtreecommitdiff
path: root/sys/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys/sys')
-rw-r--r--sys/sys/lockstat.h8
-rw-r--r--sys/sys/mutex.h21
-rw-r--r--sys/sys/sdt.h3
3 files changed, 19 insertions, 13 deletions
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index e5503539298f..958e9503185e 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -107,6 +107,10 @@ extern int lockstat_enabled;
LOCKSTAT_RECORD1(probe, lp, a); \
} while (0)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) SDT_PROBE_ENABLED(lockstat, , , probe)
+#endif
+
struct lock_object;
uint64_t lockstat_nsecs(struct lock_object *);
@@ -130,6 +134,10 @@ uint64_t lockstat_nsecs(struct lock_object *);
#define LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a) \
LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
+#ifndef LOCK_PROFILING
+#define LOCKSTAT_PROFILE_ENABLED(probe) 0
+#endif
+
#endif /* !KDTRACE_HOOKS */
#endif /* _KERNEL */
#endif /* _SYS_LOCKSTAT_H */
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index e72d23d1cac4..e4c69b76c939 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -171,10 +171,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#define _mtx_obtain_lock(mp, tid) \
atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
-#define _mtx_obtain_lock_fetch(mp, vp, tid) ({ \
- *vp = MTX_UNOWNED; \
- atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)); \
-})
+#define _mtx_obtain_lock_fetch(mp, vp, tid) \
+ atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid))
/* Try to release mtx_lock if it is unrecursed and uncontested. */
#define _mtx_release_lock(mp, tid) \
@@ -193,13 +191,11 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
/* Lock a normal mutex. */
#define __mtx_lock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
- if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\
+ !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \
_mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\
- else \
- LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \
- mp, 0, 0, file, line); \
} while (0)
/*
@@ -211,7 +207,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#ifdef SMP
#define __mtx_lock_spin(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- uintptr_t _v; \
+ uintptr_t _v = MTX_UNOWNED; \
\
spinlock_enter(); \
if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) { \
@@ -270,9 +266,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int);
#define __mtx_unlock(mp, tid, opts, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
\
- if ((mp)->mtx_recurse == 0) \
- LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \
- if (!_mtx_release_lock((mp), _tid)) \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\
+ !_mtx_release_lock((mp), _tid))) \
_mtx_unlock_sleep((mp), (opts), (file), (line)); \
} while (0)
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index 25423d764e3c..42598837fd72 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -160,6 +160,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype);
#define SDT_PROBE_DECLARE(prov, mod, func, name) \
extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1]
+#define SDT_PROBE_ENABLED(prov, mod, func, name) \
+ __predict_false((sdt_##prov##_##mod##_##func##_##name->id))
+
#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do { \
if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id)) \
(*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id, \