diff options
Diffstat (limited to 'sys/sys')
| -rw-r--r-- | sys/sys/lockstat.h | 8 | ||||
| -rw-r--r-- | sys/sys/mutex.h | 21 | ||||
| -rw-r--r-- | sys/sys/sdt.h | 3 |
3 files changed, 19 insertions, 13 deletions
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h index e5503539298f..958e9503185e 100644 --- a/sys/sys/lockstat.h +++ b/sys/sys/lockstat.h @@ -107,6 +107,10 @@ extern int lockstat_enabled; LOCKSTAT_RECORD1(probe, lp, a); \ } while (0) +#ifndef LOCK_PROFILING +#define LOCKSTAT_PROFILE_ENABLED(probe) SDT_PROBE_ENABLED(lockstat, , , probe) +#endif + struct lock_object; uint64_t lockstat_nsecs(struct lock_object *); @@ -130,6 +134,10 @@ uint64_t lockstat_nsecs(struct lock_object *); #define LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a) \ LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp) +#ifndef LOCK_PROFILING +#define LOCKSTAT_PROFILE_ENABLED(probe) 0 +#endif + #endif /* !KDTRACE_HOOKS */ #endif /* _KERNEL */ #endif /* _SYS_LOCKSTAT_H */ diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index e72d23d1cac4..e4c69b76c939 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -171,10 +171,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int); #define _mtx_obtain_lock(mp, tid) \ atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) -#define _mtx_obtain_lock_fetch(mp, vp, tid) ({ \ - *vp = MTX_UNOWNED; \ - atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)); \ -}) +#define _mtx_obtain_lock_fetch(mp, vp, tid) \ + atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)) /* Try to release mtx_lock if it is unrecursed and uncontested. */ #define _mtx_release_lock(mp, tid) \ @@ -193,13 +191,11 @@ void thread_lock_flags_(struct thread *, int, const char *, int); /* Lock a normal mutex. */ #define __mtx_lock(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ - uintptr_t _v; \ + uintptr_t _v = MTX_UNOWNED; \ \ - if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\ + !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \ _mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\ - else \ - LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \ - mp, 0, 0, file, line); \ } while (0) /* @@ -211,7 +207,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int); #ifdef SMP #define __mtx_lock_spin(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ - uintptr_t _v; \ + uintptr_t _v = MTX_UNOWNED; \ \ spinlock_enter(); \ if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) { \ @@ -270,9 +266,8 @@ void thread_lock_flags_(struct thread *, int, const char *, int); #define __mtx_unlock(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if ((mp)->mtx_recurse == 0) \ - LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \ - if (!_mtx_release_lock((mp), _tid)) \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\ + !_mtx_release_lock((mp), _tid))) \ _mtx_unlock_sleep((mp), (opts), (file), (line)); \ } while (0) diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h index 25423d764e3c..42598837fd72 100644 --- a/sys/sys/sdt.h +++ b/sys/sys/sdt.h @@ -160,6 +160,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype); #define SDT_PROBE_DECLARE(prov, mod, func, name) \ extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1] +#define SDT_PROBE_ENABLED(prov, mod, func, name) \ + __predict_false((sdt_##prov##_##mod##_##func##_##name->id)) + #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do { \ if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id)) \ (*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id, \ |