aboutsummaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
authorMateusz Guzik <mjg@FreeBSD.org>2017-02-17 14:55:59 +0000
committerMateusz Guzik <mjg@FreeBSD.org>2017-02-17 14:55:59 +0000
commit7640beb9202f1e6bf2e1a1b1cdc5a48c05045118 (patch)
tree7c36eee887fac2a3fa4f297007a676398b9394eb /sys
parent57abbcd5ad527c7a5f6dc70ab7e03d821a32640c (diff)
downloadsrc-7640beb9202f1e6bf2e1a1b1cdc5a48c05045118.tar.gz
src-7640beb9202f1e6bf2e1a1b1cdc5a48c05045118.zip
mtx: microoptimize lockstat handling in __mtx_lock_sleep
This saves a function call and multiple branches after the lock is acquired.
Notes
Notes: svn path=/head/; revision=313875
Diffstat (limited to 'sys')
-rw-r--r--sys/kern/kern_lockstat.c2
-rw-r--r--sys/kern/kern_mutex.c13
-rw-r--r--sys/sys/lockstat.h2
3 files changed, 11 insertions, 6 deletions
diff --git a/sys/kern/kern_lockstat.c b/sys/kern/kern_lockstat.c
index 10da98bbb0d6..679a01c01f5c 100644
--- a/sys/kern/kern_lockstat.c
+++ b/sys/kern/kern_lockstat.c
@@ -62,7 +62,7 @@ SDT_PROBE_DEFINE1(lockstat, , , sx__downgrade, "struct sx *");
SDT_PROBE_DEFINE2(lockstat, , , thread__spin, "struct mtx *", "uint64_t");
-int __read_mostly lockstat_enabled;
+volatile int __read_mostly lockstat_enabled;
uint64_t
lockstat_nsecs(struct lock_object *lo)
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index a6847d0fb207..36e8fa6e3d7d 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -446,6 +446,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts,
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
+ int doing_lockstat;
#endif
if (SCHEDULER_STOPPED())
@@ -484,7 +485,9 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts,
"_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
#ifdef KDTRACE_HOOKS
- all_time -= lockstat_nsecs(&m->lock_object);
+ doing_lockstat = lockstat_enabled;
+ if (__predict_false(doing_lockstat))
+ all_time -= lockstat_nsecs(&m->lock_object);
#endif
for (;;) {
@@ -591,9 +594,6 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts,
#endif
v = MTX_READ_VALUE(m);
}
-#ifdef KDTRACE_HOOKS
- all_time += lockstat_nsecs(&m->lock_object);
-#endif
#ifdef KTR
if (cont_logged) {
CTR4(KTR_CONTENTION,
@@ -601,6 +601,11 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts,
m->lock_object.lo_name, (void *)tid, file, line);
}
#endif
+#ifdef KDTRACE_HOOKS
+ if (__predict_true(!doing_lockstat))
+ return;
+ all_time += lockstat_nsecs(&m->lock_object);
+#endif
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
waittime, file, line);
#ifdef KDTRACE_HOOKS
diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index 77294fb356af..64745052974b 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -68,7 +68,7 @@ SDT_PROBE_DECLARE(lockstat, , , thread__spin);
#define LOCKSTAT_WRITER 0
#define LOCKSTAT_READER 1
-extern int lockstat_enabled;
+extern volatile int lockstat_enabled;
#ifdef KDTRACE_HOOKS