aboutsummaryrefslogtreecommitdiff
path: root/sys/kern/subr_lock.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern/subr_lock.c')
-rw-r--r--sys/kern/subr_lock.c62
1 files changed, 44 insertions, 18 deletions
diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c
index f83ba58f7aa4..2ed5fdb6bb72 100644
--- a/sys/kern/subr_lock.c
+++ b/sys/kern/subr_lock.c
@@ -56,6 +56,9 @@ __FBSDID("$FreeBSD$");
#include <machine/cpufunc.h>
+SDT_PROVIDER_DEFINE(lock);
+SDT_PROBE_DEFINE1(lock, , , starvation, "u_int");
+
CTASSERT(LOCK_CLASS_MAX == 15);
struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
@@ -103,32 +106,56 @@ lock_destroy(struct lock_object *lock)
lock->lo_flags &= ~LO_INITIALIZED;
}
+static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
+static SYSCTL_NODE(_debug_lock, OID_AUTO, delay, CTLFLAG_RD, NULL,
+ "lock delay");
+
+static u_int __read_mostly starvation_limit = 131072;
+SYSCTL_INT(_debug_lock_delay, OID_AUTO, starvation_limit, CTLFLAG_RW,
+ &starvation_limit, 0, "");
+
+static u_int __read_mostly restrict_starvation = 0;
+SYSCTL_INT(_debug_lock_delay, OID_AUTO, restrict_starvation, CTLFLAG_RW,
+ &restrict_starvation, 0, "");
+
void
lock_delay(struct lock_delay_arg *la)
{
- u_int i, delay, backoff, min, max;
struct lock_delay_config *lc = la->config;
+ u_int i;
- delay = la->delay;
+ la->delay <<= 1;
+ if (__predict_false(la->delay > lc->max))
+ la->delay = lc->max;
- if (delay == 0)
- delay = lc->initial;
- else {
- delay += lc->step;
- max = lc->max;
- if (delay > max)
- delay = max;
+ for (i = la->delay; i > 0; i++)
+ cpu_spinwait();
+
+ la->spin_cnt += la->delay;
+ if (__predict_false(la->spin_cnt > starvation_limit)) {
+ SDT_PROBE1(lock, , , starvation, la->delay);
+ if (restrict_starvation)
+ la->delay = lc->base;
}
+}
- backoff = cpu_ticks() % delay;
- min = lc->min;
- if (backoff < min)
- backoff = min;
- for (i = 0; i < backoff; i++)
- cpu_spinwait();
+static u_int
+lock_roundup_2(u_int val)
+{
+ u_int res;
+
+ for (res = 1; res <= val; res <<= 1)
+ continue;
+
+ return (res);
+}
- la->delay = delay;
- la->spin_cnt += backoff;
+void
+lock_delay_default_init(struct lock_delay_config *lc)
+{
+
+ lc->base = lock_roundup_2(mp_ncpus) / 4;
+ lc->max = lc->base * 1024;
}
#ifdef DDB
@@ -655,7 +682,6 @@ out:
critical_exit();
}
-static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL,
"lock profiling");
SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,