aboutsummaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/kern/kern_mutex.c14
-rw-r--r--sys/kern/kern_rwlock.c96
-rw-r--r--sys/kern/kern_sx.c77
-rw-r--r--sys/sys/lock.h4
-rw-r--r--sys/sys/mutex.h1
-rw-r--r--sys/sys/rwlock.h46
-rw-r--r--sys/sys/sx.h29
7 files changed, 191 insertions, 76 deletions
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 9438bf583e21..7b6b3e0549b3 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -380,9 +380,8 @@ __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file,
* is already owned, it will recursively acquire the lock.
*/
int
-_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
+_mtx_trylock_flags_int(struct mtx *m, int opts LOCK_FILE_LINE_ARG_DEF)
{
- struct mtx *m;
struct thread *td;
uintptr_t tid, v;
#ifdef LOCK_PROFILING
@@ -397,8 +396,6 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
if (SCHEDULER_STOPPED_TD(td))
return (1);
- m = mtxlock2mtx(c);
-
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td),
("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d",
curthread, m->lock_object.lo_name, file, line));
@@ -443,6 +440,15 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
return (rval);
}
+int
+_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
+{
+ struct mtx *m;
+
+ m = mtxlock2mtx(c);
+ return (_mtx_trylock_flags_int(m, opts LOCK_FILE_LINE_ARG));
+}
+
/*
* __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
*
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 393979947ce8..35c6b44e4821 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -273,7 +273,7 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
tid = (uintptr_t)curthread;
v = RW_UNLOCKED;
if (!_rw_write_lock_fetch(rw, &v, tid))
- _rw_wlock_hard(rw, v, tid, file, line);
+ _rw_wlock_hard(rw, v, file, line);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
0, 0, file, line, LOCKSTAT_WRITER);
@@ -369,8 +369,8 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ))
static bool __always_inline
-__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp,
- const char *file, int line)
+__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp
+ LOCK_FILE_LINE_ARG_DEF)
{
/*
@@ -399,10 +399,9 @@ __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp,
}
static void __noinline
-__rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
- const char *file, int line)
+__rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
volatile struct thread *owner;
@@ -434,7 +433,6 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
- rw = rwlock2rw(c);
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
@@ -454,7 +452,7 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
#endif
for (;;) {
- if (__rw_rlock_try(rw, td, &v, file, line))
+ if (__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
@@ -612,14 +610,12 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
}
void
-__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+__rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct thread *td;
uintptr_t v;
td = curthread;
- rw = rwlock2rw(c);
KASSERT(kdb_active != 0 || SCHEDULER_STOPPED_TD(td) ||
!TD_IS_IDLETHREAD(td),
@@ -634,14 +630,23 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
v = RW_READ_VALUE(rw);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
- !__rw_rlock_try(rw, td, &v, file, line)))
- __rw_rlock_hard(c, td, v, file, line);
+ !__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG)))
+ __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
WITNESS_LOCK(&rw->lock_object, 0, file, line);
TD_LOCKS_INC(curthread);
}
+void
+__rw_rlock(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ __rw_rlock_int(rw LOCK_FILE_LINE_ARG);
+}
+
int
__rw_try_rlock(volatile uintptr_t *c, const char *file, int line)
{
@@ -724,18 +729,15 @@ __rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp)
}
static void __noinline
-__rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
- const char *file, int line)
+__rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
uintptr_t x, queue;
if (SCHEDULER_STOPPED())
return;
- rw = rwlock2rw(c);
-
for (;;) {
if (__rw_runlock_try(rw, td, &v))
break;
@@ -799,17 +801,14 @@ __rw_runlock_hard(volatile uintptr_t *c, struct thread *td, uintptr_t v,
}
void
-_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+_rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct thread *td;
uintptr_t v;
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_RLOCKED, file, line);
+ __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
@@ -818,20 +817,29 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__release) ||
!__rw_runlock_try(rw, td, &v)))
- __rw_runlock_hard(c, td, v, file, line);
+ __rw_runlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
TD_LOCKS_DEC(curthread);
}
+void
+_rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ _rw_runlock_cookie_int(rw LOCK_FILE_LINE_ARG);
+}
+
/*
* This function is called when we are unable to obtain a write lock on the
* first try. This means that at least one other thread holds either a
* read or write lock.
*/
void
-__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
- const char *file, int line)
+__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
{
+ uintptr_t tid;
struct rwlock *rw;
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
@@ -857,6 +865,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
int doing_lockprof;
#endif
+ tid = (uintptr_t)curthread;
if (SCHEDULER_STOPPED())
return;
@@ -1069,8 +1078,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
* on this lock.
*/
void
-__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
- int line)
+__rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid LOCK_FILE_LINE_ARG_DEF)
{
struct rwlock *rw;
struct turnstile *ts;
@@ -1145,9 +1153,8 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
* lock. Returns true if the upgrade succeeded and false otherwise.
*/
int
-__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
+__rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
uintptr_t v, x, tid;
struct turnstile *ts;
int success;
@@ -1155,11 +1162,9 @@ __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
if (SCHEDULER_STOPPED())
return (1);
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_RLOCKED, file, line);
+ __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
/*
* Attempt to switch from one reader to a writer. If there
@@ -1217,13 +1222,21 @@ __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
return (success);
}
+int
+__rw_try_upgrade(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ return (__rw_try_upgrade_int(rw LOCK_FILE_LINE_ARG));
+}
+
/*
* Downgrade a write lock into a single read lock.
*/
void
-__rw_downgrade(volatile uintptr_t *c, const char *file, int line)
+__rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
{
- struct rwlock *rw;
struct turnstile *ts;
uintptr_t tid, v;
int rwait, wwait;
@@ -1231,11 +1244,9 @@ __rw_downgrade(volatile uintptr_t *c, const char *file, int line)
if (SCHEDULER_STOPPED())
return;
- rw = rwlock2rw(c);
-
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
- __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line);
+ __rw_assert(&rw->rw_lock, RA_WLOCKED | RA_NOTRECURSED, file, line);
#ifndef INVARIANTS
if (rw_recursed(rw))
panic("downgrade of a recursed lock");
@@ -1287,6 +1298,15 @@ out:
LOCKSTAT_RECORD0(rw__downgrade, rw);
}
+void
+__rw_downgrade(volatile uintptr_t *c, const char *file, int line)
+{
+ struct rwlock *rw;
+
+ rw = rwlock2rw(c);
+ __rw_downgrade_int(rw LOCK_FILE_LINE_ARG);
+}
+
#ifdef INVARIANT_SUPPORT
#ifndef INVARIANTS
#undef __rw_assert
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index a5c2c801b414..36481c66d0ec 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -259,7 +259,7 @@ sx_destroy(struct sx *sx)
}
int
-sx_try_slock_(struct sx *sx, const char *file, int line)
+sx_try_slock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
@@ -291,6 +291,13 @@ sx_try_slock_(struct sx *sx, const char *file, int line)
}
int
+sx_try_slock_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_slock_int(sx LOCK_FILE_LINE_ARG));
+}
+
+int
_sx_xlock(struct sx *sx, int opts, const char *file, int line)
{
uintptr_t tid, x;
@@ -307,7 +314,7 @@ _sx_xlock(struct sx *sx, int opts, const char *file, int line)
tid = (uintptr_t)curthread;
x = SX_LOCK_UNLOCKED;
if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, tid))
- error = _sx_xlock_hard(sx, x, tid, opts, file, line);
+ error = _sx_xlock_hard(sx, x, opts LOCK_FILE_LINE_ARG);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
0, 0, file, line, LOCKSTAT_WRITER);
@@ -322,7 +329,7 @@ _sx_xlock(struct sx *sx, int opts, const char *file, int line)
}
int
-sx_try_xlock_(struct sx *sx, const char *file, int line)
+sx_try_xlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
struct thread *td;
uintptr_t tid, x;
@@ -370,6 +377,13 @@ sx_try_xlock_(struct sx *sx, const char *file, int line)
return (rval);
}
+int
+sx_try_xlock_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_xlock_int(sx LOCK_FILE_LINE_ARG));
+}
+
void
_sx_xunlock(struct sx *sx, const char *file, int line)
{
@@ -394,7 +408,7 @@ _sx_xunlock(struct sx *sx, const char *file, int line)
* Return 1 if if the upgrade succeed, 0 otherwise.
*/
int
-sx_try_upgrade_(struct sx *sx, const char *file, int line)
+sx_try_upgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int success;
@@ -423,11 +437,18 @@ sx_try_upgrade_(struct sx *sx, const char *file, int line)
return (success);
}
+int
+sx_try_upgrade_(struct sx *sx, const char *file, int line)
+{
+
+ return (sx_try_upgrade_int(sx LOCK_FILE_LINE_ARG));
+}
+
/*
* Downgrade an unrecursed exclusive lock into a single shared lock.
*/
void
-sx_downgrade_(struct sx *sx, const char *file, int line)
+sx_downgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int wakeup_swapper;
@@ -490,6 +511,13 @@ sx_downgrade_(struct sx *sx, const char *file, int line)
kick_proc0();
}
+void
+sx_downgrade_(struct sx *sx, const char *file, int line)
+{
+
+ sx_downgrade_int(sx LOCK_FILE_LINE_ARG);
+}
+
/*
* This function represents the so-called 'hard case' for sx_xlock
* operation. All 'easy case' failures are redirected to this. Note
@@ -497,10 +525,10 @@ sx_downgrade_(struct sx *sx, const char *file, int line)
* accessible from at least sx.h.
*/
int
-_sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts,
- const char *file, int line)
+_sx_xlock_hard(struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
+ uintptr_t tid;
#ifdef ADAPTIVE_SX
volatile struct thread *owner;
u_int i, n, spintries = 0;
@@ -523,6 +551,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts,
#endif
int extra_work = 0;
+ tid = (uintptr_t)curthread;
if (SCHEDULER_STOPPED())
return (0);
@@ -770,7 +799,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t tid, int opts,
* accessible from at least sx.h.
*/
void
-_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
+_sx_xunlock_hard(struct sx *sx, uintptr_t tid LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x, setx;
int queue, wakeup_swapper;
@@ -835,7 +864,7 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
}
static bool __always_inline
-__sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line)
+__sx_slock_try(struct sx *sx, uintptr_t *xp LOCK_FILE_LINE_ARG_DEF)
{
/*
@@ -859,7 +888,7 @@ __sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line)
}
static int __noinline
-_sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x)
+_sx_slock_hard(struct sx *sx, int opts, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
GIANT_DECLARE;
#ifdef ADAPTIVE_SX
@@ -914,7 +943,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x)
* shared locks once there is an exclusive waiter.
*/
for (;;) {
- if (__sx_slock_try(sx, &x, file, line))
+ if (__sx_slock_try(sx, &x LOCK_FILE_LINE_ARG))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
@@ -1058,7 +1087,7 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x)
}
int
-_sx_slock(struct sx *sx, int opts, const char *file, int line)
+_sx_slock_int(struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
int error;
@@ -1074,8 +1103,8 @@ _sx_slock(struct sx *sx, int opts, const char *file, int line)
error = 0;
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__acquire) ||
- !__sx_slock_try(sx, &x, file, line)))
- error = _sx_slock_hard(sx, opts, file, line, x);
+ !__sx_slock_try(sx, &x LOCK_FILE_LINE_ARG)))
+ error = _sx_slock_hard(sx, opts, x LOCK_FILE_LINE_ARG);
if (error == 0) {
LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
WITNESS_LOCK(&sx->lock_object, 0, file, line);
@@ -1084,6 +1113,13 @@ _sx_slock(struct sx *sx, int opts, const char *file, int line)
return (error);
}
+int
+_sx_slock(struct sx *sx, int opts, const char *file, int line)
+{
+
+ return (_sx_slock_int(sx, opts LOCK_FILE_LINE_ARG));
+}
+
static bool __always_inline
_sx_sunlock_try(struct sx *sx, uintptr_t *xp)
{
@@ -1135,7 +1171,7 @@ _sx_sunlock_try(struct sx *sx, uintptr_t *xp)
}
static void __noinline
-_sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line)
+_sx_sunlock_hard(struct sx *sx, uintptr_t x LOCK_FILE_LINE_ARG_DEF)
{
int wakeup_swapper;
@@ -1182,7 +1218,7 @@ _sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line)
}
void
-_sx_sunlock(struct sx *sx, const char *file, int line)
+_sx_sunlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF)
{
uintptr_t x;
@@ -1195,11 +1231,18 @@ _sx_sunlock(struct sx *sx, const char *file, int line)
x = SX_READ_VALUE(sx);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__release) ||
!_sx_sunlock_try(sx, &x)))
- _sx_sunlock_hard(sx, x, file, line);
+ _sx_sunlock_hard(sx, x LOCK_FILE_LINE_ARG);
TD_LOCKS_DEC(curthread);
}
+void
+_sx_sunlock(struct sx *sx, const char *file, int line)
+{
+
+ _sx_sunlock_int(sx LOCK_FILE_LINE_ARG);
+}
+
#ifdef INVARIANT_SUPPORT
#ifndef INVARIANTS
#undef _sx_assert
diff --git a/sys/sys/lock.h b/sys/sys/lock.h
index fb3ad3101e88..efb30538f8ab 100644
--- a/sys/sys/lock.h
+++ b/sys/sys/lock.h
@@ -137,9 +137,13 @@ struct lock_class {
* operations. Otherwise, use default values to avoid the unneeded bloat.
*/
#if LOCK_DEBUG > 0
+#define LOCK_FILE_LINE_ARG_DEF , const char *file, int line
+#define LOCK_FILE_LINE_ARG , file, line
#define LOCK_FILE __FILE__
#define LOCK_LINE __LINE__
#else
+#define LOCK_FILE_LINE_ARG_DEF
+#define LOCK_FILE_LINE_ARG
#define LOCK_FILE NULL
#define LOCK_LINE 0
#endif
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index d2b3702f2a17..9d20bbdbef9d 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -91,6 +91,7 @@ void _mtx_init(volatile uintptr_t *c, const char *name, const char *type,
int opts);
void _mtx_destroy(volatile uintptr_t *c);
void mtx_sysinit(void *arg);
+int _mtx_trylock_flags_int(struct mtx *m, int opts LOCK_FILE_LINE_ARG_DEF);
int _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file,
int line);
void mutex_init(void);
diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index cd888259d9f3..180773cdadeb 100644
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -105,7 +105,7 @@
\
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \
!_rw_write_lock_fetch((rw), &_v, _tid))) \
- _rw_wlock_hard((rw), _v, _tid, (file), (line)); \
+ _rw_wlock_hard((rw), _v, (file), (line)); \
} while (0)
/* Release a write lock. */
@@ -128,16 +128,22 @@ void rw_sysinit(void *arg);
void rw_sysinit_flags(void *arg);
int _rw_wowned(const volatile uintptr_t *c);
void _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_wlock(volatile uintptr_t *c, const char *file, int line);
void _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line);
+void __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_rlock(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
+void _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
-void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
- const char *file, int line);
-void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
- const char *file, int line);
+void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+int __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
+void __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_downgrade(volatile uintptr_t *c, const char *file, int line);
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
void __rw_assert(const volatile uintptr_t *c, int what, const char *file,
@@ -163,20 +169,38 @@ void __rw_assert(const volatile uintptr_t *c, int what, const char *file,
__rw_try_wlock(&(rw)->rw_lock, f, l)
#define _rw_wunlock(rw, f, l) \
_rw_wunlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_rlock(rw, f, l) \
- __rw_rlock(&(rw)->rw_lock, f, l)
#define _rw_try_rlock(rw, f, l) \
__rw_try_rlock(&(rw)->rw_lock, f, l)
+#if LOCK_DEBUG > 0
+#define _rw_rlock(rw, f, l) \
+ __rw_rlock(&(rw)->rw_lock, f, l)
#define _rw_runlock(rw, f, l) \
_rw_runlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_wlock_hard(rw, v, t, f, l) \
- __rw_wlock_hard(&(rw)->rw_lock, v, t, f, l)
-#define _rw_wunlock_hard(rw, t, f, l) \
- __rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
+#else
+#define _rw_rlock(rw, f, l) \
+ __rw_rlock_int((struct rwlock *)rw)
+#define _rw_runlock(rw, f, l) \
+ _rw_runlock_cookie_int((struct rwlock *)rw)
+#endif
+#if LOCK_DEBUG > 0
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v, f, l)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v, f, l)
#define _rw_try_upgrade(rw, f, l) \
__rw_try_upgrade(&(rw)->rw_lock, f, l)
#define _rw_downgrade(rw, f, l) \
__rw_downgrade(&(rw)->rw_lock, f, l)
+#else
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v)
+#define _rw_try_upgrade(rw, f, l) \
+ __rw_try_upgrade_int(rw)
+#define _rw_downgrade(rw, f, l) \
+ __rw_downgrade_int(rw)
+#endif
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
#define _rw_assert(rw, w, f, l) \
__rw_assert(&(rw)->rw_lock, w, f, l)
diff --git a/sys/sys/sx.h b/sys/sys/sx.h
index a31c328ba60e..8f56bba24a2a 100644
--- a/sys/sys/sx.h
+++ b/sys/sys/sx.h
@@ -101,18 +101,22 @@ void sx_sysinit(void *arg);
#define sx_init(sx, desc) sx_init_flags((sx), (desc), 0)
void sx_init_flags(struct sx *sx, const char *description, int opts);
void sx_destroy(struct sx *sx);
+int sx_try_slock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_slock_(struct sx *sx, const char *file, int line);
+int sx_try_xlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_xlock_(struct sx *sx, const char *file, int line);
+int sx_try_upgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
int sx_try_upgrade_(struct sx *sx, const char *file, int line);
+void sx_downgrade_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
void sx_downgrade_(struct sx *sx, const char *file, int line);
+int _sx_slock_int(struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF);
int _sx_slock(struct sx *sx, int opts, const char *file, int line);
int _sx_xlock(struct sx *sx, int opts, const char *file, int line);
+void _sx_sunlock_int(struct sx *sx LOCK_FILE_LINE_ARG_DEF);
void _sx_sunlock(struct sx *sx, const char *file, int line);
void _sx_xunlock(struct sx *sx, const char *file, int line);
-int _sx_xlock_hard(struct sx *sx, uintptr_t v, uintptr_t tid, int opts,
- const char *file, int line);
-void _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
- line);
+int _sx_xlock_hard(struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF);
+void _sx_xunlock_hard(struct sx *sx, uintptr_t tid LOCK_FILE_LINE_ARG_DEF);
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
void _sx_assert(const struct sx *sx, int what, const char *file, int line);
#endif
@@ -157,7 +161,7 @@ __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
!atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid)))
- error = _sx_xlock_hard(sx, v, tid, opts, file, line);
+ error = _sx_xlock_hard(sx, v, opts);
return (error);
}
@@ -170,7 +174,7 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)))
- _sx_xunlock_hard(sx, tid, file, line);
+ _sx_xunlock_hard(sx, tid);
}
#endif
@@ -195,6 +199,7 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
#define sx_xunlock_(sx, file, line) \
__sx_xunlock((sx), curthread, (file), (line))
#endif /* LOCK_DEBUG > 0 || SX_NOINLINE */
+#if (LOCK_DEBUG > 0)
#define sx_slock_(sx, file, line) \
(void)_sx_slock((sx), 0, (file), (line))
#define sx_slock_sig_(sx, file, line) \
@@ -205,6 +210,18 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
#define sx_try_xlock(sx) sx_try_xlock_((sx), LOCK_FILE, LOCK_LINE)
#define sx_try_upgrade(sx) sx_try_upgrade_((sx), LOCK_FILE, LOCK_LINE)
#define sx_downgrade(sx) sx_downgrade_((sx), LOCK_FILE, LOCK_LINE)
+#else
+#define sx_slock_(sx, file, line) \
+ (void)_sx_slock_int((sx), 0)
+#define sx_slock_sig_(sx, file, line) \
+ _sx_slock_int((sx), SX_INTERRUPTIBLE)
+#define sx_sunlock_(sx, file, line) \
+ _sx_sunlock_int((sx))
+#define sx_try_slock(sx) sx_try_slock_int((sx))
+#define sx_try_xlock(sx) sx_try_xlock_int((sx))
+#define sx_try_upgrade(sx) sx_try_upgrade_int((sx))
+#define sx_downgrade(sx) sx_downgrade_int((sx))
+#endif
#ifdef INVARIANTS
#define sx_assert_(sx, what, file, line) \
_sx_assert((sx), (what), (file), (line))