aboutsummaryrefslogtreecommitdiff
path: root/sys/kern
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern')
-rw-r--r--sys/kern/kern_exit.c46
-rw-r--r--sys/kern/kern_lock.c6
-rw-r--r--sys/kern/kern_mutex.c29
-rw-r--r--sys/kern/kern_sx.c11
-rw-r--r--sys/kern/link_elf.c6
-rw-r--r--sys/kern/link_elf_obj.c8
-rw-r--r--sys/kern/sys_generic.c36
-rw-r--r--sys/kern/sys_pipe.c5
-rw-r--r--sys/kern/vfs_vnops.c22
9 files changed, 94 insertions, 75 deletions
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index a32b5a1b3354..ab8ed32ad189 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -127,6 +127,27 @@ proc_realparent(struct proc *child)
return (parent);
}
+static void
+reaper_clear(struct proc *p, struct proc *rp)
+{
+ struct proc *p1;
+ bool clear;
+
+ sx_assert(&proctree_lock, SX_XLOCKED);
+ LIST_REMOVE(p, p_reapsibling);
+ if (p->p_reapsubtree == 1)
+ return;
+ clear = true;
+ LIST_FOREACH(p1, &rp->p_reaplist, p_reapsibling) {
+ if (p1->p_reapsubtree == p->p_reapsubtree) {
+ clear = false;
+ break;
+ }
+ }
+ if (clear)
+ proc_id_clear(PROC_ID_REAP, p->p_reapsubtree);
+}
+
void
reaper_abandon_children(struct proc *p, bool exiting)
{
@@ -138,7 +159,7 @@ reaper_abandon_children(struct proc *p, bool exiting)
return;
p1 = p->p_reaper;
LIST_FOREACH_SAFE(p2, &p->p_reaplist, p_reapsibling, ptmp) {
- LIST_REMOVE(p2, p_reapsibling);
+ reaper_clear(p2, p);
p2->p_reaper = p1;
p2->p_reapsubtree = p->p_reapsubtree;
LIST_INSERT_HEAD(&p1->p_reaplist, p2, p_reapsibling);
@@ -152,27 +173,6 @@ reaper_abandon_children(struct proc *p, bool exiting)
p->p_treeflag &= ~P_TREE_REAPER;
}
-static void
-reaper_clear(struct proc *p)
-{
- struct proc *p1;
- bool clear;
-
- sx_assert(&proctree_lock, SX_LOCKED);
- LIST_REMOVE(p, p_reapsibling);
- if (p->p_reapsubtree == 1)
- return;
- clear = true;
- LIST_FOREACH(p1, &p->p_reaper->p_reaplist, p_reapsibling) {
- if (p1->p_reapsubtree == p->p_reapsubtree) {
- clear = false;
- break;
- }
- }
- if (clear)
- proc_id_clear(PROC_ID_REAP, p->p_reapsubtree);
-}
-
void
proc_clear_orphan(struct proc *p)
{
@@ -972,7 +972,7 @@ proc_reap(struct thread *td, struct proc *p, int *status, int options)
sx_xunlock(PIDHASHLOCK(p->p_pid));
LIST_REMOVE(p, p_sibling);
reaper_abandon_children(p, true);
- reaper_clear(p);
+ reaper_clear(p, p->p_reaper);
PROC_LOCK(p);
proc_clear_orphan(p);
PROC_UNLOCK(p);
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 31bff6d2c1aa..76f68677e292 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -1780,9 +1780,11 @@ lockmgr_chain(struct thread *td, struct thread **ownerp)
lk = td->td_wchan;
- if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
+ if (!TD_ON_SLEEPQ(td) || sleepq_type(td->td_wchan) != SLEEPQ_LK ||
+ LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
return (0);
- db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
+ db_printf("blocked on lock %p (%s) \"%s\" ", &lk->lock_object,
+ lock_class_lockmgr.lc_name, lk->lock_object.lo_name);
if (lk->lk_lock & LK_SHARE)
db_printf("SHARED (count %ju)\n",
(uintmax_t)LK_SHARERS(lk->lk_lock));
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 8b5908f5219a..d67c70984528 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -503,8 +503,8 @@ _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line)
/*
* __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
*
- * We call this if the lock is either contested (i.e. we need to go to
- * sleep waiting for it), or if we need to recurse on it.
+ * We get here if lock profiling is enabled, the lock is already held by
+ * someone else or we are recursing on it.
*/
#if LOCK_DEBUG > 0
void
@@ -660,13 +660,8 @@ retry_turnstile:
}
#endif
- /*
- * If the mutex isn't already contested and a failure occurs
- * setting the contested bit, the mutex was either released
- * or the state of the MTX_RECURSED bit changed.
- */
- if ((v & MTX_CONTESTED) == 0 &&
- !atomic_fcmpset_ptr(&m->mtx_lock, &v, v | MTX_CONTESTED)) {
+ if ((v & MTX_WAITERS) == 0 &&
+ !atomic_fcmpset_ptr(&m->mtx_lock, &v, v | MTX_WAITERS)) {
goto retry_turnstile;
}
@@ -869,7 +864,7 @@ _thread_lock(struct thread *td)
WITNESS_LOCK(&m->lock_object, LOP_EXCLUSIVE, file, line);
return;
}
- _mtx_release_lock_quick(m);
+ atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED);
slowpath_unlocked:
spinlock_exit();
slowpath_noirq:
@@ -959,7 +954,7 @@ retry:
}
if (m == td->td_lock)
break;
- _mtx_release_lock_quick(m);
+ atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED);
}
LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file,
line);
@@ -1029,8 +1024,8 @@ thread_lock_set(struct thread *td, struct mtx *new)
/*
* __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
*
- * We are only called here if the lock is recursed, contested (i.e. we
- * need to wake up a blocked thread) or lockstat probe is active.
+ * We get here if lock profiling is enabled, the lock is already held by
+ * someone else or we are recursing on it.
*/
#if LOCK_DEBUG > 0
void
@@ -1071,7 +1066,7 @@ __mtx_unlock_sleep(volatile uintptr_t *c, uintptr_t v)
* can be removed from the hash list if it is empty.
*/
turnstile_chain_lock(&m->lock_object);
- _mtx_release_lock_quick(m);
+ atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED);
ts = turnstile_lookup(&m->lock_object);
MPASS(ts != NULL);
if (LOCK_LOG_TEST(&m->lock_object, opts))
@@ -1207,7 +1202,7 @@ _mtx_destroy(volatile uintptr_t *c)
if (!mtx_owned(m))
MPASS(mtx_unowned(m));
else {
- MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
+ MPASS((m->mtx_lock & (MTX_RECURSED|MTX_WAITERS)) == 0);
/* Perform the non-mtx related part of mtx_unlock_spin(). */
if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) {
@@ -1359,8 +1354,8 @@ db_show_mtx(const struct lock_object *lock)
db_printf("DESTROYED");
else {
db_printf("OWNED");
- if (m->mtx_lock & MTX_CONTESTED)
- db_printf(", CONTESTED");
+ if (m->mtx_lock & MTX_WAITERS)
+ db_printf(", WAITERS");
if (m->mtx_lock & MTX_RECURSED)
db_printf(", RECURSED");
}
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index c005e112d3b9..249faf5b1ec4 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -1539,16 +1539,19 @@ sx_chain(struct thread *td, struct thread **ownerp)
/*
* Check to see if this thread is blocked on an sx lock.
- * First, we check the lock class. If that is ok, then we
- * compare the lock name against the wait message.
+ * The thread should be on a sleep queue with type SLEEPQ_SX, the
+ * purported lock should have the lock class index of sx, and the lock
+ * name should match the wait message.
*/
sx = td->td_wchan;
- if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
+ if (!TD_ON_SLEEPQ(td) || sleepq_type(td->td_wchan) != SLEEPQ_SX ||
+ LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
sx->lock_object.lo_name != td->td_wmesg)
return (0);
/* We think we have an sx lock, so output some details. */
- db_printf("blocked on sx \"%s\" ", td->td_wmesg);
+ db_printf("blocked on lock %p (%s) \"%s\" ", &sx->lock_object,
+ lock_class_sx.lc_name, td->td_wmesg);
*ownerp = sx_xholder(sx);
if (sx->sx_lock & SX_LOCK_SHARED)
db_printf("SLOCK (count %ju)\n",
diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index bbebadc4c395..ebd203858b66 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -518,9 +518,15 @@ link_elf_init(void* arg)
(void)link_elf_link_common_finish(linker_kernel_file);
linker_kernel_file->flags |= LINKER_FILE_LINKED;
TAILQ_INIT(&set_pcpu_list);
+ ef->pcpu_start = DPCPU_START;
+ ef->pcpu_stop = DPCPU_STOP;
+ ef->pcpu_base = DPCPU_START;
#ifdef VIMAGE
TAILQ_INIT(&set_vnet_list);
vnet_save_init((void *)VNET_START, VNET_STOP - VNET_START);
+ ef->vnet_start = VNET_START;
+ ef->vnet_stop = VNET_STOP;
+ ef->vnet_base = VNET_START;
#endif
}
diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c
index 151aab96f9be..a3a53a39bfd6 100644
--- a/sys/kern/link_elf_obj.c
+++ b/sys/kern/link_elf_obj.c
@@ -70,6 +70,7 @@
typedef struct {
void *addr;
+ void *origaddr; /* Used by debuggers. */
Elf_Off size;
int flags; /* Section flags. */
int sec; /* Original section number. */
@@ -492,7 +493,8 @@ link_elf_link_preload(linker_class_t cls, const char *filename,
case SHT_FINI_ARRAY:
if (shdr[i].sh_addr == 0)
break;
- ef->progtab[pb].addr = (void *)shdr[i].sh_addr;
+ ef->progtab[pb].addr = ef->progtab[pb].origaddr =
+ (void *)shdr[i].sh_addr;
if (shdr[i].sh_type == SHT_PROGBITS)
ef->progtab[pb].name = "<<PROGBITS>>";
#ifdef __amd64__
@@ -1088,6 +1090,8 @@ link_elf_load_file(linker_class_t cls, const char *filename,
ef->progtab[pb].name = "<<NOBITS>>";
if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
+ ef->progtab[pb].origaddr =
+ (void *)(uintptr_t)mapbase;
ef->progtab[pb].addr =
dpcpu_alloc(shdr[i].sh_size);
if (ef->progtab[pb].addr == NULL) {
@@ -1101,6 +1105,8 @@ link_elf_load_file(linker_class_t cls, const char *filename,
#ifdef VIMAGE
else if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
+ ef->progtab[pb].origaddr =
+ (void *)(uintptr_t)mapbase;
ef->progtab[pb].addr =
vnet_data_alloc(shdr[i].sh_size);
if (ef->progtab[pb].addr == NULL) {
diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c
index 5606b36f772f..7d666da9f88b 100644
--- a/sys/kern/sys_generic.c
+++ b/sys/kern/sys_generic.c
@@ -729,7 +729,7 @@ kern_ioctl(struct thread *td, int fd, u_long com, caddr_t data)
{
struct file *fp;
struct filedesc *fdp;
- int error, tmp, locked;
+ int error, f_flag, tmp, locked;
AUDIT_ARG_FD(fd);
AUDIT_ARG_CMD(com);
@@ -782,30 +782,36 @@ kern_ioctl(struct thread *td, int fd, u_long com, caddr_t data)
goto out;
}
+ f_flag = 0;
switch (com) {
case FIONCLEX:
fdp->fd_ofiles[fd].fde_flags &= ~UF_EXCLOSE;
- goto out;
+ break;
case FIOCLEX:
fdp->fd_ofiles[fd].fde_flags |= UF_EXCLOSE;
- goto out;
- case FIONBIO:
- if ((tmp = *(int *)data))
- atomic_set_int(&fp->f_flag, FNONBLOCK);
- else
- atomic_clear_int(&fp->f_flag, FNONBLOCK);
- data = (void *)&tmp;
break;
+ case FIONBIO:
case FIOASYNC:
- if ((tmp = *(int *)data))
- atomic_set_int(&fp->f_flag, FASYNC);
- else
- atomic_clear_int(&fp->f_flag, FASYNC);
- data = (void *)&tmp;
+ f_flag = com == FIONBIO ? FNONBLOCK : FASYNC;
+ tmp = *(int *)data;
+ fsetfl_lock(fp);
+ if (((fp->f_flag & f_flag) != 0) != (tmp != 0)) {
+ error = fo_ioctl(fp, com, (void *)&tmp, td->td_ucred,
+ td);
+ if (error == 0) {
+ if (tmp != 0)
+ atomic_set_int(&fp->f_flag, f_flag);
+ else
+ atomic_clear_int(&fp->f_flag, f_flag);
+ }
+ }
+ fsetfl_unlock(fp);
+ break;
+ default:
+ error = fo_ioctl(fp, com, data, td->td_ucred, td);
break;
}
- error = fo_ioctl(fp, com, data, td->td_ucred, td);
out:
switch (locked) {
case LA_XLOCKED:
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index 30527fdd4fd0..57ebe8dc85f0 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -567,7 +567,7 @@ pipespace_new(struct pipe *cpipe, int size)
static int curfail = 0;
static struct timeval lastfail;
- KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
+ PIPE_LOCK_ASSERT(cpipe, MA_NOTOWNED);
KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
("pipespace: resize of direct writes not allowed"));
retry:
@@ -1679,8 +1679,7 @@ static void
pipe_free_kmem(struct pipe *cpipe)
{
- KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
- ("pipe_free_kmem: pipe mutex locked"));
+ PIPE_LOCK_ASSERT(cpipe, MA_NOTOWNED);
if (cpipe->pipe_buffer.buffer != NULL) {
atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 3d4567b6ab1e..a53df50c06bd 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -806,9 +806,12 @@ file_v_lock(struct file *fp, short lock_bit, short lock_wait_bit)
flagsp = &fp->f_vflags;
state = atomic_load_16(flagsp);
- if ((state & lock_bit) == 0 &&
- atomic_cmpset_acq_16(flagsp, state, state | lock_bit))
- return;
+ for (;;) {
+ if ((state & lock_bit) != 0)
+ break;
+ if (atomic_fcmpset_acq_16(flagsp, &state, state | lock_bit))
+ return;
+ }
sleepq_lock(flagsp);
state = atomic_load_16(flagsp);
@@ -842,9 +845,12 @@ file_v_unlock(struct file *fp, short lock_bit, short lock_wait_bit)
flagsp = &fp->f_vflags;
state = atomic_load_16(flagsp);
- if ((state & lock_wait_bit) == 0 &&
- atomic_cmpset_rel_16(flagsp, state, state & ~lock_bit))
- return;
+ for (;;) {
+ if ((state & lock_wait_bit) != 0)
+ break;
+ if (atomic_fcmpset_rel_16(flagsp, &state, state & ~lock_bit))
+ return;
+ }
sleepq_lock(flagsp);
MPASS((*flagsp & lock_bit) != 0);
@@ -864,10 +870,6 @@ foffset_lock(struct file *fp, int flags)
FILE_V_FOFFSET_LOCK_WAITING);
}
- /*
- * According to McKusick the vn lock was protecting f_offset here.
- * It is now protected by the FOFFSET_LOCKED flag.
- */
return (atomic_load_long(&fp->f_offset));
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-07 22:27:28 +0000