aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--sys/kern/subr_smp.c63
1 files changed, 48 insertions, 15 deletions
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index aba6f0e1ea7e..0c2c286f1d03 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -110,6 +110,7 @@ static void (*volatile smp_rv_action_func)(void *arg);
static void (*volatile smp_rv_teardown_func)(void *arg);
static void *volatile smp_rv_func_arg;
static volatile int smp_rv_waiters[3];
+static volatile int smp_rv_generation;
/*
* Shared mutex to restrict busywaits between smp_rendezvous() and
@@ -310,39 +311,63 @@ restart_cpus(cpumask_t map)
void
smp_rendezvous_action(void)
{
- void* local_func_arg = smp_rv_func_arg;
- void (*local_setup_func)(void*) = smp_rv_setup_func;
- void (*local_action_func)(void*) = smp_rv_action_func;
- void (*local_teardown_func)(void*) = smp_rv_teardown_func;
+ void *local_func_arg;
+ void (*local_setup_func)(void*);
+ void (*local_action_func)(void*);
+ void (*local_teardown_func)(void*);
+ int generation;
/* Ensure we have up-to-date values. */
atomic_add_acq_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < smp_rv_ncpus)
cpu_spinwait();
- /* setup function */
+ /* Fetch rendezvous parameters after acquire barrier. */
+ local_func_arg = smp_rv_func_arg;
+ local_setup_func = smp_rv_setup_func;
+ local_action_func = smp_rv_action_func;
+ local_teardown_func = smp_rv_teardown_func;
+ generation = smp_rv_generation;
+
+ /*
+ * If requested, run a setup function before the main action
+ * function. Ensure all CPUs have completed the setup
+ * function before moving on to the action function.
+ */
if (local_setup_func != smp_no_rendevous_barrier) {
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
-
- /* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < smp_rv_ncpus)
cpu_spinwait();
}
- /* action function */
if (local_action_func != NULL)
local_action_func(local_func_arg);
- /* spin on exit rendezvous */
+ /*
+ * Signal that the main action has been completed. If a
+ * full exit rendezvous is requested, then all CPUs will
+ * wait here until all CPUs have finished the main action.
+ *
+ * Note that the write by the last CPU to finish the action
+ * may become visible to different CPUs at different times.
+ * As a result, the CPU that initiated the rendezvous may
+ * exit the rendezvous and drop the lock allowing another
+ * rendezvous to be initiated on the same CPU or a different
+ * CPU. In that case the exit sentinel may be cleared before
+ * all CPUs have noticed causing those CPUs to hang forever.
+ * Workaround this by using a generation count to notice when
+ * this race occurs and to exit the rendezvous in that case.
+ */
+ MPASS(generation == smp_rv_generation);
atomic_add_int(&smp_rv_waiters[2], 1);
if (local_teardown_func == smp_no_rendevous_barrier)
return;
- while (smp_rv_waiters[2] < smp_rv_ncpus)
+ while (smp_rv_waiters[2] < smp_rv_ncpus &&
+ generation == smp_rv_generation)
cpu_spinwait();
- /* teardown function */
if (local_teardown_func != NULL)
local_teardown_func(local_func_arg);
}
@@ -373,10 +398,11 @@ smp_rendezvous_cpus(cpumask_t map,
if (ncpus == 0)
panic("ncpus is 0 with map=0x%x", map);
- /* obtain rendezvous lock */
mtx_lock_spin(&smp_ipi_mtx);
- /* set static function pointers */
+ atomic_add_acq_int(&smp_rv_generation, 1);
+
+ /* Pass rendezvous parameters via global variables. */
smp_rv_ncpus = ncpus;
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
@@ -386,18 +412,25 @@ smp_rendezvous_cpus(cpumask_t map,
smp_rv_waiters[2] = 0;
atomic_store_rel_int(&smp_rv_waiters[0], 0);
- /* signal other processors, which will enter the IPI with interrupts off */
+ /*
+ * Signal other processors, which will enter the IPI with
+ * interrupts off.
+ */
ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS);
/* Check if the current CPU is in the map */
if ((map & (1 << curcpu)) != 0)
smp_rendezvous_action();
+ /*
+ * If the caller did not request an exit barrier to be enforced
+ * on each CPU, ensure that this CPU waits for all the other
+ * CPUs to finish the rendezvous.
+ */
if (teardown_func == smp_no_rendevous_barrier)
while (atomic_load_acq_int(&smp_rv_waiters[2]) < ncpus)
cpu_spinwait();
- /* release lock */
mtx_unlock_spin(&smp_ipi_mtx);
}