taskqueue: Move wakeup() out of tq_mutex

taskqueue KPI require wakeup() to be called for each completed
task.  With everything else there heavily optimized over the years,
even when doing nothing this wakeup()'s lock/unlock is significant.
Since no external taskqueue consumer can depend on the tq_mutex,
we can move the wakeup() out of it.  It creates some complications
for internal waiters, but those should be much more rare, and can
be handled with separate locked wakeups on demand.

My tests of taskqueue-intensive ZFS RAIDZ writes on 64-core system
show performance improvement from this change ~4%, while same time
reducing CPU usage by several percent due to lower lock contention,
confirmed by CPU profiler.

1 files changed, 27 insertions, 4 deletions

diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c
index b337aa83d69d..f1f6f18bccdc 100644
--- a/sys/kern/subr_taskqueue.c
+++ b/sys/kern/subr_taskqueue.c
@@ -58,6 +58,7 @@ struct taskqueue_busy {
 	struct task		*tb_running;
 	u_int			 tb_seq;
 	bool			 tb_canceling;
+	bool			 tb_wanted;
 	LIST_ENTRY(taskqueue_busy) tb_link;
 };
 
@@ -137,6 +138,15 @@ TQ_SLEEP(struct taskqueue *tq, void *p, const char *wm)
 	return (msleep(p, &tq->tq_mutex, 0, wm, 0));
 }
 
+static __inline int
+TQ_SLEEP_BUSY(struct taskqueue *tq, struct taskqueue_busy *tb, const char *wm)
+{
+
+	TQ_ASSERT_LOCKED(tq);
+	tb->tb_wanted = true;
+	return (TQ_SLEEP(tq, tb, wm));
+}
+
 static struct taskqueue_busy *
 task_get_busy(struct taskqueue *queue, struct task *task)
 {
@@ -462,7 +472,7 @@ taskqueue_drain_tq_active(struct taskqueue *queue)
 restart:
 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {
 		if ((int)(tb->tb_seq - seq) <= 0) {
-			TQ_SLEEP(queue, tb->tb_running, "tq_adrain");
+			TQ_SLEEP_BUSY(queue, tb, "tq_adrain");
 			goto restart;
 		}
 	}
@@ -506,6 +516,7 @@ taskqueue_run_locked(struct taskqueue *queue)
 	KASSERT(queue != NULL, ("tq is NULL"));
 	TQ_ASSERT_LOCKED(queue);
 	tb.tb_running = NULL;
+	tb.tb_wanted = false;
 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
 
 	epochtasks = 0;
@@ -534,8 +545,13 @@ taskqueue_run_locked(struct taskqueue *queue)
 			epochtasks = 0;
 		}
 
-		TQ_LOCK(queue);
 		wakeup(task);
+
+		TQ_LOCK(queue);
+		if (__predict_false(tb.tb_wanted)) {
+			tb.tb_wanted = false;
+			wakeup(&tb);
+		}
 	}
 	if (epochtasks > 0)
 		NET_EPOCH_EXIT(et);
@@ -628,13 +644,20 @@ taskqueue_cancel_timeout(struct taskqueue *queue,
 void
 taskqueue_drain(struct taskqueue *queue, struct task *task)
 {
+	struct taskqueue_busy *tb;
 
 	if (!queue->tq_spin)
 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__);
 
 	TQ_LOCK(queue);
-	while (task->ta_pending != 0 || task_get_busy(queue, task) != NULL)
-		TQ_SLEEP(queue, task, "tq_drain");
+	for (;;) {
+		if (task->ta_pending != 0)
+			TQ_SLEEP(queue, task, "tq_drain");
+		else if ((tb = task_get_busy(queue, task)) != NULL)
+			TQ_SLEEP_BUSY(queue, tb, "tq_drain");
+		else
+			break;
+	}
 	TQ_UNLOCK(queue);
 }