bq_insert: delay the BUF_UNLOCK if not bd_flush()ing

Reduce bufqueue lock contention by delaying the BUF_UNLOCK to after
dropping the bufqueue lock.  Still do the early BUF_UNLOCK if we
actually have to bd_flush.

Reviewed by:	kib, markj
Sponsored by:	Dell Inc.
Differential Revision:	https://reviews.freebsd.org/D56948

1 files changed, 12 insertions, 8 deletions

diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 675f6fb4e526..181f2907c245 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -2053,22 +2053,26 @@ bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)
 	bp->b_qindex = bq->bq_index;
 	bp->b_subqueue = bq->bq_subqueue;
 
-	/*
-	 * Unlock before we notify so that we don't wakeup a waiter that
-	 * fails a trylock on the buf and sleeps again.
-	 */
-	if (unlock)
-		BUF_UNLOCK(bp);
-
 	if (bp->b_qindex == QUEUE_CLEAN) {
 		/*
 		 * Flush the per-cpu queue and notify any waiters.
+		 *
+		 * Unlock before we notify so that we don't wakeup a waiter
+		 * that fails a trylock on the buf and sleeps again.
 		 */
 		if (bd->bd_wanted || (bq != bd->bd_cleanq &&
-		    bq->bq_len >= bd->bd_lim))
+		    bq->bq_len >= bd->bd_lim)) {
+			if (unlock) {
+				BUF_UNLOCK(bp);
+				unlock = false;
+			}
 			bd_flush(bd, bq);
+		}
 	}
 	BQ_UNLOCK(bq);
+
+	if (unlock)
+		BUF_UNLOCK(bp);
 }
 
 /*