Only trigger read-ahead if two adjacent blocks have been requested.

The change makes block caching algorithm to work better for remote
media on low-BW/high-delay links.

This cuts boot time over IP KVMs noticeably, since the initialization
stage reads bunch of small 4th (and now lua) files that are not in
the same cache stripe (usually), thus wasting lot of bandwidth and
increasing latency even further.

The original regression came in 2017 with revision 87ed2b7f5. We've
seen increase of time it takes for the loader to get to the kernel
loading from under a minute to 10-15 minutes in many cases.

Reviewed by:	tsoome
MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D31623

1 files changed, 15 insertions, 2 deletions

diff --git a/stand/common/bcache.c b/stand/common/bcache.c
index a020f3c3c53c..526f9fe3fa5c 100644
--- a/stand/common/bcache.c
+++ b/stand/common/bcache.c
@@ -66,6 +66,7 @@ struct bcache {
     caddr_t		bcache_data;
     size_t		bcache_nblks;
     size_t		ra;
+    daddr_t		bcache_nextblkno;
 };
 
 static u_int bcache_total_nblks;	/* set by bcache_init */
@@ -163,6 +164,7 @@ bcache_allocate(void)
     }
     bcache_units++;
     bc->ra = BCACHE_READAHEAD;	/* optimistic read ahead */
+    bc->bcache_nextblkno = -1;
     return (bc);
 }
 
@@ -291,6 +293,14 @@ read_strategy(void *devdata, int rw, daddr_t blk, size_t size,
     else
 	ra = bc->bcache_nblks - BHASH(bc, p_blk + p_size);
 
+    /*
+     * Only trigger read-ahead if we detect two blocks being read
+     * sequentially.
+     */
+    if ((bc->bcache_nextblkno != blk) && ra != 0) {
+        ra = 0;
+    }
+
     if (ra != 0 && ra != bc->bcache_nblks) { /* do we have RA space? */
 	ra = MIN(bc->ra, ra - 1);
 	ra = rounddown(ra, 16);		/* multiple of 16 blocks */
@@ -342,8 +352,11 @@ read_strategy(void *devdata, int rw, daddr_t blk, size_t size,
     }
 
  done:
-    if ((result == 0) && (rsize != NULL))
-	*rsize = size;
+    if (result == 0) {
+        if (rsize != NULL)
+	    *rsize = size;
+        bc->bcache_nextblkno = blk + (size / DEV_BSIZE);
+    }
     return(result);
 }