gnop: Introduce requests delay.

This allows to simulated disk that is responding slowly to the IO requests.

Reviewed by:	markj, bcr, pjd (previous version)
Differential Revision:	https://reviews.freebsd.org/D21052

4 files changed, 205 insertions, 31 deletions

diff --git a/lib/geom/nop/geom_nop.c b/lib/geom/nop/geom_nop.c
index 65dc07cf1a3c..12321d2179f2 100644
--- a/lib/geom/nop/geom_nop.c
+++ b/lib/geom/nop/geom_nop.c
@@ -43,29 +43,36 @@ uint32_t version = G_NOP_VERSION;
 struct g_command class_commands[] = {
 	{ "create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL,
 	    {
+		{ 'd', "delaymsec", "-1", G_TYPE_NUMBER },
 		{ 'e', "error", "-1", G_TYPE_NUMBER },
 		{ 'o', "offset", "0", G_TYPE_NUMBER },
 		{ 'p', "stripesize", "0", G_TYPE_NUMBER },
 		{ 'P', "stripeoffset", "0", G_TYPE_NUMBER },
+		{ 'q', "rdelayprob", "-1", G_TYPE_NUMBER },
 		{ 'r', "rfailprob", "-1", G_TYPE_NUMBER },
 		{ 's', "size", "0", G_TYPE_NUMBER },
 		{ 'S', "secsize", "0", G_TYPE_NUMBER },
 		{ 'w', "wfailprob", "-1", G_TYPE_NUMBER },
+		{ 'x', "wdelayprob", "1", G_TYPE_NUMBER },
 		{ 'z', "physpath", G_NOP_PHYSPATH_PASSTHROUGH, G_TYPE_STRING },
 		G_OPT_SENTINEL
 	    },
-	    "[-v] [-e error] [-o offset] [-p stripesize] [-P stripeoffset] "
-	    "[-r rfailprob] [-s size] [-S secsize] [-w wfailprob] "
-	    "[-z physpath] dev ..."
+	    "[-v] [-d delaymsec] [-e error] [-o offset] [-p stripesize] "
+	    "[-P stripeoffset] [-q rdelayprob] [-r rfailprob] [-s size] "
+	    "[-S secsize] [-w wfailprob] [-x wdelayprob] [-z physpath] dev ..."
 	},
 	{ "configure", G_FLAG_VERBOSE, NULL,
 	    {
+		{ 'd', "delaymsec", "-1", G_TYPE_NUMBER },
 		{ 'e', "error", "-1", G_TYPE_NUMBER },
+		{ 'q', "rdelayprob", "-1", G_TYPE_NUMBER },
 		{ 'r', "rfailprob", "-1", G_TYPE_NUMBER },
 		{ 'w', "wfailprob", "-1", G_TYPE_NUMBER },
+		{ 'x', "wdelayprob", "1", G_TYPE_NUMBER },
 		G_OPT_SENTINEL
 	    },
-	    "[-v] [-e error] [-r rfailprob] [-w wfailprob] prov ..."
+	    "[-v] [-d delaymsec] [-e error] [-q rdelayprob] [-r rfailprob] "
+	    "[-w wfailprob] [-x wdelayprob] prov ..."
 	},
 	{ "destroy", G_FLAG_VERBOSE, NULL,
 	    {
diff --git a/lib/geom/nop/gnop.8 b/lib/geom/nop/gnop.8
index f9b3dc2c440e..ef6c44bc5297 100644
--- a/lib/geom/nop/gnop.8
+++ b/lib/geom/nop/gnop.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 17, 2018
+.Dd July 31, 2019
 .Dt GNOP 8
 .Os
 .Sh NAME
@@ -34,22 +34,28 @@
 .Nm
 .Cm create
 .Op Fl v
+.Op Fl d Ar delaymsec
 .Op Fl e Ar error
 .Op Fl o Ar offset
 .Op Fl p Ar stripesize
 .Op Fl P Ar stripeoffset
+.Op Fl q Ar rdelayprob
 .Op Fl r Ar rfailprob
 .Op Fl s Ar size
 .Op Fl S Ar secsize
 .Op Fl w Ar wfailprob
+.Op Fl x Ar wdelayprob
 .Op Fl z Ar physpath
 .Ar dev ...
 .Nm
 .Cm configure
 .Op Fl v
+.Op Fl d Ar delaymsec
 .Op Fl e Ar error
+.Op Fl q Ar rdelayprob
 .Op Fl r Ar rfailprob
 .Op Fl w Ar wfailprob
+.Op Fl x Ar wdelayprob
 .Ar prov ...
 .Nm
 .Cm destroy
@@ -113,6 +119,9 @@ See
 .Pp
 Additional options:
 .Bl -tag -width ".Fl r Ar rfailprob"
+.It Fl d Ar delaymsec
+Specifies the delay of the requests in milliseconds.
+Note that requests will be delayed before they are sent to the backing device.
 .It Fl e Ar error
 Specifies the error number to return on failure.
 .It Fl f
@@ -123,6 +132,8 @@ Where to begin on the original provider.
 Value of the stripesize property of the transparent provider.
 .It Fl P Ar stripeoffset
 Value of the stripeoffset property of the transparent provider.
+.It Fl q Ar rdelayprob
+Specifies read delay probability in percent.
 .It Fl r Ar rfailprob
 Specifies read failure probability in percent.
 .It Fl s Ar size
@@ -133,6 +144,8 @@ Sector size of the transparent provider.
 Specifies write failure probability in percent.
 .It Fl v
 Be more verbose.
+.It Fl x Ar wdelayprob
+Specifies write delay probability in percent.
 .It Fl z Ar physpath
 Physical path of the transparent provider.
 .El
diff --git a/sys/geom/nop/g_nop.c b/sys/geom/nop/g_nop.c
index 68d652e2cec6..2f465eba3d9b 100644
--- a/sys/geom/nop/g_nop.c
+++ b/sys/geom/nop/g_nop.c
@@ -74,6 +74,12 @@ struct g_class g_nop_class = {
 	.start = g_nop_start,
 };
 
+struct g_nop_delay {
+	struct callout			 dl_cal;
+	struct bio			*dl_bio;
+	TAILQ_ENTRY(g_nop_delay)	 dl_next;
+};
+
 static void
 g_nop_orphan(struct g_consumer *cp)
 {
@@ -143,16 +149,48 @@ g_nop_kerneldump(struct bio *bp, struct g_nop_softc *sc)
 }
 
 static void
+g_nop_pass(struct bio *cbp, struct g_geom *gp)
+{
+
+	G_NOP_LOGREQ(cbp, "Sending request.");
+	g_io_request(cbp, LIST_FIRST(&gp->consumer));
+}
+
+static void
+g_nop_pass_timeout(void *data)
+{
+	struct g_nop_softc *sc;
+	struct g_geom *gp;
+	struct g_nop_delay *gndelay;
+
+	gndelay = (struct g_nop_delay *)data;
+
+	gp = gndelay->dl_bio->bio_to->geom;
+	sc = gp->softc;
+
+	mtx_lock(&sc->sc_lock);
+	TAILQ_REMOVE(&sc->sc_head_delay, gndelay, dl_next);
+	mtx_unlock(&sc->sc_lock);
+
+	g_nop_pass(gndelay->dl_bio, gp);
+
+	g_free(data);
+}
+
+static void
 g_nop_start(struct bio *bp)
 {
 	struct g_nop_softc *sc;
 	struct g_geom *gp;
 	struct g_provider *pp;
 	struct bio *cbp;
-	u_int failprob = 0;
+	u_int failprob, delayprob, delaytime;
+
+	failprob = delayprob = 0;
 
 	gp = bp->bio_to->geom;
 	sc = gp->softc;
+
 	G_NOP_LOGREQ(bp, "Request received.");
 	mtx_lock(&sc->sc_lock);
 	switch (bp->bio_cmd) {
@@ -160,11 +198,15 @@ g_nop_start(struct bio *bp)
 		sc->sc_reads++;
 		sc->sc_readbytes += bp->bio_length;
 		failprob = sc->sc_rfailprob;
+		delayprob = sc->sc_rdelayprob;
+		delaytime = sc->sc_delaymsec;
 		break;
 	case BIO_WRITE:
 		sc->sc_writes++;
 		sc->sc_wrotebytes += bp->bio_length;
 		failprob = sc->sc_wfailprob;
+		delayprob = sc->sc_wdelayprob;
+		delaytime = sc->sc_delaymsec;
 		break;
 	case BIO_DELETE:
 		sc->sc_deletes++;
@@ -208,6 +250,7 @@ g_nop_start(struct bio *bp)
 			return;
 		}
 	}
+
 	cbp = g_clone_bio(bp);
 	if (cbp == NULL) {
 		g_io_deliver(bp, ENOMEM);
@@ -218,8 +261,33 @@ g_nop_start(struct bio *bp)
 	pp = LIST_FIRST(&gp->provider);
 	KASSERT(pp != NULL, ("NULL pp"));
 	cbp->bio_to = pp;
-	G_NOP_LOGREQ(cbp, "Sending request.");
-	g_io_request(cbp, LIST_FIRST(&gp->consumer));
+
+	if (delayprob > 0) {
+		struct g_nop_delay *gndelay;
+		u_int rval;
+
+		rval = arc4random() % 100;
+		if (rval < delayprob) {
+			gndelay = g_malloc(sizeof(*gndelay), M_NOWAIT | M_ZERO);
+			if (gndelay != NULL) {
+				callout_init(&gndelay->dl_cal, 1);
+
+				gndelay->dl_bio = cbp;
+
+				mtx_lock(&sc->sc_lock);
+				TAILQ_INSERT_TAIL(&sc->sc_head_delay, gndelay,
+				    dl_next);
+				mtx_unlock(&sc->sc_lock);
+
+				callout_reset(&gndelay->dl_cal,
+				    MSEC_2_TICKS(delaytime), g_nop_pass_timeout,
+				    gndelay);
+				return;
+			}
+		}
+	}
+
+	g_nop_pass(cbp, gp);
 }
 
 static int
@@ -238,8 +306,9 @@ g_nop_access(struct g_provider *pp, int dr, int dw, int de)
 
 static int
 g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
-    int ioerror, u_int rfailprob, u_int wfailprob, off_t offset, off_t size,
-    u_int secsize, off_t stripesize, off_t stripeoffset, const char *physpath)
+    int ioerror, u_int rfailprob, u_int wfailprob, u_int delaymsec, u_int rdelayprob,
+    u_int wdelayprob, off_t offset, off_t size, u_int secsize, off_t stripesize,
+    off_t stripeoffset, const char *physpath)
 {
 	struct g_nop_softc *sc;
 	struct g_geom *gp;
@@ -317,6 +386,9 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	sc->sc_error = ioerror;
 	sc->sc_rfailprob = rfailprob;
 	sc->sc_wfailprob = wfailprob;
+	sc->sc_delaymsec = delaymsec;
+	sc->sc_rdelayprob = rdelayprob;
+	sc->sc_wdelayprob = wdelayprob;
 	sc->sc_reads = 0;
 	sc->sc_writes = 0;
 	sc->sc_deletes = 0;
@@ -327,6 +399,7 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	sc->sc_cmd2s = 0;
 	sc->sc_readbytes = 0;
 	sc->sc_wrotebytes = 0;
+	TAILQ_INIT(&sc->sc_head_delay);
 	mtx_init(&sc->sc_lock, "gnop lock", NULL, MTX_DEF);
 	gp->softc = sc;
 
@@ -367,6 +440,9 @@ g_nop_providergone(struct g_provider *pp)
 	struct g_geom *gp = pp->geom;
 	struct g_nop_softc *sc = gp->softc;
 
+	KASSERT(TAILQ_EMPTY(&sc->sc_head_delay),
+	    ("delayed request list is not empty"));
+
 	gp->softc = NULL;
 	free(sc->sc_physpath, M_GEOM);
 	mtx_destroy(&sc->sc_lock);
@@ -396,6 +472,7 @@ g_nop_destroy(struct g_geom *gp, boolean_t force)
 	} else {
 		G_NOP_DEBUG(0, "Device %s removed.", gp->name);
 	}
+
 	g_wither_geom(gp, ENXIO);
 
 	return (0);
@@ -413,7 +490,7 @@ g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 {
 	struct g_provider *pp;
 	intmax_t *error, *rfailprob, *wfailprob, *offset, *secsize, *size,
-	    *stripesize, *stripeoffset;
+	    *stripesize, *stripeoffset, *delaymsec, *rdelayprob, *wdelayprob;
 	const char *name, *physpath;
 	char param[16];
 	int i, *nargs;
@@ -452,6 +529,33 @@ g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "Invalid '%s' argument", "wfailprob");
 		return;
 	}
+	delaymsec = gctl_get_paraml(req, "delaymsec", sizeof(*delaymsec));
+	if (delaymsec == NULL) {
+		gctl_error(req, "No '%s' argument", "delaymsec");
+		return;
+	}
+	if (*delaymsec < 1 && *delaymsec != -1) {
+		gctl_error(req, "Invalid '%s' argument", "delaymsec");
+		return;
+	}
+	rdelayprob = gctl_get_paraml(req, "rdelayprob", sizeof(*rdelayprob));
+	if (rdelayprob == NULL) {
+		gctl_error(req, "No '%s' argument", "rdelayprob");
+		return;
+	}
+	if (*rdelayprob < -1 || *rdelayprob > 100) {
+		gctl_error(req, "Invalid '%s' argument", "rdelayprob");
+		return;
+	}
+	wdelayprob = gctl_get_paraml(req, "wdelayprob", sizeof(*wdelayprob));
+	if (wdelayprob == NULL) {
+		gctl_error(req, "No '%s' argument", "wdelayprob");
+		return;
+	}
+	if (*wdelayprob < -1 || *wdelayprob > 100) {
+		gctl_error(req, "Invalid '%s' argument", "wdelayprob");
+		return;
+	}
 	offset = gctl_get_paraml(req, "offset", sizeof(*offset));
 	if (offset == NULL) {
 		gctl_error(req, "No '%s' argument", "offset");
@@ -518,6 +622,9 @@ g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 		    *error == -1 ? EIO : (int)*error,
 		    *rfailprob == -1 ? 0 : (u_int)*rfailprob,
 		    *wfailprob == -1 ? 0 : (u_int)*wfailprob,
+		    *delaymsec == -1 ? 1 : (u_int)*delaymsec,
+		    *rdelayprob == -1 ? 0 : (u_int)*rdelayprob,
+		    *wdelayprob == -1 ? 0 : (u_int)*wdelayprob,
 		    (off_t)*offset, (off_t)*size, (u_int)*secsize,
 		    (off_t)*stripesize, (off_t)*stripeoffset,
 		    physpath) != 0) {
@@ -531,7 +638,7 @@ g_nop_ctl_configure(struct gctl_req *req, struct g_class *mp)
 {
 	struct g_nop_softc *sc;
 	struct g_provider *pp;
-	intmax_t *error, *rfailprob, *wfailprob;
+	intmax_t *delaymsec, *error, *rdelayprob, *rfailprob, *wdelayprob, *wfailprob;
 	const char *name;
 	char param[16];
 	int i, *nargs;
@@ -571,6 +678,34 @@ g_nop_ctl_configure(struct gctl_req *req, struct g_class *mp)
 		return;
 	}
 
+	delaymsec = gctl_get_paraml(req, "delaymsec", sizeof(*delaymsec));
+	if (delaymsec == NULL) {
+		gctl_error(req, "No '%s' argument", "delaymsec");
+		return;
+	}
+	if (*delaymsec < 1 && *delaymsec != -1) {
+		gctl_error(req, "Invalid '%s' argument", "delaymsec");
+		return;
+	}
+	rdelayprob = gctl_get_paraml(req, "rdelayprob", sizeof(*rdelayprob));
+	if (rdelayprob == NULL) {
+		gctl_error(req, "No '%s' argument", "rdelayprob");
+		return;
+	}
+	if (*rdelayprob < -1 || *rdelayprob > 100) {
+		gctl_error(req, "Invalid '%s' argument", "rdelayprob");
+		return;
+	}
+	wdelayprob = gctl_get_paraml(req, "wdelayprob", sizeof(*wdelayprob));
+	if (wdelayprob == NULL) {
+		gctl_error(req, "No '%s' argument", "wdelayprob");
+		return;
+	}
+	if (*wdelayprob < -1 || *wdelayprob > 100) {
+		gctl_error(req, "Invalid '%s' argument", "wdelayprob");
+		return;
+	}
+
 	for (i = 0; i < *nargs; i++) {
 		snprintf(param, sizeof(param), "arg%d", i);
 		name = gctl_get_asciiparam(req, param);
@@ -593,6 +728,12 @@ g_nop_ctl_configure(struct gctl_req *req, struct g_class *mp)
 			sc->sc_rfailprob = (u_int)*rfailprob;
 		if (*wfailprob != -1)
 			sc->sc_wfailprob = (u_int)*wfailprob;
+		if (*rdelayprob != -1)
+			sc->sc_rdelayprob = (u_int)*rdelayprob;
+		if (*wdelayprob != -1)
+			sc->sc_wdelayprob = (u_int)*wdelayprob;
+		if (*delaymsec != -1)
+			sc->sc_delaymsec = (u_int)*delaymsec;
 	}
 }
 
@@ -756,6 +897,11 @@ g_nop_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 	    sc->sc_rfailprob);
 	sbuf_printf(sb, "%s<WriteFailProb>%u</WriteFailProb>\n", indent,
 	    sc->sc_wfailprob);
+	sbuf_printf(sb, "%s<ReadDelayedProb>%u</ReadDelayedProb>\n", indent,
+	    sc->sc_rdelayprob);
+	sbuf_printf(sb, "%s<WriteDelayedProb>%u</WriteDelayedProb>\n", indent,
+	    sc->sc_wdelayprob);
+	sbuf_printf(sb, "%s<Delay>%d</Delay>\n", indent, sc->sc_delaymsec);
 	sbuf_printf(sb, "%s<Error>%d</Error>\n", indent, sc->sc_error);
 	sbuf_printf(sb, "%s<Reads>%ju</Reads>\n", indent, sc->sc_reads);
 	sbuf_printf(sb, "%s<Writes>%ju</Writes>\n", indent, sc->sc_writes);
diff --git a/sys/geom/nop/g_nop.h b/sys/geom/nop/g_nop.h
index 353e5c5b8c09..95894bc1283f 100644
--- a/sys/geom/nop/g_nop.h
+++ b/sys/geom/nop/g_nop.h
@@ -62,26 +62,34 @@
 	}								\
 } while (0)
 
+struct g_nop_delay;
+
+TAILQ_HEAD(g_nop_delay_head, g_nop_delay);
+
 struct g_nop_softc {
-	int		sc_error;
-	off_t		sc_offset;
-	off_t		sc_explicitsize;
-	off_t		sc_stripesize;
-	off_t		sc_stripeoffset;
-	u_int		sc_rfailprob;
-	u_int		sc_wfailprob;
-	uintmax_t	sc_reads;
-	uintmax_t	sc_writes;
-	uintmax_t	sc_deletes;
-	uintmax_t	sc_getattrs;
-	uintmax_t	sc_flushes;
-	uintmax_t	sc_cmd0s;
-	uintmax_t	sc_cmd1s;
-	uintmax_t	sc_cmd2s;
-	uintmax_t	sc_readbytes;
-	uintmax_t	sc_wrotebytes;
-	char*		sc_physpath;
-	struct mtx	sc_lock;
+	int			 sc_error;
+	off_t			 sc_offset;
+	off_t			 sc_explicitsize;
+	off_t			 sc_stripesize;
+	off_t			 sc_stripeoffset;
+	u_int			 sc_rfailprob;
+	u_int			 sc_wfailprob;
+	u_int			 sc_delaymsec;
+	u_int			 sc_rdelayprob;
+	u_int			 sc_wdelayprob;
+	uintmax_t		 sc_reads;
+	uintmax_t		 sc_writes;
+	uintmax_t		 sc_deletes;
+	uintmax_t		 sc_getattrs;
+	uintmax_t		 sc_flushes;
+	uintmax_t		 sc_cmd0s;
+	uintmax_t		 sc_cmd1s;
+	uintmax_t		 sc_cmd2s;
+	uintmax_t		 sc_readbytes;
+	uintmax_t		 sc_wrotebytes;
+	char			*sc_physpath;
+	struct mtx		 sc_lock;
+	struct g_nop_delay_head	 sc_head_delay;
 };
 #endif	/* _KERNEL */