OpenZFS merge main-gf11b09

- add dRAID support
- fix duplicate close handling
- fix memory leak in prefetch
- fix problem with SIMD benchmarking on FreeBSD boot
...

6 files changed, 162 insertions, 97 deletions

diff --git a/sys/contrib/openzfs/module/zcommon/Makefile.in b/sys/contrib/openzfs/module/zcommon/Makefile.in
index b5cdf4c0c9fe..ebc538440445 100644
--- a/sys/contrib/openzfs/module/zcommon/Makefile.in
+++ b/sys/contrib/openzfs/module/zcommon/Makefile.in
@@ -19,7 +19,6 @@ $(MODULE)-objs += zfs_fletcher_superscalar.o
 $(MODULE)-objs += zfs_fletcher_superscalar4.o
 $(MODULE)-objs += zfs_namecheck.o
 $(MODULE)-objs += zfs_prop.o
-$(MODULE)-objs += zfs_uio.o
 $(MODULE)-objs += zpool_prop.o
 $(MODULE)-objs += zprop_common.o
 
diff --git a/sys/contrib/openzfs/module/zcommon/zfeature_common.c b/sys/contrib/openzfs/module/zcommon/zfeature_common.c
index 97ddacbab9e0..34ebabcf3b3c 100644
--- a/sys/contrib/openzfs/module/zcommon/zfeature_common.c
+++ b/sys/contrib/openzfs/module/zcommon/zfeature_common.c
@@ -576,7 +576,7 @@ zpool_feature_init(void)
 
 	zfeature_register(SPA_FEATURE_DEVICE_REBUILD,
 	    "org.openzfs:device_rebuild", "device_rebuild",
-	    "Support for sequential device rebuilds",
+	    "Support for sequential mirror/dRAID device rebuilds",
 	    ZFEATURE_FLAG_READONLY_COMPAT, ZFEATURE_TYPE_BOOLEAN, NULL);
 
 	{
@@ -589,6 +589,10 @@ zpool_feature_init(void)
 	    "zstd compression algorithm support.",
 	    ZFEATURE_FLAG_PER_DATASET, ZFEATURE_TYPE_BOOLEAN, zstd_deps);
 	}
+
+	zfeature_register(SPA_FEATURE_DRAID,
+	    "org.openzfs:draid", "draid", "Support for distributed spare RAID",
+	    ZFEATURE_FLAG_MOS, ZFEATURE_TYPE_BOOLEAN, NULL);
 }
 
 #if defined(_KERNEL)
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_fletcher.c b/sys/contrib/openzfs/module/zcommon/zfs_fletcher.c
index 3e0632a32864..7a9de4a4309d 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_fletcher.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_fletcher.c
@@ -660,7 +660,7 @@ fletcher_4_kstat_addr(kstat_t *ksp, loff_t n)
 	fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \
 }
 
-#define	FLETCHER_4_BENCH_NS	(MSEC2NSEC(50))		/* 50ms */
+#define	FLETCHER_4_BENCH_NS	(MSEC2NSEC(1))		/* 1ms */
 
 typedef void fletcher_checksum_func_t(const void *, uint64_t, const void *,
 					zio_cksum_t *);
@@ -885,23 +885,26 @@ zio_abd_checksum_func_t fletcher_4_abd_ops = {
 	.acf_iter = abd_fletcher_4_iter
 };
 
+#if defined(_KERNEL)
+
+#define	IMPL_FMT(impl, i)	(((impl) == (i)) ? "[%s] " : "%s ")
 
-#if defined(_KERNEL) && defined(__linux__)
+#if defined(__linux__)
 
 static int
 fletcher_4_param_get(char *buffer, zfs_kernel_param_t *unused)
 {
 	const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
 	char *fmt;
-	int i, cnt = 0;
+	int cnt = 0;
 
 	/* list fastest */
-	fmt = (impl == IMPL_FASTEST) ? "[%s] " : "%s ";
+	fmt = IMPL_FMT(impl, IMPL_FASTEST);
 	cnt += sprintf(buffer + cnt, fmt, "fastest");
 
 	/* list all supported implementations */
-	for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
-		fmt = (i == impl) ? "[%s] " : "%s ";
+	for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) {
+		fmt = IMPL_FMT(impl, i);
 		cnt += sprintf(buffer + cnt, fmt,
 		    fletcher_4_supp_impls[i]->name);
 	}
@@ -915,14 +918,62 @@ fletcher_4_param_set(const char *val, zfs_kernel_param_t *unused)
 	return (fletcher_4_impl_set(val));
 }
 
+#else
+
+#include <sys/sbuf.h>
+
+static int
+fletcher_4_param(ZFS_MODULE_PARAM_ARGS)
+{
+	int err;
+
+	if (req->newptr == NULL) {
+		const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
+		const int init_buflen = 64;
+		const char *fmt;
+		struct sbuf *s;
+
+		s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
+
+		/* list fastest */
+		fmt = IMPL_FMT(impl, IMPL_FASTEST);
+		(void) sbuf_printf(s, fmt, "fastest");
+
+		/* list all supported implementations */
+		for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) {
+			fmt = IMPL_FMT(impl, i);
+			(void) sbuf_printf(s, fmt,
+			    fletcher_4_supp_impls[i]->name);
+		}
+
+		err = sbuf_finish(s);
+		sbuf_delete(s);
+
+		return (err);
+	}
+
+	char buf[16];
+
+	err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
+	if (err)
+		return (err);
+	return (-fletcher_4_impl_set(buf));
+}
+
+#endif
+
+#undef IMPL_FMT
+
 /*
  * Choose a fletcher 4 implementation in ZFS.
  * Users can choose "cycle" to exercise all implementations, but this is
  * for testing purpose therefore it can only be set in user space.
  */
-module_param_call(zfs_fletcher_4_impl,
-    fletcher_4_param_set, fletcher_4_param_get, NULL, 0644);
-MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 implementation.");
+/* BEGIN CSTYLED */
+ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, fletcher_4_impl,
+	fletcher_4_param_set, fletcher_4_param_get, ZMOD_RW,
+	"Select fletcher 4 implementation.");
+/* END CSTYLED */
 
 EXPORT_SYMBOL(fletcher_init);
 EXPORT_SYMBOL(fletcher_2_incremental_native);
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_namecheck.c b/sys/contrib/openzfs/module/zcommon/zfs_namecheck.c
index f8625042a74c..0011a971cacb 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_namecheck.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_namecheck.c
@@ -442,7 +442,9 @@ pool_namecheck(const char *pool, namecheck_err_t *why, char *what)
 		return (-1);
 	}
 
-	if (strcmp(pool, "mirror") == 0 || strcmp(pool, "raidz") == 0) {
+	if (strcmp(pool, "mirror") == 0 ||
+	    strcmp(pool, "raidz") == 0 ||
+	    strcmp(pool, "draid") == 0) {
 		if (why)
 			*why = NAME_ERR_RESERVED;
 		return (-1);
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_prop.c b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
index 0352b13aa240..b78331187e13 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_prop.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_prop.c
@@ -551,14 +551,14 @@ zfs_prop_init(void)
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM, "<path> | legacy | none",
 	    "MOUNTPOINT");
 	zprop_register_string(ZFS_PROP_SHARENFS, "sharenfs", "off",
-	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM, "on | off | share(1M) options",
+	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM, "on | off | NFS share options",
 	    "SHARENFS");
 	zprop_register_string(ZFS_PROP_TYPE, "type", NULL, PROP_READONLY,
 	    ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK,
 	    "filesystem | volume | snapshot | bookmark", "TYPE");
 	zprop_register_string(ZFS_PROP_SHARESMB, "sharesmb", "off",
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
-	    "on | off | sharemgr(1M) options", "SHARESMB");
+	    "on | off | SMB share options", "SHARESMB");
 	zprop_register_string(ZFS_PROP_MLSLABEL, "mlslabel",
 	    ZFS_MLSLABEL_DEFAULT, PROP_INHERIT, ZFS_TYPE_DATASET,
 	    "<sensitivity label>", "MLSLABEL");
@@ -1016,7 +1016,7 @@ zcommon_fini(void)
 	kfpu_fini();
 }
 
-module_init(zcommon_init);
+module_init_early(zcommon_init);
 module_exit(zcommon_fini);
 
 #endif
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_uio.c b/sys/contrib/openzfs/module/zcommon/zfs_uio.c
index d586e0a1220a..e435e1a9f78a 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_uio.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_uio.c
@@ -39,12 +39,6 @@
  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
-/*
- * The uio support from OpenSolaris has been added as a short term
- * work around.  The hope is to adopt native Linux type and drop the
- * use of uio's entirely.  Under Linux they only add overhead and
- * when possible we want to use native APIs for the ZPL layer.
- */
 #ifdef _KERNEL
 
 #include <sys/types.h>
@@ -71,7 +65,6 @@ uiomove_iov(void *p, size_t n, enum uio_rw rw, struct uio *uio)
 		cnt = MIN(iov->iov_len - skip, n);
 		switch (uio->uio_segflg) {
 		case UIO_USERSPACE:
-		case UIO_USERISPACE:
 			/*
 			 * p = kernel data pointer
 			 * iov->iov_base = user data pointer
@@ -165,81 +158,82 @@ uiomove_bvec(void *p, size_t n, enum uio_rw rw, struct uio *uio)
 	return (0);
 }
 
+#if defined(HAVE_VFS_IOV_ITER)
+static int
+uiomove_iter(void *p, size_t n, enum uio_rw rw, struct uio *uio,
+    boolean_t revert)
+{
+	size_t cnt = MIN(n, uio->uio_resid);
+
+	if (uio->uio_skip)
+		iov_iter_advance(uio->uio_iter, uio->uio_skip);
+
+	if (rw == UIO_READ)
+		cnt = copy_to_iter(p, cnt, uio->uio_iter);
+	else
+		cnt = copy_from_iter(p, cnt, uio->uio_iter);
+
+	/*
+	 * When operating on a full pipe no bytes are processed.
+	 * In which case return EFAULT which is converted to EAGAIN
+	 * by the kernel's generic_file_splice_read() function.
+	 */
+	if (cnt == 0)
+		return (EFAULT);
+
+	/*
+	 * Revert advancing the uio_iter.  This is set by uiocopy()
+	 * to avoid consuming the uio and its iov_iter structure.
+	 */
+	if (revert)
+		iov_iter_revert(uio->uio_iter, cnt);
+
+	uio->uio_resid -= cnt;
+	uio->uio_loffset += cnt;
+
+	return (0);
+}
+#endif
+
 int
 uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio)
 {
-	if (uio->uio_segflg != UIO_BVEC)
-		return (uiomove_iov(p, n, rw, uio));
-	else
+	if (uio->uio_segflg == UIO_BVEC)
 		return (uiomove_bvec(p, n, rw, uio));
+#if defined(HAVE_VFS_IOV_ITER)
+	else if (uio->uio_segflg == UIO_ITER)
+		return (uiomove_iter(p, n, rw, uio, B_FALSE));
+#endif
+	else
+		return (uiomove_iov(p, n, rw, uio));
 }
 EXPORT_SYMBOL(uiomove);
 
-#define	fuword8(uptr, vptr)	get_user((*vptr), (uptr))
-
-/*
- * Fault in the pages of the first n bytes specified by the uio structure.
- * 1 byte in each page is touched and the uio struct is unmodified. Any
- * error will terminate the process as this is only a best attempt to get
- * the pages resident.
- */
 int
 uio_prefaultpages(ssize_t n, struct uio *uio)
 {
-	const struct iovec *iov;
-	ulong_t cnt, incr;
-	caddr_t p;
-	uint8_t tmp;
-	int iovcnt;
-	size_t skip;
+	struct iov_iter iter, *iterp = NULL;
 
-	/* no need to fault in kernel pages */
-	switch (uio->uio_segflg) {
-		case UIO_SYSSPACE:
-		case UIO_BVEC:
-			return (0);
-		case UIO_USERSPACE:
-		case UIO_USERISPACE:
-			break;
-		default:
-			ASSERT(0);
-	}
-
-	iov = uio->uio_iov;
-	iovcnt = uio->uio_iovcnt;
-	skip = uio->uio_skip;
-
-	for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
-		cnt = MIN(iov->iov_len - skip, n);
-		/* empty iov */
-		if (cnt == 0)
-			continue;
-		n -= cnt;
-		/*
-		 * touch each page in this segment.
-		 */
-		p = iov->iov_base + skip;
-		while (cnt) {
-			if (fuword8((uint8_t *)p, &tmp))
-				return (EFAULT);
-			incr = MIN(cnt, PAGESIZE);
-			p += incr;
-			cnt -= incr;
-		}
-		/*
-		 * touch the last byte in case it straddles a page.
-		 */
-		p--;
-		if (fuword8((uint8_t *)p, &tmp))
-			return (EFAULT);
+#if defined(HAVE_IOV_ITER_FAULT_IN_READABLE)
+	if (uio->uio_segflg == UIO_USERSPACE) {
+		iterp = &iter;
+		iov_iter_init_compat(iterp, READ, uio->uio_iov,
+		    uio->uio_iovcnt, uio->uio_resid);
+#if defined(HAVE_VFS_IOV_ITER)
+	} else if (uio->uio_segflg == UIO_ITER) {
+		iterp = uio->uio_iter;
+#endif
 	}
 
+	if (iterp && iov_iter_fault_in_readable(iterp, n))
+		return (EFAULT);
+#endif
 	return (0);
 }
 EXPORT_SYMBOL(uio_prefaultpages);
 
 /*
- * same as uiomove() but doesn't modify uio structure.
+ * The same as uiomove() but doesn't modify uio structure.
  * return in cbytes how many bytes were copied.
  */
 int
@@ -249,39 +243,54 @@ uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes)
 	int ret;
 
 	bcopy(uio, &uio_copy, sizeof (struct uio));
-	ret = uiomove(p, n, rw, &uio_copy);
+
+	if (uio->uio_segflg == UIO_BVEC)
+		ret = uiomove_bvec(p, n, rw, &uio_copy);
+#if defined(HAVE_VFS_IOV_ITER)
+	else if (uio->uio_segflg == UIO_ITER)
+		ret = uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
+#endif
+	else
+		ret = uiomove_iov(p, n, rw, &uio_copy);
+
 	*cbytes = uio->uio_resid - uio_copy.uio_resid;
+
 	return (ret);
 }
 EXPORT_SYMBOL(uiocopy);
 
 /*
- * Drop the next n chars out of *uiop.
+ * Drop the next n chars out of *uio.
  */
 void
-uioskip(uio_t *uiop, size_t n)
+uioskip(uio_t *uio, size_t n)
 {
-	if (n > uiop->uio_resid)
+	if (n > uio->uio_resid)
 		return;
 
-	uiop->uio_skip += n;
-	if (uiop->uio_segflg != UIO_BVEC) {
-		while (uiop->uio_iovcnt &&
-		    uiop->uio_skip >= uiop->uio_iov->iov_len) {
-			uiop->uio_skip -= uiop->uio_iov->iov_len;
-			uiop->uio_iov++;
-			uiop->uio_iovcnt--;
+	if (uio->uio_segflg == UIO_BVEC) {
+		uio->uio_skip += n;
+		while (uio->uio_iovcnt &&
+		    uio->uio_skip >= uio->uio_bvec->bv_len) {
+			uio->uio_skip -= uio->uio_bvec->bv_len;
+			uio->uio_bvec++;
+			uio->uio_iovcnt--;
 		}
+#if defined(HAVE_VFS_IOV_ITER)
+	} else if (uio->uio_segflg == UIO_ITER) {
+		iov_iter_advance(uio->uio_iter, n);
+#endif
 	} else {
-		while (uiop->uio_iovcnt &&
-		    uiop->uio_skip >= uiop->uio_bvec->bv_len) {
-			uiop->uio_skip -= uiop->uio_bvec->bv_len;
-			uiop->uio_bvec++;
-			uiop->uio_iovcnt--;
+		uio->uio_skip += n;
+		while (uio->uio_iovcnt &&
+		    uio->uio_skip >= uio->uio_iov->iov_len) {
+			uio->uio_skip -= uio->uio_iov->iov_len;
+			uio->uio_iov++;
+			uio->uio_iovcnt--;
 		}
 	}
-	uiop->uio_loffset += n;
-	uiop->uio_resid -= n;
+	uio->uio_loffset += n;
+	uio->uio_resid -= n;
 }
 EXPORT_SYMBOL(uioskip);
 #endif /* _KERNEL */