zfs: convert z_teardown_inactive_lock to sleepable read-mostly lock

This eliminates a global serialisation point. It only gets write locked
on unmount.

Sample result doing an incremental -j 40 build:
before: 173.30s user 458.97s system 2595% cpu 24.358 total
after:  168.58s user 254.92s system 2211% cpu 19.147 total

2 files changed, 10 insertions, 9 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
index d32f14aa0774..22222358ad73 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h
@@ -33,6 +33,7 @@
 #include <sys/sa.h>
 #include <sys/rrwlock.h>
 #include <sys/zfs_ioctl.h>
+#include <sys/rmlock.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -65,7 +66,7 @@ struct zfsvfs {
 	boolean_t	z_atime;	/* enable atimes mount option */
 	boolean_t	z_unmounted;	/* unmounted */
 	rrmlock_t	z_teardown_lock;
-	krwlock_t	z_teardown_inactive_lock;
+	struct rmslock	z_teardown_inactive_lock;
 	list_t		z_all_znodes;	/* all vnodes in the fs */
 	kmutex_t	z_znodes_lock;	/* lock for z_all_znodes */
 	struct zfsctl_root	*z_ctldir;	/* .zfs directory pointer */
@@ -91,22 +92,22 @@ struct zfsvfs {
 };
 
 #define	ZFS_TRYRLOCK_TEARDOWN_INACTIVE(zfsvfs) \
-		rw_tryenter(&(zfsvfs)->z_teardown_inactive_lock, RW_READER)
+		rms_try_rlock(&(zfsvfs)->z_teardown_inactive_lock)
 
 #define	ZFS_RLOCK_TEARDOWN_INACTIVE(zfsvfs) \
-		rw_enter(&(zfsvfs)->z_teardown_inactive_lock, RW_READER)
+		rms_rlock(&(zfsvfs)->z_teardown_inactive_lock)
 
 #define	ZFS_RUNLOCK_TEARDOWN_INACTIVE(zfsvfs) \
-		rw_exit(&(zfsvfs)->z_teardown_inactive_lock)
+		rms_runlock(&(zfsvfs)->z_teardown_inactive_lock)
 
 #define	ZFS_WLOCK_TEARDOWN_INACTIVE(zfsvfs) \
-		rw_enter(&(zfsvfs)->z_teardown_inactive_lock, RW_WRITER)
+		rms_wlock(&(zfsvfs)->z_teardown_inactive_lock)
 
 #define	ZFS_WUNLOCK_TEARDOWN_INACTIVE(zfsvfs) \
-		rw_exit(&(zfsvfs)->z_teardown_inactive_lock)
+		rms_wunlock(&(zfsvfs)->z_teardown_inactive_lock)
 
 #define	ZFS_WLOCK_TEARDOWN_INACTIVE_WLOCKED(zfsvfs) \
-		RW_WRITE_HELD(&(zfsvfs)->z_teardown_inactive_lock)
+		rms_wowned(&(zfsvfs)->z_teardown_inactive_lock)
 
 /*
  * Normal filesystems (those not under .zfs/snapshot) have a total
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index 10f526735943..9364c09f163b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -1205,7 +1205,7 @@ zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os)
 #else
 	rrm_init(&zfsvfs->z_teardown_lock, B_FALSE);
 #endif
-	rw_init(&zfsvfs->z_teardown_inactive_lock, NULL, RW_DEFAULT, NULL);
+	rms_init(&zfsvfs->z_teardown_inactive_lock, "zfs teardown inactive");
 	rw_init(&zfsvfs->z_fuid_lock, NULL, RW_DEFAULT, NULL);
 	for (int i = 0; i != ZFS_OBJ_MTX_SZ; i++)
 		mutex_init(&zfsvfs->z_hold_mtx[i], NULL, MUTEX_DEFAULT, NULL);
@@ -1322,7 +1322,7 @@ zfsvfs_free(zfsvfs_t *zfsvfs)
 	mutex_destroy(&zfsvfs->z_lock);
 	list_destroy(&zfsvfs->z_all_znodes);
 	rrm_destroy(&zfsvfs->z_teardown_lock);
-	rw_destroy(&zfsvfs->z_teardown_inactive_lock);
+	rms_destroy(&zfsvfs->z_teardown_inactive_lock);
 	rw_destroy(&zfsvfs->z_fuid_lock);
 	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
 		mutex_destroy(&zfsvfs->z_hold_mtx[i]);