Add the "nocache" nullfs mount option, which disables the caching of

the free nullfs vnodes, switching nullfs behaviour to pre-r240285.
The option is mostly intended as the last-resort when higher pressure
on the vnode cache due to doubling of the vnode counts is not
desirable.

Note that disabling the cache costs more than 2x wall time in the
metadata-hungry scenarious.  The default is "cache".

Tested and benchmarked by:	pho (previous version)
MFC after:	2 weeks

4 files changed, 63 insertions, 13 deletions

diff --git a/sys/fs/nullfs/null.h b/sys/fs/nullfs/null.h
index 0878e559329d..4f37020d6da5 100644
--- a/sys/fs/nullfs/null.h
+++ b/sys/fs/nullfs/null.h
@@ -34,9 +34,15 @@
  * $FreeBSD$
  */
 
+#ifndef	FS_NULL_H
+#define	FS_NULL_H
+
+#define	NULLM_CACHE	0x0001
+
 struct null_mount {
 	struct mount	*nullm_vfs;
 	struct vnode	*nullm_rootvp;	/* Reference to root null_node */
+	uint64_t	nullm_flags;
 };
 
 #ifdef _KERNEL
@@ -80,3 +86,5 @@ MALLOC_DECLARE(M_NULLFSNODE);
 #endif /* NULLFS_DEBUG */
 
 #endif /* _KERNEL */
+
+#endif
diff --git a/sys/fs/nullfs/null_subr.c b/sys/fs/nullfs/null_subr.c
index b2c7a75d14ea..f82d73807470 100644
--- a/sys/fs/nullfs/null_subr.c
+++ b/sys/fs/nullfs/null_subr.c
@@ -224,6 +224,9 @@ null_nodeget(mp, lowervp, vpp)
 	 * provide ready to use vnode.
 	 */
 	if (VOP_ISLOCKED(lowervp) != LK_EXCLUSIVE) {
+		KASSERT((MOUNTTONULLMOUNT(mp)->nullm_flags & NULLM_CACHE) == 0,
+		    ("lowervp %p is not excl locked and cache is disabled",
+		    lowervp));
 		vn_lock(lowervp, LK_UPGRADE | LK_RETRY);
 		if ((lowervp->v_iflag & VI_DOOMED) != 0) {
 			vput(lowervp);
diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c
index 7d84d5190667..5abfa49cf14f 100644
--- a/sys/fs/nullfs/null_vfsops.c
+++ b/sys/fs/nullfs/null_vfsops.c
@@ -67,6 +67,15 @@ static vfs_vget_t	nullfs_vget;
 static vfs_extattrctl_t	nullfs_extattrctl;
 static vfs_reclaim_lowervp_t nullfs_reclaim_lowervp;
 
+/* Mount options that we support. */
+static const char *nullfs_opts[] = {
+	"cache",
+	"export",
+	"from",
+	"target",
+	NULL
+};
+
 /*
  * Mount null layer
  */
@@ -86,9 +95,11 @@ nullfs_mount(struct mount *mp)
 
 	if (!prison_allow(td->td_ucred, PR_ALLOW_MOUNT_NULLFS))
 		return (EPERM);
-
 	if (mp->mnt_flag & MNT_ROOTFS)
 		return (EOPNOTSUPP);
+	if (vfs_filteropt(mp->mnt_optnew, nullfs_opts))
+		return (EINVAL);
+
 	/*
 	 * Update is a no-op
 	 */
@@ -149,7 +160,7 @@ nullfs_mount(struct mount *mp)
 	}
 
 	xmp = (struct null_mount *) malloc(sizeof(struct null_mount),
-	    M_NULLFSMNT, M_WAITOK);
+	    M_NULLFSMNT, M_WAITOK | M_ZERO);
 
 	/*
 	 * Save reference to underlying FS
@@ -187,16 +198,27 @@ nullfs_mount(struct mount *mp)
 		mp->mnt_flag |= MNT_LOCAL;
 		MNT_IUNLOCK(mp);
 	}
+
+	xmp->nullm_flags |= NULLM_CACHE;
+	if (vfs_getopt(mp->mnt_optnew, "nocache", NULL, NULL) == 0)
+		xmp->nullm_flags &= ~NULLM_CACHE;
+
 	MNT_ILOCK(mp);
-	mp->mnt_kern_flag |= lowerrootvp->v_mount->mnt_kern_flag &
-	    (MNTK_SHARED_WRITES | MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED);
+	if ((xmp->nullm_flags & NULLM_CACHE) != 0) {
+		mp->mnt_kern_flag |= lowerrootvp->v_mount->mnt_kern_flag &
+		    (MNTK_SHARED_WRITES | MNTK_LOOKUP_SHARED |
+		    MNTK_EXTENDED_SHARED);
+	}
 	mp->mnt_kern_flag |= MNTK_LOOKUP_EXCL_DOTDOT;
 	MNT_IUNLOCK(mp);
 	mp->mnt_data = xmp;
 	vfs_getnewfsid(mp);
-	MNT_ILOCK(xmp->nullm_vfs);
-	TAILQ_INSERT_TAIL(&xmp->nullm_vfs->mnt_uppers, mp, mnt_upper_link);
-	MNT_IUNLOCK(xmp->nullm_vfs);
+	if ((xmp->nullm_flags & NULLM_CACHE) != 0) {
+		MNT_ILOCK(xmp->nullm_vfs);
+		TAILQ_INSERT_TAIL(&xmp->nullm_vfs->mnt_uppers, mp,
+		    mnt_upper_link);
+		MNT_IUNLOCK(xmp->nullm_vfs);
+	}
 
 	vfs_mountedfrom(mp, target);
 
@@ -234,13 +256,15 @@ nullfs_unmount(mp, mntflags)
 	 */
 	mntdata = mp->mnt_data;
 	ump = mntdata->nullm_vfs;
-	MNT_ILOCK(ump);
-	while ((ump->mnt_kern_flag & MNTK_VGONE_UPPER) != 0) {
-		ump->mnt_kern_flag |= MNTK_VGONE_WAITER;
-		msleep(&ump->mnt_uppers, &ump->mnt_mtx, 0, "vgnupw", 0);
+	if ((mntdata->nullm_flags & NULLM_CACHE) != 0) {
+		MNT_ILOCK(ump);
+		while ((ump->mnt_kern_flag & MNTK_VGONE_UPPER) != 0) {
+			ump->mnt_kern_flag |= MNTK_VGONE_WAITER;
+			msleep(&ump->mnt_uppers, &ump->mnt_mtx, 0, "vgnupw", 0);
+		}
+		TAILQ_REMOVE(&ump->mnt_uppers, mp, mnt_upper_link);
+		MNT_IUNLOCK(ump);
 	}
-	TAILQ_REMOVE(&ump->mnt_uppers, mp, mnt_upper_link);
-	MNT_IUNLOCK(ump);
 	mp->mnt_data = NULL;
 	free(mntdata, M_NULLFSMNT);
 	return (0);
diff --git a/sys/fs/nullfs/null_vnops.c b/sys/fs/nullfs/null_vnops.c
index f530ed217f2f..cc35d819149f 100644
--- a/sys/fs/nullfs/null_vnops.c
+++ b/sys/fs/nullfs/null_vnops.c
@@ -692,7 +692,22 @@ null_unlock(struct vop_unlock_args *ap)
 static int
 null_inactive(struct vop_inactive_args *ap __unused)
 {
+	struct vnode *vp;
+	struct mount *mp;
+	struct null_mount *xmp;
 
+	vp = ap->a_vp;
+	mp = vp->v_mount;
+	xmp = MOUNTTONULLMOUNT(mp);
+	if ((xmp->nullm_flags & NULLM_CACHE) == 0) {
+		/*
+		 * If this is the last reference and caching of the
+		 * nullfs vnodes is not enabled, then free up the
+		 * vnode so as not to tie up the lower vnodes.
+		 */
+		vp->v_object = NULL;
+		vrecycle(vp);
+	}
 	return (0);
 }