null_vput_pair(): release use reference on dvp earlier

We might own the last use reference, and then vrele() at the end would
need to take the dvp vnode lock to inactivate, which causes deadlock
with vp. We cannot vrele() dvp from start since this might unlock ldvp.

Handle it by holding the vnode and dropping use ref after lowerfs
VOP_VPUT_PAIR() ended.  This effectivaly requires unlock of the vp vnode
after VOP_VPUT_PAIR(), so the call is changed to set unlock_vp to true
unconditionally.  This opens more opportunities for vp to be reclaimed,
if lvp is still alive we reinstantiate vp with null_nodeget().

Reported and tested by:	pho
Reviewed by:	mckusick
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
Differential revision:	https://reviews.freebsd.org/D29178

1 files changed, 31 insertions, 14 deletions

diff --git a/sys/fs/nullfs/null_vnops.c b/sys/fs/nullfs/null_vnops.c
index 45065e0be7b5..bc0f5cdb7801 100644
--- a/sys/fs/nullfs/null_vnops.c
+++ b/sys/fs/nullfs/null_vnops.c
@@ -985,33 +985,50 @@ null_vput_pair(struct vop_vput_pair_args *ap)
 	vpp = ap->a_vpp;
 	vp = NULL;
 	lvp = NULL;
-	if (vpp != NULL) {
+	mp = NULL;
+	if (vpp != NULL)
 		vp = *vpp;
-		if (vp != NULL) {
+	if (vp != NULL) {
+		lvp = NULLVPTOLOWERVP(vp);
+		vref(lvp);
+		if (!ap->a_unlock_vp) {
 			vhold(vp);
+			vhold(lvp);
 			mp = vp->v_mount;
-			lvp = NULLVPTOLOWERVP(vp);
-			if (ap->a_unlock_vp)
-				vref(lvp);
+			vfs_ref(mp);
 		}
 	}
 
-	res = VOP_VPUT_PAIR(ldvp, &lvp, ap->a_unlock_vp);
+	res = VOP_VPUT_PAIR(ldvp, lvp != NULL ? &lvp : NULL, true);
+	if (vp != NULL && ap->a_unlock_vp)
+		vrele(vp);
+	vrele(dvp);
+
+	if (vp == NULL || ap->a_unlock_vp)
+		return (res);
 
-	/* lvp might have been unlocked and vp reclaimed */
-	if (vp != NULL) {
-		if (!ap->a_unlock_vp && vp->v_vnlock != lvp->v_vnlock) {
+	/* lvp has been unlocked and vp might be reclaimed */
+	VOP_LOCK(vp, LK_EXCLUSIVE | LK_RETRY);
+	if (vp->v_data == NULL && vfs_busy(mp, MBF_NOWAIT) == 0) {
+		vput(vp);
+		vget(lvp, LK_EXCLUSIVE | LK_RETRY);
+		if (VN_IS_DOOMED(lvp)) {
+			vput(lvp);
+			vget(vp, LK_EXCLUSIVE | LK_RETRY);
+		} else {
 			error = null_nodeget(mp, lvp, &vp1);
 			if (error == 0) {
-				vput(vp);
 				*vpp = vp1;
+			} else {
+				vget(vp, LK_EXCLUSIVE | LK_RETRY);
 			}
 		}
-		if (ap->a_unlock_vp)
-			vrele(vp);
-		vdrop(vp);
+		vfs_unbusy(mp);
 	}
-	vrele(dvp);
+	vdrop(lvp);
+	vdrop(vp);
+	vfs_rel(mp);
+
 	return (res);
 }