vfs: handle vfs_init() failures

Most vfs_init implementations will not fail, with the notable current
exception that tmpfs_subr_init() can fail to allocate a new swap pager
type, in which case we probably do not want to proceed and keep it
registered.  linsysfs was a potential consumer, but we opted to go a
different direction and move pseudofs init/deinit over to first mount
and last mount instead.

Reviewed by:	fuz, kib
Differential Revision:	https://reviews.freebsd.org/D52037

1 files changed, 15 insertions, 6 deletions

diff --git a/sys/kern/vfs_init.c b/sys/kern/vfs_init.c
index 93ac001af8ad..ceda770cb714 100644
--- a/sys/kern/vfs_init.c
+++ b/sys/kern/vfs_init.c
@@ -399,7 +399,7 @@ vfs_register(struct vfsconf *vfc)
 	static int once;
 	struct vfsconf *tvfc;
 	uint32_t hashval;
-	int secondpass;
+	int error, prevmaxconf, secondpass;
 
 	if (!once) {
 		vattr_null(&va_null);
@@ -417,6 +417,7 @@ vfs_register(struct vfsconf *vfc)
 		return (EEXIST);
 	}
 
+	prevmaxconf = maxvfsconf;
 	if (vfs_typenumhash != 0) {
 		/*
 		 * Calculate a hash on vfc_name to use for vfc_typenum. Unless
@@ -509,16 +510,24 @@ vfs_register(struct vfsconf *vfc)
 		vfc->vfc_vfsops = &vfsops_sigdefer;
 	}
 
-	if (vfc->vfc_flags & VFCF_JAIL)
-		prison_add_vfs(vfc);
-
 	/*
 	 * Call init function for this VFS...
 	 */
 	if ((vfc->vfc_flags & VFCF_SBDRY) != 0)
-		vfc->vfc_vfsops_sd->vfs_init(vfc);
+		error = vfc->vfc_vfsops_sd->vfs_init(vfc);
 	else
-		vfc->vfc_vfsops->vfs_init(vfc);
+		error = vfc->vfc_vfsops->vfs_init(vfc);
+
+	if (error != 0) {
+		maxvfsconf = prevmaxconf;
+		TAILQ_REMOVE(&vfsconf, vfc, vfc_list);
+		vfsconf_unlock();
+		return (error);
+	}
+
+	if ((vfc->vfc_flags & VFCF_JAIL) != 0)
+		prison_add_vfs(vfc);
+
 	vfsconf_unlock();
 
 	/*