nfs_fha_new: Fix nfs_fha_new so that sysctls work in prisons

The sysctls were in "struct fha_params", making it impractical
to vnet them.  This patch moves the sysctls out of "struct fha_params"
and vnet's them, so that they are available in vnet prisons
running nfsd(8).  It also avoids the IS_DEFAULT_VNET() macros
in the VNET_SYSINIT() function.

Reviewed by:	glebius
MFC after:	3 months
Differential Revision:	https://reviews.freebsd.org/D38748

2 files changed, 53 insertions, 68 deletions

diff --git a/sys/fs/nfsserver/nfs_fha_new.c b/sys/fs/nfsserver/nfs_fha_new.c
index 6ae6884a4b74..20343c7c846f 100644
--- a/sys/fs/nfsserver/nfs_fha_new.c
+++ b/sys/fs/nfsserver/nfs_fha_new.c
@@ -57,8 +57,46 @@ static void		fha_extract_info(struct svc_req *req,
 			    struct fha_info *i);
 
 NFSD_VNET_DEFINE_STATIC(struct fha_params *, fhanew_softc);
+NFSD_VNET_DEFINE_STATIC(struct fha_ctls, nfsfha_ctls);
 
 SYSCTL_DECL(_vfs_nfsd);
+SYSCTL_NODE(_vfs_nfsd, OID_AUTO, fha, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+    "NFS File Handle Affinity (FHA)");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, enable, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).enable, 0,
+    "Enable NFS File Handle Affinity (FHA)");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, read, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).read, 0,
+    "Enable NFS FHA read locality");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, write, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).write, 0,
+    "Enable NFS FHA write locality");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, bin_shift, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).bin_shift, 0,
+    "Maximum locality distance 2^(bin_shift) bytes");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, max_nfsds_per_fh, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).max_nfsds_per_fh, 0,
+    "Maximum nfsd threads that "
+    "should be working on requests for the same file handle");
+
+SYSCTL_UINT(_vfs_nfsd_fha,
+    OID_AUTO, max_reqs_per_nfsd, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfsfha_ctls).max_reqs_per_nfsd, 0, "Maximum requests that "
+    "single nfsd thread should be working on at any time");
+
+SYSCTL_PROC(_vfs_nfsd_fha, OID_AUTO, fhe_stats,
+    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
+    fhenew_stats_sysctl, "A", "");
 
 extern int newnfs_nfsv3_procid[];
 
@@ -78,68 +116,19 @@ fhanew_init(void *foo)
 	snprintf(softc->server_name, sizeof(softc->server_name),
 	    FHANEW_SERVER_NAME);
 
-	/*
-	 * Initialize the sysctl context list for the fha module.
-	 */
-	sysctl_ctx_init(&softc->sysctl_ctx);
-	if (IS_DEFAULT_VNET(curvnet)) {
-		softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
-		    SYSCTL_STATIC_CHILDREN(_vfs_nfsd), OID_AUTO, "fha",
-		    CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "NFS File Handle Affinity (FHA)");
-		if (softc->sysctl_tree == NULL) {
-			printf("%s: unable to allocate sysctl tree\n", __func__);
-			return;
-		}
-	}
-
 	for (i = 0; i < FHA_HASH_SIZE; i++)
 		mtx_init(&softc->fha_hash[i].mtx, "fhalock", NULL, MTX_DEF);
 
 	/*
 	 * Set the default tuning parameters.
 	 */
-	softc->ctls.enable = FHA_DEF_ENABLE;
-	softc->ctls.read = FHA_DEF_READ;
-	softc->ctls.write = FHA_DEF_WRITE;
-	softc->ctls.bin_shift = FHA_DEF_BIN_SHIFT;
-	softc->ctls.max_nfsds_per_fh = FHA_DEF_MAX_NFSDS_PER_FH;
-	softc->ctls.max_reqs_per_nfsd = FHA_DEF_MAX_REQS_PER_NFSD;
+	NFSD_VNET(nfsfha_ctls).enable = FHA_DEF_ENABLE;
+	NFSD_VNET(nfsfha_ctls).read = FHA_DEF_READ;
+	NFSD_VNET(nfsfha_ctls).write = FHA_DEF_WRITE;
+	NFSD_VNET(nfsfha_ctls).bin_shift = FHA_DEF_BIN_SHIFT;
+	NFSD_VNET(nfsfha_ctls).max_nfsds_per_fh = FHA_DEF_MAX_NFSDS_PER_FH;
+	NFSD_VNET(nfsfha_ctls).max_reqs_per_nfsd = FHA_DEF_MAX_REQS_PER_NFSD;
 
-	/*
-	 * Add sysctls so the user can change the tuning parameters.
-	 */
-	if (IS_DEFAULT_VNET(curvnet)) {
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "enable", CTLFLAG_RWTUN,
-		    &softc->ctls.enable, 0, "Enable NFS File Handle Affinity (FHA)");
-
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "read", CTLFLAG_RWTUN,
-		    &softc->ctls.read, 0, "Enable NFS FHA read locality");
-
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "write", CTLFLAG_RWTUN,
-		    &softc->ctls.write, 0, "Enable NFS FHA write locality");
-
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "bin_shift", CTLFLAG_RWTUN,
-		    &softc->ctls.bin_shift, 0,
-		    "Maximum locality distance 2^(bin_shift) bytes");
-
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "max_nfsds_per_fh", CTLFLAG_RWTUN,
-		    &softc->ctls.max_nfsds_per_fh, 0, "Maximum nfsd threads that "
-		    "should be working on requests for the same file handle");
-
-		SYSCTL_ADD_UINT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "max_reqs_per_nfsd", CTLFLAG_RWTUN,
-		    &softc->ctls.max_reqs_per_nfsd, 0, "Maximum requests that "
-		    "single nfsd thread should be working on at any time");
-
-		SYSCTL_ADD_OID(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
-		    OID_AUTO, "fhe_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
-		    0, 0, fhenew_stats_sysctl, "A", "");
-	}
 }
 
 static void
@@ -150,7 +139,6 @@ fhanew_uninit(void *foo)
 
 	softc = NFSD_VNET(fhanew_softc);
 
-	sysctl_ctx_free(&softc->sysctl_ctx);
 	for (i = 0; i < FHA_HASH_SIZE; i++)
 		mtx_destroy(&softc->fha_hash[i].mtx);
 	free(softc, M_TEMP);
@@ -489,8 +477,8 @@ fha_hash_entry_choose_thread(struct fha_params *softc,
 		}
 
 		/* Check whether we should consider locality. */
-		if ((i->read && !softc->ctls.read) ||
-		    (i->write && !softc->ctls.write))
+		if ((i->read && !NFSD_VNET(nfsfha_ctls).read) ||
+		    (i->write && !NFSD_VNET(nfsfha_ctls).write))
 			goto noloc;
 
 		/*
@@ -501,11 +489,11 @@ fha_hash_entry_choose_thread(struct fha_params *softc,
 		offset2 = thread->st_p3;
 
 		if (((offset1 >= offset2)
-		  && ((offset1 - offset2) < (1 << softc->ctls.bin_shift)))
+		  && ((offset1 - offset2) < (1 << NFSD_VNET(nfsfha_ctls).bin_shift)))
 		 || ((offset2 > offset1)
-		  && ((offset2 - offset1) < (1 << softc->ctls.bin_shift)))) {
-			if ((softc->ctls.max_reqs_per_nfsd == 0) ||
-			    (req_count < softc->ctls.max_reqs_per_nfsd)) {
+		  && ((offset2 - offset1) < (1 << NFSD_VNET(nfsfha_ctls).bin_shift)))) {
+			if ((NFSD_VNET(nfsfha_ctls).max_reqs_per_nfsd == 0) ||
+			    (req_count < NFSD_VNET(nfsfha_ctls).max_reqs_per_nfsd)) {
 #if 0
 				ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO,
 				    "fha: %p(%d)r", thread, req_count);
@@ -535,8 +523,8 @@ noloc:
 	 * We didn't find a good match yet.  See if we can add
 	 * a new thread to this file handle entry's thread list.
 	 */
-	if ((softc->ctls.max_nfsds_per_fh == 0) ||
-	    (fhe->num_threads < softc->ctls.max_nfsds_per_fh)) {
+	if ((NFSD_VNET(nfsfha_ctls).max_nfsds_per_fh == 0) ||
+	    (fhe->num_threads < NFSD_VNET(nfsfha_ctls).max_nfsds_per_fh)) {
 		thread = this_thread;
 #if 0
 		ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO,
@@ -569,7 +557,7 @@ fhanew_assign(SVCTHREAD *this_thread, struct svc_req *req)
 	NFSD_CURVNET_SET(NFSD_TD_TO_VNET(curthread));
 	softc = NFSD_VNET(fhanew_softc);
 	/* Check to see whether we're enabled. */
-	if (softc->ctls.enable == 0)
+	if (NFSD_VNET(nfsfha_ctls).enable == 0)
 		goto thist;
 
 	/*
diff --git a/sys/fs/nfsserver/nfs_fha_new.h b/sys/fs/nfsserver/nfs_fha_new.h
index 08f52539450e..5510fa2891f1 100644
--- a/sys/fs/nfsserver/nfs_fha_new.h
+++ b/sys/fs/nfsserver/nfs_fha_new.h
@@ -95,9 +95,6 @@ struct fha_info {
 
 struct fha_params {
 	struct fha_hash_slot fha_hash[FHA_HASH_SIZE];
-	struct sysctl_ctx_list sysctl_ctx;
-	struct sysctl_oid *sysctl_tree;
-	struct fha_ctls ctls;
 	char server_name[32];
 };