nfsd: Prepare the NFS server code to run in a vnet prison

This patch defines null macros that can be used to apply
the vnet macros for global variables and SYSCTL flags.
It also applies these macros to many of the global variables
and some of the SYSCTLs.  Since the macros do nothing, these
changes should not result in semantics changes, although the
changes are large in number.

The patch does change several global variables that were
arrays or structures to pointers to same.  For these variables,
modified initialization and cleanup code malloc's and free's
the arrays/structures.  This was done so that the vnet footprint
would be about 300bytes when the macros are defined as vnet macros,
allowing nfsd.ko to load dynamically.

I believe the comments in D37519 have been addressed, although
it has never been reviewed, due in part to the large size of the patch.
This is the first of a series of patches that will put D37519 in main.

Once everything is in main, the macros will be defined as front
end macros to the vnet ones.

MFC after:	3 months
Differential Revision:	https://reviews.freebsd.org/D37519

9 files changed, 524 insertions, 347 deletions

diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index f8fd2229095f..0a6d46964ffd 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -122,7 +122,7 @@ int nfsrv_delegupdate(struct nfsrv_descript *, nfsquad_t, nfsv4stateid_t *,
 int nfsrv_releaselckown(struct nfsstate *, nfsquad_t, NFSPROC_T *);
 void nfsrv_zapclient(struct nfsclient *, NFSPROC_T *);
 int nfssvc_idname(struct nfsd_idargs *);
-void nfsrv_servertimer(void);
+void nfsrv_servertimer(void * __unused);
 int nfsrv_getclientipaddr(struct nfsrv_descript *, struct nfsclient *);
 void nfsrv_setupstable(NFSPROC_T *);
 void nfsrv_updatestable(NFSPROC_T *);
diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h
index 2060ed03de34..bf59aea285f1 100644
--- a/sys/fs/nfs/nfsport.h
+++ b/sys/fs/nfs/nfsport.h
@@ -181,6 +181,20 @@
  */
 #define	NFSMUTEX_T		struct mtx
 
+/* Define the NFSD_VNET macros similar to !VIMAGE. */
+#define	NFSD_VNET_NAME(n)		n
+#define	NFSD_VNET_DECLARE(t, n)		extern t n
+#define	NFSD_VNET_DEFINE(t, n)		t n
+#define	NFSD_VNET_DEFINE_STATIC(t, n)	static t n
+#define	NFSD_VNET(n)			(n)
+
+#define	CTLFLAG_NFSD_VNET		0
+
+#define	NFSD_CURVNET_SET(n)
+#define	NFSD_CURVNET_SET_QUIET(n)
+#define	NFSD_CURVNET_RESTORE()
+#define	NFSD_TD_TO_VNET(n)		NULL
+
 #endif	/* _KERNEL */
 
 /*
diff --git a/sys/fs/nfsserver/nfs_fha_new.c b/sys/fs/nfsserver/nfs_fha_new.c
index 1f66842da5b8..59933ef01bd4 100644
--- a/sys/fs/nfsserver/nfs_fha_new.c
+++ b/sys/fs/nfsserver/nfs_fha_new.c
@@ -61,7 +61,8 @@ static struct fha_params fhanew_softc;
 SYSCTL_DECL(_vfs_nfsd);
 
 extern int newnfs_nfsv3_procid[];
-extern SVCPOOL	*nfsrvd_pool;
+
+NFSD_VNET_DECLARE(SVCPOOL *, nfsrvd_pool);
 
 SYSINIT(nfs_fhanew, SI_SUB_ROOT_CONF, SI_ORDER_ANY, fhanew_init, NULL);
 SYSUNINIT(nfs_fhanew, SI_SUB_ROOT_CONF, SI_ORDER_ANY, fhanew_uninit, NULL);
@@ -79,7 +80,7 @@ fhanew_init(void *foo)
 	snprintf(softc->server_name, sizeof(softc->server_name),
 	    FHANEW_SERVER_NAME);
 
-	softc->pool = &nfsrvd_pool;
+	softc->pool = &NFSD_VNET(nfsrvd_pool);
 
 	/*
 	 * Initialize the sysctl context list for the fha module.
diff --git a/sys/fs/nfsserver/nfs_nfsdcache.c b/sys/fs/nfsserver/nfs_nfsdcache.c
index 5efcb90411e0..e94d58481286 100644
--- a/sys/fs/nfsserver/nfs_nfsdcache.c
+++ b/sys/fs/nfsserver/nfs_nfsdcache.c
@@ -160,11 +160,15 @@ __FBSDID("$FreeBSD$");
  */
 #include <fs/nfs/nfsport.h>
 
-extern struct nfsstatsv1 nfsstatsv1;
 extern struct mtx nfsrc_udpmtx;
-extern struct nfsrchash_bucket nfsrchash_table[NFSRVCACHE_HASHSIZE];
-extern struct nfsrchash_bucket nfsrcahash_table[NFSRVCACHE_HASHSIZE];
-int nfsrc_floodlevel = NFSRVCACHE_FLOODLEVEL, nfsrc_tcpsavedreplies = 0;
+
+NFSD_VNET_DECLARE(struct nfsrvhashhead *, nfsrvudphashtbl);
+NFSD_VNET_DECLARE(struct nfsrchash_bucket *, nfsrchash_table);
+NFSD_VNET_DECLARE(struct nfsrchash_bucket *, nfsrcahash_table);
+NFSD_VNET_DECLARE(struct nfsstatsv1 *, nfsstatsv1_p);
+
+NFSD_VNET_DEFINE(int, nfsrc_floodlevel) = NFSRVCACHE_FLOODLEVEL;
+NFSD_VNET_DEFINE(int, nfsrc_tcpsavedreplies) = 0;
 
 SYSCTL_DECL(_vfs_nfsd);
 
@@ -180,8 +184,8 @@ sysctl_tcphighwater(SYSCTL_HANDLER_ARGS)
 		return (error);
 	if (newhighwater < 0)
 		return (EINVAL);
-	if (newhighwater >= nfsrc_floodlevel)
-		nfsrc_floodlevel = newhighwater + newhighwater / 5;
+	if (newhighwater >= NFSD_VNET(nfsrc_floodlevel))
+		NFSD_VNET(nfsrc_floodlevel) = newhighwater + newhighwater / 5;
 	nfsrc_tcphighwater = newhighwater;
 	return (0);
 }
@@ -202,9 +206,8 @@ SYSCTL_UINT(_vfs_nfsd, OID_AUTO, cachetcp, CTLFLAG_RW,
     &nfsrc_tcpnonidempotent, 0,
     "Enable the DRC for NFS over TCP");
 
-static int nfsrc_udpcachesize = 0;
-static TAILQ_HEAD(, nfsrvcache) nfsrvudplru;
-static struct nfsrvhashhead nfsrvudphashtbl[NFSRVCACHE_HASHSIZE];
+NFSD_VNET_DEFINE_STATIC(int, nfsrc_udpcachesize) = 0;
+NFSD_VNET_DEFINE_STATIC(TAILQ_HEAD(, nfsrvcache), nfsrvudplru);
 
 /*
  * and the reverse mapping from generic to Version 2 procedure numbers
@@ -236,10 +239,10 @@ static int newnfsv2_procid[NFS_V3NPROCS] = {
 
 #define	nfsrc_hash(xid)	(((xid) + ((xid) >> 24)) % NFSRVCACHE_HASHSIZE)
 #define	NFSRCUDPHASH(xid) \
-	(&nfsrvudphashtbl[nfsrc_hash(xid)])
+	(&NFSD_VNET(nfsrvudphashtbl)[nfsrc_hash(xid)])
 #define	NFSRCHASH(xid) \
-	(&nfsrchash_table[nfsrc_hash(xid)].tbl)
-#define	NFSRCAHASH(xid) (&nfsrcahash_table[nfsrc_hash(xid)])
+	(&NFSD_VNET(nfsrchash_table)[nfsrc_hash(xid)].tbl)
+#define	NFSRCAHASH(xid) (&NFSD_VNET(nfsrcahash_table)[nfsrc_hash(xid)])
 #define	TRUE	1
 #define	FALSE	0
 #define	NFSRVCACHE_CHECKLEN	100
@@ -295,7 +298,7 @@ nfsrc_cachemutex(struct nfsrvcache *rp)
 
 	if ((rp->rc_flag & RC_UDP) != 0)
 		return (&nfsrc_udpmtx);
-	return (&nfsrchash_table[nfsrc_hash(rp->rc_xid)].mtx);
+	return (&NFSD_VNET(nfsrchash_table)[nfsrc_hash(rp->rc_xid)].mtx);
 }
 
 /*
@@ -305,21 +308,27 @@ void
 nfsrvd_initcache(void)
 {
 	int i;
-	static int inited = 0;
 
-	if (inited)
-		return;
-	inited = 1;
+	NFSD_VNET(nfsrvudphashtbl) = malloc(sizeof(struct nfsrvhashhead) *
+	    NFSRVCACHE_HASHSIZE, M_NFSRVCACHE, M_WAITOK | M_ZERO);
+	NFSD_VNET(nfsrchash_table) = malloc(sizeof(struct nfsrchash_bucket) *
+	    NFSRVCACHE_HASHSIZE, M_NFSRVCACHE, M_WAITOK | M_ZERO);
+	NFSD_VNET(nfsrcahash_table) = malloc(sizeof(struct nfsrchash_bucket) *
+	    NFSRVCACHE_HASHSIZE, M_NFSRVCACHE, M_WAITOK | M_ZERO);
+	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
+		mtx_init(&NFSD_VNET(nfsrchash_table)[i].mtx, "nfsrtc", NULL,
+		    MTX_DEF);
+		mtx_init(&NFSD_VNET(nfsrcahash_table)[i].mtx, "nfsrtca", NULL,
+		    MTX_DEF);
+	}
 	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-		LIST_INIT(&nfsrvudphashtbl[i]);
-		LIST_INIT(&nfsrchash_table[i].tbl);
-		LIST_INIT(&nfsrcahash_table[i].tbl);
+		LIST_INIT(&NFSD_VNET(nfsrvudphashtbl)[i]);
+		LIST_INIT(&NFSD_VNET(nfsrchash_table)[i].tbl);
+		LIST_INIT(&NFSD_VNET(nfsrcahash_table)[i].tbl);
 	}
-	TAILQ_INIT(&nfsrvudplru);
-	nfsrc_tcpsavedreplies = 0;
-	nfsrc_udpcachesize = 0;
-	nfsstatsv1.srvcache_tcppeak = 0;
-	nfsstatsv1.srvcache_size = 0;
+	TAILQ_INIT(&NFSD_VNET(nfsrvudplru));
+	NFSD_VNET(nfsrc_tcpsavedreplies) = 0;
+	NFSD_VNET(nfsrc_udpcachesize) = 0;
 }
 
 /*
@@ -392,17 +401,17 @@ loop:
 			if (rp->rc_flag == 0)
 				panic("nfs udp cache0");
 			rp->rc_flag |= RC_LOCKED;
-			TAILQ_REMOVE(&nfsrvudplru, rp, rc_lru);
-			TAILQ_INSERT_TAIL(&nfsrvudplru, rp, rc_lru);
+			TAILQ_REMOVE(&NFSD_VNET(nfsrvudplru), rp, rc_lru);
+			TAILQ_INSERT_TAIL(&NFSD_VNET(nfsrvudplru), rp, rc_lru);
 			if (rp->rc_flag & RC_INPROG) {
-				nfsstatsv1.srvcache_inproghits++;
+				nfsstatsv1_p->srvcache_inproghits++;
 				mtx_unlock(mutex);
 				ret = RC_DROPIT;
 			} else if (rp->rc_flag & RC_REPSTATUS) {
 				/*
 				 * V2 only.
 				 */
-				nfsstatsv1.srvcache_nonidemdonehits++;
+				nfsstatsv1_p->srvcache_nonidemdonehits++;
 				mtx_unlock(mutex);
 				nfsrvd_rephead(nd);
 				*(nd->nd_errp) = rp->rc_status;
@@ -410,7 +419,7 @@ loop:
 				rp->rc_timestamp = NFSD_MONOSEC +
 					NFSRVCACHE_UDPTIMEOUT;
 			} else if (rp->rc_flag & RC_REPMBUF) {
-				nfsstatsv1.srvcache_nonidemdonehits++;
+				nfsstatsv1_p->srvcache_nonidemdonehits++;
 				mtx_unlock(mutex);
 				nd->nd_mreq = m_copym(rp->rc_reply, 0,
 					M_COPYALL, M_WAITOK);
@@ -425,9 +434,9 @@ loop:
 			goto out;
 		}
 	}
-	nfsstatsv1.srvcache_misses++;
-	atomic_add_int(&nfsstatsv1.srvcache_size, 1);
-	nfsrc_udpcachesize++;
+	nfsstatsv1_p->srvcache_misses++;
+	atomic_add_int(&nfsstatsv1_p->srvcache_size, 1);
+	NFSD_VNET(nfsrc_udpcachesize)++;
 
 	newrp->rc_flag |= RC_INPROG;
 	saddr = NFSSOCKADDR(nd->nd_nam, struct sockaddr_in *);
@@ -440,7 +449,7 @@ loop:
 		newrp->rc_flag |= RC_INETIPV6;
 	}
 	LIST_INSERT_HEAD(hp, newrp, rc_hash);
-	TAILQ_INSERT_TAIL(&nfsrvudplru, newrp, rc_lru);
+	TAILQ_INSERT_TAIL(&NFSD_VNET(nfsrvudplru), newrp, rc_lru);
 	mtx_unlock(mutex);
 	nd->nd_rp = newrp;
 	ret = RC_DOIT;
@@ -472,15 +481,15 @@ nfsrvd_updatecache(struct nfsrv_descript *nd)
 		panic("nfsrvd_updatecache not inprog");
 	rp->rc_flag &= ~RC_INPROG;
 	if (rp->rc_flag & RC_UDP) {
-		TAILQ_REMOVE(&nfsrvudplru, rp, rc_lru);
-		TAILQ_INSERT_TAIL(&nfsrvudplru, rp, rc_lru);
+		TAILQ_REMOVE(&NFSD_VNET(nfsrvudplru), rp, rc_lru);
+		TAILQ_INSERT_TAIL(&NFSD_VNET(nfsrvudplru), rp, rc_lru);
 	}
 
 	/*
 	 * Reply from cache is a special case returned by nfsrv_checkseqid().
 	 */
 	if (nd->nd_repstat == NFSERR_REPLYFROMCACHE) {
-		nfsstatsv1.srvcache_nonidemdonehits++;
+		nfsstatsv1_p->srvcache_nonidemdonehits++;
 		mtx_unlock(mutex);
 		nd->nd_repstat = 0;
 		if (nd->nd_mreq)
@@ -503,7 +512,7 @@ nfsrvd_updatecache(struct nfsrv_descript *nd)
 	    (rp->rc_refcnt > 0 ||
 	     ((nd->nd_flag & ND_SAVEREPLY) && (rp->rc_flag & RC_UDP)) ||
 	     ((nd->nd_flag & ND_SAVEREPLY) && !(rp->rc_flag & RC_UDP) &&
-	      nfsrc_tcpsavedreplies <= nfsrc_floodlevel &&
+	      NFSD_VNET(nfsrc_tcpsavedreplies) <= NFSD_VNET(nfsrc_floodlevel) &&
 	      nfsrc_tcpnonidempotent))) {
 		if (rp->rc_refcnt > 0) {
 			if (!(rp->rc_flag & RC_NFSV4))
@@ -517,11 +526,12 @@ nfsrvd_updatecache(struct nfsrv_descript *nd)
 			mtx_unlock(mutex);
 		} else {
 			if (!(rp->rc_flag & RC_UDP)) {
-			    atomic_add_int(&nfsrc_tcpsavedreplies, 1);
-			    if (nfsrc_tcpsavedreplies >
-				nfsstatsv1.srvcache_tcppeak)
-				nfsstatsv1.srvcache_tcppeak =
-				    nfsrc_tcpsavedreplies;
+			    atomic_add_int(&NFSD_VNET(nfsrc_tcpsavedreplies),
+				1);
+			    if (NFSD_VNET(nfsrc_tcpsavedreplies) >
+				nfsstatsv1_p->srvcache_tcppeak)
+				nfsstatsv1_p->srvcache_tcppeak =
+				    NFSD_VNET(nfsrc_tcpsavedreplies);
 			}
 			mtx_unlock(mutex);
 			m = m_copym(nd->nd_mreq, 0, M_COPYALL, M_WAITOK);
@@ -678,7 +688,7 @@ tryagain:
 			panic("nfs tcp cache0");
 		rp->rc_flag |= RC_LOCKED;
 		if (rp->rc_flag & RC_INPROG) {
-			nfsstatsv1.srvcache_inproghits++;
+			nfsstatsv1_p->srvcache_inproghits++;
 			mtx_unlock(mutex);
 			if (newrp->rc_sockref == rp->rc_sockref)
 				nfsrc_marksametcpconn(rp->rc_sockref);
@@ -687,7 +697,7 @@ tryagain:
 			/*
 			 * V2 only.
 			 */
-			nfsstatsv1.srvcache_nonidemdonehits++;
+			nfsstatsv1_p->srvcache_nonidemdonehits++;
 			mtx_unlock(mutex);
 			if (newrp->rc_sockref == rp->rc_sockref)
 				nfsrc_marksametcpconn(rp->rc_sockref);
@@ -696,7 +706,7 @@ tryagain:
 			*(nd->nd_errp) = rp->rc_status;
 			rp->rc_timestamp = NFSD_MONOSEC + nfsrc_tcptimeout;
 		} else if (rp->rc_flag & RC_REPMBUF) {
-			nfsstatsv1.srvcache_nonidemdonehits++;
+			nfsstatsv1_p->srvcache_nonidemdonehits++;
 			mtx_unlock(mutex);
 			if (newrp->rc_sockref == rp->rc_sockref)
 				nfsrc_marksametcpconn(rp->rc_sockref);
@@ -711,8 +721,8 @@ tryagain:
 		free(newrp, M_NFSRVCACHE);
 		goto out;
 	}
-	nfsstatsv1.srvcache_misses++;
-	atomic_add_int(&nfsstatsv1.srvcache_size, 1);
+	nfsstatsv1_p->srvcache_misses++;
+	atomic_add_int(&nfsstatsv1_p->srvcache_size, 1);
 
 	/*
 	 * For TCP, multiple entries for a key are allowed, so don't
@@ -785,8 +795,8 @@ nfsrc_freecache(struct nfsrvcache *rp)
 
 	LIST_REMOVE(rp, rc_hash);
 	if (rp->rc_flag & RC_UDP) {
-		TAILQ_REMOVE(&nfsrvudplru, rp, rc_lru);
-		nfsrc_udpcachesize--;
+		TAILQ_REMOVE(&NFSD_VNET(nfsrvudplru), rp, rc_lru);
+		NFSD_VNET(nfsrc_udpcachesize)--;
 	} else if (rp->rc_acked != RC_NO_SEQ) {
 		hbp = NFSRCAHASH(rp->rc_sockref);
 		mtx_lock(&hbp->mtx);
@@ -798,10 +808,10 @@ nfsrc_freecache(struct nfsrvcache *rp)
 	if (rp->rc_flag & RC_REPMBUF) {
 		m_freem(rp->rc_reply);
 		if (!(rp->rc_flag & RC_UDP))
-			atomic_add_int(&nfsrc_tcpsavedreplies, -1);
+			atomic_add_int(&NFSD_VNET(nfsrc_tcpsavedreplies), -1);
 	}
 	free(rp, M_NFSRVCACHE);
-	atomic_add_int(&nfsstatsv1.srvcache_size, -1);
+	atomic_add_int(&nfsstatsv1_p->srvcache_size, -1);
 }
 
 /*
@@ -814,20 +824,22 @@ nfsrvd_cleancache(void)
 	int i;
 
 	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-		mtx_lock(&nfsrchash_table[i].mtx);
-		LIST_FOREACH_SAFE(rp, &nfsrchash_table[i].tbl, rc_hash, nextrp)
+		mtx_lock(&NFSD_VNET(nfsrchash_table)[i].mtx);
+		LIST_FOREACH_SAFE(rp, &NFSD_VNET(nfsrchash_table)[i].tbl,
+		    rc_hash, nextrp)
 			nfsrc_freecache(rp);
-		mtx_unlock(&nfsrchash_table[i].mtx);
+		mtx_unlock(&NFSD_VNET(nfsrchash_table)[i].mtx);
 	}
 	mtx_lock(&nfsrc_udpmtx);
 	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-		LIST_FOREACH_SAFE(rp, &nfsrvudphashtbl[i], rc_hash, nextrp) {
+		LIST_FOREACH_SAFE(rp, &NFSD_VNET(nfsrvudphashtbl)[i], rc_hash,
+		    nextrp) {
 			nfsrc_freecache(rp);
 		}
 	}
-	nfsstatsv1.srvcache_size = 0;
+	nfsstatsv1_p->srvcache_size = 0;
 	mtx_unlock(&nfsrc_udpmtx);
-	nfsrc_tcpsavedreplies = 0;
+	NFSD_VNET(nfsrc_tcpsavedreplies) = 0;
 }
 
 #define HISTSIZE	16
@@ -864,25 +876,28 @@ nfsrc_trimcache(u_int64_t sockref, uint32_t snd_una, int final)
 	if (atomic_cmpset_acq_int(&onethread, 0, 1) == 0)
 		return;
 	if (NFSD_MONOSEC != udp_lasttrim ||
-	    nfsrc_udpcachesize >= (nfsrc_udphighwater +
+	    NFSD_VNET(nfsrc_udpcachesize) >= (nfsrc_udphighwater +
 	    nfsrc_udphighwater / 2)) {
 		mtx_lock(&nfsrc_udpmtx);
 		udp_lasttrim = NFSD_MONOSEC;
-		TAILQ_FOREACH_SAFE(rp, &nfsrvudplru, rc_lru, nextrp) {
+		TAILQ_FOREACH_SAFE(rp, &NFSD_VNET(nfsrvudplru), rc_lru,
+		    nextrp) {
 			if (!(rp->rc_flag & (RC_INPROG|RC_LOCKED|RC_WANTED))
 			     && rp->rc_refcnt == 0
 			     && ((rp->rc_flag & RC_REFCNT) ||
 				 udp_lasttrim > rp->rc_timestamp ||
-				 nfsrc_udpcachesize > nfsrc_udphighwater))
+				 NFSD_VNET(nfsrc_udpcachesize) >
+				 nfsrc_udphighwater))
 				nfsrc_freecache(rp);
 		}
 		mtx_unlock(&nfsrc_udpmtx);
 	}
 	if (NFSD_MONOSEC != tcp_lasttrim ||
-	    nfsrc_tcpsavedreplies >= nfsrc_tcphighwater) {
+	    NFSD_VNET(nfsrc_tcpsavedreplies) >= nfsrc_tcphighwater) {
 		force = nfsrc_tcphighwater / 4;
 		if (force > 0 &&
-		    nfsrc_tcpsavedreplies + force >= nfsrc_tcphighwater) {
+		    NFSD_VNET(nfsrc_tcpsavedreplies) + force >=
+		    nfsrc_tcphighwater) {
 			for (i = 0; i < HISTSIZE; i++)
 				time_histo[i] = 0;
 			i = 0;
@@ -901,8 +916,9 @@ nfsrc_trimcache(u_int64_t sockref, uint32_t snd_una, int final)
 		tto = nfsrc_tcptimeout;
 		tcp_lasttrim = NFSD_MONOSEC;
 		for (; i <= lastslot; i++) {
-			mtx_lock(&nfsrchash_table[i].mtx);
-			LIST_FOREACH_SAFE(rp, &nfsrchash_table[i].tbl, rc_hash,
+			mtx_lock(&NFSD_VNET(nfsrchash_table)[i].mtx);
+			LIST_FOREACH_SAFE(rp,
+			    &NFSD_VNET(nfsrchash_table)[i].tbl, rc_hash,
 			    nextrp) {
 				if (!(rp->rc_flag &
 				     (RC_INPROG|RC_LOCKED|RC_WANTED))
@@ -932,7 +948,7 @@ nfsrc_trimcache(u_int64_t sockref, uint32_t snd_una, int final)
 					time_histo[j]++;
 				}
 			}
-			mtx_unlock(&nfsrchash_table[i].mtx);
+			mtx_unlock(&NFSD_VNET(nfsrchash_table)[i].mtx);
 		}
 		if (force) {
 			/*
@@ -951,8 +967,9 @@ nfsrc_trimcache(u_int64_t sockref, uint32_t snd_una, int final)
 				k = 1;
 			thisstamp = tcp_lasttrim + k;
 			for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-				mtx_lock(&nfsrchash_table[i].mtx);
-				LIST_FOREACH_SAFE(rp, &nfsrchash_table[i].tbl,
+				mtx_lock(&NFSD_VNET(nfsrchash_table)[i].mtx);
+				LIST_FOREACH_SAFE(rp,
+				    &NFSD_VNET(nfsrchash_table)[i].tbl,
 				    rc_hash, nextrp) {
 					if (!(rp->rc_flag &
 					     (RC_INPROG|RC_LOCKED|RC_WANTED))
@@ -962,7 +979,7 @@ nfsrc_trimcache(u_int64_t sockref, uint32_t snd_una, int final)
 						 rp->rc_acked == RC_ACK))
 						nfsrc_freecache(rp);
 				}
-				mtx_unlock(&nfsrchash_table[i].mtx);
+				mtx_unlock(&NFSD_VNET(nfsrchash_table)[i].mtx);
 			}
 		}
 	}
diff --git a/sys/fs/nfsserver/nfs_nfsdkrpc.c b/sys/fs/nfsserver/nfs_nfsdkrpc.c
index d2ba7887b8c2..696af8a0e409 100644
--- a/sys/fs/nfsserver/nfs_nfsdkrpc.c
+++ b/sys/fs/nfsserver/nfs_nfsdkrpc.c
@@ -52,7 +52,6 @@ __FBSDID("$FreeBSD$");
 
 NFSDLOCKMUTEX;
 NFSV4ROOTLOCKMUTEX;
-struct nfsv4lock nfsd_suspend_lock;
 char *nfsrv_zeropnfsdat = NULL;
 
 /*
@@ -85,32 +84,39 @@ int newnfs_nfsv3_procid[NFS_V3NPROCS] = {
 
 SYSCTL_DECL(_vfs_nfsd);
 
-SVCPOOL		*nfsrvd_pool;
-
-static int	nfs_privport = 0;
-SYSCTL_INT(_vfs_nfsd, OID_AUTO, nfs_privport, CTLFLAG_RWTUN,
-    &nfs_privport, 0,
+NFSD_VNET_DEFINE_STATIC(int, nfs_privport) = 0;
+SYSCTL_INT(_vfs_nfsd, OID_AUTO, nfs_privport, CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN,
+    &NFSD_VNET_NAME(nfs_privport), 0,
     "Only allow clients using a privileged port for NFSv2, 3 and 4");
 
-static int	nfs_minvers = NFS_VER2;
-SYSCTL_INT(_vfs_nfsd, OID_AUTO, server_min_nfsvers, CTLFLAG_RWTUN,
-    &nfs_minvers, 0, "The lowest version of NFS handled by the server");
+NFSD_VNET_DEFINE_STATIC(int, nfs_minvers) = NFS_VER2;
+SYSCTL_INT(_vfs_nfsd, OID_AUTO, server_min_nfsvers,
+    CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN, &NFSD_VNET_NAME(nfs_minvers), 0,
+    "The lowest version of NFS handled by the server");
 
-static int	nfs_maxvers = NFS_VER4;
-SYSCTL_INT(_vfs_nfsd, OID_AUTO, server_max_nfsvers, CTLFLAG_RWTUN,
-    &nfs_maxvers, 0, "The highest version of NFS handled by the server");
+NFSD_VNET_DEFINE_STATIC(int, nfs_maxvers) = NFS_VER4;
+SYSCTL_INT(_vfs_nfsd, OID_AUTO, server_max_nfsvers,
+    CTLFLAG_NFSD_VNET | CTLFLAG_RWTUN, &NFSD_VNET_NAME(nfs_maxvers), 0,
+    "The highest version of NFS handled by the server");
 
 static int nfs_proc(struct nfsrv_descript *, u_int32_t, SVCXPRT *xprt,
     struct nfsrvcache **);
 
 extern u_long sb_max_adj;
 extern int newnfs_numnfsd;
-extern struct proc *nfsd_master_proc;
 extern time_t nfsdev_time;
 extern int nfsrv_writerpc[NFS_NPROCS];
 extern volatile int nfsrv_devidcnt;
 extern struct nfsv4_opflag nfsv4_opflag[NFSV42_NOPS];
 
+NFSD_VNET_DECLARE(struct proc *, nfsd_master_proc);
+
+NFSD_VNET_DEFINE(SVCPOOL *, nfsrvd_pool);
+NFSD_VNET_DEFINE(int, nfsrv_numnfsd) = 0;
+NFSD_VNET_DEFINE(struct nfsv4lock, nfsd_suspend_lock);
+
+NFSD_VNET_DEFINE_STATIC(bool, nfsrvd_inited) = false;
+
 /*
  * NFS server system calls
  */
@@ -125,6 +131,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 	u_int maxlen;
 #endif
 
+	NFSD_CURVNET_SET_QUIET(NFSD_TD_TO_VNET(curthread));
 	memset(&nd, 0, sizeof(nd));
 	if (rqst->rq_vers == NFS_VER2) {
 		if (rqst->rq_proc > NFSV2PROC_STATFS ||
@@ -169,7 +176,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 	nd.nd_mreq = NULL;
 	nd.nd_cred = NULL;
 
-	if (nfs_privport != 0) {
+	if (NFSD_VNET(nfs_privport) != 0) {
 		/* Check if source port is privileged */
 		u_short port;
 		struct sockaddr *nam = nd.nd_nam;
@@ -261,17 +268,17 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 		 * nfsv4root exports by nfsvno_v4rootexport().
 		 */
 		NFSLOCKV4ROOTMUTEX();
-		nfsv4_lock(&nfsd_suspend_lock, 0, NULL, NFSV4ROOTLOCKMUTEXPTR,
-		    NULL);
-		nfsv4_getref(&nfsd_suspend_lock, NULL, NFSV4ROOTLOCKMUTEXPTR,
-		    NULL);
+		nfsv4_lock(&NFSD_VNET(nfsd_suspend_lock), 0, NULL,
+		    NFSV4ROOTLOCKMUTEXPTR, NULL);
+		nfsv4_getref(&NFSD_VNET(nfsd_suspend_lock), NULL,
+		    NFSV4ROOTLOCKMUTEXPTR, NULL);
 		NFSUNLOCKV4ROOTMUTEX();
 
 		if ((nd.nd_flag & ND_NFSV4) != 0) {
 			nd.nd_repstat = nfsvno_v4rootexport(&nd);
 			if (nd.nd_repstat != 0) {
 				NFSLOCKV4ROOTMUTEX();
-				nfsv4_relref(&nfsd_suspend_lock);
+				nfsv4_relref(&NFSD_VNET(nfsd_suspend_lock));
 				NFSUNLOCKV4ROOTMUTEX();
 				svcerr_weakauth(rqst);
 				svc_freereq(rqst);
@@ -287,7 +294,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 #endif
 		cacherep = nfs_proc(&nd, rqst->rq_xid, xprt, &rp);
 		NFSLOCKV4ROOTMUTEX();
-		nfsv4_relref(&nfsd_suspend_lock);
+		nfsv4_relref(&NFSD_VNET(nfsd_suspend_lock));
 		NFSUNLOCKV4ROOTMUTEX();
 	} else {
 		NFSMGET(nd.nd_mreq);
@@ -327,6 +334,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 	svc_freereq(rqst);
 
 out:
+	NFSD_CURVNET_RESTORE();
 	ast_kclear(curthread);
 	NFSEXITCODE(0);
 }
@@ -440,7 +448,9 @@ nfssvc_loss(SVCXPRT *xprt)
 
 	ack = 0;
 	SVC_ACK(xprt, &ack);
+	NFSD_CURVNET_SET(NFSD_TD_TO_VNET(curthread));
 	nfsrc_trimcache(xprt->xp_sockref, ack, 1);
+	NFSD_CURVNET_RESTORE();
 }
 
 /*
@@ -467,26 +477,28 @@ nfsrvd_addsock(struct file *fp)
 	 * unexpectedly.
 	 */
 	if (so->so_type == SOCK_DGRAM)
-		xprt = svc_dg_create(nfsrvd_pool, so, 0, 0);
+		xprt = svc_dg_create(NFSD_VNET(nfsrvd_pool), so, 0, 0);
 	else
-		xprt = svc_vc_create(nfsrvd_pool, so, 0, 0);
+		xprt = svc_vc_create(NFSD_VNET(nfsrvd_pool), so, 0, 0);
 	if (xprt) {
 		fp->f_ops = &badfileops;
 		fp->f_data = NULL;
 		xprt->xp_sockref = ++sockref;
-		if (nfs_minvers == NFS_VER2)
+		if (NFSD_VNET(nfs_minvers) == NFS_VER2)
 			svc_reg(xprt, NFS_PROG, NFS_VER2, nfssvc_program,
 			    NULL);
-		if (nfs_minvers <= NFS_VER3 && nfs_maxvers >= NFS_VER3)
+		if (NFSD_VNET(nfs_minvers) <= NFS_VER3 &&
+		    NFSD_VNET(nfs_maxvers) >= NFS_VER3)
 			svc_reg(xprt, NFS_PROG, NFS_VER3, nfssvc_program,
 			    NULL);
-		if (nfs_maxvers >= NFS_VER4)
+		if (NFSD_VNET(nfs_maxvers) >= NFS_VER4)
 			svc_reg(xprt, NFS_PROG, NFS_VER4, nfssvc_program,
 			    NULL);
 		if (so->so_type == SOCK_STREAM)
 			svc_loss_reg(xprt, nfssvc_loss);
 		SVC_RELEASE(xprt);
-	}
+	} else
+		error = EPERM;
 
 out:
 	NFSEXITCODE(error);
@@ -518,13 +530,15 @@ nfsrvd_nfsd(struct thread *td, struct nfsd_nfsd_args *args)
 	 * use.
 	 */
 	NFSD_LOCK();
-	if (newnfs_numnfsd == 0) {
+	if (NFSD_VNET(nfsrv_numnfsd) == 0) {
+		nfsrvd_init(0);
 		nfsdev_time = time_second;
 		p = td->td_proc;
 		PROC_LOCK(p);
 		p->p_flag2 |= P2_AST_SU;
 		PROC_UNLOCK(p);
-		newnfs_numnfsd++;
+		newnfs_numnfsd++;	/* Total num for all vnets. */
+		NFSD_VNET(nfsrv_numnfsd)++;	/* Num for this vnet. */
 
 		NFSD_UNLOCK();
 		error = nfsrv_createdevids(args, td);
@@ -542,12 +556,15 @@ nfsrvd_nfsd(struct thread *td, struct nfsd_nfsd_args *args)
 				    NFS_VER4);
 
 				if (!ret2 || !ret3 || !ret4)
-					printf(
-					    "nfsd: can't register svc name\n");
+					printf("nfsd: can't register svc "
+					    "name %s jid:%d\n", principal,
+					    td->td_ucred->cr_prison->pr_id);
 			}
 
-			nfsrvd_pool->sp_minthreads = args->minthreads;
-			nfsrvd_pool->sp_maxthreads = args->maxthreads;
+			NFSD_VNET(nfsrvd_pool)->sp_minthreads =
+			    args->minthreads;
+			NFSD_VNET(nfsrvd_pool)->sp_maxthreads =
+			    args->maxthreads;
 				
 			/*
 			 * If this is a pNFS service, make Getattr do a
@@ -558,7 +575,7 @@ nfsrvd_nfsd(struct thread *td, struct nfsd_nfsd_args *args)
 				nfsv4_opflag[NFSV4OP_GETATTR].modifyfs = 1;
 			}
 
-			svc_run(nfsrvd_pool);
+			svc_run(NFSD_VNET(nfsrvd_pool));
 
 			/* Reset Getattr to not do a vn_start_write(). */
 			nfsrv_writerpc[NFSPROC_GETATTR] = 0;
@@ -572,6 +589,7 @@ nfsrvd_nfsd(struct thread *td, struct nfsd_nfsd_args *args)
 		}
 		NFSD_LOCK();
 		newnfs_numnfsd--;
+		NFSD_VNET(nfsrv_numnfsd)--;
 		nfsrvd_init(1);
 		PROC_LOCK(p);
 		p->p_flag2 &= ~P2_AST_SU;
@@ -596,21 +614,25 @@ nfsrvd_init(int terminating)
 	NFSD_LOCK_ASSERT();
 
 	if (terminating) {
-		nfsd_master_proc = NULL;
+		NFSD_VNET(nfsd_master_proc) = NULL;
 		NFSD_UNLOCK();
 		nfsrv_freealllayoutsanddevids();
 		nfsrv_freeallbackchannel_xprts();
-		svcpool_close(nfsrvd_pool);
+		svcpool_close(NFSD_VNET(nfsrvd_pool));
 		free(nfsrv_zeropnfsdat, M_TEMP);
 		nfsrv_zeropnfsdat = NULL;
 		NFSD_LOCK();
 	} else {
+		/* Initialize per-vnet globals once per vnet. */
+		if (NFSD_VNET(nfsrvd_inited))
+			return;
+		NFSD_VNET(nfsrvd_inited) = true;
 		NFSD_UNLOCK();
-		nfsrvd_pool = svcpool_create("nfsd",
+		NFSD_VNET(nfsrvd_pool) = svcpool_create("nfsd",
 		    SYSCTL_STATIC_CHILDREN(_vfs_nfsd));
-		nfsrvd_pool->sp_rcache = NULL;
-		nfsrvd_pool->sp_assign = fhanew_assign;
-		nfsrvd_pool->sp_done = fhanew_nd_complete;
+		NFSD_VNET(nfsrvd_pool)->sp_rcache = NULL;
+		NFSD_VNET(nfsrvd_pool)->sp_assign = fhanew_assign;
+		NFSD_VNET(nfsrvd_pool)->sp_done = fhanew_nd_complete;
 		NFSD_LOCK();
 	}
 }
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index 9cd7f9320bd5..af5327bd5a2d 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/callout.h>
 #include <sys/filio.h>
 #include <sys/hash.h>
+#include <sys/osd.h>
 #include <sys/sysctl.h>
 #include <nlm/nlm_prot.h>
 #include <nlm/nlm.h>
@@ -59,45 +60,57 @@ FEATURE(nfsd, "NFSv4 server");
 extern u_int32_t newnfs_true, newnfs_false, newnfs_xdrneg1;
 extern int nfsrv_useacl;
 extern int newnfs_numnfsd;
-extern struct mount nfsv4root_mnt;
-extern struct nfsrv_stablefirst nfsrv_stablefirst;
-extern SVCPOOL	*nfsrvd_pool;
-extern struct nfsv4lock nfsd_suspend_lock;
-extern struct nfsclienthashhead *nfsclienthash;
-extern struct nfslockhashhead *nfslockhash;
-extern struct nfssessionhash *nfssessionhash;
 extern int nfsrv_sessionhashsize;
 extern struct nfsstatsv1 nfsstatsv1;
 extern struct nfslayouthash *nfslayouthash;
 extern int nfsrv_layouthashsize;
 extern struct mtx nfsrv_dslock_mtx;
 extern int nfs_pnfsiothreads;
-extern struct nfsdontlisthead nfsrv_dontlisthead;
-extern volatile int nfsrv_dontlistlen;
 extern volatile int nfsrv_devidcnt;
 extern int nfsrv_maxpnfsmirror;
 extern uint32_t nfs_srvmaxio;
 extern int nfs_bufpackets;
 extern u_long sb_max_adj;
-struct vfsoptlist nfsv4root_opt, nfsv4root_newopt;
+
+NFSD_VNET_DECLARE(int, nfsrv_numnfsd);
+NFSD_VNET_DECLARE(struct nfsrv_stablefirst, nfsrv_stablefirst);
+NFSD_VNET_DECLARE(SVCPOOL *, nfsrvd_pool);
+NFSD_VNET_DECLARE(struct nfsclienthashhead *, nfsclienthash);
+NFSD_VNET_DECLARE(struct nfslockhashhead *, nfslockhash);
+NFSD_VNET_DECLARE(struct nfssessionhash *, nfssessionhash);
+NFSD_VNET_DECLARE(struct nfsv4lock, nfsd_suspend_lock);
+
 NFSDLOCKMUTEX;
 NFSSTATESPINLOCK;
-struct nfsrchash_bucket nfsrchash_table[NFSRVCACHE_HASHSIZE];
-struct nfsrchash_bucket nfsrcahash_table[NFSRVCACHE_HASHSIZE];
 struct mtx nfsrc_udpmtx;
 struct mtx nfs_v4root_mutex;
 struct mtx nfsrv_dontlistlock_mtx;
 struct mtx nfsrv_recalllock_mtx;
-struct nfsrvfh nfs_rootfh, nfs_pubfh;
-int nfs_pubfhset = 0, nfs_rootfhset = 0;
-struct proc *nfsd_master_proc = NULL;
+struct nfsrvfh nfs_pubfh;
+int nfs_pubfhset = 0;
 int nfsd_debuglevel = 0;
 static pid_t nfsd_master_pid = (pid_t)-1;
 static char nfsd_master_comm[MAXCOMLEN + 1];
 static struct timeval nfsd_master_start;
 static uint32_t nfsv4_sysid = 0;
 static fhandle_t zerofh;
-struct callout nfsd_callout;
+static int nfsrv_osd_jail_slot;
+
+NFSD_VNET_DEFINE(struct proc *, nfsd_master_proc) = NULL;
+NFSD_VNET_DEFINE(struct nfsrvhashhead *, nfsrvudphashtbl);
+NFSD_VNET_DEFINE(struct nfsrchash_bucket *, nfsrchash_table);
+NFSD_VNET_DEFINE(struct nfsrchash_bucket *, nfsrcahash_table);
+NFSD_VNET_DEFINE(struct nfsrvfh, nfs_rootfh);
+NFSD_VNET_DEFINE(int, nfs_rootfhset) = 0;
+NFSD_VNET_DEFINE(struct callout, nfsd_callout);
+NFSD_VNET_DEFINE(struct nfsstatsv1 *, nfsstatsv1_p);
+NFSD_VNET_DEFINE_STATIC(struct mount *, nfsv4root_mnt);
+NFSD_VNET_DEFINE_STATIC(struct vfsoptlist, nfsv4root_opt);
+NFSD_VNET_DEFINE_STATIC(struct vfsoptlist, nfsv4root_newopt);
+NFSD_VNET_DEFINE_STATIC(bool, nfsrv_suspend_nfsd) = false;
+NFSD_VNET_DEFINE_STATIC(bool, nfsrv_mntinited) = false;
+
+static void nfsrv_cleanup(struct prison *);
 
 static int nfssvc_srvcall(struct thread *, struct nfssvc_args *,
     struct ucred *);
@@ -108,7 +121,6 @@ static int nfs_commit_blks;
 static int nfs_commit_miss;
 extern int nfsrv_issuedelegs;
 extern int nfsrv_dolocallocks;
-extern int nfsd_enable_stringtouid;
 extern struct nfsdevicehead nfsrv_devidhead;
 
 static int nfsrv_createiovec(int, struct mbuf **, struct mbuf **,
@@ -170,8 +182,10 @@ SYSCTL_INT(_vfs_nfsd, OID_AUTO, enable_locallocks, CTLFLAG_RW,
     &nfsrv_dolocallocks, 0, "Enable nfsd to acquire local locks on files");
 SYSCTL_INT(_vfs_nfsd, OID_AUTO, debuglevel, CTLFLAG_RW, &nfsd_debuglevel,
     0, "Debug level for NFS server");
-SYSCTL_INT(_vfs_nfsd, OID_AUTO, enable_stringtouid, CTLFLAG_RW,
-    &nfsd_enable_stringtouid, 0, "Enable nfsd to accept numeric owner_names");
+NFSD_VNET_DECLARE(int, nfsd_enable_stringtouid);
+SYSCTL_INT(_vfs_nfsd, OID_AUTO, enable_stringtouid,
+    CTLFLAG_NFSD_VNET | CTLFLAG_RW, &NFSD_VNET_NAME(nfsd_enable_stringtouid),
+    0, "Enable nfsd to accept numeric owner_names");
 static int nfsrv_pnfsgetdsattr = 1;
 SYSCTL_INT(_vfs_nfsd, OID_AUTO, pnfsgetdsattr, CTLFLAG_RW,
     &nfsrv_pnfsgetdsattr, 0, "When set getattr gets DS attributes via RPC");
@@ -1022,7 +1036,7 @@ nfsvno_read(struct vnode *vp, off_t off, int cnt, struct ucred *cred,
 	nh = nfsrv_sequential_heuristic(uiop, vp);
 	ioflag |= nh->nh_seqcount << IO_SEQSHIFT;
 	/* XXX KDM make this more systematic? */
-	nfsstatsv1.srvbytes[NFSV4OP_READ] += uiop->uio_resid;
+	nfsstatsv1_p->srvbytes[NFSV4OP_READ] += uiop->uio_resid;
 	error = VOP_READ(vp, uiop, IO_NODELOCKED | ioflag, cred);
 	free(iv, M_TEMP);
 	if (error) {
@@ -1147,7 +1161,7 @@ nfsvno_write(struct vnode *vp, off_t off, int retlen, int *stable,
 	nh = nfsrv_sequential_heuristic(uiop, vp);
 	ioflags |= nh->nh_seqcount << IO_SEQSHIFT;
 	/* XXX KDM make this more systematic? */
-	nfsstatsv1.srvbytes[NFSV4OP_WRITE] += uiop->uio_resid;
+	nfsstatsv1_p->srvbytes[NFSV4OP_WRITE] += uiop->uio_resid;
 	error = VOP_WRITE(vp, uiop, ioflags, cred);
 	if (error == 0)
 		nh->nh_nextoff = uiop->uio_offset;
@@ -3252,7 +3266,7 @@ nfsvno_checkexp(struct mount *mp, struct sockaddr *nam, struct nfsexstuff *exp,
 	error = VFS_CHECKEXP(mp, nam, &exp->nes_exflag, credp,
 	    &exp->nes_numsecflavor, exp->nes_secflavors);
 	if (error) {
-		if (nfs_rootfhset) {
+		if (NFSD_VNET(nfs_rootfhset)) {
 			exp->nes_exflag = 0;
 			exp->nes_numsecflavor = 0;
 			error = 0;
@@ -3287,7 +3301,7 @@ nfsvno_fhtovp(struct mount *mp, fhandle_t *fhp, struct sockaddr *nam,
 		error = VFS_CHECKEXP(mp, nam, &exp->nes_exflag, credp,
 		    &exp->nes_numsecflavor, exp->nes_secflavors);
 		if (error) {
-			if (nfs_rootfhset) {
+			if (NFSD_VNET(nfs_rootfhset)) {
 				exp->nes_exflag = 0;
 				exp->nes_numsecflavor = 0;
 				error = 0;
@@ -3455,9 +3469,9 @@ nfsrv_v4rootexport(void *argp, struct ucred *cred, struct thread *p)
 	struct nameidata nd;
 	fhandle_t fh;
 
-	error = vfs_export(&nfsv4root_mnt, &nfsexargp->export);
+	error = vfs_export(NFSD_VNET(nfsv4root_mnt), &nfsexargp->export);
 	if ((nfsexargp->export.ex_flags & MNT_DELEXPORT) != 0)
-		nfs_rootfhset = 0;
+		NFSD_VNET(nfs_rootfhset) = 0;
 	else if (error == 0) {
 		if (nfsexargp->fspec == NULL) {
 			error = EPERM;
@@ -3472,11 +3486,11 @@ nfsrv_v4rootexport(void *argp, struct ucred *cred, struct thread *p)
 		error = nfsvno_getfh(nd.ni_vp, &fh, p);
 		vrele(nd.ni_vp);
 		if (!error) {
-			nfs_rootfh.nfsrvfh_len = NFSX_MYFH;
+			NFSD_VNET(nfs_rootfh).nfsrvfh_len = NFSX_MYFH;
 			NFSBCOPY((caddr_t)&fh,
-			    nfs_rootfh.nfsrvfh_data,
+			    NFSD_VNET(nfs_rootfh).nfsrvfh_data,
 			    sizeof (fhandle_t));
-			nfs_rootfhset = 1;
+			NFSD_VNET(nfs_rootfhset) = 1;
 		}
 	}
 
@@ -3512,29 +3526,53 @@ nfsrv_mallocmget_limit(void)
 void
 nfsd_mntinit(void)
 {
-	static int inited = 0;
 
-	if (inited)
+	NFSD_LOCK();
+	if (NFSD_VNET(nfsrv_mntinited)) {
+		NFSD_UNLOCK();
 		return;
-	inited = 1;
-	nfsv4root_mnt.mnt_flag = (MNT_RDONLY | MNT_EXPORTED);
-	TAILQ_INIT(&nfsv4root_mnt.mnt_nvnodelist);
-	TAILQ_INIT(&nfsv4root_mnt.mnt_lazyvnodelist);
-	nfsv4root_mnt.mnt_export = NULL;
-	TAILQ_INIT(&nfsv4root_opt);
-	TAILQ_INIT(&nfsv4root_newopt);
-	nfsv4root_mnt.mnt_opt = &nfsv4root_opt;
-	nfsv4root_mnt.mnt_optnew = &nfsv4root_newopt;
-	nfsv4root_mnt.mnt_nvnodelistsize = 0;
-	nfsv4root_mnt.mnt_lazyvnodelistsize = 0;
+	}
+	NFSD_VNET(nfsrv_mntinited) = true;
+	NFSD_UNLOCK();
+
+	if (curthread->td_ucred->cr_prison == &prison0)
+		NFSD_VNET(nfsstatsv1_p) = &nfsstatsv1;
+	else
+		NFSD_VNET(nfsstatsv1_p) = malloc(sizeof(struct nfsstatsv1),
+		    M_TEMP, M_WAITOK | M_ZERO);
+	nfsstatsv1_p->srvcache_tcppeak = 0;
+	nfsstatsv1_p->srvcache_size = 0;
+	NFSD_VNET(nfsv4root_mnt) = malloc(sizeof(struct mount), M_TEMP,
+	    M_WAITOK | M_ZERO);
+	NFSD_VNET(nfsv4root_mnt)->mnt_flag = (MNT_RDONLY | MNT_EXPORTED);
+	mtx_init(&NFSD_VNET(nfsv4root_mnt)->mnt_mtx, "nfs4mnt", NULL, MTX_DEF);
+	lockinit(&NFSD_VNET(nfsv4root_mnt)->mnt_explock, PVFS, "explock", 0, 0);
+	TAILQ_INIT(&NFSD_VNET(nfsv4root_mnt)->mnt_nvnodelist);
+	TAILQ_INIT(&NFSD_VNET(nfsv4root_mnt)->mnt_lazyvnodelist);
+	NFSD_VNET(nfsv4root_mnt)->mnt_export = NULL;
+	TAILQ_INIT(&NFSD_VNET(nfsv4root_opt));
+	TAILQ_INIT(&NFSD_VNET(nfsv4root_newopt));
+	NFSD_VNET(nfsv4root_mnt)->mnt_opt = &NFSD_VNET(nfsv4root_opt);
+	NFSD_VNET(nfsv4root_mnt)->mnt_optnew = &NFSD_VNET(nfsv4root_newopt);
+	NFSD_VNET(nfsv4root_mnt)->mnt_nvnodelistsize = 0;
+	NFSD_VNET(nfsv4root_mnt)->mnt_lazyvnodelistsize = 0;
+	callout_init(&NFSD_VNET(nfsd_callout), 1);
+
+	nfsrvd_initcache();
+	nfsd_init();
 }
 
 static void
 nfsd_timer(void *arg)
 {
-
-	nfsrv_servertimer();
-	callout_reset_sbt(&nfsd_callout, SBT_1S, SBT_1S, nfsd_timer, NULL, 0);
+	struct vnet *vnetp;
+
+	vnetp = (struct vnet *)arg;
+	NFSD_CURVNET_SET_QUIET(vnetp);
+	nfsrv_servertimer(vnetp);
+	callout_reset_sbt(&NFSD_VNET(nfsd_callout), SBT_1S, SBT_1S, nfsd_timer,
+	    arg, 0);
+	NFSD_CURVNET_RESTORE();
 }
 
 /*
@@ -3616,7 +3654,7 @@ nfsvno_v4rootexport(struct nfsrv_descript *nd)
 	int error = 0, numsecflavor, secflavors[MAXSECFLAVORS], i;
 	uint64_t exflags;
 
-	error = vfs_stdcheckexp(&nfsv4root_mnt, nd->nd_nam, &exflags,
+	error = vfs_stdcheckexp(NFSD_VNET(nfsv4root_mnt), nd->nd_nam, &exflags,
 	    &credanon, &numsecflavor, secflavors);
 	if (error) {
 		error = NFSERR_PROGUNAVAIL;
@@ -3649,6 +3687,18 @@ out:
 	return (error);
 }
 
+/* Osd entry for nfsrv_cleanup. */
+static int
+nfsrv_prison_cleanup(void *obj, void *data __unused)
+{
+	struct prison *pr = obj;
+
+	if ((pr->pr_flags & PR_VNET) == 0)
+		return (0);
+	nfsrv_cleanup(pr);
+	return (0);
+}
+
 /*
  * Nfs server pseudo system call for the nfsd's
  */
@@ -3671,6 +3721,7 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap)
 	char *buf, *cp, *cp2, *cp3;
 	char fname[PNFS_FILENAME_LEN + 1];
 
+	NFSD_CURVNET_SET(NFSD_TD_TO_VNET(td));
 	if (uap->flag & NFSSVC_NFSDADDSOCK) {
 		error = copyin(uap->argp, (caddr_t)&sockarg, sizeof (sockarg));
 		if (error)
@@ -3778,8 +3829,9 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap)
 			nfsdarg.mdspathlen = 0;
 			nfsdarg.mirrorcnt = 1;
 		}
-		nfsd_timer(NULL);
+		nfsd_timer(NFSD_TD_TO_VNET(td));
 		error = nfsrvd_nfsd(td, &nfsdarg);
+		callout_drain(&NFSD_VNET(nfsd_callout));
 		free(nfsdarg.addr, M_TEMP);
 		free(nfsdarg.dnshost, M_TEMP);
 		free(nfsdarg.dspath, M_TEMP);
@@ -3857,6 +3909,7 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap)
 	}
 
 out:
+	NFSD_CURVNET_RESTORE();
 	NFSEXITCODE(error);
 	return (error);
 }
@@ -3878,7 +3931,6 @@ nfssvc_srvcall(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 	int error = EINVAL, igotlock;
 	struct proc *procp;
 	gid_t *grps;
-	static int suspend_nfsd = 0;
 
 	if (uap->flag & NFSSVC_PUBLICFH) {
 		NFSBZERO((caddr_t)&nfs_pubfh.nfsrvfh_data,
@@ -3962,10 +4014,10 @@ nfssvc_srvcall(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 			error = fp_getfvp(p, stablefd, &fp, &vp);
 		if (!error && (NFSFPFLAG(fp) & (FREAD | FWRITE)) != (FREAD | FWRITE))
 			error = EBADF;
-		if (!error && newnfs_numnfsd != 0)
+		if (!error && NFSD_VNET(nfsrv_numnfsd) != 0)
 			error = EPERM;
 		if (!error) {
-			nfsrv_stablefirst.nsf_fp = fp;
+			NFSD_VNET(nfsrv_stablefirst).nsf_fp = fp;
 			nfsrv_setupstable(p);
 		}
 	} else if (uap->flag & NFSSVC_ADMINREVOKE) {
@@ -4012,25 +4064,27 @@ nfssvc_srvcall(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 		nfsd_master_pid = procp->p_pid;
 		bcopy(procp->p_comm, nfsd_master_comm, MAXCOMLEN + 1);
 		nfsd_master_start = procp->p_stats->p_start;
-		nfsd_master_proc = procp;
+		NFSD_VNET(nfsd_master_proc) = procp;
 		PROC_UNLOCK(procp);
 	} else if ((uap->flag & NFSSVC_SUSPENDNFSD) != 0) {
 		NFSLOCKV4ROOTMUTEX();
-		if (suspend_nfsd == 0) {
+		if (!NFSD_VNET(nfsrv_suspend_nfsd)) {
 			/* Lock out all nfsd threads */
 			do {
-				igotlock = nfsv4_lock(&nfsd_suspend_lock, 1,
-				    NULL, NFSV4ROOTLOCKMUTEXPTR, NULL);
-			} while (igotlock == 0 && suspend_nfsd == 0);
-			suspend_nfsd = 1;
+				igotlock = nfsv4_lock(
+				    &NFSD_VNET(nfsd_suspend_lock), 1, NULL,
+				    NFSV4ROOTLOCKMUTEXPTR, NULL);
+			} while (igotlock == 0 &&
+			    !NFSD_VNET(nfsrv_suspend_nfsd));
+			NFSD_VNET(nfsrv_suspend_nfsd) = true;
 		}
 		NFSUNLOCKV4ROOTMUTEX();
 		error = 0;
 	} else if ((uap->flag & NFSSVC_RESUMENFSD) != 0) {
 		NFSLOCKV4ROOTMUTEX();
-		if (suspend_nfsd != 0) {
-			nfsv4_unlock(&nfsd_suspend_lock, 0);
-			suspend_nfsd = 0;
+		if (NFSD_VNET(nfsrv_suspend_nfsd)) {
+			nfsv4_unlock(&NFSD_VNET(nfsd_suspend_lock), 0);
+			NFSD_VNET(nfsrv_suspend_nfsd) = false;
 		}
 		NFSUNLOCKV4ROOTMUTEX();
 		error = 0;
@@ -4138,10 +4192,10 @@ nfsrv_backupstable(void)
 {
 	struct proc *procp;
 
-	if (nfsd_master_proc != NULL) {
+	if (NFSD_VNET(nfsd_master_proc) != NULL) {
 		procp = pfind(nfsd_master_pid);
 		/* Try to make sure it is the correct process. */
-		if (procp == nfsd_master_proc &&
+		if (procp == NFSD_VNET(nfsd_master_proc) &&
 		    procp->p_stats->p_start.tv_sec ==
 		    nfsd_master_start.tv_sec &&
 		    procp->p_stats->p_start.tv_usec ==
@@ -4149,7 +4203,7 @@ nfsrv_backupstable(void)
 		    strcmp(procp->p_comm, nfsd_master_comm) == 0)
 			kern_psignal(procp, SIGUSR2);
 		else
-			nfsd_master_proc = NULL;
+			NFSD_VNET(nfsd_master_proc) = NULL;
 
 		if (procp != NULL)
 			PROC_UNLOCK(procp);
@@ -7060,6 +7114,76 @@ nfsrv_checknospc(void)
 	free(devid, M_TEMP);
 }
 
+/*
+ * Initialize everything that needs to be initialized for a vnet.
+ */
+static void
+nfsrv_vnetinit(const void *unused __unused)
+{
+
+	nfsd_mntinit();
+}
+VNET_SYSINIT(nfsrv_vnetinit, SI_SUB_VNET_DONE, SI_ORDER_ANY,
+    nfsrv_vnetinit, NULL);
+
+/*
+ * Clean up everything that is in a vnet and needs to be
+ * done when the jail is destroyed or the module unloaded.
+ */
+static void
+nfsrv_cleanup(struct prison *pr)
+{
+	int i;
+
+	NFSD_CURVNET_SET(pr->pr_vnet);
+	NFSD_LOCK();
+	if (!NFSD_VNET(nfsrv_mntinited)) {
+		NFSD_UNLOCK();
+		NFSD_CURVNET_RESTORE();
+		return;
+	}
+	NFSD_VNET(nfsrv_mntinited) = false;
+	NFSD_UNLOCK();
+
+	/* Clean out all NFSv4 state. */
+	nfsrv_throwawayallstate(curthread);
+
+	/* Clean the NFS server reply cache */
+	nfsrvd_cleancache();
+
+	/* Clean out v4root exports. */
+	if (NFSD_VNET(nfsv4root_mnt)->mnt_export != NULL) {
+		vfs_free_addrlist(NFSD_VNET(nfsv4root_mnt)->mnt_export);
+		free(NFSD_VNET(nfsv4root_mnt)->mnt_export, M_MOUNT);
+		NFSD_VNET(nfsv4root_mnt)->mnt_export = NULL;
+	}
+
+	/* Free up the krpc server pool. */
+	if (NFSD_VNET(nfsrvd_pool) != NULL)
+		svcpool_destroy(NFSD_VNET(nfsrvd_pool));
+
+	/* and get rid of the locks */
+	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
+		mtx_destroy(&NFSD_VNET(nfsrchash_table)[i].mtx);
+		mtx_destroy(&NFSD_VNET(nfsrcahash_table)[i].mtx);
+	}
+	mtx_destroy(&NFSD_VNET(nfsv4root_mnt)->mnt_mtx);
+	for (i = 0; i < nfsrv_sessionhashsize; i++)
+		mtx_destroy(&NFSD_VNET(nfssessionhash)[i].mtx);
+	lockdestroy(&NFSD_VNET(nfsv4root_mnt)->mnt_explock);
+	free(NFSD_VNET(nfsrvudphashtbl), M_NFSRVCACHE);
+	free(NFSD_VNET(nfsrchash_table), M_NFSRVCACHE);
+	free(NFSD_VNET(nfsrcahash_table), M_NFSRVCACHE);
+	free(NFSD_VNET(nfsclienthash), M_NFSDCLIENT);
+	free(NFSD_VNET(nfslockhash), M_NFSDLOCKFILE);
+	free(NFSD_VNET(nfssessionhash), M_NFSDSESSION);
+	free(NFSD_VNET(nfsv4root_mnt), M_TEMP);
+	NFSD_VNET(nfsv4root_mnt) = NULL;
+	if (pr != &prison0)
+		free(NFSD_VNET(nfsstatsv1_p), M_TEMP);
+	NFSD_CURVNET_RESTORE();
+}
+
 extern int (*nfsd_call_nfsd)(struct thread *, struct nfssvc_args *);
 
 /*
@@ -7069,6 +7193,9 @@ static int
 nfsd_modevent(module_t mod, int type, void *data)
 {
 	int error = 0, i;
+	osd_method_t methods[PR_MAXMETHOD] = {
+	    [PR_METHOD_REMOVE] = nfsrv_prison_cleanup,
+	};
 	static int loaded = 0;
 
 	switch (type) {
@@ -7076,30 +7203,17 @@ nfsd_modevent(module_t mod, int type, void *data)
 		if (loaded)
 			goto out;
 		newnfs_portinit();
-		for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-			mtx_init(&nfsrchash_table[i].mtx, "nfsrtc", NULL,
-			    MTX_DEF);
-			mtx_init(&nfsrcahash_table[i].mtx, "nfsrtca", NULL,
-			    MTX_DEF);
-		}
 		mtx_init(&nfsrc_udpmtx, "nfsuc", NULL, MTX_DEF);
 		mtx_init(&nfs_v4root_mutex, "nfs4rt", NULL, MTX_DEF);
-		mtx_init(&nfsv4root_mnt.mnt_mtx, "nfs4mnt", NULL, MTX_DEF);
 		mtx_init(&nfsrv_dontlistlock_mtx, "nfs4dnl", NULL, MTX_DEF);
 		mtx_init(&nfsrv_recalllock_mtx, "nfs4rec", NULL, MTX_DEF);
-		lockinit(&nfsv4root_mnt.mnt_explock, PVFS, "explock", 0, 0);
-		callout_init(&nfsd_callout, 1);
-		nfsrvd_initcache();
-		nfsd_init();
-		NFSD_LOCK();
-		nfsrvd_init(0);
-		NFSD_UNLOCK();
-		nfsd_mntinit();
 #ifdef VV_DISABLEDELEG
 		vn_deleg_ops.vndeleg_recall = nfsd_recalldelegation;
 		vn_deleg_ops.vndeleg_disable = nfsd_disabledelegation;
 #endif
 		nfsd_call_nfsd = nfssvc_nfsd;
+		/* XXX-BZ OSD to VNET? */
+		nfsrv_osd_jail_slot = osd_jail_register(NULL, methods);
 		loaded = 1;
 		break;
 
@@ -7114,39 +7228,17 @@ nfsd_modevent(module_t mod, int type, void *data)
 		vn_deleg_ops.vndeleg_disable = NULL;
 #endif
 		nfsd_call_nfsd = NULL;
-		callout_drain(&nfsd_callout);
-
-		/* Clean out all NFSv4 state. */
-		nfsrv_throwawayallstate(curthread);
-
-		/* Clean the NFS server reply cache */
-		nfsrvd_cleancache();
-
-		/* Free up the krpc server pool. */
-		if (nfsrvd_pool != NULL)
-			svcpool_destroy(nfsrvd_pool);
-
-		/* and get rid of the locks */
-		for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
-			mtx_destroy(&nfsrchash_table[i].mtx);
-			mtx_destroy(&nfsrcahash_table[i].mtx);
-		}
+		osd_jail_deregister(nfsrv_osd_jail_slot);
+		nfsrv_cleanup(&prison0);
 		mtx_destroy(&nfsrc_udpmtx);
 		mtx_destroy(&nfs_v4root_mutex);
-		mtx_destroy(&nfsv4root_mnt.mnt_mtx);
 		mtx_destroy(&nfsrv_dontlistlock_mtx);
 		mtx_destroy(&nfsrv_recalllock_mtx);
-		for (i = 0; i < nfsrv_sessionhashsize; i++)
-			mtx_destroy(&nfssessionhash[i].mtx);
 		if (nfslayouthash != NULL) {
 			for (i = 0; i < nfsrv_layouthashsize; i++)
 				mtx_destroy(&nfslayouthash[i].mtx);
 			free(nfslayouthash, M_NFSDSESSION);
 		}
-		lockdestroy(&nfsv4root_mnt.mnt_explock);
-		free(nfsclienthash, M_NFSDCLIENT);
-		free(nfslockhash, M_NFSDLOCKFILE);
-		free(nfssessionhash, M_NFSDSESSION);
 		loaded = 0;
 		break;
 	default:
diff --git a/sys/fs/nfsserver/nfs_nfsdsocket.c b/sys/fs/nfsserver/nfs_nfsdsocket.c
index 997b7f5b88e2..40f3202af93c 100644
--- a/sys/fs/nfsserver/nfs_nfsdsocket.c
+++ b/sys/fs/nfsserver/nfs_nfsdsocket.c
@@ -42,20 +42,24 @@ __FBSDID("$FreeBSD$");
 
 #include <fs/nfs/nfsport.h>
 
-extern struct nfsstatsv1 nfsstatsv1;
-extern struct nfsrvfh nfs_pubfh, nfs_rootfh;
-extern int nfs_pubfhset, nfs_rootfhset;
+extern struct nfsrvfh nfs_pubfh;
+extern int nfs_pubfhset;
 extern struct nfsv4lock nfsv4rootfs_lock;
-extern struct nfsrv_stablefirst nfsrv_stablefirst;
-extern struct nfsclienthashhead *nfsclienthash;
 extern int nfsrv_clienthashsize;
-extern int nfsrc_floodlevel, nfsrc_tcpsavedreplies;
 extern int nfsd_debuglevel;
 extern int nfsrv_layouthighwater;
 extern volatile int nfsrv_layoutcnt;
 NFSV4ROOTLOCKMUTEX;
 NFSSTATESPINLOCK;
 
+NFSD_VNET_DECLARE(struct nfsrv_stablefirst, nfsrv_stablefirst);
+NFSD_VNET_DECLARE(struct nfsclienthashhead *, nfsclienthash);
+NFSD_VNET_DECLARE(int, nfsrc_floodlevel);
+NFSD_VNET_DECLARE(int, nfsrc_tcpsavedreplies);
+NFSD_VNET_DECLARE(struct nfsrvfh, nfs_rootfh);
+NFSD_VNET_DECLARE(int, nfs_rootfhset);
+NFSD_VNET_DECLARE(struct nfsstatsv1 *, nfsstatsv1_p);
+
 int (*nfsrv3_procs0[NFS_V3NPROCS])(struct nfsrv_descript *,
     int, vnode_t , struct nfsexstuff *) = {
 	(int (*)(struct nfsrv_descript *, int, vnode_t , struct nfsexstuff *))0,
@@ -471,15 +475,15 @@ nfsrvd_statstart(int op, struct bintime *now)
 	}
 
 	mtx_lock(&nfsrvd_statmtx);
-	if (nfsstatsv1.srvstartcnt == nfsstatsv1.srvdonecnt) {
+	if (nfsstatsv1_p->srvstartcnt == nfsstatsv1_p->srvdonecnt) {
 		if (now != NULL)
-			nfsstatsv1.busyfrom = *now;
+			nfsstatsv1_p->busyfrom = *now;
 		else
-			binuptime(&nfsstatsv1.busyfrom);
+			binuptime(&nfsstatsv1_p->busyfrom);
 		
 	}
-	nfsstatsv1.srvrpccnt[op]++;
-	nfsstatsv1.srvstartcnt++;
+	nfsstatsv1_p->srvrpccnt[op]++;
+	nfsstatsv1_p->srvstartcnt++;
 	mtx_unlock(&nfsrvd_statmtx);
 
 }
@@ -502,21 +506,21 @@ nfsrvd_statend(int op, uint64_t bytes, struct bintime *now,
 
 	mtx_lock(&nfsrvd_statmtx);
 
-	nfsstatsv1.srvbytes[op] += bytes;
-	nfsstatsv1.srvops[op]++;
+	nfsstatsv1_p->srvbytes[op] += bytes;
+	nfsstatsv1_p->srvops[op]++;
 
 	if (then != NULL) {
 		dt = *now;
 		bintime_sub(&dt, then);
-		bintime_add(&nfsstatsv1.srvduration[op], &dt);
+		bintime_add(&nfsstatsv1_p->srvduration[op], &dt);
 	}
 
 	dt = *now;
-	bintime_sub(&dt, &nfsstatsv1.busyfrom);
-	bintime_add(&nfsstatsv1.busytime, &dt);
-	nfsstatsv1.busyfrom = *now;
+	bintime_sub(&dt, &nfsstatsv1_p->busyfrom);
+	bintime_add(&nfsstatsv1_p->busytime, &dt);
+	nfsstatsv1_p->busyfrom = *now;
 
-	nfsstatsv1.srvdonecnt++;
+	nfsstatsv1_p->srvdonecnt++;
 
 	mtx_unlock(&nfsrvd_statmtx);
 }
@@ -753,7 +757,7 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 	 */
 	igotlock = 0;
 	NFSLOCKV4ROOTMUTEX();
-	if (nfsrv_stablefirst.nsf_flags & NFSNSF_NEEDLOCK)
+	if (NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_NEEDLOCK)
 		igotlock = nfsv4_lock(&nfsv4rootfs_lock, 1, NULL,
 		    NFSV4ROOTLOCKMUTEXPTR, NULL);
 	else
@@ -766,8 +770,8 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 		 * Done when the grace period is over or a client has long
 		 * since expired.
 		 */
-		nfsrv_stablefirst.nsf_flags &= ~NFSNSF_NEEDLOCK;
-		if ((nfsrv_stablefirst.nsf_flags &
+		NFSD_VNET(nfsrv_stablefirst).nsf_flags &= ~NFSNSF_NEEDLOCK;
+		if ((NFSD_VNET(nfsrv_stablefirst).nsf_flags &
 		    (NFSNSF_GRACEOVER | NFSNSF_UPDATEDONE)) == NFSNSF_GRACEOVER)
 			nfsrv_updatestable(p);
 
@@ -777,11 +781,13 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 		 * stable storage file and then remove them from the client
 		 * list.
 		 */
-		if (nfsrv_stablefirst.nsf_flags & NFSNSF_EXPIREDCLIENT) {
-			nfsrv_stablefirst.nsf_flags &= ~NFSNSF_EXPIREDCLIENT;
+		if (NFSD_VNET(nfsrv_stablefirst).nsf_flags &
+		    NFSNSF_EXPIREDCLIENT) {
+			NFSD_VNET(nfsrv_stablefirst).nsf_flags &=
+			    ~NFSNSF_EXPIREDCLIENT;
 			for (i = 0; i < nfsrv_clienthashsize; i++) {
-			    LIST_FOREACH_SAFE(clp, &nfsclienthash[i], lc_hash,
-				nclp) {
+			    LIST_FOREACH_SAFE(clp, &NFSD_VNET(nfsclienthash)[i],
+				lc_hash, nclp) {
 				if (clp->lc_flags & LCL_EXPIREIT) {
 				    if (!LIST_EMPTY(&clp->lc_open) ||
 					!LIST_EMPTY(&clp->lc_deleg))
@@ -814,7 +820,7 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 	 * If flagged, search for open owners that haven't had any opens
 	 * for a long time.
 	 */
-	if (nfsrv_stablefirst.nsf_flags & NFSNSF_NOOPENS) {
+	if (NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_NOOPENS) {
 		nfsrv_throwawayopens(p);
 	}
 
@@ -941,8 +947,10 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 		if (i == 0 && (nd->nd_rp == NULL ||
 		    nd->nd_rp->rc_refcnt == 0) &&
 		    (nfsrv_mallocmget_limit() ||
-		     nfsrc_tcpsavedreplies > nfsrc_floodlevel)) {
-			if (nfsrc_tcpsavedreplies > nfsrc_floodlevel)
+		     NFSD_VNET(nfsrc_tcpsavedreplies) >
+		     NFSD_VNET(nfsrc_floodlevel))) {
+			if (NFSD_VNET(nfsrc_tcpsavedreplies) >
+			    NFSD_VNET(nfsrc_floodlevel))
 				printf("nfsd server cache flooded, try "
 				    "increasing vfs.nfsd.tcphighwater\n");
 			nd->nd_repstat = NFSERR_RESOURCE;
@@ -1033,7 +1041,7 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 			}
 			break;
 		case NFSV4OP_PUTROOTFH:
-			if (nfs_rootfhset) {
+			if (NFSD_VNET(nfs_rootfhset)) {
 				if ((nd->nd_flag & ND_LASTOP) == 0) {
 					/*
 					 * Pre-parse the next op#.  If it is
@@ -1054,8 +1062,8 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram, u_char *tag,
 					} while (nextop == NFSV4OP_SAVEFH &&
 					    i < numops - 1);
 				}
-				nfsd_fhtovp(nd, &nfs_rootfh, LK_SHARED, &nvp,
-				    &nes, NULL, 0, nextop);
+				nfsd_fhtovp(nd, &NFSD_VNET(nfs_rootfh),
+				    LK_SHARED, &nvp, &nes, NULL, 0, nextop);
 				if (!nd->nd_repstat) {
 					if (vp)
 						vrele(vp);
diff --git a/sys/fs/nfsserver/nfs_nfsdstate.c b/sys/fs/nfsserver/nfs_nfsdstate.c
index d410a808d06d..7a8527f57068 100644
--- a/sys/fs/nfsserver/nfs_nfsdstate.c
+++ b/sys/fs/nfsserver/nfs_nfsdstate.c
@@ -35,17 +35,19 @@ __FBSDID("$FreeBSD$");
 #include <sys/extattr.h>
 #include <fs/nfs/nfsport.h>
 
-struct nfsrv_stablefirst nfsrv_stablefirst;
 int nfsrv_issuedelegs = 0;
 int nfsrv_dolocallocks = 0;
 struct nfsv4lock nfsv4rootfs_lock;
 time_t nfsdev_time = 0;
 int nfsrv_layouthashsize;
 volatile int nfsrv_layoutcnt = 0;
-extern uint32_t nfs_srvmaxio;
 
-extern int newnfs_numnfsd;
-extern struct nfsstatsv1 nfsstatsv1;
+NFSD_VNET_DEFINE(struct nfsrv_stablefirst, nfsrv_stablefirst);
+
+NFSD_VNET_DECLARE(int, nfsrv_numnfsd);
+NFSD_VNET_DECLARE(struct nfsstatsv1 *, nfsstatsv1_p);
+
+extern uint32_t nfs_srvmaxio;
 extern int nfsrv_lease;
 extern struct timeval nfsboottime;
 extern u_int32_t newnfs_true, newnfs_false;
@@ -118,20 +120,22 @@ SYSCTL_INT(_vfs_nfsd, OID_AUTO, flexlinuxhack, CTLFLAG_RW,
 /*
  * Hash lists for nfs V4.
  */
-struct nfsclienthashhead	*nfsclienthash;
-struct nfslockhashhead		*nfslockhash;
-struct nfssessionhash		*nfssessionhash;
+NFSD_VNET_DEFINE(struct nfsclienthashhead *, nfsclienthash);
+NFSD_VNET_DEFINE(struct nfslockhashhead *, nfslockhash);
+NFSD_VNET_DEFINE(struct nfssessionhash *, nfssessionhash);
+
 struct nfslayouthash		*nfslayouthash;
 volatile int nfsrv_dontlistlen = 0;
 
 static u_int32_t nfsrv_openpluslock = 0, nfsrv_delegatecnt = 0;
-static time_t nfsrvboottime;
 static int nfsrv_returnoldstateid = 0, nfsrv_clients = 0;
 static int nfsrv_clienthighwater = NFSRV_CLIENTHIGHWATER;
 static int nfsrv_nogsscallback = 0;
 static volatile int nfsrv_writedelegcnt = 0;
 static int nfsrv_faildscnt;
 
+NFSD_VNET_DEFINE_STATIC(time_t, nfsrvboottime);
+
 /* local functions */
 static void nfsrv_dumpaclient(struct nfsclient *clp,
     struct nfsd_dumpclients *dumpp);
@@ -298,7 +302,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 	 */
 	gotit = i = 0;
 	while (i < nfsrv_clienthashsize && !gotit) {
-	    LIST_FOREACH(clp, &nfsclienthash[i], lc_hash) {
+	    LIST_FOREACH(clp, &NFSD_VNET(nfsclienthash)[i], lc_hash) {
 		if (new_clp->lc_idlen == clp->lc_idlen &&
 		    !NFSBCMP(new_clp->lc_id, clp->lc_id, clp->lc_idlen)) {
 			gotit = 1;
@@ -343,7 +347,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 			confirmp->qval = new_clp->lc_confirm.qval =
 			    ++confirm_index;
 		clientidp->lval[0] = new_clp->lc_clientid.lval[0] =
-		    (u_int32_t)nfsrvboottime;
+		    NFSD_VNET(nfsrvboottime);
 		clientidp->lval[1] = new_clp->lc_clientid.lval[1] =
 		    nfsrv_nextclientindex();
 		new_clp->lc_stateindex = 0;
@@ -359,7 +363,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 			LIST_INIT(&new_clp->lc_stateid[i]);
 		LIST_INSERT_HEAD(NFSCLIENTHASH(new_clp->lc_clientid), new_clp,
 		    lc_hash);
-		nfsstatsv1.srvclients++;
+		nfsstatsv1_p->srvclients++;
 		nfsrv_openpluslock++;
 		nfsrv_clients++;
 		NFSLOCKV4ROOTMUTEX();
@@ -457,7 +461,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 			confirmp->qval = new_clp->lc_confirm.qval =
 			    ++confirm_index;
 		clientidp->lval[0] = new_clp->lc_clientid.lval[0] =
-		    nfsrvboottime;
+		    NFSD_VNET(nfsrvboottime);
 		clientidp->lval[1] = new_clp->lc_clientid.lval[1] =
 		    nfsrv_nextclientindex();
 		new_clp->lc_stateindex = 0;
@@ -488,7 +492,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 		LIST_INIT(&new_clp->lc_session);
 		LIST_INSERT_HEAD(NFSCLIENTHASH(new_clp->lc_clientid), new_clp,
 		    lc_hash);
-		nfsstatsv1.srvclients++;
+		nfsstatsv1_p->srvclients++;
 		nfsrv_openpluslock++;
 		nfsrv_clients++;
 		NFSLOCKV4ROOTMUTEX();
@@ -553,7 +557,7 @@ nfsrv_setclient(struct nfsrv_descript *nd, struct nfsclient **new_clpp,
 		LIST_INIT(&new_clp->lc_session);
 		LIST_INSERT_HEAD(NFSCLIENTHASH(new_clp->lc_clientid), new_clp,
 		    lc_hash);
-		nfsstatsv1.srvclients++;
+		nfsstatsv1_p->srvclients++;
 		nfsrv_openpluslock++;
 		nfsrv_clients++;
 	}
@@ -604,7 +608,8 @@ nfsrv_getclient(nfsquad_t clientid, int opflags, struct nfsclient **clpp,
 	if (clpp)
 		*clpp = NULL;
 	if ((nd == NULL || (nd->nd_flag & ND_NFSV41) == 0 ||
-	    opflags != CLOPS_RENEW) && nfsrvboottime != clientid.lval[0]) {
+	    opflags != CLOPS_RENEW) && NFSD_VNET(nfsrvboottime) !=
+	    clientid.lval[0]) {
 		error = NFSERR_STALECLIENTID;
 		goto out;
 	}
@@ -820,7 +825,7 @@ nfsrv_destroyclient(nfsquad_t clientid, NFSPROC_T *p)
 	struct nfsclienthashhead *hp;
 	int error = 0, i, igotlock;
 
-	if (nfsrvboottime != clientid.lval[0]) {
+	if (NFSD_VNET(nfsrvboottime) != clientid.lval[0]) {
 		error = NFSERR_STALECLIENTID;
 		goto out;
 	}
@@ -912,7 +917,7 @@ nfsrv_adminrevoke(struct nfsd_clid *revokep, NFSPROC_T *p)
 	 */
 	gotit = i = 0;
 	while (i < nfsrv_clienthashsize && !gotit) {
-	    LIST_FOREACH(clp, &nfsclienthash[i], lc_hash) {
+	    LIST_FOREACH(clp, &NFSD_VNET(nfsclienthash)[i], lc_hash) {
 		if (revokep->nclid_idlen == clp->lc_idlen &&
 		    !NFSBCMP(revokep->nclid_id, clp->lc_id, clp->lc_idlen)) {
 			gotit = 1;
@@ -974,8 +979,9 @@ nfsrv_dumpclients(struct nfsd_dumpclients *dumpp, int maxcnt)
 	 * Rattle through the client lists until done.
 	 */
 	while (i < nfsrv_clienthashsize && cnt < maxcnt) {
-	    clp = LIST_FIRST(&nfsclienthash[i]);
-	    while (clp != LIST_END(&nfsclienthash[i]) && cnt < maxcnt) {
+	    clp = LIST_FIRST(&NFSD_VNET(nfsclienthash)[i]);
+	    while (clp != LIST_END(&NFSD_VNET(nfsclienthash)[i]) && cnt <
+		maxcnt) {
 		nfsrv_dumpaclient(clp, &dumpp[cnt]);
 		cnt++;
 		clp = LIST_NEXT(clp, lc_hash);
@@ -1233,7 +1239,7 @@ nfsrv_dumplocks(vnode_t vp, struct nfsd_dumplocks *ldumpp, int maxcnt,
  * Should be called once per second.
  */
 void
-nfsrv_servertimer(void)
+nfsrv_servertimer(void *arg __unused)
 {
 	struct nfsclient *clp, *nclp;
 	struct nfsstate *stp, *nstp;
@@ -1251,14 +1257,15 @@ nfsrv_servertimer(void)
 	 * If server hasn't started yet, just return.
 	 */
 	NFSLOCKSTATE();
-	if (nfsrv_stablefirst.nsf_eograce == 0) {
+	if (NFSD_VNET(nfsrv_stablefirst).nsf_eograce == 0) {
 		NFSUNLOCKSTATE();
 		return;
 	}
-	if (!(nfsrv_stablefirst.nsf_flags & NFSNSF_UPDATEDONE)) {
-		if (!(nfsrv_stablefirst.nsf_flags & NFSNSF_GRACEOVER) &&
-		    NFSD_MONOSEC > nfsrv_stablefirst.nsf_eograce)
-			nfsrv_stablefirst.nsf_flags |=
+	if (!(NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_UPDATEDONE)) {
+		if (!(NFSD_VNET(nfsrv_stablefirst).nsf_flags &
+		      NFSNSF_GRACEOVER) &&
+		    NFSD_MONOSEC > NFSD_VNET(nfsrv_stablefirst).nsf_eograce)
+			NFSD_VNET(nfsrv_stablefirst).nsf_flags |=
 			    (NFSNSF_GRACEOVER | NFSNSF_NEEDLOCK);
 		NFSUNLOCKSTATE();
 		return;
@@ -1281,8 +1288,8 @@ nfsrv_servertimer(void)
 	 * For each client...
 	 */
 	for (i = 0; i < nfsrv_clienthashsize; i++) {
-	    clp = LIST_FIRST(&nfsclienthash[i]);
-	    while (clp != LIST_END(&nfsclienthash[i])) {
+	    clp = LIST_FIRST(&NFSD_VNET(nfsclienthash)[i]);
+	    while (clp != LIST_END(&NFSD_VNET(nfsclienthash)[i])) {
 		nclp = LIST_NEXT(clp, lc_hash);
 		if (!(clp->lc_flags & LCL_EXPIREIT)) {
 		    if (((clp->lc_expiry + NFSRV_STALELEASE) < NFSD_MONOSEC
@@ -1313,7 +1320,7 @@ nfsrv_servertimer(void)
 			 *     by an nfsd sometime soon.
 			 */
 			clp->lc_flags |= LCL_EXPIREIT;
-			nfsrv_stablefirst.nsf_flags |=
+			NFSD_VNET(nfsrv_stablefirst).nsf_flags |=
 			    (NFSNSF_NEEDLOCK | NFSNSF_EXPIREDCLIENT);
 		    } else {
 			/*
@@ -1331,7 +1338,7 @@ nfsrv_servertimer(void)
 					if (stp->ls_noopens > NFSNOOPEN ||
 					    (nfsrv_openpluslock * 2) >
 					    nfsrv_v4statelimit)
-						nfsrv_stablefirst.nsf_flags |=
+						NFSD_VNET(nfsrv_stablefirst).nsf_flags |=
 							NFSNSF_NOOPENS;
 				} else {
 					stp->ls_noopens = 0;
@@ -1397,7 +1404,7 @@ nfsrv_zapclient(struct nfsclient *clp, NFSPROC_T *p)
 	free(clp->lc_stateid, M_NFSDCLIENT);
 	free(clp, M_NFSDCLIENT);
 	NFSLOCKSTATE();
-	nfsstatsv1.srvclients--;
+	nfsstatsv1_p->srvclients--;
 	nfsrv_openpluslock--;
 	nfsrv_clients--;
 	NFSUNLOCKSTATE();
@@ -1440,7 +1447,7 @@ nfsrv_freedeleg(struct nfsstate *stp)
 	    nfsv4_testlock(&lfp->lf_locallock_lck) == 0)
 		nfsrv_freenfslockfile(lfp);
 	free(stp, M_NFSDSTATE);
-	nfsstatsv1.srvdelegates--;
+	nfsstatsv1_p->srvdelegates--;
 	nfsrv_openpluslock--;
 	nfsrv_delegatecnt--;
 }
@@ -1466,7 +1473,7 @@ nfsrv_freeopenowner(struct nfsstate *stp, int cansleep, NFSPROC_T *p)
 	if (stp->ls_op)
 		nfsrvd_derefcache(stp->ls_op);
 	free(stp, M_NFSDSTATE);
-	nfsstatsv1.srvopenowners--;
+	nfsstatsv1_p->srvopenowners--;
 	nfsrv_openpluslock--;
 }
 
@@ -1516,7 +1523,7 @@ nfsrv_freeopen(struct nfsstate *stp, vnode_t vp, int cansleep, NFSPROC_T *p)
 	if (cansleep != 0)
 		NFSUNLOCKSTATE();
 	free(stp, M_NFSDSTATE);
-	nfsstatsv1.srvopens--;
+	nfsstatsv1_p->srvopens--;
 	nfsrv_openpluslock--;
 	return (ret);
 }
@@ -1535,7 +1542,7 @@ nfsrv_freelockowner(struct nfsstate *stp, vnode_t vp, int cansleep,
 	if (stp->ls_op)
 		nfsrvd_derefcache(stp->ls_op);
 	free(stp, M_NFSDSTATE);
-	nfsstatsv1.srvlockowners--;
+	nfsstatsv1_p->srvlockowners--;
 	nfsrv_openpluslock--;
 }
 
@@ -1611,7 +1618,7 @@ nfsrv_freenfslock(struct nfslock *lop)
 
 	if (lop->lo_lckfile.le_prev != NULL) {
 		LIST_REMOVE(lop, lo_lckfile);
-		nfsstatsv1.srvlocks--;
+		nfsstatsv1_p->srvlocks--;
 		nfsrv_openpluslock--;
 	}
 	LIST_REMOVE(lop, lo_lckowner);
@@ -2388,7 +2395,7 @@ tryagain:
 		LIST_INSERT_HEAD(&stp->ls_open, new_stp, ls_list);
 		*new_lopp = NULL;
 		*new_stpp = NULL;
-		nfsstatsv1.srvlockowners++;
+		nfsstatsv1_p->srvlockowners++;
 		nfsrv_openpluslock++;
 	}
 	if (filestruct_locked != 0) {
@@ -3040,12 +3047,12 @@ tryagain:
 			LIST_INSERT_HEAD(&new_stp->ls_open, new_open, ls_list);
 			LIST_INSERT_HEAD(&clp->lc_open, new_stp, ls_list);
 			*new_stpp = NULL;
-			nfsstatsv1.srvopenowners++;
+			nfsstatsv1_p->srvopenowners++;
 			nfsrv_openpluslock++;
 		    }
 		    openstp = new_open;
 		    new_open = NULL;
-		    nfsstatsv1.srvopens++;
+		    nfsstatsv1_p->srvopens++;
 		    nfsrv_openpluslock++;
 		    break;
 		}
@@ -3106,7 +3113,7 @@ tryagain:
 		    NFSRV_V4DELEGLIMIT(nfsrv_delegatecnt) ||
 		    !NFSVNO_DELEGOK(vp))
 		    *rflagsp |= NFSV4OPEN_RECALL;
-		nfsstatsv1.srvdelegates++;
+		nfsstatsv1_p->srvdelegates++;
 		nfsrv_openpluslock++;
 		nfsrv_delegatecnt++;
 
@@ -3146,12 +3153,12 @@ tryagain:
 		    LIST_INSERT_HEAD(&new_stp->ls_open, new_open, ls_list);
 		    LIST_INSERT_HEAD(&clp->lc_open, new_stp, ls_list);
 		    *new_stpp = NULL;
-		    nfsstatsv1.srvopenowners++;
+		    nfsstatsv1_p->srvopenowners++;
 		    nfsrv_openpluslock++;
 		}
 		openstp = new_open;
 		new_open = NULL;
-		nfsstatsv1.srvopens++;
+		nfsstatsv1_p->srvopens++;
 		nfsrv_openpluslock++;
 	    } else {
 		error = NFSERR_RECLAIMCONFLICT;
@@ -3223,7 +3230,7 @@ tryagain:
 			    new_deleg->ls_stateid), new_deleg, ls_hash);
 			LIST_INSERT_HEAD(&clp->lc_deleg, new_deleg, ls_list);
 			new_deleg = NULL;
-			nfsstatsv1.srvdelegates++;
+			nfsstatsv1_p->srvdelegates++;
 			nfsrv_openpluslock++;
 			nfsrv_delegatecnt++;
 		    }
@@ -3245,7 +3252,7 @@ tryagain:
 			new_open, ls_hash);
 		    openstp = new_open;
 		    new_open = NULL;
-		    nfsstatsv1.srvopens++;
+		    nfsstatsv1_p->srvopens++;
 		    nfsrv_openpluslock++;
 
 		    /*
@@ -3293,7 +3300,7 @@ tryagain:
 			    new_deleg->ls_stateid), new_deleg, ls_hash);
 			LIST_INSERT_HEAD(&clp->lc_deleg, new_deleg, ls_list);
 			new_deleg = NULL;
-			nfsstatsv1.srvdelegates++;
+			nfsstatsv1_p->srvdelegates++;
 			nfsrv_openpluslock++;
 			nfsrv_delegatecnt++;
 		    }
@@ -3374,7 +3381,7 @@ tryagain:
 				LIST_INSERT_HEAD(&clp->lc_deleg, new_deleg,
 				    ls_list);
 				new_deleg = NULL;
-				nfsstatsv1.srvdelegates++;
+				nfsstatsv1_p->srvdelegates++;
 				nfsrv_openpluslock++;
 				nfsrv_delegatecnt++;
 			}
@@ -3402,9 +3409,9 @@ tryagain:
 		openstp = new_open;
 		new_open = NULL;
 		*new_stpp = NULL;
-		nfsstatsv1.srvopens++;
+		nfsstatsv1_p->srvopens++;
 		nfsrv_openpluslock++;
-		nfsstatsv1.srvopenowners++;
+		nfsstatsv1_p->srvopenowners++;
 		nfsrv_openpluslock++;
 	}
 	if (!error) {
@@ -3880,7 +3887,7 @@ nfsrv_insertlock(struct nfslock *new_lop, struct nfslock *insert_lop,
 	else
 		LIST_INSERT_AFTER(insert_lop, new_lop, lo_lckowner);
 	if (stp != NULL) {
-		nfsstatsv1.srvlocks++;
+		nfsstatsv1_p->srvlocks++;
 		nfsrv_openpluslock++;
 	}
 }
@@ -4357,11 +4364,11 @@ nfsrv_checkrestart(nfsquad_t clientid, u_int32_t flags,
 	 */
 	if (flags &
 	    (NFSLCK_OPEN | NFSLCK_TEST | NFSLCK_RELEASE | NFSLCK_DELEGPURGE)) {
-		if (clientid.lval[0] != nfsrvboottime) {
+		if (clientid.lval[0] != NFSD_VNET(nfsrvboottime)) {
 			ret = NFSERR_STALECLIENTID;
 			goto out;
 		}
-	} else if (stateidp->other[0] != nfsrvboottime &&
+	} else if (stateidp->other[0] != NFSD_VNET(nfsrvboottime) &&
 		specialid == 0) {
 		ret = NFSERR_STALESTATEID;
 		goto out;
@@ -4394,25 +4401,26 @@ nfsrv_checkgrace(struct nfsrv_descript *nd, struct nfsclient *clp,
 	int error = 0, notreclaimed;
 	struct nfsrv_stable *sp;
 
-	if ((nfsrv_stablefirst.nsf_flags & (NFSNSF_UPDATEDONE |
+	if ((NFSD_VNET(nfsrv_stablefirst).nsf_flags & (NFSNSF_UPDATEDONE |
 	     NFSNSF_GRACEOVER)) == 0) {
 		/*
 		 * First, check to see if all of the clients have done a
 		 * ReclaimComplete.  If so, grace can end now.
 		 */
 		notreclaimed = 0;
-		LIST_FOREACH(sp, &nfsrv_stablefirst.nsf_head, nst_list) {
+		LIST_FOREACH(sp, &NFSD_VNET(nfsrv_stablefirst).nsf_head,
+		    nst_list) {
 			if ((sp->nst_flag & NFSNST_RECLAIMED) == 0) {
 				notreclaimed = 1;
 				break;
 			}
 		}
 		if (notreclaimed == 0)
-			nfsrv_stablefirst.nsf_flags |= (NFSNSF_GRACEOVER |
-			    NFSNSF_NEEDLOCK);
+			NFSD_VNET(nfsrv_stablefirst).nsf_flags |=
+			    (NFSNSF_GRACEOVER | NFSNSF_NEEDLOCK);
 	}
 
-	if ((nfsrv_stablefirst.nsf_flags & NFSNSF_GRACEOVER) != 0) {
+	if ((NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_GRACEOVER) != 0) {
 		if (flags & NFSLCK_RECLAIM) {
 			error = NFSERR_NOGRACE;
 			goto out;
@@ -4434,9 +4442,9 @@ nfsrv_checkgrace(struct nfsrv_descript *nd, struct nfsclient *clp,
 		 * extend grace a bit.
 		 */
 		if ((NFSD_MONOSEC + NFSRV_LEASEDELTA) >
-		    nfsrv_stablefirst.nsf_eograce)
-			nfsrv_stablefirst.nsf_eograce = NFSD_MONOSEC +
-				NFSRV_LEASEDELTA;
+		    NFSD_VNET(nfsrv_stablefirst).nsf_eograce)
+			NFSD_VNET(nfsrv_stablefirst).nsf_eograce =
+				NFSD_MONOSEC + NFSRV_LEASEDELTA;
 	}
 
 out:
@@ -4870,7 +4878,7 @@ tryagain:
 void
 nfsrv_setupstable(NFSPROC_T *p)
 {
-	struct nfsrv_stablefirst *sf = &nfsrv_stablefirst;
+	struct nfsrv_stablefirst *sf = &NFSD_VNET(nfsrv_stablefirst);
 	struct nfsrv_stable *sp, *nsp;
 	struct nfst_rec *tsp;
 	int error, i, tryagain;
@@ -4886,7 +4894,7 @@ nfsrv_setupstable(NFSPROC_T *p)
 	/*
 	 * Set Grace over just until the file reads successfully.
 	 */
-	nfsrvboottime = time_second;
+	NFSD_VNET(nfsrvboottime) = time_second;
 	LIST_INIT(&sf->nsf_head);
 	sf->nsf_flags = (NFSNSF_GRACEOVER | NFSNSF_NEEDLOCK);
 	sf->nsf_eograce = NFSD_MONOSEC + NFSRV_LEASEDELTA;
@@ -4903,7 +4911,7 @@ nfsrv_setupstable(NFSPROC_T *p)
 	 * Now, read in the boottimes.
 	 */
 	sf->nsf_bootvals = (time_t *)malloc((sf->nsf_numboots + 1) *
-		sizeof (time_t), M_TEMP, M_WAITOK);
+		sizeof(time_t), M_TEMP, M_WAITOK);
 	off = sizeof (struct nfsf_rec);
 	error = NFSD_RDWR(UIO_READ, NFSFPVNODE(sf->nsf_fp),
 	    (caddr_t)sf->nsf_bootvals, sf->nsf_numboots * sizeof (time_t), off,
@@ -4921,8 +4929,8 @@ nfsrv_setupstable(NFSPROC_T *p)
 	do {
 		tryagain = 0;
 		for (i = 0; i < sf->nsf_numboots; i++) {
-			if (nfsrvboottime == sf->nsf_bootvals[i]) {
-				nfsrvboottime++;
+			if (NFSD_VNET(nfsrvboottime) == sf->nsf_bootvals[i]) {
+				NFSD_VNET(nfsrvboottime)++;
 				tryagain = 1;
 				break;
 			}
@@ -5005,12 +5013,14 @@ nfsrv_setupstable(NFSPROC_T *p)
 void
 nfsrv_updatestable(NFSPROC_T *p)
 {
-	struct nfsrv_stablefirst *sf = &nfsrv_stablefirst;
+	struct nfsrv_stablefirst *sf = &NFSD_VNET(nfsrv_stablefirst);
 	struct nfsrv_stable *sp, *nsp;
 	int i;
 	struct nfsvattr nva;
 	vnode_t vp;
+#if defined(__FreeBSD_version) && (__FreeBSD_version >= 500000)
 	mount_t mp = NULL;
+#endif
 	int error;
 
 	if (sf->nsf_fp == NULL || (sf->nsf_flags & NFSNSF_UPDATEDONE))
@@ -5029,10 +5039,10 @@ nfsrv_updatestable(NFSPROC_T *p)
 			sf->nsf_bootvals[i + 1] = sf->nsf_bootvals[i];
 	} else {
 		sf->nsf_numboots = 1;
-		sf->nsf_bootvals = (time_t *)malloc(sizeof (time_t),
+		sf->nsf_bootvals = (time_t *)malloc(sizeof(time_t),
 			M_TEMP, M_WAITOK);
 	}
-	sf->nsf_bootvals[0] = nfsrvboottime;
+	sf->nsf_bootvals[0] = NFSD_VNET(nfsrvboottime);
 	sf->nsf_lease = nfsrv_lease;
 	NFSVNO_ATTRINIT(&nva);
 	NFSVNO_SETATTRVAL(&nva, size, 0);
@@ -5086,7 +5096,7 @@ nfsrv_updatestable(NFSPROC_T *p)
 void
 nfsrv_writestable(u_char *client, int len, int flag, NFSPROC_T *p)
 {
-	struct nfsrv_stablefirst *sf = &nfsrv_stablefirst;
+	struct nfsrv_stablefirst *sf = &NFSD_VNET(nfsrv_stablefirst);
 	struct nfst_rec *sp;
 	int error;
 
@@ -5119,12 +5129,12 @@ nfsrv_markstable(struct nfsclient *clp)
 	/*
 	 * First find the client structure.
 	 */
-	LIST_FOREACH(sp, &nfsrv_stablefirst.nsf_head, nst_list) {
+	LIST_FOREACH(sp, &NFSD_VNET(nfsrv_stablefirst).nsf_head, nst_list) {
 		if (sp->nst_len == clp->lc_idlen &&
 		    !NFSBCMP(sp->nst_client, clp->lc_id, sp->nst_len))
 			break;
 	}
-	if (sp == LIST_END(&nfsrv_stablefirst.nsf_head))
+	if (sp == LIST_END(&NFSD_VNET(nfsrv_stablefirst).nsf_head))
 		return;
 
 	/*
@@ -5146,12 +5156,12 @@ nfsrv_markreclaim(struct nfsclient *clp)
 	/*
 	 * First find the client structure.
 	 */
-	LIST_FOREACH(sp, &nfsrv_stablefirst.nsf_head, nst_list) {
+	LIST_FOREACH(sp, &NFSD_VNET(nfsrv_stablefirst).nsf_head, nst_list) {
 		if (sp->nst_len == clp->lc_idlen &&
 		    !NFSBCMP(sp->nst_client, clp->lc_id, sp->nst_len))
 			break;
 	}
-	if (sp == LIST_END(&nfsrv_stablefirst.nsf_head))
+	if (sp == LIST_END(&NFSD_VNET(nfsrv_stablefirst).nsf_head))
 		return;
 
 	/*
@@ -5172,7 +5182,7 @@ nfsrv_checkstable(struct nfsclient *clp)
 	/*
 	 * First, find the entry for the client.
 	 */
-	LIST_FOREACH(sp, &nfsrv_stablefirst.nsf_head, nst_list) {
+	LIST_FOREACH(sp, &NFSD_VNET(nfsrv_stablefirst).nsf_head, nst_list) {
 		if (sp->nst_len == clp->lc_idlen &&
 		    !NFSBCMP(sp->nst_client, clp->lc_id, sp->nst_len))
 			break;
@@ -5182,9 +5192,9 @@ nfsrv_checkstable(struct nfsclient *clp)
 	 * If not in the list, state was revoked or no state was issued
 	 * since the previous reboot, a reclaim is denied.
 	 */
-	if (sp == LIST_END(&nfsrv_stablefirst.nsf_head) ||
+	if (sp == LIST_END(&NFSD_VNET(nfsrv_stablefirst).nsf_head) ||
 	    (sp->nst_flag & NFSNST_REVOKE) ||
-	    !(nfsrv_stablefirst.nsf_flags & NFSNSF_OK))
+	    !(NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_OK))
 		return (1);
 	return (0);
 }
@@ -5212,7 +5222,7 @@ nfsrv_clientconflict(struct nfsclient *clp, int *haslockp, vnode_t vp,
 	 * If lease hasn't expired, we can't fix it.
 	 */
 	if (clp->lc_expiry >= NFSD_MONOSEC ||
-	    !(nfsrv_stablefirst.nsf_flags & NFSNSF_UPDATEDONE))
+	    !(NFSD_VNET(nfsrv_stablefirst).nsf_flags & NFSNSF_UPDATEDONE))
 		return (0);
 	if (*haslockp == 0) {
 		NFSUNLOCKSTATE();
@@ -5611,7 +5621,7 @@ nfsd_recalldelegation(vnode_t vp, NFSPROC_T *p)
 	 * First, check to see if the server is currently running and it has
 	 * been called for a regular file when issuing delegations.
 	 */
-	if (newnfs_numnfsd == 0 || vp->v_type != VREG ||
+	if (NFSD_VNET(nfsrv_numnfsd) == 0 || vp->v_type != VREG ||
 	    nfsrv_issuedelegs == 0)
 		return;
 
@@ -5845,12 +5855,13 @@ nfsrv_throwawayopens(NFSPROC_T *p)
 	int i;
 
 	NFSLOCKSTATE();
-	nfsrv_stablefirst.nsf_flags &= ~NFSNSF_NOOPENS;
+	NFSD_VNET(nfsrv_stablefirst).nsf_flags &= ~NFSNSF_NOOPENS;
 	/*
 	 * For each client...
 	 */
 	for (i = 0; i < nfsrv_clienthashsize; i++) {
-	    LIST_FOREACH_SAFE(clp, &nfsclienthash[i], lc_hash, nclp) {
+	    LIST_FOREACH_SAFE(clp, &NFSD_VNET(nfsclienthash)[i], lc_hash,
+		nclp) {
 		LIST_FOREACH_SAFE(stp, &clp->lc_open, ls_list, nstp) {
 			if (LIST_EMPTY(&stp->ls_open) &&
 			    (stp->ls_noopens > NFSNOOPEN ||
@@ -5911,7 +5922,7 @@ static time_t
 nfsrv_leaseexpiry(void)
 {
 
-	if (nfsrv_stablefirst.nsf_eograce > NFSD_MONOSEC)
+	if (NFSD_VNET(nfsrv_stablefirst).nsf_eograce > NFSD_MONOSEC)
 		return (NFSD_MONOSEC + 2 * (nfsrv_lease + NFSRV_LEASEDELTA));
 	return (NFSD_MONOSEC + nfsrv_lease + NFSRV_LEASEDELTA);
 }
@@ -6232,7 +6243,8 @@ nfsrv_throwawayallstate(NFSPROC_T *p)
 	 * For each client, clean out the state and then free the structure.
 	 */
 	for (i = 0; i < nfsrv_clienthashsize; i++) {
-		LIST_FOREACH_SAFE(clp, &nfsclienthash[i], lc_hash, nclp) {
+		LIST_FOREACH_SAFE(clp, &NFSD_VNET(nfsclienthash)[i], lc_hash,
+		    nclp) {
 			nfsrv_cleanclient(clp, p);
 			nfsrv_freedeleglist(&clp->lc_deleg);
 			nfsrv_freedeleglist(&clp->lc_olddeleg);
@@ -6245,7 +6257,8 @@ nfsrv_throwawayallstate(NFSPROC_T *p)
 	 * Also, free up any remaining lock file structures.
 	 */
 	for (i = 0; i < nfsrv_lockhashsize; i++) {
-		LIST_FOREACH_SAFE(lfp, &nfslockhash[i], lf_hash, nlfp) {
+		LIST_FOREACH_SAFE(lfp, &NFSD_VNET(nfslockhash)[i], lf_hash,
+		    nlfp) {
 			printf("nfsd unload: fnd a lock file struct\n");
 			nfsrv_freenfslockfile(lfp);
 		}
@@ -6363,7 +6376,8 @@ nfsrv_cache_session(struct nfsrv_descript *nd, struct mbuf **m)
 	sep = nfsrv_findsession(nd->nd_sessionid);
 	if (sep == NULL) {
 		NFSUNLOCKSESSION(shp);
-		if ((nfsrv_stablefirst.nsf_flags & NFSNSF_GRACEOVER) != 0) {
+		if ((NFSD_VNET(nfsrv_stablefirst).nsf_flags &
+		     NFSNSF_GRACEOVER) != 0) {
 			buf = malloc(INET6_ADDRSTRLEN, M_TEMP, M_WAITOK);
 			switch (nd->nd_nam->sa_family) {
 #ifdef INET
@@ -6725,7 +6739,7 @@ nfsrv_freeallbackchannel_xprts(void)
 	int i;
 
 	for (i = 0; i < nfsrv_clienthashsize; i++) {
-		LIST_FOREACH(clp, &nfsclienthash[i], lc_hash) {
+		LIST_FOREACH(clp, &NFSD_VNET(nfsclienthash)[i], lc_hash) {
 			LIST_FOREACH(sep, &clp->lc_session, sess_list) {
 				xprt = sep->sess_cbsess.nfsess_xprt;
 				sep->sess_cbsess.nfsess_xprt = NULL;
@@ -7441,7 +7455,7 @@ nfsrv_addlayout(struct nfsrv_descript *nd, struct nfslayout **lypp,
 	/* Insert the new layout in the lists. */
 	*lypp = NULL;
 	atomic_add_int(&nfsrv_layoutcnt, 1);
-	nfsstatsv1.srvlayouts++;
+	nfsstatsv1_p->srvlayouts++;
 	NFSBCOPY(lyp->lay_xdr, layp, lyp->lay_layoutlen);
 	*layoutlenp = lyp->lay_layoutlen;
 	TAILQ_INSERT_HEAD(&lhyp->list, lyp, lay_list);
@@ -7534,7 +7548,7 @@ nfsrv_freelayout(struct nfslayouthead *lhp, struct nfslayout *lyp)
 
 	NFSD_DEBUG(4, "Freelayout=%p\n", lyp);
 	atomic_add_int(&nfsrv_layoutcnt, -1);
-	nfsstatsv1.srvlayouts--;
+	nfsstatsv1_p->srvlayouts--;
 	TAILQ_REMOVE(lhp, lyp, lay_list);
 	free(lyp, M_NFSDSTATE);
 }
@@ -7684,6 +7698,10 @@ nfsrv_setdsserver(char *dspathp, char *mdspathp, NFSPROC_T *p,
 
 	NFSD_DEBUG(4, "setdssrv path=%s\n", dspathp);
 	*dsp = NULL;
+	if (jailed(p->td_ucred)) {
+		printf("A pNFS nfsd cannot run in a jail\n");
+		return (EPERM);
+	}
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, UIO_SYSSPACE,
 	    dspathp);
 	error = namei(&nd);
diff --git a/sys/fs/nfsserver/nfs_nfsdsubs.c b/sys/fs/nfsserver/nfs_nfsdsubs.c
index 20f62211f53e..fa5747c9c741 100644
--- a/sys/fs/nfsserver/nfs_nfsdsubs.c
+++ b/sys/fs/nfsserver/nfs_nfsdsubs.c
@@ -44,18 +44,24 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfs/nfsport.h>
 
 extern u_int32_t newnfs_true, newnfs_false;
-extern int nfs_rootfhset;
 extern int nfs_pubfhset;
-extern struct nfsclienthashhead *nfsclienthash;
 extern int nfsrv_clienthashsize;
-extern struct nfslockhashhead *nfslockhash;
 extern int nfsrv_lockhashsize;
-extern struct nfssessionhash *nfssessionhash;
 extern int nfsrv_sessionhashsize;
 extern int nfsrv_useacl;
 extern uid_t nfsrv_defaultuid;
 extern gid_t nfsrv_defaultgid;
 
+NFSD_VNET_DECLARE(struct nfsclienthashhead *, nfsclienthash);
+NFSD_VNET_DECLARE(struct nfslockhashhead *, nfslockhash);
+NFSD_VNET_DECLARE(struct nfssessionhash *, nfssessionhash);
+NFSD_VNET_DECLARE(int, nfs_rootfhset);
+NFSD_VNET_DECLARE(uid_t, nfsrv_defaultuid);
+NFSD_VNET_DECLARE(gid_t, nfsrv_defaultgid);
+
+NFSD_VNET_DEFINE(struct nfsdontlisthead, nfsrv_dontlisthead);
+
+
 char nfs_v2pubfh[NFSX_V2FH];
 struct nfsdontlisthead nfsrv_dontlisthead;
 struct nfslayouthead nfsrv_recalllisthead;
@@ -1607,10 +1613,12 @@ nfsrv_checkuidgid(struct nfsrv_descript *nd, struct nfsvattr *nvap)
 	 */
 	if (NFSVNO_NOTSETUID(nvap) && NFSVNO_NOTSETGID(nvap))
 		goto out;
-	if ((NFSVNO_ISSETUID(nvap) && nvap->na_uid == nfsrv_defaultuid &&
-           enable_nobodycheck == 1)
-	    || (NFSVNO_ISSETGID(nvap) && nvap->na_gid == nfsrv_defaultgid &&
-           enable_nogroupcheck == 1)) {
+	if ((NFSVNO_ISSETUID(nvap) &&
+	     nvap->na_uid == NFSD_VNET(nfsrv_defaultuid) &&
+             enable_nobodycheck == 1) ||
+	    (NFSVNO_ISSETGID(nvap) &&
+	     nvap->na_gid == NFSD_VNET(nfsrv_defaultgid) &&
+             enable_nogroupcheck == 1)) {
 		error = NFSERR_BADOWNER;
 		goto out;
 	}
@@ -2080,29 +2088,26 @@ void
 nfsd_init(void)
 {
 	int i;
-	static int inited = 0;
 
-	if (inited)
-		return;
-	inited = 1;
 
 	/*
 	 * Initialize client queues. Don't free/reinitialize
 	 * them when nfsds are restarted.
 	 */
-	nfsclienthash = malloc(sizeof(struct nfsclienthashhead) *
+	NFSD_VNET(nfsclienthash) = malloc(sizeof(struct nfsclienthashhead) *
 	    nfsrv_clienthashsize, M_NFSDCLIENT, M_WAITOK | M_ZERO);
 	for (i = 0; i < nfsrv_clienthashsize; i++)
-		LIST_INIT(&nfsclienthash[i]);
-	nfslockhash = malloc(sizeof(struct nfslockhashhead) *
+		LIST_INIT(&NFSD_VNET(nfsclienthash)[i]);
+	NFSD_VNET(nfslockhash) = malloc(sizeof(struct nfslockhashhead) *
 	    nfsrv_lockhashsize, M_NFSDLOCKFILE, M_WAITOK | M_ZERO);
 	for (i = 0; i < nfsrv_lockhashsize; i++)
-		LIST_INIT(&nfslockhash[i]);
-	nfssessionhash = malloc(sizeof(struct nfssessionhash) *
+		LIST_INIT(&NFSD_VNET(nfslockhash)[i]);
+	NFSD_VNET(nfssessionhash) = malloc(sizeof(struct nfssessionhash) *
 	    nfsrv_sessionhashsize, M_NFSDSESSION, M_WAITOK | M_ZERO);
 	for (i = 0; i < nfsrv_sessionhashsize; i++) {
-		mtx_init(&nfssessionhash[i].mtx, "nfssm", NULL, MTX_DEF);
-		LIST_INIT(&nfssessionhash[i].list);
+		mtx_init(&NFSD_VNET(nfssessionhash)[i].mtx, "nfssm", NULL,
+		    MTX_DEF);
+		LIST_INIT(&NFSD_VNET(nfssessionhash)[i].list);
 	}
 	LIST_INIT(&nfsrv_dontlisthead);
 	TAILQ_INIT(&nfsrv_recalllisthead);
@@ -2119,7 +2124,7 @@ int
 nfsd_checkrootexp(struct nfsrv_descript *nd)
 {
 
-	if (nfs_rootfhset == 0)
+	if (NFSD_VNET(nfs_rootfhset) == 0)
 		return (NFSERR_AUTHERR | AUTH_FAILED);
 	/*
 	 * For NFSv4.1/4.2, if the client specifies SP4_NONE, then these