vfs: group mount per-cpu vars into one struct

While here move frequently read stuff into the same cacheline.

This shrinks struct mount by 64 bytes.

Tested by:	pho

6 files changed, 144 insertions, 110 deletions

diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index 170805b64d6f..7a9fb0aada6e 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -4249,6 +4249,7 @@ static int __noinline
 cache_fplookup_climb_mount(struct cache_fpl *fpl)
 {
 	struct mount *mp, *prev_mp;
+	struct mount_pcpu *mpcpu, *prev_mpcpu;
 	struct vnode *vp;
 	seqc_t vp_seqc;
 
@@ -4262,38 +4263,39 @@ cache_fplookup_climb_mount(struct cache_fpl *fpl)
 
 	prev_mp = NULL;
 	for (;;) {
-		if (!vfs_op_thread_enter_crit(mp)) {
+		if (!vfs_op_thread_enter_crit(mp, mpcpu)) {
 			if (prev_mp != NULL)
-				vfs_op_thread_exit_crit(prev_mp);
+				vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
 			return (cache_fpl_partial(fpl));
 		}
 		if (prev_mp != NULL)
-			vfs_op_thread_exit_crit(prev_mp);
+			vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
 		if (!vn_seqc_consistent(vp, vp_seqc)) {
-			vfs_op_thread_exit_crit(mp);
+			vfs_op_thread_exit_crit(mp, mpcpu);
 			return (cache_fpl_partial(fpl));
 		}
 		if (!cache_fplookup_mp_supported(mp)) {
-			vfs_op_thread_exit_crit(mp);
+			vfs_op_thread_exit_crit(mp, mpcpu);
 			return (cache_fpl_partial(fpl));
 		}
 		vp = atomic_load_ptr(&mp->mnt_rootvnode);
 		if (vp == NULL || VN_IS_DOOMED(vp)) {
-			vfs_op_thread_exit_crit(mp);
+			vfs_op_thread_exit_crit(mp, mpcpu);
 			return (cache_fpl_partial(fpl));
 		}
 		vp_seqc = vn_seqc_read_any(vp);
 		if (seqc_in_modify(vp_seqc)) {
-			vfs_op_thread_exit_crit(mp);
+			vfs_op_thread_exit_crit(mp, mpcpu);
 			return (cache_fpl_partial(fpl));
 		}
 		prev_mp = mp;
+		prev_mpcpu = mpcpu;
 		mp = atomic_load_ptr(&vp->v_mountedhere);
 		if (mp == NULL)
 			break;
 	}
 
-	vfs_op_thread_exit_crit(prev_mp);
+	vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
 	fpl->tvp = vp;
 	fpl->tvp_seqc = vp_seqc;
 	return (0);
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index 64de62cca79a..d9f44e1dc6b9 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -663,6 +663,7 @@ vop_stdgetwritemount(ap)
 	} */ *ap;
 {
 	struct mount *mp;
+	struct mount_pcpu *mpcpu;
 	struct vnode *vp;
 
 	/*
@@ -680,12 +681,12 @@ vop_stdgetwritemount(ap)
 		*(ap->a_mpp) = NULL;
 		return (0);
 	}
-	if (vfs_op_thread_enter(mp)) {
+	if (vfs_op_thread_enter(mp, mpcpu)) {
 		if (mp == vp->v_mount) {
-			vfs_mp_count_add_pcpu(mp, ref, 1);
-			vfs_op_thread_exit(mp);
+			vfs_mp_count_add_pcpu(mpcpu, ref, 1);
+			vfs_op_thread_exit(mp, mpcpu);
 		} else {
-			vfs_op_thread_exit(mp);
+			vfs_op_thread_exit(mp, mpcpu);
 			mp = NULL;
 		}
 	} else {
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 7ab3b6274491..925f84b192d0 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -127,14 +127,7 @@ mount_init(void *mem, int size, int flags)
 	mtx_init(&mp->mnt_mtx, "struct mount mtx", NULL, MTX_DEF);
 	mtx_init(&mp->mnt_listmtx, "struct mount vlist mtx", NULL, MTX_DEF);
 	lockinit(&mp->mnt_explock, PVFS, "explock", 0, 0);
-	mp->mnt_thread_in_ops_pcpu = uma_zalloc_pcpu(pcpu_zone_4,
-	    M_WAITOK | M_ZERO);
-	mp->mnt_ref_pcpu = uma_zalloc_pcpu(pcpu_zone_4,
-	    M_WAITOK | M_ZERO);
-	mp->mnt_lockref_pcpu = uma_zalloc_pcpu(pcpu_zone_4,
-	    M_WAITOK | M_ZERO);
-	mp->mnt_writeopcount_pcpu = uma_zalloc_pcpu(pcpu_zone_4,
-	    M_WAITOK | M_ZERO);
+	mp->mnt_pcpu = uma_zalloc_pcpu(pcpu_zone_16, M_WAITOK | M_ZERO);
 	mp->mnt_ref = 0;
 	mp->mnt_vfs_ops = 1;
 	mp->mnt_rootvnode = NULL;
@@ -147,10 +140,7 @@ mount_fini(void *mem, int size)
 	struct mount *mp;
 
 	mp = (struct mount *)mem;
-	uma_zfree_pcpu(pcpu_zone_4, mp->mnt_writeopcount_pcpu);
-	uma_zfree_pcpu(pcpu_zone_4, mp->mnt_lockref_pcpu);
-	uma_zfree_pcpu(pcpu_zone_4, mp->mnt_ref_pcpu);
-	uma_zfree_pcpu(pcpu_zone_4, mp->mnt_thread_in_ops_pcpu);
+	uma_zfree_pcpu(pcpu_zone_16, mp->mnt_pcpu);
 	lockdestroy(&mp->mnt_explock);
 	mtx_destroy(&mp->mnt_listmtx);
 	mtx_destroy(&mp->mnt_mtx);
@@ -462,11 +452,12 @@ sys_nmount(struct thread *td, struct nmount_args *uap)
 void
 vfs_ref(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 
 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
-	if (vfs_op_thread_enter(mp)) {
-		vfs_mp_count_add_pcpu(mp, ref, 1);
-		vfs_op_thread_exit(mp);
+	if (vfs_op_thread_enter(mp, mpcpu)) {
+		vfs_mp_count_add_pcpu(mpcpu, ref, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		return;
 	}
 
@@ -478,11 +469,12 @@ vfs_ref(struct mount *mp)
 void
 vfs_rel(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 
 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
-	if (vfs_op_thread_enter(mp)) {
-		vfs_mp_count_sub_pcpu(mp, ref, 1);
-		vfs_op_thread_exit(mp);
+	if (vfs_op_thread_enter(mp, mpcpu)) {
+		vfs_mp_count_sub_pcpu(mpcpu, ref, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		return;
 	}
 
@@ -503,6 +495,12 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf *vfsp, const char *fspath,
 	mp = uma_zalloc(mount_zone, M_WAITOK);
 	bzero(&mp->mnt_startzero,
 	    __rangeof(struct mount, mnt_startzero, mnt_endzero));
+	mp->mnt_kern_flag = 0;
+	mp->mnt_flag = 0;
+	mp->mnt_rootvnode = NULL;
+	mp->mnt_vnodecovered = NULL;
+	mp->mnt_op = NULL;
+	mp->mnt_vfc = NULL;
 	TAILQ_INIT(&mp->mnt_nvnodelist);
 	mp->mnt_nvnodelistsize = 0;
 	TAILQ_INIT(&mp->mnt_lazyvnodelist);
@@ -1513,6 +1511,7 @@ dounmount_cleanup(struct mount *mp, struct vnode *coveredvp, int mntkflags)
 void
 vfs_op_enter(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 	int cpu;
 
 	MNT_ILOCK(mp);
@@ -1523,12 +1522,16 @@ vfs_op_enter(struct mount *mp)
 	}
 	vfs_op_barrier_wait(mp);
 	CPU_FOREACH(cpu) {
-		mp->mnt_ref +=
-		    zpcpu_replace_cpu(mp->mnt_ref_pcpu, 0, cpu);
-		mp->mnt_lockref +=
-		    zpcpu_replace_cpu(mp->mnt_lockref_pcpu, 0, cpu);
-		mp->mnt_writeopcount +=
-		    zpcpu_replace_cpu(mp->mnt_writeopcount_pcpu, 0, cpu);
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+
+		mp->mnt_ref += mpcpu->mntp_ref;
+		mpcpu->mntp_ref = 0;
+
+		mp->mnt_lockref += mpcpu->mntp_lockref;
+		mpcpu->mntp_lockref = 0;
+
+		mp->mnt_writeopcount += mpcpu->mntp_writeopcount;
+		mpcpu->mntp_writeopcount = 0;
 	}
 	if (mp->mnt_ref <= 0 || mp->mnt_lockref < 0 || mp->mnt_writeopcount < 0)
 		panic("%s: invalid count(s) on mp %p: ref %d lockref %d writeopcount %d\n",
@@ -1581,13 +1584,13 @@ vfs_op_wait_func(void *arg, int cpu)
 {
 	struct vfs_op_barrier_ipi *vfsopipi;
 	struct mount *mp;
-	int *in_op;
+	struct mount_pcpu *mpcpu;
 
 	vfsopipi = __containerof(arg, struct vfs_op_barrier_ipi, srcra);
 	mp = vfsopipi->mp;
 
-	in_op = zpcpu_get_cpu(mp->mnt_thread_in_ops_pcpu, cpu);
-	while (atomic_load_int(in_op))
+	mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+	while (atomic_load_int(&mpcpu->mntp_thread_in_ops))
 		cpu_spinwait();
 }
 
@@ -1610,15 +1613,17 @@ vfs_op_barrier_wait(struct mount *mp)
 void
 vfs_assert_mount_counters(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 	int cpu;
 
 	if (mp->mnt_vfs_ops == 0)
 		return;
 
 	CPU_FOREACH(cpu) {
-		if (*zpcpu_get_cpu(mp->mnt_ref_pcpu, cpu) != 0 ||
-		    *zpcpu_get_cpu(mp->mnt_lockref_pcpu, cpu) != 0 ||
-		    *zpcpu_get_cpu(mp->mnt_writeopcount_pcpu, cpu) != 0)
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+		if (mpcpu->mntp_ref != 0 ||
+		    mpcpu->mntp_lockref != 0 ||
+		    mpcpu->mntp_writeopcount != 0)
 			vfs_dump_mount_counters(mp);
 	}
 }
@@ -1626,33 +1631,34 @@ vfs_assert_mount_counters(struct mount *mp)
 void
 vfs_dump_mount_counters(struct mount *mp)
 {
-	int cpu, *count;
+	struct mount_pcpu *mpcpu;
 	int ref, lockref, writeopcount;
+	int cpu;
 
 	printf("%s: mp %p vfs_ops %d\n", __func__, mp, mp->mnt_vfs_ops);
 
 	printf("        ref : ");
 	ref = mp->mnt_ref;
 	CPU_FOREACH(cpu) {
-		count = zpcpu_get_cpu(mp->mnt_ref_pcpu, cpu);
-		printf("%d ", *count);
-		ref += *count;
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+		printf("%d ", mpcpu->mntp_ref);
+		ref += mpcpu->mntp_ref;
 	}
 	printf("\n");
 	printf("    lockref : ");
 	lockref = mp->mnt_lockref;
 	CPU_FOREACH(cpu) {
-		count = zpcpu_get_cpu(mp->mnt_lockref_pcpu, cpu);
-		printf("%d ", *count);
-		lockref += *count;
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+		printf("%d ", mpcpu->mntp_lockref);
+		lockref += mpcpu->mntp_lockref;
 	}
 	printf("\n");
 	printf("writeopcount: ");
 	writeopcount = mp->mnt_writeopcount;
 	CPU_FOREACH(cpu) {
-		count = zpcpu_get_cpu(mp->mnt_writeopcount_pcpu, cpu);
-		printf("%d ", *count);
-		writeopcount += *count;
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+		printf("%d ", mpcpu->mntp_writeopcount);
+		writeopcount += mpcpu->mntp_writeopcount;
 	}
 	printf("\n");
 
@@ -1668,27 +1674,34 @@ vfs_dump_mount_counters(struct mount *mp)
 int
 vfs_mount_fetch_counter(struct mount *mp, enum mount_counter which)
 {
-	int *base, *pcpu;
+	struct mount_pcpu *mpcpu;
 	int cpu, sum;
 
 	switch (which) {
 	case MNT_COUNT_REF:
-		base = &mp->mnt_ref;
-		pcpu = mp->mnt_ref_pcpu;
+		sum = mp->mnt_ref;
 		break;
 	case MNT_COUNT_LOCKREF:
-		base = &mp->mnt_lockref;
-		pcpu = mp->mnt_lockref_pcpu;
+		sum = mp->mnt_lockref;
 		break;
 	case MNT_COUNT_WRITEOPCOUNT:
-		base = &mp->mnt_writeopcount;
-		pcpu = mp->mnt_writeopcount_pcpu;
+		sum = mp->mnt_writeopcount;
 		break;
 	}
 
-	sum = *base;
 	CPU_FOREACH(cpu) {
-		sum += *zpcpu_get_cpu(pcpu, cpu);
+		mpcpu = vfs_mount_pcpu_remote(mp, cpu);
+		switch (which) {
+		case MNT_COUNT_REF:
+			sum += mpcpu->mntp_ref;
+			break;
+		case MNT_COUNT_LOCKREF:
+			sum += mpcpu->mntp_lockref;
+			break;
+		case MNT_COUNT_WRITEOPCOUNT:
+			sum += mpcpu->mntp_writeopcount;
+			break;
+		}
 	}
 	return (sum);
 }
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 6045c704b227..93f5c4979925 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -734,17 +734,18 @@ SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
 int
 vfs_busy(struct mount *mp, int flags)
 {
+	struct mount_pcpu *mpcpu;
 
 	MPASS((flags & ~MBF_MASK) == 0);
 	CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);
 
-	if (vfs_op_thread_enter(mp)) {
+	if (vfs_op_thread_enter(mp, mpcpu)) {
 		MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
 		MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0);
 		MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0);
-		vfs_mp_count_add_pcpu(mp, ref, 1);
-		vfs_mp_count_add_pcpu(mp, lockref, 1);
-		vfs_op_thread_exit(mp);
+		vfs_mp_count_add_pcpu(mpcpu, ref, 1);
+		vfs_mp_count_add_pcpu(mpcpu, lockref, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		if (flags & MBF_MNTLSTLOCK)
 			mtx_unlock(&mountlist_mtx);
 		return (0);
@@ -794,15 +795,16 @@ vfs_busy(struct mount *mp, int flags)
 void
 vfs_unbusy(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 	int c;
 
 	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
 
-	if (vfs_op_thread_enter(mp)) {
+	if (vfs_op_thread_enter(mp, mpcpu)) {
 		MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
-		vfs_mp_count_sub_pcpu(mp, lockref, 1);
-		vfs_mp_count_sub_pcpu(mp, ref, 1);
-		vfs_op_thread_exit(mp);
+		vfs_mp_count_sub_pcpu(mpcpu, lockref, 1);
+		vfs_mp_count_sub_pcpu(mpcpu, ref, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		return;
 	}
 
@@ -6399,18 +6401,19 @@ restart:
 int
 vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp)
 {
+	struct mount_pcpu *mpcpu;
 	struct vnode *vp;
 	int error;
 
-	if (!vfs_op_thread_enter(mp))
+	if (!vfs_op_thread_enter(mp, mpcpu))
 		return (vfs_cache_root_fallback(mp, flags, vpp));
 	vp = atomic_load_ptr(&mp->mnt_rootvnode);
 	if (vp == NULL || VN_IS_DOOMED(vp)) {
-		vfs_op_thread_exit(mp);
+		vfs_op_thread_exit(mp, mpcpu);
 		return (vfs_cache_root_fallback(mp, flags, vpp));
 	}
 	vrefact(vp);
-	vfs_op_thread_exit(mp);
+	vfs_op_thread_exit(mp, mpcpu);
 	error = vn_lock(vp, flags);
 	if (error != 0) {
 		vrele(vp);
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 9c3b52bf4cc4..4aec027f2c7d 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -1761,13 +1761,14 @@ vn_closefile(struct file *fp, struct thread *td)
 static int
 vn_start_write_refed(struct mount *mp, int flags, bool mplocked)
 {
+	struct mount_pcpu *mpcpu;
 	int error, mflags;
 
 	if (__predict_true(!mplocked) && (flags & V_XSLEEP) == 0 &&
-	    vfs_op_thread_enter(mp)) {
+	    vfs_op_thread_enter(mp, mpcpu)) {
 		MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
-		vfs_mp_count_add_pcpu(mp, writeopcount, 1);
-		vfs_op_thread_exit(mp);
+		vfs_mp_count_add_pcpu(mpcpu, writeopcount, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		return (0);
 	}
 
@@ -1917,15 +1918,16 @@ vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
 void
 vn_finished_write(struct mount *mp)
 {
+	struct mount_pcpu *mpcpu;
 	int c;
 
 	if (mp == NULL)
 		return;
 
-	if (vfs_op_thread_enter(mp)) {
-		vfs_mp_count_sub_pcpu(mp, writeopcount, 1);
-		vfs_mp_count_sub_pcpu(mp, ref, 1);
-		vfs_op_thread_exit(mp);
+	if (vfs_op_thread_enter(mp, mpcpu)) {
+		vfs_mp_count_sub_pcpu(mpcpu, writeopcount, 1);
+		vfs_mp_count_sub_pcpu(mpcpu, ref, 1);
+		vfs_op_thread_exit(mp, mpcpu);
 		return;
 	}
 
diff --git a/sys/sys/mount.h b/sys/sys/mount.h
index ee8d916c2432..f965dd72d7ba 100644
--- a/sys/sys/mount.h
+++ b/sys/sys/mount.h
@@ -177,6 +177,16 @@ struct vfsopt {
 	int	seen;
 };
 
+struct mount_pcpu {
+	int		mntp_thread_in_ops;
+	int		mntp_ref;
+	int		mntp_lockref;
+	int		mntp_writeopcount;
+};
+
+_Static_assert(sizeof(struct mount_pcpu) == 16,
+    "the struct is allocated from pcpu 16 zone");
+
 /*
  * Structure per mounted filesystem.  Each mounted filesystem has an
  * array of operations and an instance record.  The filesystems are
@@ -192,20 +202,23 @@ struct vfsopt {
  *
  */
 struct mount {
-	struct mtx	mnt_mtx;		/* mount structure interlock */
+	int 		mnt_vfs_ops;		/* (i) pending vfs ops */
+	int		mnt_kern_flag;		/* (i) kernel only flags */
+	uint64_t	mnt_flag;		/* (i) flags shared with user */
+	struct mount_pcpu *mnt_pcpu;		/* per-CPU data */
+	struct vnode	*mnt_rootvnode;
+	struct vnode	*mnt_vnodecovered;	/* vnode we mounted on */
+	struct vfsops	*mnt_op;		/* operations on fs */
+	struct vfsconf	*mnt_vfc;		/* configuration info */
+	struct mtx __aligned(CACHE_LINE_SIZE)	mnt_mtx; /* mount structure interlock */
 	int		mnt_gen;		/* struct mount generation */
 #define	mnt_startzero	mnt_list
 	TAILQ_ENTRY(mount) mnt_list;		/* (m) mount list */
-	struct vfsops	*mnt_op;		/* operations on fs */
-	struct vfsconf	*mnt_vfc;		/* configuration info */
-	struct vnode	*mnt_vnodecovered;	/* vnode we mounted on */
 	struct vnode	*mnt_syncer;		/* syncer vnode */
 	int		mnt_ref;		/* (i) Reference count */
 	struct vnodelst	mnt_nvnodelist;		/* (i) list of vnodes */
 	int		mnt_nvnodelistsize;	/* (i) # of vnodes */
 	int		mnt_writeopcount;	/* (i) write syscalls pending */
-	int		mnt_kern_flag;		/* (i) kernel only flags */
-	uint64_t	mnt_flag;		/* (i) flags shared with user */
 	struct vfsoptlist *mnt_opt;		/* current mount options */
 	struct vfsoptlist *mnt_optnew;		/* new options passed to fs */
 	int		mnt_maxsymlinklen;	/* max size of short symlink */
@@ -229,12 +242,6 @@ struct mount {
 	struct lock	mnt_explock;		/* vfs_export walkers lock */
 	TAILQ_ENTRY(mount) mnt_upper_link;	/* (m) we in the all uppers */
 	TAILQ_HEAD(, mount) mnt_uppers;		/* (m) upper mounts over us*/
-	int __aligned(CACHE_LINE_SIZE)	mnt_vfs_ops;/* (i) pending vfs ops */
-	int		*mnt_thread_in_ops_pcpu;
-	int		*mnt_ref_pcpu;
-	int		*mnt_lockref_pcpu;
-	int		*mnt_writeopcount_pcpu;
-	struct vnode	*mnt_rootvnode;
 };
 
 /*
@@ -1054,7 +1061,7 @@ void resume_all_fs(void);
 
 /*
  * Code transitioning mnt_vfs_ops to > 0 issues IPIs until it observes
- * all CPUs not executing code enclosed by mnt_thread_in_ops_pcpu.
+ * all CPUs not executing code enclosed by thread_in_ops_pcpu variable.
  *
  * This provides an invariant that by the time the last CPU is observed not
  * executing, everyone else entering will see the counter > 0 and exit.
@@ -1064,52 +1071,58 @@ void resume_all_fs(void);
  * before making any changes or only make changes safe while the section is
  * executed.
  */
+#define	vfs_mount_pcpu(mp)		zpcpu_get(mp->mnt_pcpu)
+#define	vfs_mount_pcpu_remote(mp, cpu)	zpcpu_get_cpu(mp->mnt_pcpu, cpu)
+
 #define vfs_op_thread_entered(mp) ({				\
 	MPASS(curthread->td_critnest > 0);			\
-	*zpcpu_get(mp->mnt_thread_in_ops_pcpu) == 1;		\
+	struct mount_pcpu *_mpcpu = vfs_mount_pcpu(mp);		\
+	_mpcpu->mntp_thread_in_ops == 1;			\
 })
 
-#define vfs_op_thread_enter_crit(mp) ({				\
+#define vfs_op_thread_enter_crit(mp, _mpcpu) ({			\
 	bool _retval_crit = true;				\
 	MPASS(curthread->td_critnest > 0);			\
-	MPASS(!vfs_op_thread_entered(mp));			\
-	zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 1);	\
+	_mpcpu = vfs_mount_pcpu(mp);				\
+	MPASS(mpcpu->mntp_thread_in_ops == 0);			\
+	_mpcpu->mntp_thread_in_ops = 1;				\
 	__compiler_membar();					\
 	if (__predict_false(mp->mnt_vfs_ops > 0)) {		\
-		vfs_op_thread_exit_crit(mp);			\
+		vfs_op_thread_exit_crit(mp, _mpcpu);		\
 		_retval_crit = false;				\
 	}							\
 	_retval_crit;						\
 })
 
-#define vfs_op_thread_enter(mp) ({				\
+#define vfs_op_thread_enter(mp, _mpcpu) ({			\
 	bool _retval;						\
 	critical_enter();					\
-	_retval = vfs_op_thread_enter_crit(mp);			\
+	_retval = vfs_op_thread_enter_crit(mp, _mpcpu);		\
 	if (__predict_false(!_retval))				\
 		critical_exit();				\
 	_retval;						\
 })
 
-#define vfs_op_thread_exit_crit(mp) do {			\
-	MPASS(vfs_op_thread_entered(mp));			\
+#define vfs_op_thread_exit_crit(mp, _mpcpu) do {		\
+	MPASS(_mpcpu == vfs_mount_pcpu(mp));			\
+	MPASS(_mpcpu->mntp_thread_in_ops == 1);			\
 	__compiler_membar();					\
-	zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 0);	\
+	_mpcpu->mntp_thread_in_ops = 0;				\
 } while (0)
 
-#define vfs_op_thread_exit(mp) do {				\
-	vfs_op_thread_exit_crit(mp);				\
+#define vfs_op_thread_exit(mp, _mpcpu) do {			\
+	vfs_op_thread_exit_crit(mp, _mpcpu);			\
 	critical_exit();					\
 } while (0)
 
-#define vfs_mp_count_add_pcpu(mp, count, val) do {		\
-	MPASS(vfs_op_thread_entered(mp));			\
-	zpcpu_add_protected(mp->mnt_##count##_pcpu, val);	\
+#define vfs_mp_count_add_pcpu(_mpcpu, count, val) do {		\
+	MPASS(_mpcpu->mntp_thread_in_ops == 1);			\
+	_mpcpu->mntp_##count += val;				\
 } while (0)
 
-#define vfs_mp_count_sub_pcpu(mp, count, val) do {		\
-	MPASS(vfs_op_thread_entered(mp));			\
-	zpcpu_sub_protected(mp->mnt_##count##_pcpu, val);	\
+#define vfs_mp_count_sub_pcpu(_mpcpu, count, val) do {		\
+	MPASS(_mpcpu->mntp_thread_in_ops == 1);			\
+	_mpcpu->mntp_##count -= val;				\
 } while (0)
 
 #else /* !_KERNEL */