aboutsummaryrefslogtreecommitdiff
path: root/sys/kern/subr_smr.c
diff options
context:
space:
mode:
authorJeff Roberson <jeff@FreeBSD.org>2020-01-31 22:21:15 +0000
committerJeff Roberson <jeff@FreeBSD.org>2020-01-31 22:21:15 +0000
commit915c367e8e9b718e5146401375d2188f14d39ac3 (patch)
treef28d2dbc991f9f0b8ed3c60e85a58d232426b23b /sys/kern/subr_smr.c
parent82952c40657eb7957689581e4e204d117197f36b (diff)
downloadsrc-915c367e8e9b718e5146401375d2188f14d39ac3.tar.gz
src-915c367e8e9b718e5146401375d2188f14d39ac3.zip
Add two missing fences with comments describing them. These were found by
inspection and after a lengthy discussion with jhb and kib. They have not produced test failures. Don't pointer chase through cpu0's smr. Use cpu correct smr even when not in a critical section to reduce the likelihood of false sharing.
Notes
Notes: svn path=/head/; revision=357355
Diffstat (limited to 'sys/kern/subr_smr.c')
-rw-r--r--sys/kern/subr_smr.c17
1 files changed, 14 insertions, 3 deletions
diff --git a/sys/kern/subr_smr.c b/sys/kern/subr_smr.c
index c6a8f0e89015..c344fdcb7f5f 100644
--- a/sys/kern/subr_smr.c
+++ b/sys/kern/subr_smr.c
@@ -195,7 +195,7 @@ smr_advance(smr_t smr)
* odd and an observed value of 0 in a particular CPU means
* it is not currently in a read section.
*/
- s = smr->c_shared;
+ s = zpcpu_get(smr)->c_shared;
goal = atomic_fetchadd_int(&s->s_wr_seq, SMR_SEQ_INCR) + SMR_SEQ_INCR;
/*
@@ -242,14 +242,19 @@ smr_poll(smr_t smr, smr_seq_t goal, bool wait)
*/
success = true;
critical_enter();
- s = smr->c_shared;
+ s = zpcpu_get(smr)->c_shared;
/*
* Acquire barrier loads s_wr_seq after s_rd_seq so that we can not
* observe an updated read sequence that is larger than write.
*/
s_rd_seq = atomic_load_acq_int(&s->s_rd_seq);
- s_wr_seq = smr_current(smr);
+
+ /*
+ * wr_seq must be loaded prior to any c_seq value so that a stale
+ * c_seq can only reference time after this wr_seq.
+ */
+ s_wr_seq = atomic_load_acq_int(&s->s_wr_seq);
/*
* Detect whether the goal is valid and has already been observed.
@@ -336,6 +341,12 @@ smr_poll(smr_t smr, smr_seq_t goal, bool wait)
out:
critical_exit();
+ /*
+ * Serialize with smr_advance()/smr_exit(). The caller is now free
+ * to modify memory as expected.
+ */
+ atomic_thread_fence_acq();
+
return (success);
}