aboutsummaryrefslogtreecommitdiff
path: root/sys/sys/smr.h
diff options
context:
space:
mode:
authorJeff Roberson <jeff@FreeBSD.org>2020-02-22 03:44:10 +0000
committerJeff Roberson <jeff@FreeBSD.org>2020-02-22 03:44:10 +0000
commit226dd6db470570bcba4db850760d485921d55fc3 (patch)
tree9ff79bd40bdba43edc0d0d0521198fe051605b6e /sys/sys/smr.h
parentb63d2d683ddbbfc066fb4653b2e37e6c58875d6d (diff)
downloadsrc-226dd6db470570bcba4db850760d485921d55fc3.tar.gz
src-226dd6db470570bcba4db850760d485921d55fc3.zip
Add an atomic-free tick moderated lazy update variant of SMR.
This enables very cheap read sections with free-to-use latencies and memory overhead similar to epoch. On a recent AMD platform a read section cost 1ns vs 5ns for the default SMR. On Xeon the numbers should be more like 1 ns vs 11. The memory consumption should be proportional to the product of the free rate and 2*1/hz while normal SMR consumption is proportional to the product of free rate and maximum read section time. While here refactor the code to make future additions more straightforward. Name the overall technique Global Unbound Sequences (GUS) and adjust some comments accordingly. This helps distinguish discussions of the general technique (SMR) vs this specific implementation (GUS). Discussed with: rlibby, markj
Notes
Notes: svn path=/head/; revision=358236
Diffstat (limited to 'sys/sys/smr.h')
-rw-r--r--sys/sys/smr.h93
1 files changed, 78 insertions, 15 deletions
diff --git a/sys/sys/smr.h b/sys/sys/smr.h
index 378088f37cad..9e9cc3dec3a5 100644
--- a/sys/sys/smr.h
+++ b/sys/sys/smr.h
@@ -45,11 +45,13 @@
* Modular arithmetic for comparing sequence numbers that have
* potentially wrapped. Copied from tcp_seq.h.
*/
-#define SMR_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0)
-#define SMR_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0)
-#define SMR_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0)
-#define SMR_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0)
-#define SMR_SEQ_DELTA(a, b) ((int32_t)((a)-(b)))
+#define SMR_SEQ_LT(a, b) ((smr_delta_t)((a)-(b)) < 0)
+#define SMR_SEQ_LEQ(a, b) ((smr_delta_t)((a)-(b)) <= 0)
+#define SMR_SEQ_GT(a, b) ((smr_delta_t)((a)-(b)) > 0)
+#define SMR_SEQ_GEQ(a, b) ((smr_delta_t)((a)-(b)) >= 0)
+#define SMR_SEQ_DELTA(a, b) ((smr_delta_t)((a)-(b)))
+#define SMR_SEQ_MIN(a, b) (SMR_SEQ_LT((a), (b)) ? (a) : (b))
+#define SMR_SEQ_MAX(a, b) (SMR_SEQ_GT((a), (b)) ? (a) : (b))
#define SMR_SEQ_INVALID 0
@@ -66,8 +68,13 @@ struct smr {
smr_seq_t c_seq; /* Current observed sequence. */
smr_shared_t c_shared; /* Shared SMR state. */
int c_deferred; /* Deferred advance counter. */
+ int c_limit; /* Deferred advance limit. */
+ int c_flags; /* SMR Configuration */
};
+#define SMR_LAZY 0x0001 /* Higher latency write, fast read. */
+#define SMR_DEFERRED 0x0002 /* Aggregate updates to wr_seq. */
+
#define SMR_ENTERED(smr) \
(curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID)
@@ -94,7 +101,7 @@ struct smr {
* All acceses include a parameter for an assert to verify the required
* synchronization. For example, a writer might use:
*
- * smr_serilized_store(pointer, value, mtx_owned(&writelock));
+ * smr_serialized_store(pointer, value, mtx_owned(&writelock));
*
* These are only enabled in INVARIANTS kernels.
*/
@@ -127,6 +134,9 @@ typedef struct { \
* Store 'v' to an SMR protected pointer while serialized by an
* external mechanism. 'ex' should contain an assert that the
* external mechanism is held. i.e. mtx_owned()
+ *
+ * Writers that are serialized with mutual exclusion or on a single
+ * thread should use smr_serialized_store() rather than swap.
*/
#define smr_serialized_store(p, v, ex) do { \
SMR_ASSERT(ex, "smr_serialized_store"); \
@@ -138,6 +148,8 @@ typedef struct { \
* swap 'v' with an SMR protected pointer and return the old value
* while serialized by an external mechanism. 'ex' should contain
* an assert that the external mechanism is provided. i.e. mtx_owned()
+ *
+ * Swap permits multiple writers to update a pointer concurrently.
*/
#define smr_serialized_swap(p, v, ex) ({ \
SMR_ASSERT(ex, "smr_serialized_swap"); \
@@ -170,7 +182,8 @@ typedef struct { \
} while (0)
/*
- * Return the current write sequence number.
+ * Return the current write sequence number. This is not the same as the
+ * current goal which may be in the future.
*/
static inline smr_seq_t
smr_shared_current(smr_shared_t s)
@@ -195,6 +208,8 @@ smr_enter(smr_t smr)
critical_enter();
smr = zpcpu_get(smr);
+ KASSERT((smr->c_flags & SMR_LAZY) == 0,
+ ("smr_enter(%s) lazy smr.", smr->c_shared->s_name));
KASSERT(smr->c_seq == 0,
("smr_enter(%s) does not support recursion.",
smr->c_shared->s_name));
@@ -228,6 +243,8 @@ smr_exit(smr_t smr)
smr = zpcpu_get(smr);
CRITICAL_ASSERT(curthread);
+ KASSERT((smr->c_flags & SMR_LAZY) == 0,
+ ("smr_exit(%s) lazy smr.", smr->c_shared->s_name));
KASSERT(smr->c_seq != SMR_SEQ_INVALID,
("smr_exit(%s) not in a smr section.", smr->c_shared->s_name));
@@ -243,17 +260,61 @@ smr_exit(smr_t smr)
}
/*
- * Advances the write sequence number. Returns the sequence number
- * required to ensure that all modifications are visible to readers.
+ * Enter a lazy smr section. This is used for read-mostly state that
+ * can tolerate a high free latency.
*/
-smr_seq_t smr_advance(smr_t smr);
+static inline void
+smr_lazy_enter(smr_t smr)
+{
+
+ critical_enter();
+ smr = zpcpu_get(smr);
+ KASSERT((smr->c_flags & SMR_LAZY) != 0,
+ ("smr_lazy_enter(%s) non-lazy smr.", smr->c_shared->s_name));
+ KASSERT(smr->c_seq == 0,
+ ("smr_lazy_enter(%s) does not support recursion.",
+ smr->c_shared->s_name));
+
+ /*
+ * This needs no serialization. If an interrupt occurs before we
+ * assign sr_seq to c_seq any speculative loads will be discarded.
+ * If we assign a stale wr_seq value due to interrupt we use the
+ * same algorithm that renders smr_enter() safe.
+ */
+ smr->c_seq = smr_shared_current(smr->c_shared);
+}
/*
- * Advances the write sequence number only after N calls. Returns
- * the correct goal for a wr_seq that has not yet occurred. Used to
- * minimize shared cacheline invalidations for frequent writers.
+ * Exit a lazy smr section. This is used for read-mostly state that
+ * can tolerate a high free latency.
*/
-smr_seq_t smr_advance_deferred(smr_t smr, int limit);
+static inline void
+smr_lazy_exit(smr_t smr)
+{
+
+ smr = zpcpu_get(smr);
+ CRITICAL_ASSERT(curthread);
+ KASSERT((smr->c_flags & SMR_LAZY) != 0,
+ ("smr_lazy_enter(%s) non-lazy smr.", smr->c_shared->s_name));
+ KASSERT(smr->c_seq != SMR_SEQ_INVALID,
+ ("smr_lazy_exit(%s) not in a smr section.", smr->c_shared->s_name));
+
+ /*
+ * All loads/stores must be retired before the sequence becomes
+ * visible. The fence compiles away on amd64. Another
+ * alternative would be to omit the fence but store the exit
+ * time and wait 1 tick longer.
+ */
+ atomic_thread_fence_rel();
+ smr->c_seq = SMR_SEQ_INVALID;
+ critical_exit();
+}
+
+/*
+ * Advances the write sequence number. Returns the sequence number
+ * required to ensure that all modifications are visible to readers.
+ */
+smr_seq_t smr_advance(smr_t smr);
/*
* Returns true if a goal sequence has been reached. If
@@ -262,7 +323,9 @@ smr_seq_t smr_advance_deferred(smr_t smr, int limit);
bool smr_poll(smr_t smr, smr_seq_t goal, bool wait);
/* Create a new SMR context. */
-smr_t smr_create(const char *name);
+smr_t smr_create(const char *name, int limit, int flags);
+
+/* Destroy the context. */
void smr_destroy(smr_t smr);
/*