IPsec RFC6479 support for replay window sizes up to 2^32 - 32 packets.

Since the previous algorithm, based on bit shifting, does not scale
with large replay windows, the algorithm used here is based on
RFC 6479: IPsec Anti-Replay Algorithm without Bit Shifting.
The replay window will be fast to be updated, but will cost as many bits
in RAM as its size.

The previous implementation did not provide a lock on the replay window,
which may lead to replay issues.

Reviewed by:	ae
Obtained from:	emeric.poupon@stormshield.eu
Sponsored by:	Stormshield
Differential Revision:	https://reviews.freebsd.org/D8468

7 files changed, 227 insertions, 120 deletions

diff --git a/sys/net/pfkeyv2.h b/sys/net/pfkeyv2.h
index c9b276954a36..353488197c9d 100644
--- a/sys/net/pfkeyv2.h
+++ b/sys/net/pfkeyv2.h
@@ -283,6 +283,14 @@ struct sadb_x_nat_t_frag {
 };
 _Static_assert(sizeof(struct sadb_x_nat_t_frag) == 8, "struct size mismatch");
 
+/* Additional large replay window support
+ */
+struct sadb_x_sa_replay {
+  u_int16_t sadb_x_sa_replay_len;
+  u_int16_t sadb_x_sa_replay_exttype;
+  u_int32_t sadb_x_sa_replay_replay;	/* in packets */
+};
+_Static_assert(sizeof(struct sadb_x_sa_replay) == 8, "struct size mismatch");
 
 #define SADB_EXT_RESERVED             0
 #define SADB_EXT_SA                   1
@@ -311,7 +319,8 @@ _Static_assert(sizeof(struct sadb_x_nat_t_frag) == 8, "struct size mismatch");
 #define SADB_X_EXT_NAT_T_OAI          23	/* Peer's NAT_OA for src of SA. */
 #define SADB_X_EXT_NAT_T_OAR          24	/* Peer's NAT_OA for dst of SA. */
 #define SADB_X_EXT_NAT_T_FRAG         25	/* Manual MTU override. */
-#define SADB_EXT_MAX                  25
+#define SADB_X_EXT_SA_REPLAY          26	/* Replay window override. */
+#define SADB_EXT_MAX                  26
 
 #define SADB_SATYPE_UNSPEC	0
 #define SADB_SATYPE_AH		2
diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index 6cdfc653c297..2e735393f100 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -251,7 +251,6 @@ static int ipsec6_setspidx_ipaddr(const struct mbuf *, struct secpolicyindex *);
 #endif
 static void ipsec_delpcbpolicy(struct inpcbpolicy *);
 static struct secpolicy *ipsec_deepcopy_policy(struct secpolicy *src);
-static void vshiftl(unsigned char *, int, int);
 
 MALLOC_DEFINE(M_IPSEC_INPCB, "inpcbpolicy", "inpcb-resident ipsec policy");
 
@@ -1476,57 +1475,70 @@ ipsec_hdrsiz(const struct mbuf *m, u_int dir, struct inpcb *inp)
  * beforehand).
  * 0 (zero) is returned if packet disallowed, 1 if packet permitted.
  *
- * Based on RFC 2401.
+ * Based on RFC 6479. Blocks are 32 bits unsigned integers
  */
+
+#define IPSEC_BITMAP_INDEX_MASK(w)	(w - 1)
+#define IPSEC_REDUNDANT_BIT_SHIFTS	5
+#define IPSEC_REDUNDANT_BITS		(1 << IPSEC_REDUNDANT_BIT_SHIFTS)
+#define IPSEC_BITMAP_LOC_MASK		(IPSEC_REDUNDANT_BITS - 1)
+
 int
 ipsec_chkreplay(u_int32_t seq, struct secasvar *sav)
 {
 	const struct secreplay *replay;
-	u_int32_t diff;
-	int fr;
-	u_int32_t wsizeb;	/* Constant: bits of window size. */
-	int frlast;		/* Constant: last frame. */
+	u_int32_t wsizeb;		/* Constant: window size. */
+	int ret, index, bit_location;
 
 	IPSEC_ASSERT(sav != NULL, ("Null SA"));
 	IPSEC_ASSERT(sav->replay != NULL, ("Null replay state"));
 
+	SECASVAR_LOCK(sav);
+
+	ret = 0;
 	replay = sav->replay;
 
+	/* No need to check replay if disabled. */
 	if (replay->wsize == 0)
-		return (1);	/* No need to check replay. */
+		goto allowed;
 
 	/* Constant. */
-	frlast = replay->wsize - 1;
 	wsizeb = replay->wsize << 3;
 
 	/* Sequence number of 0 is invalid. */
 	if (seq == 0)
-		return (0);
+		goto end;
 
 	/* First time is always okay. */
 	if (replay->count == 0)
-		return (1);
+		goto allowed;
 
-	if (seq > replay->lastseq) {
-		/* Larger sequences are okay. */
-		return (1);
-	} else {
-		/* seq is equal or less than lastseq. */
-		diff = replay->lastseq - seq;
+	/* Larger sequences are okay. */
+	if (seq > replay->lastseq)
+		goto allowed;
 
-		/* Over range to check, i.e. too old or wrapped. */
-		if (diff >= wsizeb)
-			return (0);
+	/* Over range to check, i.e. too old or wrapped. */
+	if (replay->lastseq - seq >= wsizeb)
+		goto end;
 
-		fr = frlast - diff / 8;
+	/* The sequence is inside the sliding window
+	 * now check the bit in the bitmap
+	 * bit location only depends on the sequence number
+	 */
+	bit_location = seq & IPSEC_BITMAP_LOC_MASK;
+	index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS)
+		& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
 
-		/* This packet already seen? */
-		if ((replay->bitmap)[fr] & (1 << (diff % 8)))
-			return (0);
+	/* This packet already seen? */
+	if ((replay->bitmap)[index] & (1 << bit_location))
+		goto end;
 
-		/* Out of order but good. */
-		return (1);
-	}
+allowed:
+	ret = 1;
+end:
+	SECASVAR_UNLOCK(sav);
+
+	return (ret);
 }
 
 /*
@@ -1539,72 +1551,61 @@ ipsec_updatereplay(u_int32_t seq, struct secasvar *sav)
 {
 	char buf[128];
 	struct secreplay *replay;
-	u_int32_t diff;
-	int fr;
-	u_int32_t wsizeb;	/* Constant: bits of window size. */
-	int frlast;		/* Constant: last frame. */
+	u_int32_t wsizeb;		/* Constant: window size. */
+	int ret, diff, index, bit_location;
 
 	IPSEC_ASSERT(sav != NULL, ("Null SA"));
 	IPSEC_ASSERT(sav->replay != NULL, ("Null replay state"));
 
+	SECASVAR_LOCK(sav);
+
+	ret = 1;
 	replay = sav->replay;
 
 	if (replay->wsize == 0)
 		goto ok;	/* No need to check replay. */
 
 	/* Constant. */
-	frlast = replay->wsize - 1;
 	wsizeb = replay->wsize << 3;
 
 	/* Sequence number of 0 is invalid. */
 	if (seq == 0)
-		return (1);
+		goto end;
 
-	/* First time. */
-	if (replay->count == 0) {
-		replay->lastseq = seq;
-		bzero(replay->bitmap, replay->wsize);
-		(replay->bitmap)[frlast] = 1;
+	/* The packet is too old, no need to update */
+	if (wsizeb + seq < replay->lastseq)
 		goto ok;
-	}
 
+	/* Now update the bit */
+	index = (seq >> IPSEC_REDUNDANT_BIT_SHIFTS);
+
+	/* First check if the sequence number is in the range */
 	if (seq > replay->lastseq) {
-		/* seq is larger than lastseq. */
-		diff = seq - replay->lastseq;
-
-		/* New larger sequence number. */
-		if (diff < wsizeb) {
-			/* In window. */
-			/* Set bit for this packet. */
-			vshiftl(replay->bitmap, diff, replay->wsize);
-			(replay->bitmap)[frlast] |= 1;
-		} else {
-			/* This packet has a "way larger". */
-			bzero(replay->bitmap, replay->wsize);
-			(replay->bitmap)[frlast] = 1;
-		}
-		replay->lastseq = seq;
+		int id;
+		int index_cur = replay->lastseq >> IPSEC_REDUNDANT_BIT_SHIFTS;
 
-		/* Larger is good. */
-	} else {
-		/* seq is equal or less than lastseq. */
-		diff = replay->lastseq - seq;
+		diff = index - index_cur;
+		if (diff > replay->bitmap_size) {
+			/* something unusual in this case */
+			diff = replay->bitmap_size;
+		}
 
-		/* Over range to check, i.e. too old or wrapped. */
-		if (diff >= wsizeb)
-			return (1);
+		for (id = 0; id < diff; ++id) {
+			replay->bitmap[(id + index_cur + 1)
+			& IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size)] = 0;
+		}
 
-		fr = frlast - diff / 8;
+		replay->lastseq = seq;
+	}
 
-		/* This packet already seen? */
-		if ((replay->bitmap)[fr] & (1 << (diff % 8)))
-			return (1);
+	index &= IPSEC_BITMAP_INDEX_MASK(replay->bitmap_size);
+	bit_location = seq & IPSEC_BITMAP_LOC_MASK;
 
-		/* Mark as seen. */
-		(replay->bitmap)[fr] |= (1 << (diff % 8));
+	/* this packet has already been received */
+	if (replay->bitmap[index] & (1 << bit_location))
+		goto end;
 
-		/* Out of order but good. */
-	}
+	replay->bitmap[index] |= (1 << bit_location);
 
 ok:
 	if (replay->count == ~0) {
@@ -1614,39 +1615,18 @@ ok:
 
 		/* Don't increment, no more packets accepted. */
 		if ((sav->flags & SADB_X_EXT_CYCSEQ) == 0)
-			return (1);
+			goto end;
 
 		ipseclog((LOG_WARNING, "%s: replay counter made %d cycle. %s\n",
 		    __func__, replay->overflow,
 		    ipsec_logsastr(sav, buf, sizeof(buf))));
 	}
 
-	replay->count++;
+	ret = 0;
 
-	return (0);
-}
-
-/*
- * Shift variable length buffer to left.
- * IN:	bitmap: pointer to the buffer
- * 	nbit:	the number of to shift.
- *	wsize:	buffer size (bytes).
- */
-static void
-vshiftl(unsigned char *bitmap, int nbit, int wsize)
-{
-	int s, j, i;
-	unsigned char over;
-
-	for (j = 0; j < nbit; j += 8) {
-		s = (nbit - j < 8) ? (nbit - j): 8;
-		bitmap[0] <<= s;
-		for (i = 1; i < wsize; i++) {
-			over = (bitmap[i] >> (8 - s));
-			bitmap[i] <<= s;
-			bitmap[i-1] |= over;
-		}
-	}
+end:
+	SECASVAR_UNLOCK(sav);
+	return (ret);
 }
 
 /* Return a printable string for the address. */
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index bacab2b753b9..2a5e0a60e626 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -243,7 +243,10 @@ static const int minsize[] = {
 	sizeof(struct sadb_address),	/* SADB_X_EXT_NAT_T_OAI */
 	sizeof(struct sadb_address),	/* SADB_X_EXT_NAT_T_OAR */
 	sizeof(struct sadb_x_nat_t_frag),/* SADB_X_EXT_NAT_T_FRAG */
+	sizeof(struct sadb_x_sa_replay), /* SADB_X_EXT_SA_REPLAY */
 };
+_Static_assert(sizeof(minsize)/sizeof(int) == SADB_EXT_MAX + 1, "minsize size mismatch");
+
 static const int maxsize[] = {
 	sizeof(struct sadb_msg),	/* SADB_EXT_RESERVED */
 	sizeof(struct sadb_sa),		/* SADB_EXT_SA */
@@ -271,7 +274,9 @@ static const int maxsize[] = {
 	0,				/* SADB_X_EXT_NAT_T_OAI */
 	0,				/* SADB_X_EXT_NAT_T_OAR */
 	sizeof(struct sadb_x_nat_t_frag),/* SADB_X_EXT_NAT_T_FRAG */
+	sizeof(struct sadb_x_sa_replay), /* SADB_X_EXT_SA_REPLAY */
 };
+_Static_assert(sizeof(maxsize)/sizeof(int) == SADB_EXT_MAX + 1, "minsize size mismatch");
 
 static VNET_DEFINE(int, ipsec_esp_keymin) = 256;
 static VNET_DEFINE(int, ipsec_esp_auth) = 0;
@@ -472,6 +477,7 @@ static void key_porttosaddr(struct sockaddr *, u_int16_t);
 #define	KEY_PORTTOSADDR(saddr, port)				\
 	key_porttosaddr((struct sockaddr *)(saddr), (port))
 static struct mbuf *key_setsadbxsa2(u_int8_t, u_int32_t, u_int32_t);
+static struct mbuf *key_setsadbxsareplay(u_int32_t);
 static struct mbuf *key_setsadbxpolicy(u_int16_t, u_int8_t,
 	u_int32_t, u_int32_t);
 static struct seckey *key_dup_keymsg(const struct sadb_key *, u_int, 
@@ -2940,6 +2946,8 @@ key_cleansav(struct secasvar *sav)
 		sav->sched = NULL;
 	}
 	if (sav->replay != NULL) {
+		if (sav->replay->bitmap != NULL)
+			free(sav->replay->bitmap, M_IPSEC_MISC);
 		free(sav->replay, M_IPSEC_MISC);
 		sav->replay = NULL;
 	}
@@ -3108,6 +3116,7 @@ key_setsaval(struct secasvar *sav, struct mbuf *m,
 	/* SA */
 	if (mhp->ext[SADB_EXT_SA] != NULL) {
 		const struct sadb_sa *sa0;
+		u_int32_t replay;
 
 		sa0 = (const struct sadb_sa *)mhp->ext[SADB_EXT_SA];
 		if (mhp->extlen[SADB_EXT_SA] < sizeof(*sa0)) {
@@ -3119,19 +3128,57 @@ key_setsaval(struct secasvar *sav, struct mbuf *m,
 		sav->alg_enc = sa0->sadb_sa_encrypt;
 		sav->flags = sa0->sadb_sa_flags;
 
-		/* replay window */
-		if ((sa0->sadb_sa_flags & SADB_X_EXT_OLD) == 0) {
-			sav->replay = (struct secreplay *)
-				malloc(sizeof(struct secreplay)+sa0->sadb_sa_replay, M_IPSEC_MISC, M_NOWAIT|M_ZERO);
-			if (sav->replay == NULL) {
+		/* Optional replay window */
+		replay = 0;
+		if ((sa0->sadb_sa_flags & SADB_X_EXT_OLD) == 0)
+			replay = sa0->sadb_sa_replay;
+		if ((mhp->ext[SADB_X_EXT_SA_REPLAY]) != NULL) {
+			replay = ((const struct sadb_x_sa_replay *)
+				mhp->ext[SADB_X_EXT_SA_REPLAY])->sadb_x_sa_replay_replay;
+
+			if (replay > UINT32_MAX - 32) {
+				ipseclog((LOG_DEBUG, "%s: replay window too big.\n",
+					__func__));
+				error = EINVAL;
+				goto fail;
+			}
+
+			replay = (replay + 7) >> 3;
+		}
+
+		sav->replay = (struct secreplay *)
+			malloc(sizeof(struct secreplay),
+					M_IPSEC_MISC, M_NOWAIT|M_ZERO);
+		if (sav->replay == NULL) {
+			ipseclog((LOG_DEBUG, "%s: No more memory.\n",
+						__func__));
+			error = ENOBUFS;
+			goto fail;
+		}
+
+		if (replay != 0) {
+			/* number of 32b blocks to be allocated */
+			u_int32_t bitmap_size;
+
+			/* RFC 6479:
+			 * - the allocated replay window size must be a power of two
+			 * - use an extra 32b block as a redundant window
+			 */
+			bitmap_size = 1;
+			while (replay + 4 > bitmap_size)
+				bitmap_size <<= 1;
+			bitmap_size = bitmap_size / 4;
+
+			sav->replay->bitmap = malloc(bitmap_size*sizeof(u_int32_t),
+					M_IPSEC_MISC, M_NOWAIT|M_ZERO);
+			if (sav->replay->bitmap == NULL) {
 				ipseclog((LOG_DEBUG, "%s: No more memory.\n",
 					__func__));
 				error = ENOBUFS;
 				goto fail;
 			}
-			if (sa0->sadb_sa_replay != 0)
-				sav->replay->bitmap = (caddr_t)(sav->replay+1);
-			sav->replay->wsize = sa0->sadb_sa_replay;
+			sav->replay->bitmap_size = bitmap_size;
+			sav->replay->wsize = replay;
 		}
 	}
 
@@ -3406,7 +3453,7 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
 	struct mbuf *result = NULL, *tres = NULL, *m;
 	int i;
 	int dumporder[] = {
-		SADB_EXT_SA, SADB_X_EXT_SA2,
+		SADB_EXT_SA, SADB_X_EXT_SA2, SADB_X_EXT_SA_REPLAY,
 		SADB_EXT_LIFETIME_HARD, SADB_EXT_LIFETIME_SOFT,
 		SADB_EXT_LIFETIME_CURRENT, SADB_EXT_ADDRESS_SRC,
 		SADB_EXT_ADDRESS_DST, SADB_EXT_ADDRESS_PROXY, SADB_EXT_KEY_AUTH,
@@ -3419,6 +3466,7 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
 		SADB_X_EXT_NAT_T_FRAG,
 #endif
 	};
+	u_int32_t replay_count;
 
 	m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
 	if (m == NULL)
@@ -3435,13 +3483,25 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
 			break;
 
 		case SADB_X_EXT_SA2:
-			m = key_setsadbxsa2(sav->sah->saidx.mode,
-					sav->replay ? sav->replay->count : 0,
+			SECASVAR_LOCK(sav);
+			replay_count = sav->replay ? sav->replay->count : 0;
+			SECASVAR_UNLOCK(sav);
+			m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
 					sav->sah->saidx.reqid);
 			if (!m)
 				goto fail;
 			break;
 
+		case SADB_X_EXT_SA_REPLAY:
+			if (sav->replay == NULL ||
+				sav->replay->wsize <= UINT8_MAX)
+				continue;
+
+			m = key_setsadbxsareplay(sav->replay->wsize);
+			if (!m)
+				goto fail;
+			break;
+
 		case SADB_EXT_ADDRESS_SRC:
 			m = key_setsadbaddr(SADB_EXT_ADDRESS_SRC,
 			    &sav->sah->saidx.src.sa,
@@ -3634,7 +3694,9 @@ key_setsadbsa(struct secasvar *sav)
 	p->sadb_sa_len = PFKEY_UNIT64(len);
 	p->sadb_sa_exttype = SADB_EXT_SA;
 	p->sadb_sa_spi = sav->spi;
-	p->sadb_sa_replay = (sav->replay != NULL ? sav->replay->wsize : 0);
+	p->sadb_sa_replay = sav->replay ?
+		(sav->replay->wsize > UINT8_MAX ?
+		 	UINT8_MAX : sav->replay->wsize) : 0;
 	p->sadb_sa_state = sav->state;
 	p->sadb_sa_auth = sav->alg_auth;
 	p->sadb_sa_encrypt = sav->alg_enc;
@@ -3719,6 +3781,32 @@ key_setsadbxsa2(u_int8_t mode, u_int32_t seq, u_int32_t reqid)
 	return m;
 }
 
+/*
+ * Set data into sadb_x_sa_replay.
+ */
+static struct mbuf *
+key_setsadbxsareplay(u_int32_t replay)
+{
+	struct mbuf *m;
+	struct sadb_x_sa_replay *p;
+	size_t len;
+
+	len = PFKEY_ALIGN8(sizeof(struct sadb_x_sa_replay));
+	m = m_get2(len, M_NOWAIT, MT_DATA, 0);
+	if (m == NULL)
+		return (NULL);
+	m_align(m, len);
+	m->m_len = len;
+	p = mtod(m, struct sadb_x_sa_replay *);
+
+	bzero(p, len);
+	p->sadb_x_sa_replay_len = PFKEY_UNIT64(len);
+	p->sadb_x_sa_replay_exttype = SADB_X_EXT_SA_REPLAY;
+	p->sadb_x_sa_replay_replay = (replay << 3);
+
+	return m;
+}
+
 #ifdef IPSEC_NAT_T
 /*
  * Set a type in sadb_x_nat_t_type.
@@ -6853,6 +6941,7 @@ key_expire(struct secasvar *sav, int hard)
 	int len;
 	int error = -1;
 	struct sadb_lifetime *lt;
+	u_int32_t replay_count;
 
 	IPSEC_ASSERT (sav != NULL, ("null sav"));
 	IPSEC_ASSERT (sav->sah != NULL, ("null sa header"));
@@ -6876,8 +6965,11 @@ key_expire(struct secasvar *sav, int hard)
 	m_cat(result, m);
 
 	/* create SA extension */
-	m = key_setsadbxsa2(sav->sah->saidx.mode,
-			sav->replay ? sav->replay->count : 0,
+	SECASVAR_LOCK(sav);
+	replay_count = sav->replay ? sav->replay->count : 0;
+	SECASVAR_UNLOCK(sav);
+
+	m = key_setsadbxsa2(sav->sah->saidx.mode, replay_count,
 			sav->sah->saidx.reqid);
 	if (!m) {
 		error = ENOBUFS;
@@ -6885,6 +6977,15 @@ key_expire(struct secasvar *sav, int hard)
 	}
 	m_cat(result, m);
 
+	if (sav->replay && sav->replay->wsize > UINT8_MAX) {
+		m = key_setsadbxsareplay(sav->replay->wsize);
+		if (!m) {
+			error = ENOBUFS;
+			goto fail;
+		}
+		m_cat(result, m);
+	}
+
 	/* create lifetime extension (current and soft) */
 	len = PFKEY_ALIGN8(sizeof(*lt)) * 2;
 	m = m_get2(len, M_NOWAIT, MT_DATA, 0);
@@ -7560,6 +7661,7 @@ key_align(struct mbuf *m, struct sadb_msghdr *mhp)
 		case SADB_X_EXT_NAT_T_OAR:
 		case SADB_X_EXT_NAT_T_FRAG:
 #endif
+		case SADB_X_EXT_SA_REPLAY:
 			/* duplicate check */
 			/*
 			 * XXX Are there duplication payloads of either
diff --git a/sys/netipsec/key_debug.c b/sys/netipsec/key_debug.c
index a3ff8be01d0d..39b5cb63874c 100644
--- a/sys/netipsec/key_debug.c
+++ b/sys/netipsec/key_debug.c
@@ -570,8 +570,11 @@ kdebug_secasv(struct secasvar *sav)
 	if (sav->key_enc != NULL)
 		kdebug_sadb_key((struct sadb_ext *)sav->key_enc);
 
-	if (sav->replay != NULL)
+	if (sav->replay != NULL) {
+		SECASVAR_LOCK(sav);
 		kdebug_secreplay(sav->replay);
+		SECASVAR_UNLOCK(sav);
+	}
 	if (sav->lft_c != NULL)
 		kdebug_sec_lifetime(sav->lft_c);
 	if (sav->lft_h != NULL)
@@ -595,8 +598,8 @@ kdebug_secreplay(struct secreplay *rpl)
 	if (rpl == NULL)
 		panic("%s: NULL pointer was passed.\n", __func__);
 
-	printf(" secreplay{ count=%u wsize=%u seq=%u lastseq=%u",
-	    rpl->count, rpl->wsize, rpl->seq, rpl->lastseq);
+	printf(" secreplay{ count=%u bitmap_size=%u wsize=%u seq=%u lastseq=%u",
+	    rpl->count, rpl->bitmap_size, rpl->wsize, rpl->seq, rpl->lastseq);
 
 	if (rpl->bitmap == NULL) {
 		printf(" }\n");
@@ -605,7 +608,7 @@ kdebug_secreplay(struct secreplay *rpl)
 
 	printf("\n   bitmap { ");
 
-	for (len = 0; len < rpl->wsize; len++) {
+	for (len = 0; len < rpl->bitmap_size*4; len++) {
 		for (l = 7; l >= 0; l--)
 			printf("%u", (((rpl->bitmap)[len] >> l) & 1) ? 1 : 0);
 	}
diff --git a/sys/netipsec/keydb.h b/sys/netipsec/keydb.h
index 3fe28eb6c155..19e9aebdf286 100644
--- a/sys/netipsec/keydb.h
+++ b/sys/netipsec/keydb.h
@@ -35,6 +35,8 @@
 
 #ifdef _KERNEL
 
+#include <sys/mutex.h>
+
 #include <netipsec/key_var.h>
 
 #ifndef _SOCKADDR_UNION_DEFINED
@@ -170,14 +172,18 @@ struct secasvar {
 #define	SAV_ISCTR(_sav) ((_sav)->alg_enc == SADB_X_EALG_AESCTR)
 #define SAV_ISCTRORGCM(_sav)	(SAV_ISCTR((_sav)) || SAV_ISGCM((_sav)))
 
-/* replay prevention */
+/* Replay prevention, protected by SECASVAR_LOCK:
+ *  (m) locked by mtx
+ *  (c) read only except during creation / free
+ */
 struct secreplay {
-	u_int32_t count;
-	u_int wsize;		/* window size, i.g. 4 bytes */
-	u_int32_t seq;		/* used by sender */
-	u_int32_t lastseq;	/* used by receiver */
-	caddr_t bitmap;		/* used by receiver */
-	int overflow;		/* overflow flag */
+	u_int32_t count;	/* (m) */
+	u_int wsize;		/* (c) window size, i.g. 4 bytes */
+	u_int32_t seq;		/* (m) used by sender */
+	u_int32_t lastseq;	/* (m) used by receiver */
+	u_int32_t *bitmap;	/* (m) used by receiver */
+	u_int bitmap_size;	/* (c) size of the bitmap array */
+	int overflow;		/* (m) overflow flag */
 };
 
 /* socket table due to send PF_KEY messages. */
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 0d39eeba49ad..6fcca719a696 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -961,8 +961,11 @@ ah_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 
 	/* Insert packet replay counter, as requested.  */
 	if (sav->replay) {
+		SECASVAR_LOCK(sav);
+
 		if (sav->replay->count == ~0 &&
 		    (sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
+			SECASVAR_UNLOCK(sav);
 			DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
 			    __func__, ipsec_address(&sav->sah->saidx.dst, buf,
 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
@@ -976,6 +979,8 @@ ah_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 #endif
 			sav->replay->count++;
 		ah->ah_seq = htonl(sav->replay->count);
+
+		SECASVAR_UNLOCK(sav);
 	}
 
 	/* Get crypto descriptors. */
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index c8b5f16417f1..bae8571f5bfd 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -762,12 +762,14 @@ esp_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 	if (sav->replay) {
 		u_int32_t replay;
 
+		SECASVAR_LOCK(sav);
 #ifdef REGRESSION
 		/* Emulate replay attack when ipsec_replay is TRUE. */
 		if (!V_ipsec_replay)
 #endif
 			sav->replay->count++;
 		replay = htonl(sav->replay->count);
+		SECASVAR_UNLOCK(sav);
 		bcopy((caddr_t) &replay,
 		    mtod(mo, caddr_t) + roff + sizeof(u_int32_t),
 		    sizeof(u_int32_t));