4 files changed, 443 insertions, 37 deletions

diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index e55f4fd0cb62..5855e62bd983 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -299,6 +299,7 @@ static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS");
 
 static void ktls_cleanup(struct ktls_session *tls);
 #if defined(INET) || defined(INET6)
+static void ktls_reset_receive_tag(void *context, int pending);
 static void ktls_reset_send_tag(void *context, int pending);
 #endif
 static void ktls_work_thread(void *ctx);
@@ -503,7 +504,7 @@ start:
 #if defined(INET) || defined(INET6)
 static int
 ktls_create_session(struct socket *so, struct tls_enable *en,
-    struct ktls_session **tlsp)
+    struct ktls_session **tlsp, int direction)
 {
 	struct ktls_session *tls;
 	int error;
@@ -619,7 +620,10 @@ ktls_create_session(struct socket *so, struct tls_enable *en,
 	counter_u64_add(ktls_offload_active, 1);
 
 	refcount_init(&tls->refcount, 1);
-	TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
+	if (direction == KTLS_RX)
+		TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls);
+	else
+		TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
 
 	tls->wq_index = ktls_get_cpu(so);
 
@@ -745,7 +749,7 @@ out:
 }
 
 static struct ktls_session *
-ktls_clone_session(struct ktls_session *tls)
+ktls_clone_session(struct ktls_session *tls, int direction)
 {
 	struct ktls_session *tls_new;
 
@@ -754,7 +758,12 @@ ktls_clone_session(struct ktls_session *tls)
 	counter_u64_add(ktls_offload_active, 1);
 
 	refcount_init(&tls_new->refcount, 1);
-	TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new);
+	if (direction == KTLS_RX)
+		TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_receive_tag,
+		    tls_new);
+	else
+		TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag,
+		    tls_new);
 
 	/* Copy fields from existing session. */
 	tls_new->params = tls->params;
@@ -810,6 +819,8 @@ ktls_cleanup(struct ktls_session *tls)
 		}
 		if (tls->snd_tag != NULL)
 			m_snd_tag_rele(tls->snd_tag);
+		if (tls->rx_ifp != NULL)
+			if_rele(tls->rx_ifp);
 		break;
 #ifdef TCP_OFFLOAD
 	case TCP_TLS_MODE_TOE:
@@ -983,28 +994,135 @@ out:
 	return (error);
 }
 
+/*
+ * Allocate an initial TLS receive tag for doing HW decryption of TLS
+ * data.
+ *
+ * This function allocates a new TLS receive tag on whatever interface
+ * the connection is currently routed over.  If the connection ends up
+ * using a different interface for receive this will get fixed up via
+ * ktls_input_ifp_mismatch as future packets arrive.
+ */
 static int
-ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force)
+ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls,
+    struct m_snd_tag **mstp)
 {
-	struct m_snd_tag *mst;
+	union if_snd_tag_alloc_params params;
+	struct ifnet *ifp;
+	struct nhop_object *nh;
 	int error;
 
-	error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
-	if (error == 0) {
-		tls->mode = TCP_TLS_MODE_IFNET;
-		tls->snd_tag = mst;
-		switch (tls->params.cipher_algorithm) {
-		case CRYPTO_AES_CBC:
-			counter_u64_add(ktls_ifnet_cbc, 1);
-			break;
-		case CRYPTO_AES_NIST_GCM_16:
-			counter_u64_add(ktls_ifnet_gcm, 1);
-			break;
-		case CRYPTO_CHACHA20_POLY1305:
-			counter_u64_add(ktls_ifnet_chacha20, 1);
-			break;
+	if (!ktls_ocf_recrypt_supported(tls))
+		return (ENXIO);
+
+	INP_RLOCK(inp);
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
+		INP_RUNLOCK(inp);
+		return (ECONNRESET);
+	}
+	if (inp->inp_socket == NULL) {
+		INP_RUNLOCK(inp);
+		return (ECONNRESET);
+	}
+
+	/*
+	 * Check administrative controls on ifnet TLS to determine if
+	 * ifnet TLS should be denied.
+	 */
+	if (ktls_ifnet_permitted == 0) {
+		INP_RUNLOCK(inp);
+		return (ENXIO);
+	}
+
+	/*
+	 * XXX: As with ktls_alloc_snd_tag, use the cached route in
+	 * the inpcb to find the interface.
+	 */
+	nh = inp->inp_route.ro_nh;
+	if (nh == NULL) {
+		INP_RUNLOCK(inp);
+		return (ENXIO);
+	}
+	ifp = nh->nh_ifp;
+	if_ref(ifp);
+	tls->rx_ifp = ifp;
+
+	params.hdr.type = IF_SND_TAG_TYPE_TLS_RX;
+	params.hdr.flowid = inp->inp_flowid;
+	params.hdr.flowtype = inp->inp_flowtype;
+	params.hdr.numa_domain = inp->inp_numa_domain;
+	params.tls_rx.inp = inp;
+	params.tls_rx.tls = tls;
+	params.tls_rx.vlan_id = 0;
+
+	INP_RUNLOCK(inp);
+
+	if (inp->inp_vflag & INP_IPV6) {
+		if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0) {
+			error = EOPNOTSUPP;
+			goto out;
 		}
+	} else {
+		if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0) {
+			error = EOPNOTSUPP;
+			goto out;
+		}
+	}
+	error = m_snd_tag_alloc(ifp, &params, mstp);
+
+	/*
+	 * If this connection is over a vlan, vlan_snd_tag_alloc
+	 * rewrites vlan_id with the saved interface.  Save the VLAN
+	 * ID for use in ktls_reset_receive_tag which allocates new
+	 * receive tags directly from the leaf interface bypassing
+	 * if_vlan.
+	 */
+	if (error == 0)
+		tls->rx_vlan_id = params.tls_rx.vlan_id;
+out:
+	return (error);
+}
+
+static int
+ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction,
+    bool force)
+{
+	struct m_snd_tag *mst;
+	int error;
+
+	switch (direction) {
+	case KTLS_TX:
+		error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
+		if (__predict_false(error != 0))
+			goto done;
+		break;
+	case KTLS_RX:
+		KASSERT(!force, ("%s: forced receive tag", __func__));
+		error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst);
+		if (__predict_false(error != 0))
+			goto done;
+		break;
+	default:
+		__assert_unreachable();
+	}
+
+	tls->mode = TCP_TLS_MODE_IFNET;
+	tls->snd_tag = mst;
+
+	switch (tls->params.cipher_algorithm) {
+	case CRYPTO_AES_CBC:
+		counter_u64_add(ktls_ifnet_cbc, 1);
+		break;
+	case CRYPTO_AES_NIST_GCM_16:
+		counter_u64_add(ktls_ifnet_gcm, 1);
+		break;
+	case CRYPTO_CHACHA20_POLY1305:
+		counter_u64_add(ktls_ifnet_chacha20, 1);
+		break;
+	default:
+		break;
 	}
+done:
 	return (error);
 }
 
@@ -1185,7 +1303,7 @@ ktls_enable_rx(struct socket *so, struct tls_enable *en)
 	if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
 		return (ENOTSUP);
 
-	error = ktls_create_session(so, en, &tls);
+	error = ktls_create_session(so, en, &tls, KTLS_RX);
 	if (error)
 		return (error);
 
@@ -1206,10 +1324,13 @@ ktls_enable_rx(struct socket *so, struct tls_enable *en)
 	ktls_check_rx(&so->so_rcv);
 	SOCKBUF_UNLOCK(&so->so_rcv);
 
+	/* Prefer TOE -> ifnet TLS -> software TLS. */
 #ifdef TCP_OFFLOAD
 	error = ktls_try_toe(so, tls, KTLS_RX);
 	if (error)
 #endif
+		error = ktls_try_ifnet(so, tls, KTLS_RX, false);
+	if (error)
 		ktls_use_sw(tls);
 
 	counter_u64_add(ktls_offload_total, 1);
@@ -1252,7 +1373,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 	if (mb_use_ext_pgs == 0)
 		return (ENXIO);
 
-	error = ktls_create_session(so, en, &tls);
+	error = ktls_create_session(so, en, &tls, KTLS_TX);
 	if (error)
 		return (error);
 
@@ -1261,7 +1382,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 	error = ktls_try_toe(so, tls, KTLS_TX);
 	if (error)
 #endif
-		error = ktls_try_ifnet(so, tls, false);
+		error = ktls_try_ifnet(so, tls, KTLS_TX, false);
 	if (error)
 		error = ktls_try_sw(so, tls, KTLS_TX);
 
@@ -1418,10 +1539,10 @@ ktls_set_tx_mode(struct socket *so, int mode)
 	SOCKBUF_UNLOCK(&so->so_snd);
 	INP_WUNLOCK(inp);
 
-	tls_new = ktls_clone_session(tls);
+	tls_new = ktls_clone_session(tls, KTLS_TX);
 
 	if (mode == TCP_TLS_MODE_IFNET)
-		error = ktls_try_ifnet(so, tls_new, true);
+		error = ktls_try_ifnet(so, tls_new, KTLS_TX, true);
 	else
 		error = ktls_try_sw(so, tls_new, KTLS_TX);
 	if (error) {
@@ -1480,6 +1601,89 @@ ktls_set_tx_mode(struct socket *so, int mode)
 }
 
 /*
+ * Try to allocate a new TLS receive tag.  This task is scheduled when
+ * sbappend_ktls_rx detects an input path change.  If a new tag is
+ * allocated, replace the tag in the TLS session.  If a new tag cannot
+ * be allocated, let the session fall back to software decryption.
+ */
+static void
+ktls_reset_receive_tag(void *context, int pending)
+{
+	union if_snd_tag_alloc_params params;
+	struct ktls_session *tls;
+	struct m_snd_tag *mst;
+	struct inpcb *inp;
+	struct ifnet *ifp;
+	struct socket *so;
+	int error;
+
+	MPASS(pending == 1);
+
+	tls = context;
+	so = tls->so;
+	inp = so->so_pcb;
+	ifp = NULL;
+
+	INP_RLOCK(inp);
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
+		INP_RUNLOCK(inp);
+		goto out;
+	}
+
+	SOCKBUF_LOCK(&so->so_rcv);
+	m_snd_tag_rele(tls->snd_tag);
+	tls->snd_tag = NULL;
+
+	ifp = tls->rx_ifp;
+	if_ref(ifp);
+	SOCKBUF_UNLOCK(&so->so_rcv);
+
+	params.hdr.type = IF_SND_TAG_TYPE_TLS_RX;
+	params.hdr.flowid = inp->inp_flowid;
+	params.hdr.flowtype = inp->inp_flowtype;
+	params.hdr.numa_domain = inp->inp_numa_domain;
+	params.tls_rx.inp = inp;
+	params.tls_rx.tls = tls;
+	params.tls_rx.vlan_id = tls->rx_vlan_id;
+	INP_RUNLOCK(inp);
+
+	if (inp->inp_vflag & INP_IPV6) {
+		if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0)
+			goto out;
+	} else {
+		if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0)
+			goto out;
+	}
+
+	error = m_snd_tag_alloc(ifp, &params, &mst);
+	if (error == 0) {
+		SOCKBUF_LOCK(&so->so_rcv);
+		tls->snd_tag = mst;
+		SOCKBUF_UNLOCK(&so->so_rcv);
+
+		counter_u64_add(ktls_ifnet_reset, 1);
+	} else {
+		/*
+		 * Just fall back to software decryption if a tag
+		 * cannot be allocated leaving the connection intact.
+		 * If a future input path change switches to another
+		 * interface this connection will resume ifnet TLS.
+		 */
+		counter_u64_add(ktls_ifnet_reset_failed, 1);
+	}
+
+out:
+	mtx_pool_lock(mtxpool_sleep, tls);
+	tls->reset_pending = false;
+	mtx_pool_unlock(mtxpool_sleep, tls);
+
+	if (ifp != NULL)
+		if_rele(ifp);
+	sorele(so);
+	ktls_free(tls);
+}
+
+/*
  * Try to allocate a new TLS send tag.  This task is scheduled when
  * ip_output detects a route change while trying to transmit a packet
  * holding a TLS record.  If a new tag is allocated, replace the tag
@@ -1565,6 +1769,37 @@ ktls_reset_send_tag(void *context, int pending)
 	ktls_free(tls);
 }
 
+void
+ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp)
+{
+	struct ktls_session *tls;
+	struct socket *so;
+
+	SOCKBUF_LOCK_ASSERT(sb);
+	KASSERT(sb->sb_flags & SB_TLS_RX, ("%s: sockbuf %p isn't TLS RX",
+	    __func__, sb));
+	so = __containerof(sb, struct socket, so_rcv);
+
+	tls = sb->sb_tls_info;
+	if_rele(tls->rx_ifp);
+	if_ref(ifp);
+	tls->rx_ifp = ifp;
+
+	/*
+	 * See if we should schedule a task to update the receive tag for
+	 * this session.
+	 */
+	mtx_pool_lock(mtxpool_sleep, tls);
+	if (!tls->reset_pending) {
+		(void) ktls_hold(tls);
+		soref(so);
+		tls->so = so;
+		tls->reset_pending = true;
+		taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);
+	}
+	mtx_pool_unlock(mtxpool_sleep, tls);
+}
+
 int
 ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls)
 {
@@ -1908,7 +2143,7 @@ ktls_detach_record(struct sockbuf *sb, int len)
 			return (NULL);
 		}
 	}
-	n->m_flags |= M_NOTREADY;
+	n->m_flags |= (m->m_flags & (M_NOTREADY | M_DECRYPTED));
 
 	/* Store remainder in 'n'. */
 	n->m_len = m->m_len - remain;
@@ -1993,6 +2228,86 @@ tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len,
 	return (0);
 }
 
+/*
+ * Check if a mbuf chain is fully decrypted at the given offset and
+ * length. Returns KTLS_MBUF_CRYPTO_ST_DECRYPTED if all data is
+ * decrypted. KTLS_MBUF_CRYPTO_ST_MIXED if there is a mix of encrypted
+ * and decrypted data. Else KTLS_MBUF_CRYPTO_ST_ENCRYPTED if all data
+ * is encrypted.
+ */
+ktls_mbuf_crypto_st_t
+ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len)
+{
+	int m_flags_ored = 0;
+	int m_flags_anded = -1;
+
+	for (; mb != NULL; mb = mb->m_next) {
+		if (offset < mb->m_len)
+			break;
+		offset -= mb->m_len;
+	}
+	offset += len;
+
+	for (; mb != NULL; mb = mb->m_next) {
+		m_flags_ored |= mb->m_flags;
+		m_flags_anded &= mb->m_flags;
+
+		if (offset <= mb->m_len)
+			break;
+		offset -= mb->m_len;
+	}
+	MPASS(mb != NULL || offset == 0);
+
+	if ((m_flags_ored ^ m_flags_anded) & M_DECRYPTED)
+		return (KTLS_MBUF_CRYPTO_ST_MIXED);
+	else
+		return ((m_flags_ored & M_DECRYPTED) ?
+		    KTLS_MBUF_CRYPTO_ST_DECRYPTED :
+		    KTLS_MBUF_CRYPTO_ST_ENCRYPTED);
+}
+
+/*
+ * ktls_resync_ifnet - get HW TLS RX back on track after packet loss
+ */
+static int
+ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num)
+{
+	union if_snd_tag_modify_params params;
+	struct m_snd_tag *mst;
+	struct inpcb *inp;
+	struct tcpcb *tp;
+
+	mst = so->so_rcv.sb_tls_info->snd_tag;
+	if (__predict_false(mst == NULL))
+		return (EINVAL);
+
+	inp = sotoinpcb(so);
+	if (__predict_false(inp == NULL))
+		return (EINVAL);
+
+	INP_RLOCK(inp);
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
+		INP_RUNLOCK(inp);
+		return (ECONNRESET);
+	}
+
+	tp = intotcpcb(inp);
+	MPASS(tp != NULL);
+
+	/* Get the TCP sequence number of the next valid TLS header. */
+	SOCKBUF_LOCK(&so->so_rcv);
+	params.tls_rx.tls_hdr_tcp_sn =
+	    tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len;
+	params.tls_rx.tls_rec_length = tls_len;
+	params.tls_rx.tls_seq_number = tls_rcd_num;
+	SOCKBUF_UNLOCK(&so->so_rcv);
+
+	INP_RUNLOCK(inp);
+
+	MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RX);
+	return (mst->sw->snd_tag_modify(mst, &params));
+}
+
 static void
 ktls_decrypt(struct socket *so)
 {
@@ -2002,6 +2317,7 @@ ktls_decrypt(struct socket *so)
 	struct tls_record_layer *hdr;
 	struct tls_get_record tgr;
 	struct mbuf *control, *data, *m;
+	ktls_mbuf_crypto_st_t state;
 	uint64_t seqno;
 	int error, remain, tls_len, trail_len;
 	bool tls13;
@@ -2074,13 +2390,46 @@ ktls_decrypt(struct socket *so)
 		SBCHECK(sb);
 		SOCKBUF_UNLOCK(sb);
 
-		error = ktls_ocf_decrypt(tls, hdr, data, seqno, &trail_len);
-		if (error == 0) {
-			if (tls13)
+		/* get crypto state for this TLS record */
+		state = ktls_mbuf_crypto_state(data, 0, tls_len);
+
+		switch (state) {
+		case KTLS_MBUF_CRYPTO_ST_MIXED:
+			error = ktls_ocf_recrypt(tls, hdr, data, seqno);
+			if (error)
+				break;
+			/* FALLTHROUGH */
+		case KTLS_MBUF_CRYPTO_ST_ENCRYPTED:
+			error = ktls_ocf_decrypt(tls, hdr, data, seqno,
+			    &trail_len);
+			if (__predict_true(error == 0)) {
+				if (tls13) {
+					error = tls13_find_record_type(tls, data,
+					    tls_len, &trail_len, &record_type);
+				} else {
+					record_type = hdr->tls_type;
+				}
+			}
+			break;
+		case KTLS_MBUF_CRYPTO_ST_DECRYPTED:
+			/*
+			 * NIC TLS is only supported for AEAD
+			 * ciphersuites which used a fixed sized
+			 * trailer.
+			 */
+			if (tls13) {
+				trail_len = tls->params.tls_tlen - 1;
 				error = tls13_find_record_type(tls, data,
 				    tls_len, &trail_len, &record_type);
-			else
+			} else {
+				trail_len = tls->params.tls_tlen;
+				error = 0;
 				record_type = hdr->tls_type;
+			}
+			break;
+		default:
+			error = EINVAL;
+			break;
 		}
 		if (error) {
 			counter_u64_add(ktls_offload_failed_crypto, 1);
@@ -2161,19 +2510,31 @@ ktls_decrypt(struct socket *so)
 			remain = be16toh(tgr.tls_length);
 			m = data;
 			for (m = data; remain > m->m_len; m = m->m_next) {
-				m->m_flags &= ~M_NOTREADY;
+				m->m_flags &= ~(M_NOTREADY | M_DECRYPTED);
 				remain -= m->m_len;
 			}
 			m->m_len = remain;
 			m_freem(m->m_next);
 			m->m_next = NULL;
-			m->m_flags &= ~M_NOTREADY;
+			m->m_flags &= ~(M_NOTREADY | M_DECRYPTED);
 
 			/* Set EOR on the final mbuf. */
 			m->m_flags |= M_EOR;
 		}
 
 		sbappendcontrol_locked(sb, data, control, 0);
+
+		if (__predict_false(state != KTLS_MBUF_CRYPTO_ST_DECRYPTED)) {
+			sb->sb_flags |= SB_TLS_RX_RESYNC;
+			SOCKBUF_UNLOCK(sb);
+			ktls_resync_ifnet(so, tls_len, seqno);
+			SOCKBUF_LOCK(sb);
+		} else if (__predict_false(sb->sb_flags & SB_TLS_RX_RESYNC)) {
+			sb->sb_flags &= ~SB_TLS_RX_RESYNC;
+			SOCKBUF_UNLOCK(sb);
+			ktls_resync_ifnet(so, 0, seqno);
+			SOCKBUF_LOCK(sb);
+		}
 	}
 
 	sb->sb_flags &= ~SB_TLS_RX_RUNNING;
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index 07c21d3b7802..da02de3e3301 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -54,6 +54,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/sx.h>
 #include <sys/sysctl.h>
 
+#include <netinet/in.h>
+
 /*
  * Function pointer set by the AIO routines so that the socket buffer code
  * can call back into the AIO module if it is loaded.
@@ -946,17 +948,45 @@ sbappend(struct sockbuf *sb, struct mbuf *m, int flags)
 static void
 sbappend_ktls_rx(struct sockbuf *sb, struct mbuf *m)
 {
+	struct ifnet *ifp;
 	struct mbuf *n;
+	int flags;
+
+	ifp = NULL;
+	flags = M_NOTREADY;
 
 	SBLASTMBUFCHK(sb);
 
+	/* Mbuf chain must start with a packet header. */
+	MPASS((m->m_flags & M_PKTHDR) != 0);
+
 	/* Remove all packet headers and mbuf tags to get a pure data chain. */
-	m_demote(m, 1, 0);
+	for (n = m; n != NULL; n = n->m_next) {
+		if (n->m_flags & M_PKTHDR) {
+			ifp = m->m_pkthdr.leaf_rcvif;
+			if ((n->m_pkthdr.csum_flags & CSUM_TLS_MASK) ==
+			    CSUM_TLS_DECRYPTED) {
+				/* Mark all mbufs in this packet decrypted. */
+				flags = M_NOTREADY | M_DECRYPTED;
+			} else {
+				flags = M_NOTREADY;
+			}
+			m_demote_pkthdr(n);
+		}
+
+		n->m_flags &= M_DEMOTEFLAGS;
+		n->m_flags |= flags;
+
+		MPASS((n->m_flags & M_NOTREADY) != 0);
+	}
 
-	for (n = m; n != NULL; n = n->m_next)
-		n->m_flags |= M_NOTREADY;
 	sbcompress_ktls_rx(sb, m, sb->sb_mtlstail);
 	ktls_check_rx(sb);
+
+	/* Check for incoming packet route changes: */
+	if (ifp != NULL && sb->sb_tls_info->rx_ifp != NULL &&
+	    sb->sb_tls_info->rx_ifp != ifp)
+		ktls_input_ifp_mismatch(sb, ifp);
 }
 #endif
 
@@ -1410,7 +1440,8 @@ sbcompress_ktls_rx(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
 		if (n &&
 		    M_WRITABLE(n) &&
 		    ((sb->sb_flags & SB_NOCOALESCE) == 0) &&
-		    !(n->m_flags & (M_EXTPG)) &&
+		    !((m->m_flags ^ n->m_flags) & M_DECRYPTED) &&
+		    !(n->m_flags & M_EXTPG) &&
 		    m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
 		    m->m_len <= M_TRAILINGSPACE(n)) {
 			m_copydata(m, 0, m->m_len, mtodo(n, n->m_len));
diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
index 885e4dc38ef9..66b510ddfe40 100644
--- a/sys/sys/ktls.h
+++ b/sys/sys/ktls.h
@@ -187,7 +187,12 @@ struct ktls_session {
 
 	struct task reset_tag_task;
 	struct task disable_ifnet_task;
-	struct inpcb *inp;
+	union {
+		struct inpcb *inp;	/* Used by transmit tasks. */
+		struct socket *so;	/* Used by receive task. */
+	};
+	struct ifnet *rx_ifp;
+	u_short rx_vlan_id;
 	bool reset_pending;
 	bool disable_ifnet_pending;
 	bool sync_dispatch;
@@ -200,7 +205,14 @@ struct ktls_session {
 
 extern unsigned int ktls_ifnet_max_rexmit_pct;
 
+typedef enum {
+	KTLS_MBUF_CRYPTO_ST_MIXED = 0,
+	KTLS_MBUF_CRYPTO_ST_ENCRYPTED = 1,
+	KTLS_MBUF_CRYPTO_ST_DECRYPTED = -1,
+} ktls_mbuf_crypto_st_t;
+
 void ktls_check_rx(struct sockbuf *sb);
+ktls_mbuf_crypto_st_t ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len);
 void ktls_disable_ifnet(void *arg);
 int ktls_enable_rx(struct socket *so, struct tls_enable *en);
 int ktls_enable_tx(struct socket *so, struct tls_enable *en);
@@ -215,6 +227,7 @@ int ktls_get_rx_mode(struct socket *so, int *modep);
 int ktls_set_tx_mode(struct socket *so, int mode);
 int ktls_get_tx_mode(struct socket *so, int *modep);
 int ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq);
+void ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp);
 int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls);
 #ifdef RATELIMIT
 int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate);
diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h
index a108e19b3ff3..31c351860a94 100644
--- a/sys/sys/sockbuf.h
+++ b/sys/sys/sockbuf.h
@@ -53,6 +53,7 @@
 #define	SB_STOP		0x1000		/* backpressure indicator */
 #define	SB_AIO_RUNNING	0x2000		/* AIO operation running */
 #define	SB_TLS_IFNET	0x4000		/* has used / is using ifnet KTLS */
+#define	SB_TLS_RX_RESYNC 0x8000		/* KTLS RX lost HW sync */
 
 #define	SBS_CANTSENDMORE	0x0010	/* can't send more data to peer */
 #define	SBS_CANTRCVMORE		0x0020	/* can't receive more data from peer */