cxgbe: De-duplicate some of the code for managing TLS key contexts.

The NIC TLS and TOE TLS modes in cxgbe(4) both work with TLS key
contexts.  Previously, TOE TLS supported TLS key contexts created by
two different methods, and NIC TLS had a separate bit of code copied
from NIC TLS but specific to KTLS.  Now that TOE TLS only supports
KTLS, pull common code for creating TLS key contexts and programming
them into on-card memory into t4_keyctx.c.

Sponsored by:	Chelsio Communications

6 files changed, 626 insertions, 840 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 27655ec2fe59..edcc6e3f5249 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -1291,11 +1291,28 @@ int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int, u_int);
 /* t4_keyctx.c */
 struct auth_hash;
 union authctx;
+#ifdef KERN_TLS
+struct ktls_session;
+struct tls_key_req;
+struct tls_keyctx;
+#endif
 
 void t4_aes_getdeckey(void *, const void *, unsigned int);
 void t4_copy_partial_hash(int, union authctx *, void *);
 void t4_init_gmac_hash(const char *, int, char *);
 void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *);
+#ifdef KERN_TLS
+u_int t4_tls_key_info_size(const struct ktls_session *);
+int t4_tls_proto_ver(const struct ktls_session *);
+int t4_tls_cipher_mode(const struct ktls_session *);
+int t4_tls_auth_mode(const struct ktls_session *);
+int t4_tls_hmac_ctrl(const struct ktls_session *);
+void t4_tls_key_ctx(const struct ktls_session *, int, struct tls_keyctx *);
+int t4_alloc_tls_keyid(struct adapter *);
+void t4_free_tls_keyid(struct adapter *, int);
+void t4_write_tlskey_wr(const struct ktls_session *, int, int, int, int,
+    struct tls_key_req *);
+#endif
 
 #ifdef DEV_NETMAP
 /* t4_netmap.c */
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.h b/sys/dev/cxgbe/crypto/t4_crypto.h
index 161bb2aea520..83ea0fecc6c8 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.h
+++ b/sys/dev/cxgbe/crypto/t4_crypto.h
@@ -191,4 +191,59 @@ struct phys_sge_pairs {
 #define CHCR_HASH_MAX_BLOCK_SIZE_64  64
 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128
 
+/*
+ * TODO: Should coalesce ccr's key context with the TLS key context.
+ * Lookaside requests use the TX context header.
+ */
+
+struct tls_key_req {
+	/* FW_ULPTX_WR */
+	__be32 wr_hi;
+	__be32 wr_mid;
+        __be32 ftid;
+        __u8   reneg_to_write_rx;
+        __u8   protocol;
+        __be16 mfs;
+	/* master command */
+	__be32 cmd;
+	__be32 len16;             /* command length */
+	__be32 dlen;              /* data length in 32-byte units */
+	__be32 kaddr;
+	/* sub-command */
+	__be32 sc_more;
+	__be32 sc_len;
+}__packed;
+
+struct tls_keyctx {
+        union key_ctx {
+                struct tx_keyctx_hdr {
+                        __u8   ctxlen;
+                        __u8   r2;
+                        __be16 dualck_to_txvalid;
+                        __u8   txsalt[4];
+                        __be64 r5;
+                } txhdr;
+                struct rx_keyctx_hdr {
+                        __u8   flitcnt_hmacctrl;
+                        __u8   protover_ciphmode;
+                        __u8   authmode_to_rxvalid;
+                        __u8   ivpresent_to_rxmk_size;
+                        __u8   rxsalt[4];
+                        __be64 ivinsert_to_authinsrt;
+                } rxhdr;
+        } u;
+        struct keys {
+                __u8   edkey[32];
+                __u8   ipad[64];
+                __u8   opad[64];
+        } keys;
+};
+
+#define SALT_SIZE		4
+
+#define	TLS_KEY_CONTEXT_SZ	roundup2(sizeof(struct tls_keyctx), 32)
+
+#define	TLS_KEY_WR_SZ							\
+	roundup2(sizeof(struct tls_key_req) + TLS_KEY_CONTEXT_SZ, 16)
+
 #endif /* !__T4_CRYPTO_H__ */
diff --git a/sys/dev/cxgbe/crypto/t4_kern_tls.c b/sys/dev/cxgbe/crypto/t4_kern_tls.c
index 99d0d33cf128..a20c3045b5b3 100644
--- a/sys/dev/cxgbe/crypto/t4_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t4_kern_tls.c
@@ -60,101 +60,13 @@ __FBSDID("$FreeBSD$");
 
 #if defined(INET) || defined(INET6)
 
-#define SALT_SIZE		4
-
-#define GCM_TAG_SIZE			16
 #define TLS_HEADER_LENGTH		5
 
-#define	TLS_KEY_CONTEXT_SZ	roundup2(sizeof(struct tls_keyctx), 32)
-
 struct tls_scmd {
 	__be32 seqno_numivs;
 	__be32 ivgen_hdrlen;
 };
 
-struct tls_key_req {
-	/* FW_ULPTX_WR */
-	__be32 wr_hi;
-	__be32 wr_mid;
-        __be32 ftid;
-        __u8   reneg_to_write_rx;
-        __u8   protocol;
-        __be16 mfs;
-	/* master command */
-	__be32 cmd;
-	__be32 len16;             /* command length */
-	__be32 dlen;              /* data length in 32-byte units */
-	__be32 kaddr;
-	/* sub-command */
-	__be32 sc_more;
-	__be32 sc_len;
-}__packed;
-
-struct tls_keyctx {
-	struct tx_keyctx_hdr {
-		__u8   ctxlen;
-		__u8   r2;
-		__be16 dualck_to_txvalid;
-		__u8   txsalt[4];
-		__be64 r5;
-	} txhdr;
-        struct keys {
-                __u8   edkey[32];
-                __u8   ipad[64];
-                __u8   opad[64];
-        } keys;
-};
-
-#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11
-#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
-#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \
-     M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
-#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \
-    V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10
-#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT)
-#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \
-     M_TLS_KEYCTX_TX_WR_SALT_PRESENT)
-#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \
-    V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6
-#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf
-#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_TXCK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2
-#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf
-#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_TXMK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_TXVALID   0
-#define M_TLS_KEYCTX_TX_WR_TXVALID   0x1
-#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXVALID)
-#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID)
-#define F_TLS_KEYCTX_TX_WR_TXVALID   V_TLS_KEYCTX_TX_WR_TXVALID(1U)
-
-/* Key Context Programming Operation type */
-#define KEY_WRITE_RX			0x1
-#define KEY_WRITE_TX			0x2
-#define KEY_DELETE_RX			0x4
-#define KEY_DELETE_TX			0x8
-
 struct tlspcb {
 	struct m_snd_tag com;
 	struct vi_info *vi;	/* virtual interface */
@@ -187,15 +99,6 @@ struct tlspcb {
 	struct sge_wrq *ctrlq;
 	struct clip_entry *ce;	/* CLIP table entry used by this tid */
 
-	unsigned char auth_mode;
-	unsigned char hmac_ctrl;
-	unsigned char mac_first;
-	unsigned char iv_size;
-
-	unsigned int frag_size;
-	unsigned int cipher_secret_size;
-	int proto_ver;
-
 	bool open_pending;
 };
 
@@ -208,27 +111,6 @@ mst_to_tls(struct m_snd_tag *t)
 	return (__containerof(t, struct tlspcb, com));
 }
 
-/* XXX: There are similar versions of these two in tom/t4_tls.c. */
-static int
-get_new_keyid(struct tlspcb *tlsp)
-{
-	vmem_addr_t addr;
-
-	if (vmem_alloc(tlsp->sc->key_map, TLS_KEY_CONTEXT_SZ,
-	    M_NOWAIT | M_FIRSTFIT, &addr) != 0)
-		return (-1);
-
-	return (addr);
-}
-
-static void
-free_keyid(struct tlspcb *tlsp, int keyid)
-{
-
-	CTR3(KTR_CXGBE, "%s: tid %d key addr %#x", __func__, tlsp->tid, keyid);
-	vmem_free(tlsp->sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
-}
-
 static struct tlspcb *
 alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
 {
@@ -250,50 +132,6 @@ alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
 	return (tlsp);
 }
 
-static void
-init_ktls_key_params(struct tlspcb *tlsp, const struct ktls_session *tls)
-{
-	int mac_key_size;
-
-	if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
-		tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_1;
-	else
-		tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_2;
-	tlsp->cipher_secret_size = tls->params.cipher_key_len;
-	tlsp->tx_key_info_size = sizeof(struct tx_keyctx_hdr) +
-	    tlsp->cipher_secret_size;
-	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
-		tlsp->auth_mode = SCMD_AUTH_MODE_GHASH;
-		tlsp->enc_mode = SCMD_CIPH_MODE_AES_GCM;
-		tlsp->iv_size = 4;
-		tlsp->mac_first = 0;
-		tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NOP;
-		tlsp->tx_key_info_size += GMAC_BLOCK_LEN;
-	} else {
-		switch (tls->params.auth_algorithm) {
-		case CRYPTO_SHA1_HMAC:
-			mac_key_size = roundup2(SHA1_HASH_LEN, 16);
-			tlsp->auth_mode = SCMD_AUTH_MODE_SHA1;
-			break;
-		case CRYPTO_SHA2_256_HMAC:
-			mac_key_size = SHA2_256_HASH_LEN;
-			tlsp->auth_mode = SCMD_AUTH_MODE_SHA256;
-			break;
-		case CRYPTO_SHA2_384_HMAC:
-			mac_key_size = SHA2_512_HASH_LEN;
-			tlsp->auth_mode = SCMD_AUTH_MODE_SHA512_384;
-			break;
-		}
-		tlsp->enc_mode = SCMD_CIPH_MODE_AES_CBC;
-		tlsp->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
-		tlsp->mac_first = 1;
-		tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
-		tlsp->tx_key_info_size += mac_key_size * 2;
-	}
-
-	tlsp->frag_size = tls->params.max_frame_len;
-}
-
 static int
 ktls_act_open_cpl_size(bool isipv6)
 {
@@ -539,7 +377,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct sge_txq *txq;
-	int atid, error, keyid;
+	int atid, error, explicit_iv_size, keyid, mac_first;
 
 	tls = params->tls.tls;
 
@@ -569,6 +407,8 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 		default:
 			return (EPROTONOSUPPORT);
 		}
+		explicit_iv_size = AES_BLOCK_LEN;
+		mac_first = 1;
 		break;
 	case CRYPTO_AES_NIST_GCM_16:
 		if (tls->params.iv_len != SALT_SIZE)
@@ -581,6 +421,8 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 		default:
 			return (EINVAL);
 		}
+		explicit_iv_size = 8;
+		mac_first = 0;
 		break;
 	default:
 		return (EPROTONOSUPPORT);
@@ -600,7 +442,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 	if (sc->tlst.inline_keys)
 		keyid = -1;
 	else
-		keyid = get_new_keyid(tlsp);
+		keyid = t4_alloc_tls_keyid(sc);
 	if (keyid < 0) {
 		CTR2(KTR_CXGBE, "%s: atid %d using immediate key ctx", __func__,
 		    atid);
@@ -673,21 +515,22 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 	if (error)
 		goto failed;
 
-	init_ktls_key_params(tlsp, tls);
-
 	error = ktls_setup_keys(tlsp, tls, txq);
 	if (error)
 		goto failed;
 
+	tlsp->enc_mode = t4_tls_cipher_mode(tls);
+	tlsp->tx_key_info_size = t4_tls_key_info_size(tls);
+
 	/* The SCMD fields used when encrypting a full TLS record. */
 	tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) |
-	    V_SCMD_PROTO_VERSION(tlsp->proto_ver) |
+	    V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) |
 	    V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
-	    V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) |
+	    V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
 	    V_SCMD_CIPH_MODE(tlsp->enc_mode) |
-	    V_SCMD_AUTH_MODE(tlsp->auth_mode) |
-	    V_SCMD_HMAC_CTRL(tlsp->hmac_ctrl) |
-	    V_SCMD_IV_SIZE(tlsp->iv_size) | V_SCMD_NUM_IVS(1));
+	    V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) |
+	    V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) |
+	    V_SCMD_IV_SIZE(explicit_iv_size / 2) | V_SCMD_NUM_IVS(1));
 
 	tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
 	    V_SCMD_TLS_FRAG_ENABLE(0);
@@ -702,7 +545,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
 	tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
-	    V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) |
+	    V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
 	    V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
 	    V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
 	    V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0);
@@ -741,14 +584,11 @@ static int
 ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,
     struct sge_txq *txq)
 {
-	struct auth_hash *axf;
-	int error, keyid, kwrlen, kctxlen, len;
 	struct tls_key_req *kwr;
 	struct tls_keyctx *kctx;
-	void *items[1], *key;
-	struct tx_keyctx_hdr *khdr;
-	unsigned int ck_size, mk_size, partial_digest_len;
+	void *items[1];
 	struct mbuf *m;
+	int error;
 
 	/*
 	 * Store the salt and keys in the key context.  For
@@ -757,80 +597,12 @@ ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,
 	 * storing the key in DDR, a work request is used to store a
 	 * copy of the key context in DDR.
 	 */
-	kctx = &tlsp->keyctx;
-	khdr = &kctx->txhdr;
-
-	switch (tlsp->cipher_secret_size) {
-	case 128 / 8:
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-		break;
-	case 192 / 8:
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
-		break;
-	case 256 / 8:
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
-		break;
-	default:
-		panic("bad key size");
-	}
-	axf = NULL;
-	partial_digest_len = 0;
-	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
-		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
-	else {
-		switch (tlsp->auth_mode) {
-		case SCMD_AUTH_MODE_SHA1:
-			axf = &auth_hash_hmac_sha1;
-			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
-			partial_digest_len = SHA1_HASH_LEN;
-			break;
-		case SCMD_AUTH_MODE_SHA256:
-			axf = &auth_hash_hmac_sha2_256;
-			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
-			partial_digest_len = SHA2_256_HASH_LEN;
-			break;
-		case SCMD_AUTH_MODE_SHA512_384:
-			axf = &auth_hash_hmac_sha2_384;
-			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
-			partial_digest_len = SHA2_512_HASH_LEN;
-			break;
-		default:
-			panic("bad auth mode");
-		}
-	}
-
-	khdr->ctxlen = (tlsp->tx_key_info_size >> 4);
-	khdr->dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
-	    V_TLS_KEYCTX_TX_WR_TXCK_SIZE(ck_size) |
-	    V_TLS_KEYCTX_TX_WR_TXMK_SIZE(mk_size) |
-	    V_TLS_KEYCTX_TX_WR_TXVALID(1);
-	if (tlsp->enc_mode != SCMD_CIPH_MODE_AES_GCM)
-		khdr->dualck_to_txvalid |= V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1);
-	khdr->dualck_to_txvalid = htobe16(khdr->dualck_to_txvalid);
-	key = kctx->keys.edkey;
-	memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len);
-	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
-		memcpy(khdr->txsalt, tls->params.iv, SALT_SIZE);
-		t4_init_gmac_hash(tls->params.cipher_key,
-		    tls->params.cipher_key_len,
-		    (char *)key + tls->params.cipher_key_len);
-	} else {
-		t4_init_hmac_digest(axf, partial_digest_len,
-		    tls->params.auth_key, tls->params.auth_key_len,
-		    (char *)key + tls->params.cipher_key_len);
-	}
-
+	t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx);
 	if (tlsp->inline_key)
 		return (0);
 
-	keyid = tlsp->tx_key_addr;
-
 	/* Populate key work request. */
-	kwrlen = sizeof(*kwr);
-	kctxlen = roundup2(sizeof(*kctx), 32);
-	len = kwrlen + kctxlen;
-
-        m = alloc_wr_mbuf(len, M_NOWAIT);
+        m = alloc_wr_mbuf(TLS_KEY_WR_SZ, M_NOWAIT);
 	if (m == NULL) {
 		CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__,
 		    tlsp->tid);
@@ -839,27 +611,9 @@ ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,
 	m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
 	m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
 	kwr = mtod(m, void *);
-	memset(kwr, 0, len);
-
-	kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) |
-	    F_FW_WR_ATOMIC);
-	kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)));
-	kwr->protocol = tlsp->proto_ver;
-	kwr->mfs = htons(tlsp->frag_size);
-	kwr->reneg_to_write_rx = KEY_WRITE_TX;
-
-	/* master command */
-	kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
-	    V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
-	kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
-	kwr->len16 = htobe32((tlsp->tid << 8) |
-	    DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
-	kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
-
-	/* sub command */
-	kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
-	kwr->sc_len = htobe32(kctxlen);
+	memset(kwr, 0, TLS_KEY_WR_SZ);
 
+	t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr);
 	kctx = (struct tls_keyctx *)(kwr + 1);
 	memcpy(kctx, &tlsp->keyctx, sizeof(*kctx));
 
@@ -2029,7 +1783,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
 	if (plen == tlen) {
 		iv = out;
 		if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
-			memcpy(iv, tlsp->keyctx.txhdr.txsalt, SALT_SIZE);
+			memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE);
 			memcpy(iv + 4, hdr + 1, 8);
 			*(uint32_t *)(iv + 12) = htobe32(2 +
 			    offset / AES_BLOCK_LEN);
@@ -2335,7 +2089,7 @@ cxgbe_tls_tag_free(struct m_snd_tag *mst)
 	if (tlsp->ce)
 		t4_release_clip_entry(sc, tlsp->ce);
 	if (tlsp->tx_key_addr >= 0)
-		free_keyid(tlsp, tlsp->tx_key_addr);
+		t4_free_tls_keyid(sc, tlsp->tx_key_addr);
 
 	zfree(tlsp, M_CXGBE);
 }
diff --git a/sys/dev/cxgbe/crypto/t4_keyctx.c b/sys/dev/cxgbe/crypto/t4_keyctx.c
index 91bd5f28321a..b64eb4ff23d7 100644
--- a/sys/dev/cxgbe/crypto/t4_keyctx.c
+++ b/sys/dev/cxgbe/crypto/t4_keyctx.c
@@ -25,10 +25,13 @@
  * SUCH DAMAGE.
  */
 
+#include "opt_kern_tls.h"
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
+#include <sys/ktls.h>
 #include <sys/malloc.h>
 
 #include <opencrypto/cryptodev.h>
@@ -61,6 +64,234 @@ __FBSDID("$FreeBSD$");
  * +-------------------------------+ -
  */
 
+/* Fields in the key context header. */
+#define S_TLS_KEYCTX_TX_WR_DUALCK    12
+#define M_TLS_KEYCTX_TX_WR_DUALCK    0x1
+#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK)
+#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK)
+#define F_TLS_KEYCTX_TX_WR_DUALCK    V_TLS_KEYCTX_TX_WR_DUALCK(1U)
+
+#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11
+#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \
+     M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \
+    V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10
+#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \
+     M_TLS_KEYCTX_TX_WR_SALT_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \
+    V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6
+#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf
+#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \
+     M_TLS_KEYCTX_TX_WR_TXCK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2
+#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf
+#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \
+     M_TLS_KEYCTX_TX_WR_TXMK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_TXVALID   0
+#define M_TLS_KEYCTX_TX_WR_TXVALID   0x1
+#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_TXVALID)
+#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID)
+#define F_TLS_KEYCTX_TX_WR_TXVALID   V_TLS_KEYCTX_TX_WR_TXVALID(1U)
+
+#define S_TLS_KEYCTX_TX_WR_FLITCNT   3
+#define M_TLS_KEYCTX_TX_WR_FLITCNT   0x1f
+#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT)
+#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT)
+
+#define S_TLS_KEYCTX_TX_WR_HMACCTRL  0
+#define M_TLS_KEYCTX_TX_WR_HMACCTRL  0x7
+#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL)
+#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL)
+
+#define S_TLS_KEYCTX_TX_WR_PROTOVER  4
+#define M_TLS_KEYCTX_TX_WR_PROTOVER  0xf
+#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER)
+#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHMODE  0
+#define M_TLS_KEYCTX_TX_WR_CIPHMODE  0xf
+#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE)
+#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHMODE  4
+#define M_TLS_KEYCTX_TX_WR_AUTHMODE  0xf
+#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE)
+#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3
+#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1
+#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
+#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \
+     M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
+#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \
+    V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U)
+
+#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1
+#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3
+#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
+#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \
+     M_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
+
+#define S_TLS_KEYCTX_TX_WR_RXVALID   0
+#define M_TLS_KEYCTX_TX_WR_RXVALID   0x1
+#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_RXVALID)
+#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID)
+#define F_TLS_KEYCTX_TX_WR_RXVALID   V_TLS_KEYCTX_TX_WR_RXVALID(1U)
+
+#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7
+#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT)
+#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \
+     M_TLS_KEYCTX_TX_WR_IVPRESENT)
+#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6
+#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \
+     M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \
+    V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3
+#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7
+#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \
+     M_TLS_KEYCTX_TX_WR_RXCK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0
+#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7
+#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \
+     M_TLS_KEYCTX_TX_WR_RXMK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_IVINSERT  55
+#define M_TLS_KEYCTX_TX_WR_IVINSERT  0x1ffULL
+#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT)
+#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT)
+
+#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47
+#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL
+#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST)
+#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \
+     M_TLS_KEYCTX_TX_WR_AADSTRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39
+#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL
+#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \
+     M_TLS_KEYCTX_TX_WR_AADSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30
+#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL
+#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
+#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \
+     M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23
+#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f
+#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \
+     M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14
+#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff
+#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
+#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \
+     M_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7
+#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f
+#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \
+     M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0
+#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f
+#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
+    ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT)
+#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
+    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \
+     M_TLS_KEYCTX_TX_WR_AUTHINSRT)
+
+/* Key Context Programming Operation type */
+#define KEY_WRITE_RX			0x1
+#define KEY_WRITE_TX			0x2
+#define KEY_DELETE_RX			0x4
+#define KEY_DELETE_TX			0x8
+
+#define S_KEY_CLR_LOC		4
+#define M_KEY_CLR_LOC		0xf
+#define V_KEY_CLR_LOC(x)	((x) << S_KEY_CLR_LOC)
+#define G_KEY_CLR_LOC(x)	(((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC)
+#define F_KEY_CLR_LOC		V_KEY_CLR_LOC(1U)
+
+#define S_KEY_GET_LOC           0
+#define M_KEY_GET_LOC           0xf
+#define V_KEY_GET_LOC(x)        ((x) << S_KEY_GET_LOC)
+#define G_KEY_GET_LOC(x)        (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC)
+
 /*
  * Generate the initial GMAC hash state for a AES-GCM key.
  *
@@ -172,3 +403,278 @@ t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits)
 	MPASS(dkey == dec_key);
 	explicit_bzero(ek, sizeof(ek));
 }
+
+#ifdef KERN_TLS
+/*
+ * - keyid management
+ * - request to program key?
+ */
+u_int
+t4_tls_key_info_size(const struct ktls_session *tls)
+{
+	u_int key_info_size, mac_key_size;
+
+	key_info_size = sizeof(struct tx_keyctx_hdr) +
+	    tls->params.cipher_key_len;
+	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
+		key_info_size += GMAC_BLOCK_LEN;
+	} else {
+		switch (tls->params.auth_algorithm) {
+		case CRYPTO_SHA1_HMAC:
+			mac_key_size = SHA1_HASH_LEN;
+			break;
+		case CRYPTO_SHA2_256_HMAC:
+			mac_key_size = SHA2_256_HASH_LEN;
+			break;
+		case CRYPTO_SHA2_384_HMAC:
+			mac_key_size = SHA2_512_HASH_LEN;
+			break;
+		default:
+			__assert_unreachable();
+		}
+		key_info_size += roundup2(mac_key_size, 16) * 2;
+	}
+	return (key_info_size);
+}
+
+int
+t4_tls_proto_ver(const struct ktls_session *tls)
+{
+	if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
+		return (SCMD_PROTO_VERSION_TLS_1_1);
+	else
+		return (SCMD_PROTO_VERSION_TLS_1_2);
+}
+
+int
+t4_tls_cipher_mode(const struct ktls_session *tls)
+{
+	switch (tls->params.cipher_algorithm) {
+	case CRYPTO_AES_CBC:
+		return (SCMD_CIPH_MODE_AES_CBC);
+	case CRYPTO_AES_NIST_GCM_16:
+		return (SCMD_CIPH_MODE_AES_GCM);
+	default:
+		return (SCMD_CIPH_MODE_NOP);
+	}
+}
+
+int
+t4_tls_auth_mode(const struct ktls_session *tls)
+{
+	switch (tls->params.cipher_algorithm) {
+	case CRYPTO_AES_CBC:
+		switch (tls->params.auth_algorithm) {
+		case CRYPTO_SHA1_HMAC:
+			return (SCMD_AUTH_MODE_SHA1);
+		case CRYPTO_SHA2_256_HMAC:
+			return (SCMD_AUTH_MODE_SHA256);
+		case CRYPTO_SHA2_384_HMAC:
+			return (SCMD_AUTH_MODE_SHA512_384);
+		default:
+			return (SCMD_AUTH_MODE_NOP);
+		}
+	case CRYPTO_AES_NIST_GCM_16:
+		return (SCMD_AUTH_MODE_GHASH);
+	default:
+		return (SCMD_AUTH_MODE_NOP);
+	}
+}
+
+int
+t4_tls_hmac_ctrl(const struct ktls_session *tls)
+{
+	switch (tls->params.cipher_algorithm) {
+	case CRYPTO_AES_CBC:
+		return (SCMD_HMAC_CTRL_NO_TRUNC);
+	case CRYPTO_AES_NIST_GCM_16:
+		return (SCMD_HMAC_CTRL_NOP);
+	default:
+		return (SCMD_HMAC_CTRL_NOP);
+	}
+}
+
+static int
+tls_cipher_key_size(const struct ktls_session *tls)
+{
+	switch (tls->params.cipher_key_len) {
+	case 128 / 8:
+		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128);
+	case 192 / 8:
+		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192);
+	case 256 / 8:
+		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256);
+	default:
+		__assert_unreachable();
+	}
+}
+
+static int
+tls_mac_key_size(const struct ktls_session *tls)
+{
+	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
+		return (CHCR_KEYCTX_MAC_KEY_SIZE_512);
+	else {
+		switch (tls->params.auth_algorithm) {
+		case CRYPTO_SHA1_HMAC:
+			return (CHCR_KEYCTX_MAC_KEY_SIZE_160);
+		case CRYPTO_SHA2_256_HMAC:
+			return (CHCR_KEYCTX_MAC_KEY_SIZE_256);
+		case CRYPTO_SHA2_384_HMAC:
+			return (CHCR_KEYCTX_MAC_KEY_SIZE_512);
+		default:
+			__assert_unreachable();
+		}
+	}
+}
+
+void
+t4_tls_key_ctx(const struct ktls_session *tls, int direction,
+    struct tls_keyctx *kctx)
+{
+	struct auth_hash *axf;
+	u_int mac_key_size;
+	char *hash;
+
+	/* Key context header. */
+	if (direction == KTLS_TX) {
+		kctx->u.txhdr.ctxlen = t4_tls_key_info_size(tls) / 16;
+		kctx->u.txhdr.dualck_to_txvalid =
+		    V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
+		    V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) |
+		    V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) |
+		    V_TLS_KEYCTX_TX_WR_TXVALID(1);
+		if (tls->params.cipher_algorithm == CRYPTO_AES_CBC)
+			kctx->u.txhdr.dualck_to_txvalid |=
+			    V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1);
+		kctx->u.txhdr.dualck_to_txvalid =
+		    htobe16(kctx->u.txhdr.dualck_to_txvalid);
+	} else {
+		kctx->u.rxhdr.flitcnt_hmacctrl =
+		    V_TLS_KEYCTX_TX_WR_FLITCNT(t4_tls_key_info_size(tls) / 16) |
+		    V_TLS_KEYCTX_TX_WR_HMACCTRL(t4_tls_hmac_ctrl(tls));
+
+		kctx->u.rxhdr.protover_ciphmode =
+		    V_TLS_KEYCTX_TX_WR_PROTOVER(t4_tls_proto_ver(tls)) |
+		    V_TLS_KEYCTX_TX_WR_CIPHMODE(t4_tls_cipher_mode(tls));
+
+		kctx->u.rxhdr.authmode_to_rxvalid =
+		    V_TLS_KEYCTX_TX_WR_AUTHMODE(t4_tls_auth_mode(tls)) |
+		    V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
+		    V_TLS_KEYCTX_TX_WR_RXVALID(1);
+
+		kctx->u.rxhdr.ivpresent_to_rxmk_size =
+		    V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
+		    V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) |
+		    V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls));
+
+		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
+			kctx->u.rxhdr.ivinsert_to_authinsrt =
+			    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
+				V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
+				V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
+				V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
+				V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
+		} else {
+			kctx->u.rxhdr.authmode_to_rxvalid |=
+			    V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1);
+			kctx->u.rxhdr.ivpresent_to_rxmk_size |=
+			    V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1);
+			kctx->u.rxhdr.ivinsert_to_authinsrt =
+			    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
+				V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
+				V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
+				V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
+				V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
+				V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
+		}
+	}
+
+	/* Key. */
+	if (direction == KTLS_RX &&
+	    tls->params.cipher_algorithm == CRYPTO_AES_CBC)
+		t4_aes_getdeckey(kctx->keys.edkey, tls->params.cipher_key,
+		    tls->params.cipher_key_len * 8);
+	else
+		memcpy(kctx->keys.edkey, tls->params.cipher_key,
+		    tls->params.cipher_key_len);
+
+	/* Auth state and implicit IV (salt). */
+	hash = kctx->keys.edkey + tls->params.cipher_key_len;
+	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
+		_Static_assert(offsetof(struct tx_keyctx_hdr, txsalt) ==
+		    offsetof(struct rx_keyctx_hdr, rxsalt),
+		    "salt offset mismatch");
+		memcpy(kctx->u.txhdr.txsalt, tls->params.iv, SALT_SIZE);
+		t4_init_gmac_hash(tls->params.cipher_key,
+		    tls->params.cipher_key_len, hash);
+	} else {
+		switch (tls->params.auth_algorithm) {
+		case CRYPTO_SHA1_HMAC:
+			axf = &auth_hash_hmac_sha1;
+			mac_key_size = SHA1_HASH_LEN;
+			break;
+		case CRYPTO_SHA2_256_HMAC:
+			axf = &auth_hash_hmac_sha2_256;
+			mac_key_size = SHA2_256_HASH_LEN;
+			break;
+		case CRYPTO_SHA2_384_HMAC:
+			axf = &auth_hash_hmac_sha2_384;
+			mac_key_size = SHA2_512_HASH_LEN;
+			break;
+		default:
+			__assert_unreachable();
+		}
+		t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
+		    tls->params.auth_key_len, hash);
+	}
+}
+
+int
+t4_alloc_tls_keyid(struct adapter *sc)
+{
+	vmem_addr_t addr;
+
+	if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
+	    &addr) != 0)
+		return (-1);
+
+	return (addr);
+}
+
+void
+t4_free_tls_keyid(struct adapter *sc, int keyid)
+{
+	vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
+}
+
+void
+t4_write_tlskey_wr(const struct ktls_session *tls, int direction, int tid,
+    int flags, int keyid, struct tls_key_req *kwr)
+{
+	kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_ATOMIC | flags);
+	kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(TLS_KEY_WR_SZ, 16)) |
+	    V_FW_WR_FLOWID(tid));
+	kwr->protocol = t4_tls_proto_ver(tls);
+	kwr->mfs = htobe16(tls->params.max_frame_len);
+	kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ?
+	    KEY_WRITE_TX : KEY_WRITE_RX);
+
+	/* master command */
+	kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
+	    V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
+	kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(TLS_KEY_CONTEXT_SZ >> 5));
+	kwr->len16 = htobe32((tid << 8) |
+	    DIV_ROUND_UP(TLS_KEY_WR_SZ - sizeof(struct work_request_hdr), 16));
+	kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
+
+	/* sub command */
+	kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+	kwr->sc_len = htobe32(TLS_KEY_CONTEXT_SZ);
+}
+#endif
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index be47dbac7ae5..95fe96b9024a 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -147,274 +147,19 @@ tls_content_type(unsigned char content_type)
 	return CPL_TX_TLS_SFO_TYPE_DATA;
 }
 
-static int
-tls_key_info_size(struct ktls_session *tls)
-{
-	u_int key_info_size, mac_key_size;
-
-	key_info_size = sizeof(struct tx_keyctx_hdr) +
-	    tls->params.cipher_key_len;
-	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
-		key_info_size += GMAC_BLOCK_LEN;
-	} else {
-		switch (tls->params.auth_algorithm) {
-		case CRYPTO_SHA1_HMAC:
-			mac_key_size = SHA1_HASH_LEN;
-			break;
-		case CRYPTO_SHA2_256_HMAC:
-			mac_key_size = SHA2_256_HASH_LEN;
-			break;
-		case CRYPTO_SHA2_384_HMAC:
-			mac_key_size = SHA2_512_HASH_LEN;
-			break;
-		default:
-			__assert_unreachable();
-		}
-		key_info_size += roundup2(mac_key_size, 16) * 2;
-	}
-	return (key_info_size);
-}
-
-static int
-tls_proto_ver(struct ktls_session *tls)
-{
-	if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
-		return (SCMD_PROTO_VERSION_TLS_1_1);
-	else
-		return (SCMD_PROTO_VERSION_TLS_1_2);
-}
-
-static int
-tls_cipher_mode(struct ktls_session *tls)
-{
-	switch (tls->params.cipher_algorithm) {
-	case CRYPTO_AES_CBC:
-		return (SCMD_CIPH_MODE_AES_CBC);
-	case CRYPTO_AES_NIST_GCM_16:
-		return (SCMD_CIPH_MODE_AES_GCM);
-	default:
-		return (SCMD_CIPH_MODE_NOP);
-	}
-}
-
-static int
-tls_auth_mode(struct ktls_session *tls)
-{
-	switch (tls->params.cipher_algorithm) {
-	case CRYPTO_AES_CBC:
-		switch (tls->params.auth_algorithm) {
-		case CRYPTO_SHA1_HMAC:
-			return (SCMD_AUTH_MODE_SHA1);
-		case CRYPTO_SHA2_256_HMAC:
-			return (SCMD_AUTH_MODE_SHA256);
-		case CRYPTO_SHA2_384_HMAC:
-			return (SCMD_AUTH_MODE_SHA512_384);
-		default:
-			return (SCMD_AUTH_MODE_NOP);
-		}
-	case CRYPTO_AES_NIST_GCM_16:
-		return (SCMD_AUTH_MODE_GHASH);
-	default:
-		return (SCMD_AUTH_MODE_NOP);
-	}
-}
-
-static int
-tls_hmac_ctrl(struct ktls_session *tls)
-{
-	switch (tls->params.cipher_algorithm) {
-	case CRYPTO_AES_CBC:
-		return (SCMD_HMAC_CTRL_NO_TRUNC);
-	case CRYPTO_AES_NIST_GCM_16:
-		return (SCMD_HMAC_CTRL_NOP);
-	default:
-		return (SCMD_HMAC_CTRL_NOP);
-	}
-}
-
-static int
-tls_cipher_key_size(struct ktls_session *tls)
-{
-	switch (tls->params.cipher_key_len) {
-	case 128 / 8:
-		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128);
-	case 192 / 8:
-		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192);
-	case 256 / 8:
-		return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256);
-	default:
-		__assert_unreachable();
-	}
-}
-
-static int
-tls_mac_key_size(struct ktls_session *tls)
-{
-	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
-		/*
-		 * XXX: This used to use 128 (SHA_NOP) for TOE,
-		 * but NIC TLS has always used 512.
-		 */
-		return (CHCR_KEYCTX_MAC_KEY_SIZE_512);
-	else {
-		switch (tls->params.auth_algorithm) {
-		case CRYPTO_SHA1_HMAC:
-			return (CHCR_KEYCTX_MAC_KEY_SIZE_160);
-		case CRYPTO_SHA2_256_HMAC:
-			return (CHCR_KEYCTX_MAC_KEY_SIZE_256);
-		case CRYPTO_SHA2_384_HMAC:
-			return (CHCR_KEYCTX_MAC_KEY_SIZE_512);
-		default:
-			__assert_unreachable();
-		}
-	}
-}
-
-static void
-prepare_tls_keys(char *key, char *salt, struct ktls_session *tls,
-    int direction)
-{
-	struct auth_hash *axf;
-	u_int mac_key_size;
-	char *hash;
-
-	if (direction == KTLS_RX &&
-	    tls->params.cipher_algorithm == CRYPTO_AES_CBC)
-		t4_aes_getdeckey(key, tls->params.cipher_key,
-		    tls->params.cipher_key_len * 8);
-	else
-		memcpy(key, tls->params.cipher_key,
-		    tls->params.cipher_key_len);
-	hash = key + tls->params.cipher_key_len;
-	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
-		memcpy(salt, tls->params.iv, SALT_SIZE);
-		t4_init_gmac_hash(tls->params.cipher_key,
-		    tls->params.cipher_key_len, hash);
-	} else {
-		switch (tls->params.auth_algorithm) {
-		case CRYPTO_SHA1_HMAC:
-			axf = &auth_hash_hmac_sha1;
-			mac_key_size = SHA1_HASH_LEN;
-			break;
-		case CRYPTO_SHA2_256_HMAC:
-			axf = &auth_hash_hmac_sha2_256;
-			mac_key_size = SHA2_256_HASH_LEN;
-			break;
-		case CRYPTO_SHA2_384_HMAC:
-			axf = &auth_hash_hmac_sha2_384;
-			mac_key_size = SHA2_512_HASH_LEN;
-			break;
-		default:
-			__assert_unreachable();
-		}
-		t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
-		    tls->params.auth_key_len, hash);
-	}
-}
-
-/* Rx key */
-static void
-prepare_rxkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls)
-{
-
-	kwr->u.rxhdr.flitcnt_hmacctrl =
-		((tls_key_info_size(tls) / 16) << 3) | tls_hmac_ctrl(tls);
-
-	kwr->u.rxhdr.protover_ciphmode =
-		V_TLS_KEYCTX_TX_WR_PROTOVER(tls_proto_ver(tls)) |
-		V_TLS_KEYCTX_TX_WR_CIPHMODE(tls_cipher_mode(tls));
-
-	kwr->u.rxhdr.authmode_to_rxvalid =
-	        V_TLS_KEYCTX_TX_WR_AUTHMODE(tls_auth_mode(tls)) |
-		V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
-		V_TLS_KEYCTX_TX_WR_RXVALID(1);
-
-	kwr->u.rxhdr.ivpresent_to_rxmk_size =
-		V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
-		V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) |
-		V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls));
-
-	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
-		kwr->u.rxhdr.ivinsert_to_authinsrt =
-		    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
-			V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
-			V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
-			V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
-			V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
-	} else {
-		kwr->u.rxhdr.authmode_to_rxvalid |=
-			V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1);
-		kwr->u.rxhdr.ivpresent_to_rxmk_size |=
-			V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1);
-		kwr->u.rxhdr.ivinsert_to_authinsrt =
-		    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
-			V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
-			V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
-			V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
-			V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
-			V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
-	}
-
-	prepare_tls_keys(kwr->keys.edkey, kwr->u.rxhdr.rxsalt, tls, KTLS_RX);
-}
-
-/* Tx key */
-static void
-prepare_txkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls)
-{
-
-	kwr->u.txhdr.ctxlen = tls_key_info_size(tls) / 16;
-	kwr->u.txhdr.dualck_to_txvalid =
-		V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
-		V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) |
-		V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) |
-		V_TLS_KEYCTX_TX_WR_TXVALID(1);
-	if (tls->params.cipher_algorithm == CRYPTO_AES_CBC)
-		kwr->u.txhdr.dualck_to_txvalid |=
-		    V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1);
-	kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid);
-
-	prepare_tls_keys(kwr->keys.edkey, kwr->u.txhdr.txsalt, tls, KTLS_TX);
-}
-
 /* TLS Key memory management */
-static int
-get_new_keyid(struct toepcb *toep)
-{
-	struct adapter *sc = td_adapter(toep->td);
-	vmem_addr_t addr;
-
-	if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
-	    &addr) != 0)
-		return (-1);
-
-	return (addr);
-}
-
-static void
-free_keyid(struct toepcb *toep, int keyid)
-{
-	struct adapter *sc = td_adapter(toep->td);
-
-	vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
-}
-
 static void
 clear_tls_keyid(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
+	struct adapter *sc = td_adapter(toep->td);
 
 	if (tls_ofld->rx_key_addr >= 0) {
-		free_keyid(toep, tls_ofld->rx_key_addr);
+		t4_free_tls_keyid(sc, tls_ofld->rx_key_addr);
 		tls_ofld->rx_key_addr = -1;
 	}
 	if (tls_ofld->tx_key_addr >= 0) {
-		free_keyid(toep, tls_ofld->tx_key_addr);
+		t4_free_tls_keyid(sc, tls_ofld->tx_key_addr);
 		tls_ofld->tx_key_addr = -1;
 	}
 }
@@ -435,64 +180,45 @@ tls_program_key_id(struct toepcb *toep, struct ktls_session *tls,
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct adapter *sc = td_adapter(toep->td);
 	struct ofld_tx_sdesc *txsd;
-	int kwrlen, kctxlen, keyid, len;
+	int keyid;
 	struct wrqe *wr;
 	struct tls_key_req *kwr;
 	struct tls_keyctx *kctx;
 
+#ifdef INVARIANTS
+	int kwrlen, kctxlen, len;
+
 	kwrlen = sizeof(*kwr);
 	kctxlen = roundup2(sizeof(*kctx), 32);
 	len = roundup2(kwrlen + kctxlen, 16);
-
+	MPASS(TLS_KEY_WR_SZ == len);
+#endif
 	if (toep->txsd_avail == 0)
 		return (EAGAIN);
 
-	if ((keyid = get_new_keyid(toep)) < 0) {
+	if ((keyid = t4_alloc_tls_keyid(sc)) < 0) {
 		return (ENOSPC);
 	}
 
-	wr = alloc_wrqe(len, &toep->ofld_txq->wrq);
+	wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq);
 	if (wr == NULL) {
-		free_keyid(toep, keyid);
+		t4_free_tls_keyid(sc, keyid);
 		return (ENOMEM);
 	}
 	kwr = wrtod(wr);
-	memset(kwr, 0, kwrlen);
-
-	kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL |
-	    F_FW_WR_ATOMIC);
-	kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) |
-	    V_FW_WR_FLOWID(toep->tid));
-	kwr->protocol = tls_proto_ver(tls);
-	kwr->mfs = htons(tls->params.max_frame_len);
-	kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ?
-	    KEY_WRITE_TX : KEY_WRITE_RX);
-
-	/* master command */
-	kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
-	    V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
-	kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
-	kwr->len16 = htobe32((toep->tid << 8) |
-	    DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
-	kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
-
-	/* sub command */
-	kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
-	kwr->sc_len = htobe32(kctxlen);
+	memset(kwr, 0, TLS_KEY_WR_SZ);
 
+	t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid,
+	    kwr);
 	kctx = (struct tls_keyctx *)(kwr + 1);
-	memset(kctx, 0, kctxlen);
-
-	if (direction == KTLS_TX) {
+	if (direction == KTLS_TX)
 		tls_ofld->tx_key_addr = keyid;
-		prepare_txkey_wr(kctx, tls);
-	} else {
+	else
 		tls_ofld->rx_key_addr = keyid;
-		prepare_rxkey_wr(kctx, tls);
-	}
+	t4_tls_key_ctx(tls, direction, kctx);
 
 	txsd = &toep->txsd[toep->txsd_pidx];
-	txsd->tx_credits = DIV_ROUND_UP(len, 16);
+	txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16);
 	txsd->plen = 0;
 	toep->tx_credits -= txsd->tx_credits;
 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
@@ -661,12 +387,12 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
 	if (direction == KTLS_TX) {
 		toep->tls.scmd0.seqno_numivs =
 			(V_SCMD_SEQ_NO_CTRL(3) |
-			 V_SCMD_PROTO_VERSION(tls_proto_ver(tls)) |
+			 V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) |
 			 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
 			 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
-			 V_SCMD_CIPH_MODE(tls_cipher_mode(tls)) |
-			 V_SCMD_AUTH_MODE(tls_auth_mode(tls)) |
-			 V_SCMD_HMAC_CTRL(tls_hmac_ctrl(tls)) |
+			 V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) |
+			 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) |
+			 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) |
 			 V_SCMD_IV_SIZE(explicit_iv_size / 2));
 
 		toep->tls.scmd0.ivgen_hdrlen =
@@ -686,7 +412,7 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
 		toep->tls.pdus_per_ulp = 1;
 		toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
 		    tls->params.max_frame_len;
-		toep->tls.tx_key_info_size = tls_key_info_size(tls);
+		toep->tls.tx_key_info_size = t4_tls_key_info_size(tls);
 	} else {
 		/* Stop timer on handshake completion */
 		tls_stop_handshake_timer(toep);
diff --git a/sys/dev/cxgbe/tom/t4_tls.h b/sys/dev/cxgbe/tom/t4_tls.h
index 2770526704aa..e5166bf6d9bd 100644
--- a/sys/dev/cxgbe/tom/t4_tls.h
+++ b/sys/dev/cxgbe/tom/t4_tls.h
@@ -33,26 +33,8 @@
 #ifndef __T4_TLS_H__
 #define __T4_TLS_H__
 
-#define SALT_SIZE		4
-
-/* Key Context Programming Operation type */
-#define KEY_WRITE_RX			0x1
-#define KEY_WRITE_TX			0x2
-#define KEY_DELETE_RX			0x4
-#define KEY_DELETE_TX			0x8
-
-#define S_KEY_CLR_LOC		4
-#define M_KEY_CLR_LOC		0xf
-#define V_KEY_CLR_LOC(x)	((x) << S_KEY_CLR_LOC)
-#define G_KEY_CLR_LOC(x)	(((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC)
-#define F_KEY_CLR_LOC		V_KEY_CLR_LOC(1U)
-
-#define S_KEY_GET_LOC           0
-#define M_KEY_GET_LOC           0xf
-#define V_KEY_GET_LOC(x)        ((x) << S_KEY_GET_LOC)
-#define G_KEY_GET_LOC(x)        (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC)
-
 #ifdef _KERNEL
+
 /* Timeouts for handshake timer in seconds. */
 #define TLS_SRV_HELLO_DONE		9
 #define TLS_SRV_HELLO_RD_TM		5
@@ -70,8 +52,6 @@
 #define TP_TX_PG_SZ			65536
 #define FC_TP_PLEN_MAX			17408
 
-#define	TLS_KEY_CONTEXT_SZ	roundup2(sizeof(struct tls_keyctx), 32)
-
 enum {
 	TLS_SFO_WR_CONTEXTLOC_DSGL,
 	TLS_SFO_WR_CONTEXTLOC_IMMEDIATE,
@@ -108,258 +88,6 @@ struct tls_ofld_info {
 	struct callout handshake_timer;
 };
 
-struct tls_key_req {
-	__be32 wr_hi;
-	__be32 wr_mid;
-        __be32 ftid;
-        __u8   reneg_to_write_rx;
-        __u8   protocol;
-        __be16 mfs;
-	/* master command */
-	__be32 cmd;
-	__be32 len16;             /* command length */
-	__be32 dlen;              /* data length in 32-byte units */
-	__be32 kaddr;
-	/* sub-command */
-	__be32 sc_more;
-	__be32 sc_len;
-}__packed;
-
-struct tls_keyctx {
-        union key_ctx {
-                struct tx_keyctx_hdr {
-                        __u8   ctxlen;
-                        __u8   r2;
-                        __be16 dualck_to_txvalid;
-                        __u8   txsalt[4];
-                        __be64 r5;
-                } txhdr;
-                struct rx_keyctx_hdr {
-                        __u8   flitcnt_hmacctrl;
-                        __u8   protover_ciphmode;
-                        __u8   authmode_to_rxvalid;
-                        __u8   ivpresent_to_rxmk_size;
-                        __u8   rxsalt[4];
-                        __be64 ivinsert_to_authinsrt;
-                } rxhdr;
-        } u;
-        struct keys {
-                __u8   edkey[32];
-                __u8   ipad[64];
-                __u8   opad[64];
-        } keys;
-};
-
-#define S_TLS_KEYCTX_TX_WR_DUALCK    12
-#define M_TLS_KEYCTX_TX_WR_DUALCK    0x1
-#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK)
-#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK)
-#define F_TLS_KEYCTX_TX_WR_DUALCK    V_TLS_KEYCTX_TX_WR_DUALCK(1U)
-
-#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11
-#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
-#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \
-     M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
-#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \
-    V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10
-#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT)
-#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \
-     M_TLS_KEYCTX_TX_WR_SALT_PRESENT)
-#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \
-    V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6
-#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf
-#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_TXCK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2
-#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf
-#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_TXMK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_TXVALID   0
-#define M_TLS_KEYCTX_TX_WR_TXVALID   0x1
-#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_TXVALID)
-#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID)
-#define F_TLS_KEYCTX_TX_WR_TXVALID   V_TLS_KEYCTX_TX_WR_TXVALID(1U)
-
-#define S_TLS_KEYCTX_TX_WR_FLITCNT   3
-#define M_TLS_KEYCTX_TX_WR_FLITCNT   0x1f
-#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT)
-#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT)
-
-#define S_TLS_KEYCTX_TX_WR_HMACCTRL  0
-#define M_TLS_KEYCTX_TX_WR_HMACCTRL  0x7
-#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL)
-#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL)
-
-#define S_TLS_KEYCTX_TX_WR_PROTOVER  4
-#define M_TLS_KEYCTX_TX_WR_PROTOVER  0xf
-#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER)
-#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER)
-
-#define S_TLS_KEYCTX_TX_WR_CIPHMODE  0
-#define M_TLS_KEYCTX_TX_WR_CIPHMODE  0xf
-#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE)
-#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE)
-
-#define S_TLS_KEYCTX_TX_WR_AUTHMODE  4
-#define M_TLS_KEYCTX_TX_WR_AUTHMODE  0xf
-#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE)
-#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE)
-
-#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3
-#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1
-#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
-#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \
-     M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
-#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \
-    V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U)
-
-#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1
-#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3
-#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
-#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \
-     M_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
-
-#define S_TLS_KEYCTX_TX_WR_RXVALID   0
-#define M_TLS_KEYCTX_TX_WR_RXVALID   0x1
-#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_RXVALID)
-#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID)
-#define F_TLS_KEYCTX_TX_WR_RXVALID   V_TLS_KEYCTX_TX_WR_RXVALID(1U)
-
-#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7
-#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT)
-#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \
-     M_TLS_KEYCTX_TX_WR_IVPRESENT)
-#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6
-#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1
-#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
-#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \
-     M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
-#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \
-    V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U)
-
-#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3
-#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7
-#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_RXCK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0
-#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7
-#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE)
-#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \
-     M_TLS_KEYCTX_TX_WR_RXMK_SIZE)
-
-#define S_TLS_KEYCTX_TX_WR_IVINSERT  55
-#define M_TLS_KEYCTX_TX_WR_IVINSERT  0x1ffULL
-#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT)
-#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT)
-
-#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47
-#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL
-#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST)
-#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \
-     M_TLS_KEYCTX_TX_WR_AADSTRTOFST)
-
-#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39
-#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL
-#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST)
-#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \
-     M_TLS_KEYCTX_TX_WR_AADSTOPOFST)
-
-#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30
-#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL
-#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
-#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \
-     M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
-
-#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23
-#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f
-#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
-#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \
-     M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
-
-#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14
-#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff
-#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
-#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \
-     M_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
-
-#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7
-#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f
-#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
-#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \
-     M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
-
-#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0
-#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f
-#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
-    ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT)
-#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
-    (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \
-     M_TLS_KEYCTX_TX_WR_AUTHINSRT)
-
 struct tls_hdr {
 	__u8   type;
 	__be16 version;