cxgbe tom: Enable ULP_MODE_TCPDDP on demand

Most ULP modes in cxgbe's TOE are enabled on the fly when a protocol
is needed (e.g. ULP_MODE_ISCSI is enabled by cxgbei when offloading a
connection using iSCSI, and ULP_MODE_TLS is enabled when RX TLS keys
are programmed for a TOE connection).  The one exception to this is
ULP_MODE_TCPDDP.

Currently the cxgbe driver enables ULP_MODE_TCPDDP when a TOE
connection is first created.  However, since DDP connections cannot be
converted to other connection types, this requires some special
handling in the driver.  For example, iSCSI daemons use the SO_NO_DDP
socket option to ensure TOE connections use ULP_MODE_NONE so they can
be converted to ULP_MODE_ISCSI.  Similarly, using TLS receive offload
(ULP_MODE_TLS) requires disabling TCP DDP for new connections by
default.

This commit changes cxgbe to instead switch a connection from
ULP_MODE_NONE to ULP_MODE_TCPDDP when a connection first attempts to
use TCP DDP via aio_read(2).  This permits connections to always start
as ULP_MODE_NONE and switch to a protocol-specific mode as needed.

Reviewed by:	np
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D43670

4 files changed, 95 insertions, 20 deletions

diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index 2f425c7b5c6d..4d61189f5fe3 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -1753,17 +1753,18 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 		if (changed) {
 			if (toep->ddp.flags & DDP_SC_REQ)
 				toep->ddp.flags ^= DDP_ON | DDP_SC_REQ;
-			else {
-				KASSERT(cpl->ddp_off == 1,
-				    ("%s: DDP switched on by itself.",
-				    __func__));
-
+			else if (cpl->ddp_off == 1) {
 				/* Fell out of DDP mode */
 				toep->ddp.flags &= ~DDP_ON;
 				CTR1(KTR_CXGBE, "%s: fell out of DDP mode",
 				    __func__);
 
 				insert_ddp_data(toep, ddp_placed);
+			} else {
+				/*
+				 * Data was received while still
+				 * ULP_MODE_NONE, just fall through.
+				 */
 			}
 		}
 
diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c
index c50446d98e2b..89cb7269e218 100644
--- a/sys/dev/cxgbe/tom/t4_ddp.c
+++ b/sys/dev/cxgbe/tom/t4_ddp.c
@@ -192,7 +192,7 @@ free_ddp_buffer(struct tom_data *td, struct ddp_buffer *db)
 		free_pageset(td, db->ps);
 }
 
-void
+static void
 ddp_init_toep(struct toepcb *toep)
 {
 
@@ -810,6 +810,78 @@ do_rx_ddp_complete(struct sge_iq *iq, const struct rss_header *rss,
 	return (0);
 }
 
+static bool
+set_ddp_ulp_mode(struct toepcb *toep)
+{
+	struct adapter *sc = toep->vi->adapter;
+	struct wrqe *wr;
+	struct work_request_hdr *wrh;
+	struct ulp_txpkt *ulpmc;
+	int fields, len;
+
+	if (!sc->tt.ddp)
+		return (false);
+
+	fields = 0;
+
+	/* Overlay region including W_TCB_RX_DDP_FLAGS */
+	fields += 3;
+
+	/* W_TCB_ULP_TYPE */
+	fields++;
+
+#ifdef USE_DDP_RX_FLOW_CONTROL
+	/* W_TCB_T_FLAGS */
+	fields++;
+#endif
+
+	len = sizeof(*wrh) + fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16);
+	KASSERT(len <= SGE_MAX_WR_LEN,
+	    ("%s: WR with %d TCB field updates too large", __func__, fields));
+
+	wr = alloc_wrqe(len, toep->ctrlq);
+	if (wr == NULL)
+		return (false);
+
+	CTR(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
+
+	wrh = wrtod(wr);
+	INIT_ULPTX_WRH(wrh, len, 1, 0);	/* atomic */
+	ulpmc = (struct ulp_txpkt *)(wrh + 1);
+
+	/*
+	 * Words 26/27 are zero except for the DDP_OFF flag in
+	 * W_TCB_RX_DDP_FLAGS (27).
+	 */
+	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 26,
+	    0xffffffffffffffff, (uint64_t)V_TF_DDP_OFF(1) << 32);
+
+	/* Words 28/29 are zero. */
+	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 28,
+	    0xffffffffffffffff, 0);
+
+	/* Words 30/31 are zero. */
+	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 30,
+	    0xffffffffffffffff, 0);
+
+	/* Set the ULP mode to ULP_MODE_TCPDDP. */
+	toep->params.ulp_mode = ULP_MODE_TCPDDP;
+	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_ULP_TYPE,
+	    V_TCB_ULP_TYPE(M_TCB_ULP_TYPE),
+	    V_TCB_ULP_TYPE(ULP_MODE_TCPDDP));
+
+#ifdef USE_DDP_RX_FLOW_CONTROL
+	/* Set TF_RX_FLOW_CONTROL_DDP. */
+	ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_T_FLAGS,
+	    V_TF_RX_FLOW_CONTROL_DDP(1), V_TF_RX_FLOW_CONTROL_DDP(1));
+#endif
+
+	ddp_init_toep(toep);
+
+	t4_wrq_tx(sc, wr);
+	return (true);
+}
+
 static void
 enable_ddp(struct adapter *sc, struct toepcb *toep)
 {
@@ -2203,7 +2275,8 @@ t4_aio_cancel_queued(struct kaiocb *job)
 int
 t4_aio_queue_ddp(struct socket *so, struct kaiocb *job)
 {
-	struct tcpcb *tp = sototcpcb(so);
+	struct inpcb *inp = sotoinpcb(so);
+	struct tcpcb *tp = intotcpcb(inp);
 	struct toepcb *toep = tp->t_toe;
 
 
@@ -2211,6 +2284,15 @@ t4_aio_queue_ddp(struct socket *so, struct kaiocb *job)
 	if (job->uaiocb.aio_lio_opcode != LIO_READ)
 		return (EOPNOTSUPP);
 
+	INP_WLOCK(inp);
+	if (__predict_false(ulp_mode(toep) == ULP_MODE_NONE)) {
+		if (!set_ddp_ulp_mode(toep)) {
+			INP_WUNLOCK(inp);
+			return (EOPNOTSUPP);
+		}
+	}
+	INP_WUNLOCK(inp);
+
 	DDP_LOCK(toep);
 
 	/*
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 77b6ba5d4032..9bfe6ec818e4 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -179,8 +179,7 @@ init_toepcb(struct vi_info *vi, struct toepcb *toep)
 	toep->ctrlq = &sc->sge.ctrlq[pi->port_id];
 
 	tls_init_toep(toep);
-	if (ulp_mode(toep) == ULP_MODE_TCPDDP)
-		ddp_init_toep(toep);
+	MPASS(ulp_mode(toep) != ULP_MODE_TCPDDP);
 
 	toep->flags |= TPF_INITIALIZED;
 
@@ -1216,10 +1215,7 @@ calc_options2(struct vi_info *vi, struct conn_params *cp)
 		opt2 |= V_RX_COALESCE(M_RX_COALESCE);
 
 	opt2 |= V_RX_FC_DDP(0) | V_RX_FC_DISABLE(0);
-#ifdef USE_DDP_RX_FLOW_CONTROL
-	if (cp->ulp_mode == ULP_MODE_TCPDDP)
-		opt2 |= F_RX_FC_DDP;
-#endif
+	MPASS(cp->ulp_mode != ULP_MODE_TCPDDP);
 
 	return (htobe32(opt2));
 }
@@ -1327,11 +1323,7 @@ init_conn_params(struct vi_info *vi , struct offload_settings *s,
 		cp->tx_align = 0;
 
 	/* ULP mode. */
-	if (s->ddp > 0 ||
-	    (s->ddp < 0 && sc->tt.ddp && (so_options_get(so) & SO_NO_DDP) == 0))
-		cp->ulp_mode = ULP_MODE_TCPDDP;
-	else
-		cp->ulp_mode = ULP_MODE_NONE;
+	cp->ulp_mode = ULP_MODE_NONE;
 
 	/* Rx coalescing. */
 	if (s->rx_coalesce >= 0)
@@ -1972,7 +1964,8 @@ t4_aio_queue_tom(struct socket *so, struct kaiocb *job)
 	if (SOLISTENING(so))
 		return (EINVAL);
 
-	if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
+	if (ulp_mode(toep) == ULP_MODE_TCPDDP ||
+	    ulp_mode(toep) == ULP_MODE_NONE) {
 		error = t4_aio_queue_ddp(so, job);
 		if (error != EOPNOTSUPP)
 			return (error);
diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h
index b492fe86ae3a..d74f3f908286 100644
--- a/sys/dev/cxgbe/tom/t4_tom.h
+++ b/sys/dev/cxgbe/tom/t4_tom.h
@@ -505,7 +505,6 @@ int t4_aio_queue_ddp(struct socket *, struct kaiocb *);
 void t4_ddp_mod_load(void);
 void t4_ddp_mod_unload(void);
 void ddp_assert_empty(struct toepcb *);
-void ddp_init_toep(struct toepcb *);
 void ddp_uninit_toep(struct toepcb *);
 void ddp_queue_toep(struct toepcb *);
 void release_ddp_resources(struct toepcb *toep);