cxgbe(4): Updates to the base NIC driver and t4_tom to support the iSCSI

offload driver.  These changes come from projects/cxl_iscsi.

7 files changed, 176 insertions, 244 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index b3553cae81a2..d7a6f76199a9 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -766,7 +766,7 @@ struct adapter {
 	void *tom_softc;	/* (struct tom_data *) */
 	struct tom_tunables tt;
 	void *iwarp_softc;	/* (struct c4iw_dev *) */
-	void *iscsi_softc;
+	void *iscsi_ulp_softc;	/* (struct cxgbei_data *) */
 #endif
 	struct l2t_data *l2t;	/* L2 table */
 	struct tid_info tids;
diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index 2a1228332b23..2b5e4dcdfd4b 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -156,7 +156,7 @@ int t4_register_uld(struct uld_info *);
 int t4_unregister_uld(struct uld_info *);
 int t4_activate_uld(struct adapter *, int);
 int t4_deactivate_uld(struct adapter *, int);
-void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *);
+void t4_iscsi_init(struct adapter *, u_int, const u_int *);
 int uld_active(struct adapter *, int);
 #endif
 #endif
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index edac7098535f..0d00a774de4d 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -8801,11 +8801,8 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
 
 #ifdef TCP_OFFLOAD
 void
-t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
-    const unsigned int *pgsz_order)
+t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
 {
-	struct vi_info *vi = ifp->if_softc;
-	struct adapter *sc = vi->pi->adapter;
 
 	t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
 	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index c91c99461d61..d58592e85b49 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2012 Chelsio Communications, Inc.
+ * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
@@ -71,33 +71,6 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc);
 VNET_DECLARE(int, tcp_autorcvbuf_max);
 #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
 
-/*
- * For ULP connections HW may add headers, e.g., for digests, that aren't part
- * of the messages sent by the host but that are part of the TCP payload and
- * therefore consume TCP sequence space.  Tx connection parameters that
- * operate in TCP sequence space are affected by the HW additions and need to
- * compensate for them to accurately track TCP sequence numbers. This array
- * contains the compensating extra lengths for ULP packets.  It is indexed by
- * a packet's ULP submode.
- */
-const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8};
-
-/*
- * Return the length of any HW additions that will be made to a Tx packet.
- * Such additions can happen for some types of ULP packets.
- */
-static inline unsigned int
-ulp_extra_len(struct mbuf *m, int *ulp_mode)
-{
-	struct m_tag    *mtag;
-
-	if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL)
-		return (0);
-	*ulp_mode = *((int *)(mtag + 1));
-
-	return (t4_ulp_extra_len[*ulp_mode & 3]);
-}
-
 void
 send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
 {
@@ -384,13 +357,10 @@ t4_rcvd(struct toedev *tod, struct tcpcb *tp)
 	KASSERT(toep->sb_cc >= sbused(sb),
 	    ("%s: sb %p has more data (%d) than last time (%d).",
 	    __func__, sb, sbused(sb), toep->sb_cc));
-	if (toep->ulp_mode == ULP_MODE_ISCSI) {
-		toep->rx_credits += toep->sb_cc;
-		toep->sb_cc = 0;
-	} else {
-		toep->rx_credits += toep->sb_cc - sbused(sb);
-		toep->sb_cc = sbused(sb);
-	}
+
+	toep->rx_credits += toep->sb_cc - sbused(sb);
+	toep->sb_cc = sbused(sb);
+
 	if (toep->rx_credits > 0 &&
 	    (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
 	    (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
@@ -490,25 +460,16 @@ max_dsgl_nsegs(int tx_credits)
 
 static inline void
 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
-    unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign)
+    unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign)
 {
 	struct fw_ofld_tx_data_wr *txwr = dst;
-	unsigned int wr_ulp_mode;
 
 	txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
 	    V_FW_WR_IMMDLEN(immdlen));
 	txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
 	    V_FW_WR_LEN16(credits));
-
-	/* for iscsi, the mode & submode setting is per-packet */
-	if (toep->ulp_mode == ULP_MODE_ISCSI)
-		wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) |
-		    V_TX_ULP_SUBMODE(ulp_mode & 3);
-	else
-		wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode);
-
-	txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/
-	    V_TX_SHOVE(shove));
+	txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) |
+	    V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
 	txwr->plen = htobe32(plen);
 
 	if (txalign > 0) {
@@ -616,6 +577,9 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 	    toep->ulp_mode == ULP_MODE_RDMA,
 	    ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
 
+	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
+		return;
+
 	/*
 	 * This function doesn't resume by itself.  Someone else must clear the
 	 * flag and call this function.
@@ -802,56 +766,70 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 		close_conn(sc, toep);
 }
 
-/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */
+static inline void
+rqdrop_locked(struct mbufq *q, int plen)
+{
+	struct mbuf *m;
+
+	while (plen > 0) {
+		m = mbufq_dequeue(q);
+
+		/* Too many credits. */
+		MPASS(m != NULL);
+		M_ASSERTPKTHDR(m);
+
+		/* Partial credits. */
+		MPASS(plen >= m->m_pkthdr.len);
+
+		plen -= m->m_pkthdr.len;
+		m_freem(m);
+	}
+}
+
 void
-t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
+t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
 {
-	struct mbuf *sndptr, *m = NULL;
+	struct mbuf *sndptr, *m;
 	struct fw_ofld_tx_data_wr *txwr;
 	struct wrqe *wr;
-	unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+	u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+	u_int adjusted_plen, ulp_submode;
 	struct inpcb *inp = toep->inp;
-	struct tcpcb *tp;
-	struct socket *so;
-	struct sockbuf *sb;
-	int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0;
-	int shove, compl;
-	struct ofld_tx_sdesc *txsd;
+	struct tcpcb *tp = intotcpcb(inp);
+	int tx_credits, shove;
+	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
+	struct mbufq *pduq = &toep->ulp_pduq;
+	static const u_int ulp_extra_len[] = {0, 4, 4, 8};
 
 	INP_WLOCK_ASSERT(inp);
-	if (toep->flags & TPF_ABORT_SHUTDOWN)
-		return;
-
-	tp = intotcpcb(inp);
-	so = inp->inp_socket;
-	sb = &so->so_snd;
-	txsd = &toep->txsd[toep->txsd_pidx];
-
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
+	KASSERT(toep->ulp_mode == ULP_MODE_ISCSI,
+	    ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
+
+	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
+		return;
 
 	/*
 	 * This function doesn't resume by itself.  Someone else must clear the
 	 * flag and call this function.
 	 */
-	if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
+	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
+		KASSERT(drop == 0,
+		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
 		return;
+	}
 
-	sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
-	if (!qlen)
-		return;
+	if (drop)
+		rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
+
+	while ((sndptr = mbufq_first(pduq)) != NULL) {
+		M_ASSERTPKTHDR(sndptr);
 
-	do {
 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
 		max_imm = max_imm_payload(tx_credits);
 		max_nsegs = max_dsgl_nsegs(tx_credits);
 
-		if (drop) {
-			t4_cpl_iscsi_callback(toep->td, toep, &drop,
-			    CPL_FW4_ACK);
-			drop = 0;
-		}
-
 		plen = 0;
 		nsegs = 0;
 		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
@@ -861,7 +839,10 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 			nsegs += n;
 			plen += m->m_len;
 
-			/* This mbuf sent us _over_ the nsegs limit, return */
+			/*
+			 * This mbuf would send us _over_ the nsegs limit.
+			 * Suspend tx because the PDU can't be sent out.
+			 */
 			if (plen > max_imm && nsegs > max_nsegs) {
 				toep->flags |= TPF_TX_SUSPENDED;
 				return;
@@ -869,30 +850,35 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 
 			if (max_nsegs_1mbuf < n)
 				max_nsegs_1mbuf = n;
-
-			/* This mbuf put us right at the max_nsegs limit */
-			if (plen > max_imm && nsegs == max_nsegs) {
-				toep->flags |= TPF_TX_SUSPENDED;
-				return;
-			}
-		}
-
-		shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
-		/* nothing to send */
-		if (plen == 0) {
-			KASSERT(m == NULL,
-			    ("%s: nothing to send, but m != NULL", __func__));
-			break;
 		}
 
 		if (__predict_false(toep->flags & TPF_FIN_SENT))
 			panic("%s: excess tx.", __func__);
 
-		ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode);
+		/*
+		 * We have a PDU to send.  All of it goes out in one WR so 'm'
+		 * is NULL.  A PDU's length is always a multiple of 4.
+		 */
+		MPASS(m == NULL);
+		MPASS((plen & 3) == 0);
+		MPASS(sndptr->m_pkthdr.len == plen);
+
+		shove = !(tp->t_flags & TF_MORETOCOME);
+		ulp_submode = mbuf_ulp_submode(sndptr);
+		MPASS(ulp_submode < nitems(ulp_extra_len));
+
+		/*
+		 * plen doesn't include header and data digests, which are
+		 * generated and inserted in the right places by the TOE, but
+		 * they do occupy TCP sequence space and need to be accounted
+		 * for.
+		 */
+		adjusted_plen = plen + ulp_extra_len[ulp_submode];
 		if (plen <= max_imm) {
 
 			/* Immediate data tx */
-			wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
+
+			wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
 					toep->ofld_txq);
 			if (wr == NULL) {
 				/* XXX: how will we recover from this? */
@@ -901,16 +887,17 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 			}
 			txwr = wrtod(wr);
 			credits = howmany(wr->wr_len, 16);
-			write_tx_wr(txwr, toep, plen, ulp_len, credits, shove,
-								ulp_mode, 0);
+			write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
+			    shove, ulp_submode, sc->tt.tx_align);
 			m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
+			nsegs = 0;
 		} else {
 			int wr_len;
 
 			/* DSGL tx */
 			wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
 			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
-			wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
+			wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
 			if (wr == NULL) {
 				/* XXX: how will we recover from this? */
 				toep->flags |= TPF_TX_SUSPENDED;
@@ -918,8 +905,8 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 			}
 			txwr = wrtod(wr);
 			credits = howmany(wr_len, 16);
-			write_tx_wr(txwr, toep, 0, ulp_len, credits, shove,
-								ulp_mode, 0);
+			write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
+			    shove, ulp_submode, sc->tt.tx_align);
 			write_tx_sgl(txwr + 1, sndptr, m, nsegs,
 			    max_nsegs_1mbuf);
 			if (wr_len & 0xf) {
@@ -932,28 +919,26 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));
 
+		m = mbufq_dequeue(pduq);
+		MPASS(m == sndptr);
+		mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
+
 		toep->tx_credits -= credits;
 		toep->tx_nocompl += credits;
 		toep->plen_nocompl += plen;
 		if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
-			toep->tx_nocompl >= toep->tx_total / 4)
-			compl = 1;
-
-		if (compl) {
+		    toep->tx_nocompl >= toep->tx_total / 4) {
 			txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
 			toep->tx_nocompl = 0;
 			toep->plen_nocompl = 0;
 		}
-		tp->snd_nxt += ulp_len;
-		tp->snd_max += ulp_len;
 
-                /* goto next mbuf */
-		sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
+		tp->snd_nxt += adjusted_plen;
+		tp->snd_max += adjusted_plen;
 
 		toep->flags |= TPF_TX_DATA_SENT;
-		if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
+		if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
 			toep->flags |= TPF_TX_SUSPENDED;
-		}
 
 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
 		txsd->plen = plen;
@@ -966,10 +951,10 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
 		toep->txsd_avail--;
 
 		t4_l2t_send(sc, wr, toep->l2te);
-	} while (m != NULL);
+	}
 
-	/* Send a FIN if requested, but only if there's no more data to send */
-	if (m == NULL && toep->flags & TPF_SEND_FIN)
+	/* Send a FIN if requested, but only if there are no more PDUs to send */
+	if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
 		close_conn(sc, toep);
 }
 
@@ -987,7 +972,10 @@ t4_tod_output(struct toedev *tod, struct tcpcb *tp)
 	    ("%s: inp %p dropped.", __func__, inp));
 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
 
-	t4_push_frames(sc, toep, 0);
+	if (toep->ulp_mode == ULP_MODE_ISCSI)
+		t4_push_pdus(sc, toep, 0);
+	else
+		t4_push_frames(sc, toep, 0);
 
 	return (0);
 }
@@ -1009,7 +997,7 @@ t4_send_fin(struct toedev *tod, struct tcpcb *tp)
 	toep->flags |= TPF_SEND_FIN;
 	if (tp->t_state >= TCPS_ESTABLISHED) {
 		if (toep->ulp_mode == ULP_MODE_ISCSI)
-			t4_ulp_push_frames(sc, toep, 0);
+			t4_push_pdus(sc, toep, 0);
 		else
 			t4_push_frames(sc, toep, 0);
 	}
@@ -1250,91 +1238,6 @@ abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
 	}
 }
 
-int
-cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
-/*
- * tom_cpl_iscsi_callback -
- * iscsi and tom would share the following cpl messages, so when any of these
- * message is received, after tom is done with processing it, the messages
- * needs to be forwarded to iscsi for further processing:
- * - CPL_SET_TCB_RPL
- * - CPL_RX_DATA_DDP
- */
-void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
-    unsigned int);
-
-struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
-/*
- * Check if the handler function is set for a given CPL
- * return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
- */
-int
-t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
-{
-
-	MPASS(opcode < nitems(sc->cpl_handler));
-
-	return (sc->cpl_handler[opcode] &&
-	    sc->cpl_handler[opcode] != cpl_not_handled);
-}
-
-/*
- * set the tom_cpl_iscsi_callback function, this function should be used
- * whenever both toe and iscsi need to process the same cpl msg.
- */
-void
-t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *,
-    void *, unsigned int))
-{
-
-	tom_cpl_iscsi_callback = fp;
-}
-
-void
-t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
-    unsigned int, int *qlen))
-{
-
-	tom_queue_iscsi_callback = fp;
-}
-
-int
-t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
-    unsigned int opcode)
-{
-	struct socket *so;
-
-	if (opcode == CPL_FW4_ACK)
-		so = toep->inp->inp_socket;
-	else {
-		INP_WLOCK(toep->inp);
-		so = toep->inp->inp_socket;
-		INP_WUNLOCK(toep->inp);
-	}
-
-	if (tom_cpl_iscsi_callback && so) {
-		if (toep->ulp_mode == ULP_MODE_ISCSI) {
-			tom_cpl_iscsi_callback(td, so, m, opcode);
-			return (0);
-		}
-	}
-
-	return (1);
-}
-
-struct mbuf *
-t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
-    unsigned int cmd, int *qlen)
-{
-
-	if (tom_queue_iscsi_callback && so) {
-		if (toep->ulp_mode == ULP_MODE_ISCSI)
-			return (tom_queue_iscsi_callback(so, cmd, qlen));
-	}
-
-	return (NULL);
-}
-
 /*
  * TCP RST from the peer, timeout, or some other such critical error.
  */
@@ -1733,21 +1636,34 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 	    toep->tx_credits >= toep->tx_total / 4) {
 		toep->flags &= ~TPF_TX_SUSPENDED;
 		if (toep->ulp_mode == ULP_MODE_ISCSI)
-			t4_ulp_push_frames(sc, toep, plen);
+			t4_push_pdus(sc, toep, plen);
 		else
 			t4_push_frames(sc, toep, plen);
 	} else if (plen > 0) {
 		struct sockbuf *sb = &so->so_snd;
+		int sbu;
 
-		if (toep->ulp_mode == ULP_MODE_ISCSI)
-			t4_cpl_iscsi_callback(toep->td, toep, &plen,
-			    CPL_FW4_ACK);
-		else {
-			SOCKBUF_LOCK(sb);
+		SOCKBUF_LOCK(sb);
+		sbu = sbused(sb);
+		if (toep->ulp_mode == ULP_MODE_ISCSI) {
+
+			if (__predict_false(sbu > 0)) {
+				/*
+				 * The data trasmitted before the tid's ULP mode
+				 * changed to ISCSI is still in so_snd.
+				 * Incoming credits should account for so_snd
+				 * first.
+				 */
+				sbdrop_locked(sb, min(sbu, plen));
+				plen -= min(sbu, plen);
+			}
+			sowwakeup_locked(so);	/* unlocks so_snd */
+			rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
+		} else {
 			sbdrop_locked(sb, plen);
-			sowwakeup_locked(so);
-			SOCKBUF_UNLOCK_ASSERT(sb);
+			sowwakeup_locked(so);	/* unlocks so_snd */
 		}
+		SOCKBUF_UNLOCK_ASSERT(sb);
 	}
 
 	INP_WUNLOCK(inp);
@@ -1771,14 +1687,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 
 	if (is_ftid(sc, tid))
 		return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
-	else {
-		struct toepcb *toep = lookup_tid(sc, tid);
 
-		t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
-		return (0);
-	}
+	/*
+	 * TOM and/or other ULPs don't request replies for CPL_SET_TCB or
+	 * CPL_SET_TCB_FIELD requests.  This can easily change and when it does
+	 * the dispatch code will go here.
+	 */
+#ifdef INVARIANTS
+	panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
+	    tid, iq);
+#else
+	log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
+	    __func__, tid, iq);
+#endif
 
-	CXGBE_UNIMPLEMENTED(__func__);
+	return (0);
 }
 
 void
diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c
index 244287fcb3aa..2df025777ddb 100644
--- a/sys/dev/cxgbe/tom/t4_ddp.c
+++ b/sys/dev/cxgbe/tom/t4_ddp.c
@@ -513,7 +513,6 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 	unsigned int tid = GET_TID(cpl);
 	uint32_t vld;
 	struct toepcb *toep = lookup_tid(sc, tid);
-	struct tom_data *td = toep->td;
 
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 	KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
@@ -525,16 +524,11 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 		panic("%s: DDP error 0x%x (tid %d, toep %p)",
 		    __func__, vld, tid, toep);
 	}
+
 	if (toep->ulp_mode == ULP_MODE_ISCSI) {
-		m = m_get(M_NOWAIT, MT_DATA);
-		if (m == NULL)
-			CXGBE_UNIMPLEMENTED("mbuf alloc failure");
-		memcpy(mtod(m, unsigned char *), cpl,
-		    sizeof(struct cpl_rx_data_ddp));
-        	if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP))
-			return (0);
-		m_freem(m);
-        }
+		sc->cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m);
+		return (0);
+	}
 
 	handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));
 
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index bcf322e1ecc2..c136834019b8 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/lock.h>
+#include <sys/limits.h>
 #include <sys/module.h>
 #include <sys/protosw.h>
 #include <sys/domain.h>
@@ -158,6 +159,8 @@ alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags)
 	toep->ofld_txq = &sc->sge.ofld_txq[txqid];
 	toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid];
 	toep->ctrlq = &sc->sge.ctrlq[pi->port_id];
+	mbufq_init(&toep->ulp_pduq, INT_MAX);
+	mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX);
 	toep->txsd_total = txsd_total;
 	toep->txsd_avail = txsd_total;
 	toep->txsd_pidx = 0;
@@ -273,6 +276,14 @@ release_offload_resources(struct toepcb *toep)
 	CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)",
 	    __func__, toep, tid, toep->l2te, toep->ce);
 
+	/*
+	 * These queues should have been emptied at approximately the same time
+	 * that a normal connection's socket's so_snd would have been purged or
+	 * drained.  Do _not_ clean up here.
+	 */
+	MPASS(mbufq_len(&toep->ulp_pduq) == 0);
+	MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0);
+
 	if (toep->ulp_mode == ULP_MODE_TCPDDP)
 		release_ddp_resources(toep);
 
@@ -380,6 +391,7 @@ final_cpl_received(struct toepcb *toep)
 
 	toep->inp = NULL;
 	toep->flags &= ~TPF_CPL_PENDING;
+	mbufq_drain(&toep->ulp_pdu_reclaimq);
 
 	if (!(toep->flags & TPF_ATTACHED))
 		release_offload_resources(toep);
diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h
index f24e147ec26e..f61888deb3d8 100644
--- a/sys/dev/cxgbe/tom/t4_tom.h
+++ b/sys/dev/cxgbe/tom/t4_tom.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2012 Chelsio Communications, Inc.
+ * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
@@ -115,6 +115,10 @@ struct toepcb {
 	int rx_credits;		/* rx credits (in bytes) to be returned to hw */
 
 	u_int ulp_mode;	/* ULP mode */
+	void *ulpcb;
+	void *ulpcb2;
+	struct mbufq ulp_pduq;	/* PDUs waiting to be sent out. */
+	struct mbufq ulp_pdu_reclaimq;
 
 	u_int ddp_flags;
 	struct ddp_buffer *db[2];
@@ -220,6 +224,22 @@ td_adapter(struct tom_data *td)
 	return (td->tod.tod_softc);
 }
 
+static inline void
+set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode)
+{
+
+	M_ASSERTPKTHDR(m);
+	m->m_pkthdr.PH_per.eight[0] = ulp_submode;
+}
+
+static inline uint8_t
+mbuf_ulp_submode(struct mbuf *m)
+{
+
+	M_ASSERTPKTHDR(m);
+	return (m->m_pkthdr.PH_per.eight[0]);
+}
+
 /* t4_tom.c */
 struct toepcb *alloc_toepcb(struct vi_info *, int, int, int);
 void free_toepcb(struct toepcb *);
@@ -275,6 +295,7 @@ int t4_send_rst(struct toedev *, struct tcpcb *);
 void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t,
     uint64_t, uint64_t);
 void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
+void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop);
 
 /* t4_ddp.c */
 void t4_init_ddp(struct adapter *, struct tom_data *);
@@ -287,19 +308,4 @@ void handle_ddp_close(struct toepcb *, struct tcpcb *, struct sockbuf *,
     uint32_t);
 void insert_ddp_data(struct toepcb *, uint32_t);
 
-/* ULP related */
-#define CXGBE_ISCSI_MBUF_TAG          50
-int t4tom_cpl_handler_registered(struct adapter *, unsigned int);
-void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *,
-    struct socket *, void *, unsigned int));
-void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
-    unsigned int, int *));
-void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int);
-int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t);
-struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t,
-    int *);
-extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *,
-    void *, unsigned int);
-extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int,
-    int *);
 #endif