diff options
Diffstat (limited to 'sys/dev/cxgbe/tom')
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_connect.c | 43 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_cpl_io.c | 151 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_ddp.c | 22 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_listen.c | 7 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tls.c | 357 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tls.h | 1 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tom.c | 143 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tom.h | 22 | ||||
| -rw-r--r-- | sys/dev/cxgbe/tom/t4_tom_l2t.c | 2 |
9 files changed, 593 insertions, 155 deletions
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c index 99e4c222996d..c236ee060bc2 100644 --- a/sys/dev/cxgbe/tom/t4_connect.c +++ b/sys/dev/cxgbe/tom/t4_connect.c @@ -89,6 +89,12 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss, INP_WLOCK(inp); toep->tid = tid; insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1); + if (sc->params.tid_qid_sel_mask != 0) { + update_tid_qid_sel(toep->vi, &toep->params, tid); + toep->ofld_txq = &sc->sge.ofld_txq[toep->params.txq_idx]; + toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx]; + } + if (inp->inp_flags & INP_DROPPED) { /* socket closed by the kernel before hw told us it connected */ @@ -205,7 +211,7 @@ static inline int act_open_cpl_size(struct adapter *sc, int isipv6) { int idx; - static const int sz_table[3][2] = { + static const int sz_table[4][2] = { { sizeof (struct cpl_act_open_req), sizeof (struct cpl_act_open_req6) @@ -218,10 +224,14 @@ act_open_cpl_size(struct adapter *sc, int isipv6) sizeof (struct cpl_t6_act_open_req), sizeof (struct cpl_t6_act_open_req6) }, + { + sizeof (struct cpl_t7_act_open_req), + sizeof (struct cpl_t7_act_open_req6) + }, }; MPASS(chip_id(sc) >= CHELSIO_T4); - idx = min(chip_id(sc) - CHELSIO_T4, 2); + idx = min(chip_id(sc) - CHELSIO_T4, 3); return (sz_table[idx][!!isipv6]); } @@ -255,6 +265,7 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh, struct offload_settings settings; struct epoch_tracker et; uint16_t vid = 0xfff, pcp = 0; + uint64_t ntuple; INP_WLOCK_ASSERT(inp); KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6, @@ -308,10 +319,12 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh, qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) | V_TID_COOKIE(CPL_COOKIE_TOM); + ntuple = select_ntuple(vi, toep->l2te); if (isipv6) { struct cpl_act_open_req6 *cpl = wrtod(wr); struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl; struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl; + struct cpl_t7_act_open_req6 *cpl7 = (void *)cpl; if ((inp->inp_vflag & INP_IPV6) == 0) DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP); @@ -323,18 +336,23 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh, switch (chip_id(sc)) { case CHELSIO_T4: INIT_TP_WR(cpl, 0); - cpl->params = select_ntuple(vi, toep->l2te); + cpl->params = htobe32((uint32_t)ntuple); break; case CHELSIO_T5: INIT_TP_WR(cpl5, 0); cpl5->iss = htobe32(tp->iss); - cpl5->params = select_ntuple(vi, toep->l2te); + cpl5->params = htobe64(V_FILTER_TUPLE(ntuple)); break; case CHELSIO_T6: - default: INIT_TP_WR(cpl6, 0); cpl6->iss = htobe32(tp->iss); - cpl6->params = select_ntuple(vi, toep->l2te); + cpl6->params = htobe64(V_FILTER_TUPLE(ntuple)); + break; + case CHELSIO_T7: + default: + INIT_TP_WR(cpl7, 0); + cpl7->iss = htobe32(tp->iss); + cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple)); break; } OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, @@ -356,23 +374,28 @@ t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh, struct cpl_act_open_req *cpl = wrtod(wr); struct cpl_t5_act_open_req *cpl5 = (void *)cpl; struct cpl_t6_act_open_req *cpl6 = (void *)cpl; + struct cpl_t7_act_open_req *cpl7 = (void *)cpl; switch (chip_id(sc)) { case CHELSIO_T4: INIT_TP_WR(cpl, 0); - cpl->params = select_ntuple(vi, toep->l2te); + cpl->params = htobe32((uint32_t)ntuple); break; case CHELSIO_T5: INIT_TP_WR(cpl5, 0); cpl5->iss = htobe32(tp->iss); - cpl5->params = select_ntuple(vi, toep->l2te); + cpl5->params = htobe64(V_FILTER_TUPLE(ntuple)); break; case CHELSIO_T6: - default: INIT_TP_WR(cpl6, 0); cpl6->iss = htobe32(tp->iss); - cpl6->params = select_ntuple(vi, toep->l2te); + cpl6->params = htobe64(V_FILTER_TUPLE(ntuple)); break; + case CHELSIO_T7: + default: + INIT_TP_WR(cpl7, 0); + cpl7->iss = htobe32(tp->iss); + cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple)); } OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 7a6b1cbdd736..84e31efa8b58 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -127,8 +127,9 @@ send_flowc_wr(struct toepcb *toep, struct tcpcb *tp) paramidx = 0; FLOWC_PARAM(PFNVFN, pfvf); - FLOWC_PARAM(CH, pi->tx_chan); - FLOWC_PARAM(PORT, pi->tx_chan); + /* Firmware expects hw port and will translate to channel itself. */ + FLOWC_PARAM(CH, pi->hw_port); + FLOWC_PARAM(PORT, pi->hw_port); FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); FLOWC_PARAM(SNDBUF, toep->params.sndbuf); if (tp) { @@ -148,6 +149,8 @@ send_flowc_wr(struct toepcb *toep, struct tcpcb *tp) KASSERT(paramidx == nparams, ("nparams mismatch")); + KASSERT(howmany(flowclen, 16) <= MAX_OFLD_TX_SDESC_CREDITS, + ("%s: tx_credits %u too large", __func__, howmany(flowclen, 16))); txsd->tx_credits = howmany(flowclen, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, @@ -215,6 +218,8 @@ update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) else flowc->mnemval[0].val = htobe32(tc_idx); + KASSERT(flowclen16 <= MAX_OFLD_TX_SDESC_CREDITS, + ("%s: tx_credits %u too large", __func__, flowclen16)); txsd->tx_credits = flowclen16; txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; @@ -491,6 +496,9 @@ t4_close_conn(struct adapter *sc, struct toepcb *toep) #define MIN_TX_CREDITS(iso) \ (MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0)) +_Static_assert(MAX_OFLD_TX_CREDITS <= MAX_OFLD_TX_SDESC_CREDITS, + "MAX_OFLD_TX_SDESC_CREDITS too small"); + /* Maximum amount of immediate data we could stuff in a WR */ static inline int max_imm_payload(int tx_credits, int iso) @@ -612,6 +620,48 @@ write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) __func__, nsegs, start, stop)); } +bool +t4_push_raw_wr(struct adapter *sc, struct toepcb *toep, struct mbuf *m) +{ +#ifdef INVARIANTS + struct inpcb *inp = toep->inp; +#endif + struct wrqe *wr; + struct ofld_tx_sdesc *txsd; + u_int credits, plen; + + INP_WLOCK_ASSERT(inp); + MPASS(mbuf_raw_wr(m)); + plen = m->m_pkthdr.len; + credits = howmany(plen, 16); + if (credits > toep->tx_credits) + return (false); + + wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq); + if (wr == NULL) + return (false); + + m_copydata(m, 0, plen, wrtod(wr)); + m_freem(m); + + toep->tx_credits -= credits; + if (toep->tx_credits < MIN_OFLD_TX_CREDITS) + toep->flags |= TPF_TX_SUSPENDED; + + KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); + KASSERT(credits <= MAX_OFLD_TX_SDESC_CREDITS, + ("%s: tx_credits %u too large", __func__, credits)); + txsd = &toep->txsd[toep->txsd_pidx]; + txsd->plen = 0; + txsd->tx_credits = credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + + t4_wrq_tx(sc, wr); + return (true); +} + /* * Max number of SGL entries an offload tx work request can have. This is 41 * (1 + 40) for a full 512B work request. @@ -644,6 +694,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) struct tcpcb *tp = intotcpcb(inp); struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; + struct mbufq *pduq = &toep->ulp_pduq; int tx_credits, shove, compl, sowwakeup; struct ofld_tx_sdesc *txsd; bool nomap_mbuf_seen; @@ -688,6 +739,19 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) max_imm = max_imm_payload(tx_credits, 0); max_nsegs = max_dsgl_nsegs(tx_credits, 0); + if (__predict_false((sndptr = mbufq_first(pduq)) != NULL)) { + if (!t4_push_raw_wr(sc, toep, sndptr)) { + toep->flags |= TPF_TX_SUSPENDED; + return; + } + + m = mbufq_dequeue(pduq); + MPASS(m == sndptr); + + txsd = &toep->txsd[toep->txsd_pidx]; + continue; + } + SOCKBUF_LOCK(sb); sowwakeup = drop; if (drop) { @@ -705,6 +769,8 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) if ((m->m_flags & M_NOTREADY) != 0) break; + if (plen + m->m_len > MAX_OFLD_TX_SDESC_PLEN) + break; if (m->m_flags & M_EXTPG) { #ifdef KERN_TLS if (m->m_epg_tls != NULL) { @@ -870,6 +936,8 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); + KASSERT(plen <= MAX_OFLD_TX_SDESC_PLEN, + ("%s: plen %u too large", __func__, plen)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; @@ -1211,6 +1279,8 @@ t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); + KASSERT(plen <= MAX_OFLD_TX_SDESC_PLEN, + ("%s: plen %u too large", __func__, plen)); txsd->plen = plen; txsd->tx_credits = credits; txsd++; @@ -1240,6 +1310,35 @@ t4_push_data(struct adapter *sc, struct toepcb *toep, int drop) t4_push_frames(sc, toep, drop); } +void +t4_raw_wr_tx(struct adapter *sc, struct toepcb *toep, struct mbuf *m) +{ +#ifdef INVARIANTS + struct inpcb *inp = toep->inp; +#endif + + INP_WLOCK_ASSERT(inp); + + /* + * If there are other raw WRs enqueued, enqueue to preserve + * FIFO ordering. + */ + if (!mbufq_empty(&toep->ulp_pduq)) { + mbufq_enqueue(&toep->ulp_pduq, m); + return; + } + + /* + * Cannot call t4_push_data here as that will lock so_snd and + * some callers of this run in rx handlers with so_rcv locked. + * Instead, just try to transmit this WR. + */ + if (!t4_push_raw_wr(sc, toep, m)) { + mbufq_enqueue(&toep->ulp_pduq, m); + toep->flags |= TPF_TX_SUSPENDED; + } +} + int t4_tod_output(struct toedev *tod, struct tcpcb *tp) { @@ -1941,35 +2040,55 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) } void -t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, +write_set_tcb_field(struct adapter *sc, void *dst, struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) { - struct wrqe *wr; - struct cpl_set_tcb_field *req; - struct ofld_tx_sdesc *txsd; + struct cpl_set_tcb_field *req = dst; MPASS((cookie & ~M_COOKIE) == 0); if (reply) { MPASS(cookie != CPL_COOKIE_RESERVED); } - wr = alloc_wrqe(sizeof(*req), wrq); + INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); + if (reply == 0) { + req->reply_ctrl = htobe16(F_NO_REPLY); + } else { + const int qid = toep->ofld_rxq->iq.abs_id; + if (chip_id(sc) >= CHELSIO_T7) { + req->reply_ctrl = htobe16(V_T7_QUEUENO(qid) | + V_T7_REPLY_CHAN(0) | V_NO_REPLY(0)); + } else { + req->reply_ctrl = htobe16(V_QUEUENO(qid) | + V_REPLY_CHAN(0) | V_NO_REPLY(0)); + } + } + req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); + req->mask = htobe64(mask); + req->val = htobe64(val); +} + +void +t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, + uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) +{ + struct wrqe *wr; + struct ofld_tx_sdesc *txsd; + const u_int len = sizeof(struct cpl_set_tcb_field); + + wr = alloc_wrqe(len, wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } - req = wrtod(wr); + write_set_tcb_field(sc, wrtod(wr), toep, word, mask, val, reply, + cookie); - INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); - req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id)); - if (reply == 0) - req->reply_ctrl |= htobe16(F_NO_REPLY); - req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); - req->mask = htobe64(mask); - req->val = htobe64(val); if (wrq->eq.type == EQ_OFLD) { txsd = &toep->txsd[toep->txsd_pidx]; - txsd->tx_credits = howmany(sizeof(*req), 16); + _Static_assert(howmany(len, 16) <= MAX_OFLD_TX_SDESC_CREDITS, + "MAX_OFLD_TX_SDESC_CREDITS too small"); + txsd->tx_credits = howmany(len, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index 2fee8fa91dac..35fb1061d867 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -1655,7 +1655,10 @@ t4_write_page_pods_for_ps(struct adapter *sc, struct sge_wrq *wrq, int tid, INIT_ULPTX_WR(ulpmc, len, 0, 0); ulpmc->cmd = cmd; - ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); + if (chip_id(sc) >= CHELSIO_T7) + ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5)); + else + ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); @@ -1785,7 +1788,7 @@ t4_write_page_pods_for_rcvbuf(struct adapter *sc, struct sge_wrq *wrq, int tid, return (0); } -static struct mbuf * +struct mbuf * alloc_raw_wr_mbuf(int len) { struct mbuf *m; @@ -1842,7 +1845,10 @@ t4_write_page_pods_for_bio(struct adapter *sc, struct toepcb *toep, ulpmc = mtod(m, struct ulp_mem_io *); INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; - ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); + if (chip_id(sc) >= CHELSIO_T7) + ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5)); + else + ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); @@ -1922,7 +1928,10 @@ t4_write_page_pods_for_buf(struct adapter *sc, struct toepcb *toep, INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; - ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); + if (chip_id(sc) >= CHELSIO_T7) + ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5)); + else + ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); @@ -2013,7 +2022,10 @@ t4_write_page_pods_for_sgl(struct adapter *sc, struct toepcb *toep, INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; - ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); + if (chip_id(sc) >= CHELSIO_T7) + ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5)); + else + ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c index 06c495dcafc3..b879f6883f25 100644 --- a/sys/dev/cxgbe/tom/t4_listen.c +++ b/sys/dev/cxgbe/tom/t4_listen.c @@ -508,10 +508,11 @@ send_flowc_wr_synqe(struct adapter *sc, struct synq_entry *synqe) V_FW_WR_FLOWID(synqe->tid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; flowc->mnemval[0].val = htobe32(pfvf); + /* Firmware expects hw port and will translate to channel itself. */ flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; - flowc->mnemval[1].val = htobe32(pi->tx_chan); + flowc->mnemval[1].val = htobe32(pi->hw_port); flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; - flowc->mnemval[2].val = htobe32(pi->tx_chan); + flowc->mnemval[2].val = htobe32(pi->hw_port); flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; flowc->mnemval[3].val = htobe32(ofld_rxq->iq.abs_id); flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF; @@ -1507,6 +1508,8 @@ found: init_conn_params(vi, &settings, &inc, so, &cpl->tcpopt, e->idx, &synqe->params); + if (sc->params.tid_qid_sel_mask != 0) + update_tid_qid_sel(vi, &synqe->params, tid); /* * If all goes well t4_syncache_respond will get called during diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c index 27c16b9988ae..bbcc1c88c3db 100644 --- a/sys/dev/cxgbe/tom/t4_tls.c +++ b/sys/dev/cxgbe/tom/t4_tls.c @@ -61,11 +61,21 @@ static void t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, - uint64_t val) + uint64_t val, int reply, int cookie) { struct adapter *sc = td_adapter(toep->td); + struct mbuf *m; + + m = alloc_raw_wr_mbuf(sizeof(struct cpl_set_tcb_field)); + if (m == NULL) { + /* XXX */ + panic("%s: out of memory", __func__); + } - t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0); + write_set_tcb_field(sc, mtod(m, void *), toep, word, mask, val, reply, + cookie); + + t4_raw_wr_tx(sc, toep, m); } /* TLS and DTLS common routines */ @@ -88,10 +98,9 @@ tls_tx_key(struct toepcb *toep) static void t4_set_rx_quiesce(struct toepcb *toep) { - struct adapter *sc = td_adapter(toep->td); - t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, W_TCB_T_FLAGS, - V_TF_RX_QUIESCE(1), V_TF_RX_QUIESCE(1), 1, CPL_COOKIE_TOM); + t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), + V_TF_RX_QUIESCE(1), 1, CPL_COOKIE_TOM); } /* Clear TF_RX_QUIESCE to re-enable receive. */ @@ -99,7 +108,7 @@ static void t4_clear_rx_quiesce(struct toepcb *toep) { - t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); + t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0, 0, 0); } /* TLS/DTLS content type for CPL SFO */ @@ -145,16 +154,15 @@ get_tp_plen_max(struct ktls_session *tls) return (tls->params.max_frame_len <= 8192 ? plen : FC_TP_PLEN_MAX); } -/* Send request to get the key-id */ +/* Send request to save the key in on-card memory. */ static int tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, int direction) { struct tls_ofld_info *tls_ofld = &toep->tls; struct adapter *sc = td_adapter(toep->td); - struct ofld_tx_sdesc *txsd; int keyid; - struct wrqe *wr; + struct mbuf *m; struct tls_key_req *kwr; struct tls_keyctx *kctx; @@ -173,12 +181,12 @@ tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, return (ENOSPC); } - wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq); - if (wr == NULL) { + m = alloc_raw_wr_mbuf(TLS_KEY_WR_SZ); + if (m == NULL) { t4_free_tls_keyid(sc, keyid); return (ENOMEM); } - kwr = wrtod(wr); + kwr = mtod(m, struct tls_key_req *); memset(kwr, 0, TLS_KEY_WR_SZ); t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid, @@ -190,15 +198,7 @@ tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, tls_ofld->rx_key_addr = keyid; t4_tls_key_ctx(tls, direction, kctx); - txsd = &toep->txsd[toep->txsd_pidx]; - txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16); - txsd->plen = 0; - toep->tx_credits -= txsd->tx_credits; - if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) - toep->txsd_pidx = 0; - toep->txsd_avail--; - - t4_wrq_tx(sc, wr); + t4_raw_wr_tx(sc, toep, m); return (0); } @@ -207,7 +207,7 @@ int tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) { struct adapter *sc = td_adapter(toep->td); - int error, explicit_iv_size, mac_first; + int error, iv_size, mac_first; if (!can_tls_offload(sc)) return (EINVAL); @@ -228,6 +228,21 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) } } + /* TLS 1.1 through TLS 1.3 are currently supported. */ + if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || + tls->params.tls_vminor < TLS_MINOR_VER_ONE || + tls->params.tls_vminor > TLS_MINOR_VER_THREE) { + return (EPROTONOSUPPORT); + } + + /* TLS 1.3 is only supported on T7+. */ + if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) { + if (is_t6(sc)) { + return (EPROTONOSUPPORT); + } + } + + /* Sanity check values in *tls. */ switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: /* XXX: Explicitly ignore any provided IV. */ @@ -247,13 +262,10 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) default: return (EPROTONOSUPPORT); } - explicit_iv_size = AES_BLOCK_LEN; + iv_size = AES_BLOCK_LEN; mac_first = 1; break; case CRYPTO_AES_NIST_GCM_16: - if (tls->params.iv_len != SALT_SIZE) { - return (EINVAL); - } switch (tls->params.cipher_key_len) { case 128 / 8: case 192 / 8: @@ -262,20 +274,19 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) default: return (EINVAL); } - explicit_iv_size = 8; + + /* + * The IV size for TLS 1.2 is the explicit IV in the + * record header. For TLS 1.3 it is the size of the + * sequence number. + */ + iv_size = 8; mac_first = 0; break; default: return (EPROTONOSUPPORT); } - /* Only TLS 1.1 and TLS 1.2 are currently supported. */ - if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || - tls->params.tls_vminor < TLS_MINOR_VER_ONE || - tls->params.tls_vminor > TLS_MINOR_VER_TWO) { - return (EPROTONOSUPPORT); - } - /* Bail if we already have a key. */ if (direction == KTLS_TX) { if (toep->tls.tx_key_addr != -1) @@ -289,6 +300,7 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) if (error) return (error); + toep->tls.tls13 = tls->params.tls_vminor == TLS_MINOR_VER_THREE; if (direction == KTLS_TX) { toep->tls.scmd0.seqno_numivs = (V_SCMD_SEQ_NO_CTRL(3) | @@ -298,14 +310,14 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) | V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | - V_SCMD_IV_SIZE(explicit_iv_size / 2)); + V_SCMD_IV_SIZE(iv_size / 2)); toep->tls.scmd0.ivgen_hdrlen = (V_SCMD_IV_GEN_CTRL(1) | V_SCMD_KEY_CTX_INLINE(0) | V_SCMD_TLS_FRAG_ENABLE(1)); - toep->tls.iv_len = explicit_iv_size; + toep->tls.iv_len = iv_size; toep->tls.frag_size = tls->params.max_frame_len; toep->tls.fcplenmax = get_tp_plen_max(tls); toep->tls.expn_per_ulp = tls->params.tls_hlen + @@ -352,7 +364,8 @@ tls_uninit_toep(struct toepcb *toep) static void write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, - unsigned int plen, unsigned int expn, uint8_t credits, int shove) + unsigned int plen, unsigned int expn, uint8_t credits, int shove, + int num_ivs) { struct tls_ofld_info *tls_ofld = &toep->tls; unsigned int len = plen + expn; @@ -365,7 +378,7 @@ write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, txwr->plen = htobe32(len); txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); - txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(1) | + txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(num_ivs) | V_FW_TLSTX_DATA_WR_EXP(expn) | V_FW_TLSTX_DATA_WR_CTXLOC(TLS_SFO_WR_CONTEXTLOC_DDR) | V_FW_TLSTX_DATA_WR_IVDSGL(0) | @@ -381,20 +394,20 @@ write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, static void write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, - struct tls_hdr *tls_hdr, unsigned int plen, uint64_t seqno) + struct tls_hdr *tls_hdr, unsigned int plen, uint8_t rec_type, + uint64_t seqno) { struct tls_ofld_info *tls_ofld = &toep->tls; int data_type, seglen; seglen = plen; - data_type = tls_content_type(tls_hdr->type); + data_type = tls_content_type(rec_type); cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); cpl->pld_len = htobe32(plen); if (data_type == CPL_TX_TLS_SFO_TYPE_CUSTOM) - cpl->type_protover = htobe32( - V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); + cpl->type_protover = htobe32(V_CPL_TX_TLS_SFO_TYPE(rec_type)); cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | V_SCMD_NUM_IVS(1)); cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); @@ -494,9 +507,11 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) struct tcpcb *tp = intotcpcb(inp); struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; + struct mbufq *pduq = &toep->ulp_pduq; int tls_size, tx_credits, shove, sowwakeup; struct ofld_tx_sdesc *txsd; char *buf; + bool tls13; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, @@ -532,10 +547,23 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) return; } + tls13 = toep->tls.tls13; txsd = &toep->txsd[toep->txsd_pidx]; for (;;) { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); + if (__predict_false((m = mbufq_first(pduq)) != NULL)) { + if (!t4_push_raw_wr(sc, toep, m)) { + toep->flags |= TPF_TX_SUSPENDED; + return; + } + + (void)mbufq_dequeue(pduq); + + txsd = &toep->txsd[toep->txsd_pidx]; + continue; + } + SOCKBUF_LOCK(sb); sowwakeup = drop; if (drop) { @@ -586,9 +614,11 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + sizeof(struct ulptx_sc_memrd); - /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ - MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); - wr_len += AES_BLOCK_LEN; + if (!tls13) { + /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ + MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); + wr_len += AES_BLOCK_LEN; + } /* Account for SGL in work request length. */ nsegs = count_ext_pgs_segs(m); @@ -658,8 +688,10 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) expn_size = m->m_epg_hdrlen + m->m_epg_trllen; tls_size = m->m_len - expn_size; - write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove); - write_tlstx_cpl(cpl, toep, thdr, tls_size, m->m_epg_seqno); + write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove, + tls13 ? 0 : 1); + write_tlstx_cpl(cpl, toep, thdr, tls_size, + tls13 ? m->m_epg_record_type : thdr->type, m->m_epg_seqno); idata = (struct ulptx_idata *)(cpl + 1); idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); @@ -670,10 +702,12 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4)); memrd->addr = htobe32(toep->tls.tx_key_addr >> 5); - /* Copy IV. */ buf = (char *)(memrd + 1); - memcpy(buf, thdr + 1, toep->tls.iv_len); - buf += AES_BLOCK_LEN; + if (!tls13) { + /* Copy IV. */ + memcpy(buf, thdr + 1, toep->tls.iv_len); + buf += AES_BLOCK_LEN; + } write_ktlstx_sgl(buf, m, nsegs); @@ -694,6 +728,8 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); + KASSERT(m->m_len <= MAX_OFLD_TX_SDESC_PLEN, + ("%s: plen %u too large", __func__, m->m_len)); txsd->plen = m->m_len; txsd->tx_credits = credits; txsd++; @@ -793,8 +829,8 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) struct sockbuf *sb; struct mbuf *tls_data; struct tls_get_record *tgr; - struct mbuf *control; - int pdu_length, trailer_len; + struct mbuf *control, *n; + int pdu_length, resid, trailer_len; #if defined(KTR) || defined(INVARIANTS) int len; #endif @@ -842,7 +878,9 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) /* * The payload of this CPL is the TLS header followed by - * additional fields. + * additional fields. For TLS 1.3 the type field holds the + * inner record type and the length field has been updated to + * strip the inner record type, padding, and MAC. */ KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), ("%s: payload too small", __func__)); @@ -854,7 +892,14 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) ("%s: sequence mismatch", __func__)); } - /* Report decryption errors as EBADMSG. */ + /* + * Report decryption errors as EBADMSG. + * + * XXX: To support rekeying for TLS 1.3 this will eventually + * have to be updated to recrypt the data with the old key and + * then decrypt with the new key. Punt for now as KTLS + * doesn't yet support rekeying. + */ if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) { CTR4(KTR_CXGBE, "%s: tid %u TLS error %#x ddp_vld %#x", __func__, toep->tid, tls_hdr_pkt->res_to_mac_error, @@ -872,6 +917,33 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) return (0); } + /* For TLS 1.3 trim the header and trailer. */ + if (toep->tls.tls13) { + KASSERT(tls_data != NULL, ("%s: TLS 1.3 record without data", + __func__)); + MPASS(tls_data->m_pkthdr.len == pdu_length); + m_adj(tls_data, sizeof(struct tls_record_layer)); + if (tls_data->m_pkthdr.len > be16toh(tls_hdr_pkt->length)) + tls_data->m_pkthdr.len = be16toh(tls_hdr_pkt->length); + resid = tls_data->m_pkthdr.len; + if (resid == 0) { + m_freem(tls_data); + tls_data = NULL; + } else { + for (n = tls_data;; n = n->m_next) { + if (n->m_len < resid) { + resid -= n->m_len; + continue; + } + + n->m_len = resid; + m_freem(n->m_next); + n->m_next = NULL; + break; + } + } + } + /* Handle data received after the socket is closed. */ sb = &so->so_rcv; SOCKBUF_LOCK(sb); @@ -1076,33 +1148,60 @@ out: } /* - * Send a work request setting multiple TCB fields to enable - * ULP_MODE_TLS. + * Send a work request setting one or more TCB fields to partially or + * fully enable ULP_MODE_TLS. + * + * - If resid == 0, the socket buffer ends at a record boundary + * (either empty or contains one or more complete records). Switch + * to ULP_MODE_TLS (if not already) and enable TLS decryption. + * + * - If resid != 0, the socket buffer contains a partial record. In + * this case, switch to ULP_MODE_TLS partially and configure the TCB + * to pass along the remaining resid bytes undecrypted. Once they + * arrive, this is called again with resid == 0 and enables TLS + * decryption. */ static void -tls_update_tcb(struct adapter *sc, struct toepcb *toep, uint64_t seqno) +tls_update_tcb(struct adapter *sc, struct toepcb *toep, uint64_t seqno, + size_t resid) { - struct wrqe *wr; + struct mbuf *m; struct work_request_hdr *wrh; struct ulp_txpkt *ulpmc; int fields, key_offset, len; - KASSERT(ulp_mode(toep) == ULP_MODE_NONE, - ("%s: tid %d already ULP_MODE_TLS", __func__, toep->tid)); + /* + * If we are already in ULP_MODE_TLS, then we should now be at + * a record boundary and ready to finish enabling TLS RX. + */ + KASSERT(resid == 0 || ulp_mode(toep) == ULP_MODE_NONE, + ("%s: tid %d needs %zu more data but already ULP_MODE_TLS", + __func__, toep->tid, resid)); fields = 0; + if (ulp_mode(toep) == ULP_MODE_NONE) { + /* 2 writes for the overlay region */ + fields += 2; + } - /* 2 writes for the overlay region */ - fields += 2; + if (resid == 0) { + /* W_TCB_TLS_SEQ */ + fields++; - /* W_TCB_TLS_SEQ */ - fields++; + /* W_TCB_ULP_RAW */ + fields++; + } else { + /* W_TCB_PDU_LEN */ + fields++; - /* W_TCB_ULP_RAW */ - fields++; + /* W_TCB_ULP_RAW */ + fields++; + } - /* W_TCB_ULP_TYPE */ - fields ++; + if (ulp_mode(toep) == ULP_MODE_NONE) { + /* W_TCB_ULP_TYPE */ + fields ++; + } /* W_TCB_T_FLAGS */ fields++; @@ -1111,59 +1210,94 @@ tls_update_tcb(struct adapter *sc, struct toepcb *toep, uint64_t seqno) KASSERT(len <= SGE_MAX_WR_LEN, ("%s: WR with %d TCB field updates too large", __func__, fields)); - wr = alloc_wrqe(len, toep->ctrlq); - if (wr == NULL) { + m = alloc_raw_wr_mbuf(len); + if (m == NULL) { /* XXX */ panic("%s: out of memory", __func__); } - wrh = wrtod(wr); - INIT_ULPTX_WRH(wrh, len, 1, 0); /* atomic */ + wrh = mtod(m, struct work_request_hdr *); + INIT_ULPTX_WRH(wrh, len, 1, toep->tid); /* atomic */ ulpmc = (struct ulp_txpkt *)(wrh + 1); - /* - * Clear the TLS overlay region: 1023:832. - * - * Words 26/27 are always set to zero. Words 28/29 - * contain seqno and are set when enabling TLS - * decryption. Word 30 is zero and Word 31 contains - * the keyid. - */ - ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26, - 0xffffffffffffffff, 0); + if (ulp_mode(toep) == ULP_MODE_NONE) { + /* + * Clear the TLS overlay region: 1023:832. + * + * Words 26/27 are always set to zero. Words 28/29 + * contain seqno and are set when enabling TLS + * decryption. Word 30 is zero and Word 31 contains + * the keyid. + */ + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26, + 0xffffffffffffffff, 0); - /* - * RX key tags are an index into the key portion of MA - * memory stored as an offset from the base address in - * units of 64 bytes. - */ - key_offset = toep->tls.rx_key_addr - sc->vres.key.start; - ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30, - 0xffffffffffffffff, - (uint64_t)V_TCB_RX_TLS_KEY_TAG(key_offset / 64) << 32); - - CTR3(KTR_CXGBE, "%s: tid %d enable TLS seqno %lu", __func__, - toep->tid, seqno); - ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_TLS_SEQ, - V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), V_TCB_TLS_SEQ(seqno)); - ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_RAW, - V_TCB_ULP_RAW(M_TCB_ULP_RAW), - V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | V_TF_TLS_CONTROL(1) | - V_TF_TLS_ACTIVE(1) | V_TF_TLS_ENABLE(1)))); - - toep->flags &= ~TPF_TLS_STARTING; - toep->flags |= TPF_TLS_RECEIVE; - - /* Set the ULP mode to ULP_MODE_TLS. */ - toep->params.ulp_mode = ULP_MODE_TLS; - ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_TYPE, - V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_TLS)); + /* + * RX key tags are an index into the key portion of MA + * memory stored as an offset from the base address in + * units of 64 bytes. + */ + key_offset = toep->tls.rx_key_addr - sc->vres.key.start; + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30, + 0xffffffffffffffff, + (uint64_t)V_TCB_RX_TLS_KEY_TAG(key_offset / 64) << 32); + } + + if (resid == 0) { + /* + * The socket buffer is empty or only contains + * complete TLS records: Set the sequence number and + * enable TLS decryption. + */ + CTR3(KTR_CXGBE, "%s: tid %d enable TLS seqno %lu", __func__, + toep->tid, seqno); + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, + W_TCB_RX_TLS_SEQ, V_TCB_RX_TLS_SEQ(M_TCB_RX_TLS_SEQ), + V_TCB_RX_TLS_SEQ(seqno)); + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, + W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | V_TF_TLS_CONTROL(1) | + V_TF_TLS_ACTIVE(1) | V_TF_TLS_ENABLE(1)))); + + toep->flags &= ~TPF_TLS_STARTING; + toep->flags |= TPF_TLS_RECEIVE; + } else { + /* + * The socket buffer ends with a partial record with a + * full header and needs at least 6 bytes. + * + * Set PDU length. This is treating the 'resid' bytes + * as a TLS PDU, so the first 5 bytes are a fake + * header and the rest are the PDU length. + */ + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, + W_TCB_PDU_LEN, V_TCB_PDU_LEN(M_TCB_PDU_LEN), + V_TCB_PDU_LEN(resid - sizeof(struct tls_hdr))); + CTR3(KTR_CXGBE, "%s: tid %d setting PDU_LEN to %zu", + __func__, toep->tid, resid - sizeof(struct tls_hdr)); + + /* Clear all bits in ULP_RAW except for ENABLE. */ + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, + W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), + V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); + + /* Wait for 'resid' bytes to be delivered as CPL_RX_DATA. */ + toep->tls.rx_resid = resid; + } + + if (ulp_mode(toep) == ULP_MODE_NONE) { + /* Set the ULP mode to ULP_MODE_TLS. */ + toep->params.ulp_mode = ULP_MODE_TLS; + ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, + W_TCB_ULP_TYPE, V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), + V_TCB_ULP_TYPE(ULP_MODE_TLS)); + } /* Clear TF_RX_QUIESCE. */ ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); - t4_wrq_tx(sc, wr); + t4_raw_wr_tx(sc, toep, m); } /* @@ -1190,7 +1324,8 @@ tls_check_rx_sockbuf(struct adapter *sc, struct toepcb *toep, * size of a TLS record, re-enable receive and pause again once * we get more data to try again. */ - if (!have_header || resid != 0) { + if (!have_header || (resid != 0 && (resid < sizeof(struct tls_hdr) || + is_t6(sc)))) { CTR(KTR_CXGBE, "%s: tid %d waiting for more data", __func__, toep->tid); toep->flags &= ~TPF_TLS_RX_QUIESCED; @@ -1198,7 +1333,7 @@ tls_check_rx_sockbuf(struct adapter *sc, struct toepcb *toep, return; } - tls_update_tcb(sc, toep, seqno); + tls_update_tcb(sc, toep, seqno, resid); } void diff --git a/sys/dev/cxgbe/tom/t4_tls.h b/sys/dev/cxgbe/tom/t4_tls.h index 753a30890fdc..6faf946e9e3c 100644 --- a/sys/dev/cxgbe/tom/t4_tls.h +++ b/sys/dev/cxgbe/tom/t4_tls.h @@ -74,6 +74,7 @@ struct tls_ofld_info { unsigned short adjusted_plen; unsigned short expn_per_ulp; unsigned short pdus_per_ulp; + bool tls13; struct tls_scmd scmd0; u_int iv_len; unsigned int tx_key_info_size; diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c index 9b09facd05a7..53a945f8b4cc 100644 --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -182,7 +182,7 @@ init_toepcb(struct vi_info *vi, struct toepcb *toep) } toep->ofld_txq = &sc->sge.ofld_txq[cp->txq_idx]; toep->ofld_rxq = &sc->sge.ofld_rxq[cp->rxq_idx]; - toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; + toep->ctrlq = &sc->sge.ctrlq[cp->ctrlq_idx]; tls_init_toep(toep); MPASS(ulp_mode(toep) != ULP_MODE_TCPDDP); @@ -494,8 +494,15 @@ send_get_tcb(struct adapter *sc, u_int tid) bzero(cpl, sizeof(*cpl)); INIT_TP_WR(cpl, tid); OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_GET_TCB, tid)); - cpl->reply_ctrl = htobe16(V_REPLY_CHAN(0) | - V_QUEUENO(sc->sge.ofld_rxq[0].iq.cntxt_id)); + if (chip_id(sc) >= CHELSIO_T7) { + cpl->reply_ctrl = + htobe16(V_T7_QUEUENO(sc->sge.ofld_rxq[0].iq.cntxt_id) | + V_T7_REPLY_CHAN(0) | V_NO_REPLY(0)); + } else { + cpl->reply_ctrl = + htobe16(V_QUEUENO(sc->sge.ofld_rxq[0].iq.cntxt_id) | + V_REPLY_CHAN(0) | V_NO_REPLY(0)); + } cpl->cookie = 0xff; commit_wrq_wr(&sc->sge.ctrlq[0], cpl, &cookie); @@ -882,6 +889,8 @@ send_mss_flowc_wr(struct adapter *sc, struct toepcb *toep) flowc->mnemval[0].val = htobe32(toep->params.emss); txsd = &toep->txsd[toep->txsd_pidx]; + _Static_assert(flowclen16 <= MAX_OFLD_TX_SDESC_CREDITS, + "MAX_OFLD_TX_SDESC_CREDITS too small"); txsd->tx_credits = flowclen16; txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; @@ -1219,7 +1228,7 @@ select_ntuple(struct vi_info *vi, struct l2t_entry *e) ntuple |= (uint64_t)(F_FT_VLAN_VLD | e->vlan) << tp->vlan_shift; if (tp->port_shift >= 0) - ntuple |= (uint64_t)e->lport << tp->port_shift; + ntuple |= (uint64_t)e->hw_port << tp->port_shift; if (tp->protocol_shift >= 0) ntuple |= (uint64_t)IPPROTO_TCP << tp->protocol_shift; @@ -1230,10 +1239,7 @@ select_ntuple(struct vi_info *vi, struct l2t_entry *e) tp->vnic_shift; } - if (is_t4(sc)) - return (htobe32((uint32_t)ntuple)); - else - return (htobe64(V_FILTER_TUPLE(ntuple))); + return (ntuple); } /* @@ -1324,6 +1330,9 @@ init_conn_params(struct vi_info *vi , struct offload_settings *s, */ cp->mtu_idx = find_best_mtu_idx(sc, inc, s); + /* Control queue. */ + cp->ctrlq_idx = vi->pi->port_id; + /* Tx queue for this connection. */ if (s->txq == QUEUE_RANDOM) q_idx = arc4random(); @@ -1436,6 +1445,32 @@ init_conn_params(struct vi_info *vi , struct offload_settings *s, cp->emss = 0; } +void +update_tid_qid_sel(struct vi_info *vi, struct conn_params *cp, int tid) +{ + struct adapter *sc = vi->adapter; + const int mask = sc->params.tid_qid_sel_mask; + struct sge_ofld_txq *ofld_txq = &sc->sge.ofld_txq[cp->txq_idx]; + uint32_t ngroup; + int g, nqpg; + + cp->ctrlq_idx = ofld_txq_group(tid, mask); + CTR(KTR_CXGBE, "tid %u is on core %u", tid, cp->ctrlq_idx); + if ((ofld_txq->wrq.eq.cntxt_id & mask) == (tid & mask)) + return; + + ngroup = 1 << bitcount32(mask); + MPASS(vi->nofldtxq % ngroup == 0); + g = ofld_txq_group(tid, mask); + nqpg = vi->nofldtxq / ngroup; + cp->txq_idx = vi->first_ofld_txq + g * nqpg + arc4random() % nqpg; +#ifdef INVARIANTS + MPASS(cp->txq_idx < vi->first_ofld_txq + vi->nofldtxq); + ofld_txq = &sc->sge.ofld_txq[cp->txq_idx]; + MPASS((ofld_txq->wrq.eq.cntxt_id & mask) == (tid & mask)); +#endif +} + int negative_advice(int status) { @@ -2231,6 +2266,98 @@ t4_aio_queue_tom(struct socket *so, struct kaiocb *job) return (0); } +/* + * Request/response structure used to find out the adapter offloading + * a socket. + */ +struct find_offload_adapter_data { + struct socket *so; + struct adapter *sc; /* result */ +}; + +static void +find_offload_adapter_cb(struct adapter *sc, void *arg) +{ + struct find_offload_adapter_data *fa = arg; + struct socket *so = fa->so; + struct tom_data *td = sc->tom_softc; + struct tcpcb *tp; + struct inpcb *inp; + + /* Non-TCP were filtered out earlier. */ + MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); + + if (fa->sc != NULL) + return; /* Found already. */ + + if (td == NULL) + return; /* TOE not enabled on this adapter. */ + + inp = sotoinpcb(so); + INP_WLOCK(inp); + if ((inp->inp_flags & INP_DROPPED) == 0) { + tp = intotcpcb(inp); + if (tp->t_flags & TF_TOE && tp->tod == &td->tod) + fa->sc = sc; /* Found. */ + } + INP_WUNLOCK(inp); +} + +struct adapter * +find_offload_adapter(struct socket *so) +{ + struct find_offload_adapter_data fa; + + fa.sc = NULL; + fa.so = so; + t4_iterate(find_offload_adapter_cb, &fa); + return (fa.sc); +} + +void +send_txdataplen_max_flowc_wr(struct adapter *sc, struct toepcb *toep, + int maxlen) +{ + struct wrqe *wr; + struct fw_flowc_wr *flowc; + const u_int nparams = 1; + u_int flowclen; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + + CTR(KTR_CXGBE, "%s: tid %u maxlen=%d", __func__, toep->tid, maxlen); + + flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); + + wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq); + if (wr == NULL) { + /* XXX */ + panic("%s: allocation failure.", __func__); + } + flowc = wrtod(wr); + memset(flowc, 0, wr->wr_len); + + flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | + V_FW_FLOWC_WR_NPARAMS(nparams)); + flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | + V_FW_WR_FLOWID(toep->tid)); + + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; + flowc->mnemval[0].val = htobe32(maxlen); + + KASSERT(howmany(flowclen, 16) <= MAX_OFLD_TX_SDESC_CREDITS, + ("%s: tx_credits %u too large", __func__, howmany(flowclen, 16))); + txsd->tx_credits = howmany(flowclen, 16); + txsd->plen = 0; + KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, + ("%s: not enough credits (%d)", __func__, toep->tx_credits)); + toep->tx_credits -= txsd->tx_credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + + t4_wrq_tx(sc, wr); +} + static int t4_tom_mod_load(void) { diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index 6295a3484b9f..c8c2d432b8f1 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -113,6 +113,7 @@ struct conn_params { int8_t mtu_idx; int8_t ulp_mode; int8_t tx_align; + int8_t ctrlq_idx; /* ctrlq = &sc->sge.ctrlq[ctrlq_idx] */ int16_t txq_idx; /* ofld_txq = &sc->sge.ofld_txq[txq_idx] */ int16_t rxq_idx; /* ofld_rxq = &sc->sge.ofld_rxq[rxq_idx] */ int16_t l2t_idx; @@ -122,10 +123,13 @@ struct conn_params { }; struct ofld_tx_sdesc { - uint32_t plen; /* payload length */ - uint8_t tx_credits; /* firmware tx credits (unit is 16B) */ + uint32_t plen : 26; /* payload length */ + uint32_t tx_credits : 6; /* firmware tx credits (unit is 16B) */ }; +#define MAX_OFLD_TX_SDESC_PLEN ((1u << 26) - 1) +#define MAX_OFLD_TX_SDESC_CREDITS ((1u << 6) - 1) + struct ppod_region { u_int pr_start; u_int pr_len; @@ -474,11 +478,14 @@ int select_rcv_wscale(void); void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp); +void update_tid_qid_sel(struct vi_info *, struct conn_params *, int); __be64 calc_options0(struct vi_info *, struct conn_params *); __be32 calc_options2(struct vi_info *, struct conn_params *); uint64_t select_ntuple(struct vi_info *, struct l2t_entry *); int negative_advice(int); int add_tid_to_history(struct adapter *, u_int); +struct adapter *find_offload_adapter(struct socket *); +void send_txdataplen_max_flowc_wr(struct adapter *, struct toepcb *, int); void t4_pcb_detach(struct toedev *, struct tcpcb *); /* t4_connect.c */ @@ -526,6 +533,10 @@ int t4_send_rst(struct toedev *, struct tcpcb *); void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *, uint16_t, uint64_t, uint64_t, int, int); void t4_push_pdus(struct adapter *, struct toepcb *, int); +bool t4_push_raw_wr(struct adapter *, struct toepcb *, struct mbuf *); +void t4_raw_wr_tx(struct adapter *, struct toepcb *, struct mbuf *); +void write_set_tcb_field(struct adapter *, void *, struct toepcb *, uint16_t, + uint64_t, uint64_t, int, int); /* t4_ddp.c */ int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, @@ -551,6 +562,7 @@ int t4_aio_queue_ddp(struct socket *, struct kaiocb *); int t4_enable_ddp_rcv(struct socket *, struct toepcb *); void t4_ddp_mod_load(void); void t4_ddp_mod_unload(void); +struct mbuf *alloc_raw_wr_mbuf(int); void ddp_assert_empty(struct toepcb *); void ddp_uninit_toep(struct toepcb *); void ddp_queue_toep(struct toepcb *); @@ -574,4 +586,10 @@ int tls_tx_key(struct toepcb *); void tls_uninit_toep(struct toepcb *); int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int); +/* t4_tpt.c */ +uint32_t t4_pblpool_alloc(struct adapter *, int); +void t4_pblpool_free(struct adapter *, uint32_t, int); +int t4_pblpool_create(struct adapter *); +void t4_pblpool_destroy(struct adapter *); + #endif diff --git a/sys/dev/cxgbe/tom/t4_tom_l2t.c b/sys/dev/cxgbe/tom/t4_tom_l2t.c index 3fd0d5ca41d4..e245c2b6fd5b 100644 --- a/sys/dev/cxgbe/tom/t4_tom_l2t.c +++ b/sys/dev/cxgbe/tom/t4_tom_l2t.c @@ -403,7 +403,7 @@ t4_l2t_get(struct port_info *pi, if_t ifp, struct sockaddr *sa) l2_store(sa, e); e->ifp = ifp; e->hash = hash; - e->lport = pi->lport; + e->hw_port = pi->hw_port; e->wrq = &sc->sge.ctrlq[pi->port_id]; e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id; atomic_store_rel_int(&e->refcnt, 1); |