aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/cxgbe
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/cxgbe')
-rw-r--r--sys/dev/cxgbe/common/t4_hw.c19
-rw-r--r--sys/dev/cxgbe/crypto/t6_kern_tls.c6
-rw-r--r--sys/dev/cxgbe/crypto/t7_kern_tls.c2
-rw-r--r--sys/dev/cxgbe/cxgbei/cxgbei.c17
-rw-r--r--sys/dev/cxgbe/cxgbei/icl_cxgbei.c19
-rw-r--r--sys/dev/cxgbe/iw_cxgbe/qp.c24
-rw-r--r--sys/dev/cxgbe/nvmf/nvmf_che.c24
-rw-r--r--sys/dev/cxgbe/t4_main.c65
-rw-r--r--sys/dev/cxgbe/t4_netmap.c4
-rw-r--r--sys/dev/cxgbe/t4_sge.c27
-rw-r--r--sys/dev/cxgbe/tom/t4_connect.c3
-rw-r--r--sys/dev/cxgbe/tom/t4_cpl_io.c156
-rw-r--r--sys/dev/cxgbe/tom/t4_ddp.c18
-rw-r--r--sys/dev/cxgbe/tom/t4_listen.c14
-rw-r--r--sys/dev/cxgbe/tom/t4_tls.c18
-rw-r--r--sys/dev/cxgbe/tom/t4_tom.c10
16 files changed, 199 insertions, 227 deletions
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index 494f83a47135..f4eef54e5c6b 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -4797,7 +4797,6 @@ struct intr_info {
static inline char
intr_alert_char(u32 cause, u32 enable, u32 fatal)
{
-
if (cause & fatal)
return ('!');
if (cause & enable)
@@ -4817,7 +4816,7 @@ show_intr_info(struct adapter *sc, const struct intr_info *ii, uint32_t cause,
if (verbose || ucause != 0 || flags & IHF_RUN_ALL_ACTIONS) {
alert = intr_alert_char(cause, enabled, fatal);
CH_ALERT(sc, "%c %s 0x%x = 0x%08x, E 0x%08x, F 0x%08x\n", alert,
- ii->name, ii->cause_reg, cause, enabled, fatal);
+ ii->name, ii->cause_reg, cause, enabled, ii->fatal);
}
leftover = verbose ? cause : ucause;
@@ -4829,8 +4828,10 @@ show_intr_info(struct adapter *sc, const struct intr_info *ii, uint32_t cause,
CH_ALERT(sc, " %c [0x%08x] %s\n", alert, msgbits, details->msg);
leftover &= ~msgbits;
}
- if (leftover != 0 && leftover != (verbose ? cause : ucause))
- CH_ALERT(sc, " ? [0x%08x]\n", leftover);
+ if (leftover != 0 && leftover != (verbose ? cause : ucause)) {
+ alert = intr_alert_char(leftover, enabled, fatal);
+ CH_ALERT(sc, " %c [0x%08x]\n", alert, leftover);
+ }
}
/*
@@ -6102,6 +6103,14 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags)
{ F_PERR_INT_CAUSE, "FIFO parity error" },
{ 0 }
};
+ static const struct intr_details t7_mem_intr_details[] = {
+ { F_DDRPHY_INT_CAUSE, "DDRPHY" },
+ { F_DDRCTL_INT_CAUSE, "DDRCTL" },
+ { F_T7_ECC_CE_INT_CAUSE, "Correctable ECC data error(s)" },
+ { F_T7_ECC_UE_INT_CAUSE, "Uncorrectable ECC data error(s)" },
+ { F_PERR_INT_CAUSE, "FIFO parity error" },
+ { 0 }
+ };
char rname[32];
struct intr_info ii = {
.name = &rname[0],
@@ -6156,6 +6165,8 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags)
} else {
ii.cause_reg = MC_T7_REG(A_T7_MC_P_INT_CAUSE, i);
ii.enable_reg = MC_T7_REG(A_T7_MC_P_INT_ENABLE, i);
+ ii.fatal = F_PERR_INT_CAUSE | F_T7_ECC_UE_INT_CAUSE;
+ ii.details = t7_mem_intr_details;
count_reg = MC_T7_REG(A_T7_MC_P_ECC_STATUS, i);
}
fatal |= t4_handle_intr(adap, &ii, 0, flags);
diff --git a/sys/dev/cxgbe/crypto/t6_kern_tls.c b/sys/dev/cxgbe/crypto/t6_kern_tls.c
index 454b2e264a0e..584e5015acfa 100644
--- a/sys/dev/cxgbe/crypto/t6_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t6_kern_tls.c
@@ -458,15 +458,15 @@ t6_tls_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
}
inp = params->tls.inp;
+ tp = intotcpcb(inp);
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
}
tlsp->inp = inp;
- tp = intotcpcb(inp);
if (tp->t_flags & TF_REQ_TSTMP) {
tlsp->using_timestamps = true;
if ((tp->ts_offset & 0xfffffff) != 0) {
@@ -501,7 +501,7 @@ t6_tls_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
goto failed;
}
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
diff --git a/sys/dev/cxgbe/crypto/t7_kern_tls.c b/sys/dev/cxgbe/crypto/t7_kern_tls.c
index d9710b5bd13f..b6078b9b53b6 100644
--- a/sys/dev/cxgbe/crypto/t7_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t7_kern_tls.c
@@ -246,7 +246,7 @@ t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
inp = params->tls.inp;
INP_RLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) {
INP_RUNLOCK(inp);
error = ECONNRESET;
goto failed;
diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.c b/sys/dev/cxgbe/cxgbei/cxgbei.c
index ccca45f5f761..4b341c9d37b2 100644
--- a/sys/dev/cxgbe/cxgbei/cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/cxgbei.c
@@ -499,10 +499,11 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len;
}
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, pdu_len, inp->inp_flags);
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, pdu_len, tp->t_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
toep->ulpcb2 = NULL;
@@ -513,7 +514,6 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
* T6+ does not report data PDUs received via DDP without F
* set. This can result in gaps in the TCP sequence space.
*/
- tp = intotcpcb(inp);
MPASS(chip_id(sc) >= CHELSIO_T6 || icp->icp_seq == tp->rcv_nxt);
tp->rcv_nxt = icp->icp_seq + pdu_len;
tp->t_rcvtime = ticks;
@@ -652,10 +652,11 @@ do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
toep->ofld_rxq->rx_iscsi_data_digest_errors++;
}
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, pdu_len, inp->inp_flags);
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, pdu_len, tp->t_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
toep->ulpcb2 = NULL;
@@ -663,8 +664,6 @@ do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
- tp = intotcpcb(inp);
-
/*
* If icc is NULL, the connection is being closed in
* icl_cxgbei_conn_close(), just drop this data.
diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
index 9cdfd0fb9652..09023b00248e 100644
--- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
@@ -164,7 +164,7 @@ static kobj_method_t icl_cxgbei_methods[] = {
KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done),
KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup),
KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done),
- { 0, 0 }
+ KOBJMETHOD_END
};
DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn));
@@ -434,6 +434,7 @@ icl_cxgbei_tx_main(void *arg)
struct toepcb *toep = icc->toep;
struct socket *so = ic->ic_socket;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct icl_pdu *ip;
struct mbuf *m;
struct mbufq mq;
@@ -476,7 +477,7 @@ icl_cxgbei_tx_main(void *arg)
INP_WLOCK(inp);
ICL_CONN_UNLOCK(ic);
- if (__predict_false(inp->inp_flags & INP_DROPPED) ||
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED) ||
__predict_false((toep->flags & TPF_ATTACHED) == 0)) {
mbufq_drain(&mq);
} else {
@@ -621,7 +622,7 @@ icl_cxgbei_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *ip,
{
struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
struct mbuf *m, *m_tail;
- vm_offset_t vaddr;
+ void *vaddr;
size_t page_offset, todo, mtodo;
bool mapped;
int i;
@@ -809,7 +810,7 @@ icl_cxgbei_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip,
size_t pdu_off, struct bio *bp, size_t bio_off, size_t len)
{
struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
- vm_offset_t vaddr;
+ void *vaddr;
size_t page_offset, todo;
bool mapped;
int i;
@@ -1080,7 +1081,7 @@ icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
inp = sotoinpcb(so);
INP_WLOCK(inp);
tp = intotcpcb(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
error = ENOTCONN;
goto out;
@@ -1334,6 +1335,7 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
struct cxgbei_ddp_state *ddp;
struct ppod_reservation *prsv;
struct inpcb *inp;
+ struct tcpcb *tp;
struct mbufq mq;
uint32_t itt;
int rc = 0;
@@ -1421,8 +1423,9 @@ no_ddp:
* detached already.
*/
inp = sotoinpcb(ic->ic_socket);
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
mbufq_drain(&mq);
t4_free_page_pods(prsv);
@@ -1497,6 +1500,7 @@ icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, struct icl_pdu *ip,
struct ppod_reservation *prsv;
struct ctl_sg_entry *sgl, sg_entry;
struct inpcb *inp;
+ struct tcpcb *tp;
struct mbufq mq;
int sg_entries = ctsio->kern_sg_entries;
uint32_t ttt;
@@ -1597,9 +1601,10 @@ no_ddp:
return (ECONNRESET);
}
inp = sotoinpcb(ic->ic_socket);
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
ICL_CONN_UNLOCK(ic);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
mbufq_drain(&mq);
t4_free_page_pods(prsv);
diff --git a/sys/dev/cxgbe/iw_cxgbe/qp.c b/sys/dev/cxgbe/iw_cxgbe/qp.c
index cbf4bae00a60..372fc5418b91 100644
--- a/sys/dev/cxgbe/iw_cxgbe/qp.c
+++ b/sys/dev/cxgbe/iw_cxgbe/qp.c
@@ -64,7 +64,7 @@ struct cpl_set_tcb_rpl;
#include "iw_cxgbe.h"
#include "user.h"
-static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize);
+static int creds(struct toepcb *toep, struct tcpcb *tp, size_t wrsize);
static int max_fr_immd = T4_MAX_FR_IMMD;//SYSCTL parameter later...
static int alloc_ird(struct c4iw_dev *dev, u32 ird)
@@ -1149,7 +1149,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
term->ecode = qhp->attr.ecode;
} else
build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
return;
@@ -1253,8 +1253,7 @@ rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep)
int ret;
struct wrqe *wr;
struct socket *so = ep->com.so;
- struct inpcb *inp = sotoinpcb(so);
- struct tcpcb *tp = intotcpcb(inp);
+ struct tcpcb *tp = intotcpcb(sotoinpcb(so));
struct toepcb *toep = tp->t_toe;
KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__));
@@ -1277,7 +1276,7 @@ rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep)
c4iw_init_wr_wait(&ep->com.wr_wait);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
return ret;
@@ -1315,14 +1314,14 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
}
static int
-creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize)
+creds(struct toepcb *toep, struct tcpcb *tp, size_t wrsize)
{
struct ofld_tx_sdesc *txsd;
CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize);
- INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
- INP_WUNLOCK(inp);
+ INP_WLOCK(tptoinpcb(tp));
+ if (tp->t_flags & TF_DISCONNECTED) {
+ INP_WUNLOCK(tptoinpcb(tp));
return (EINVAL);
}
txsd = &toep->txsd[toep->txsd_pidx];
@@ -1336,7 +1335,7 @@ creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize)
if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;
toep->txsd_avail--;
- INP_WUNLOCK(inp);
+ INP_WUNLOCK(tptoinpcb(tp));
CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep ,
txsd->tx_credits, toep->tx_credits, toep->txsd_pidx);
return (0);
@@ -1351,8 +1350,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
struct c4iw_rdev *rdev = &qhp->rhp->rdev;
struct adapter *sc = rdev->adap;
struct socket *so = ep->com.so;
- struct inpcb *inp = sotoinpcb(so);
- struct tcpcb *tp = intotcpcb(inp);
+ struct tcpcb *tp = intotcpcb(sotoinpcb(so));
struct toepcb *toep = tp->t_toe;
CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp,
@@ -1416,7 +1414,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
c4iw_init_wr_wait(&ep->com.wr_wait);
- ret = creds(toep, inp, sizeof(*wqe));
+ ret = creds(toep, tp, sizeof(*wqe));
if (ret) {
free_wrqe(wr);
free_ird(rhp, qhp->attr.max_ird);
diff --git a/sys/dev/cxgbe/nvmf/nvmf_che.c b/sys/dev/cxgbe/nvmf/nvmf_che.c
index 5c2174b8a40b..be54e7bebfea 100644
--- a/sys/dev/cxgbe/nvmf/nvmf_che.c
+++ b/sys/dev/cxgbe/nvmf/nvmf_che.c
@@ -555,6 +555,7 @@ che_write_adapter_mem(struct nvmf_che_qpair *qp, uint32_t addr, uint32_t len,
struct toepcb *toep = qp->toep;
struct socket *so = qp->so;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct mbufq mq;
int error;
@@ -568,7 +569,7 @@ che_write_adapter_mem(struct nvmf_che_qpair *qp, uint32_t addr, uint32_t len,
goto error;
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) != 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) != 0) {
INP_WUNLOCK(inp);
error = ECONNRESET;
goto error;
@@ -862,12 +863,13 @@ nvmf_che_write_pdu(struct nvmf_che_qpair *qp, struct mbuf *m)
struct epoch_tracker et;
struct socket *so = qp->so;
struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
struct toepcb *toep = qp->toep;
CURVNET_SET(so->so_vnet);
NET_EPOCH_ENTER(et);
INP_WLOCK(inp);
- if (__predict_false(inp->inp_flags & INP_DROPPED) ||
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED) ||
__predict_false((toep->flags & TPF_ATTACHED) == 0)) {
m_freem(m);
} else {
@@ -2052,10 +2054,11 @@ do_nvmt_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
inp = toep->inp;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -2070,7 +2073,6 @@ do_nvmt_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
mbufq_enqueue(&qp->rx_data, m);
SOCKBUF_UNLOCK(&so->so_rcv);
- tp = intotcpcb(inp);
tp->t_rcvtime = ticks;
#ifdef VERBOSE_TRACES
@@ -2092,6 +2094,7 @@ do_nvmt_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
struct nvmf_che_qpair *qp = toep->ulpcb;
struct socket *so = qp->so;
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
u_int hlen __diagused;
bool empty;
@@ -2107,9 +2110,9 @@ do_nvmt_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR(KTR_CXGBE, "%s: tid %u, rx (hlen %u), inp_flags 0x%x",
- __func__, tid, hlen, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (hlen %u), t_flags 0x%x",
+ __func__, tid, hlen, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -2505,7 +2508,7 @@ che_allocate_qpair(bool controller, const nvlist_t *nvl)
inp = sotoinpcb(so);
INP_WLOCK(inp);
tp = intotcpcb(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
free(qp->fl_cid_set, M_NVMF_CHE);
free(qp->fl_cids, M_NVMF_CHE);
@@ -2602,6 +2605,7 @@ che_allocate_qpair(bool controller, const nvlist_t *nvl)
free(qp->fl_cids, M_NVMF_CHE);
free(qp->open_fl_ttags, M_NVMF_CHE);
free(qp, M_NVMF_CHE);
+ soclose(so);
return (NULL);
}
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 3237c6649713..69ecfce1dade 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -128,7 +128,7 @@ device_method_t cxgbe_methods[] = {
DEVMETHOD(device_probe, cxgbe_probe),
DEVMETHOD(device_attach, cxgbe_attach),
DEVMETHOD(device_detach, cxgbe_detach),
- { 0, 0 }
+ DEVMETHOD_END
};
static driver_t cxgbe_driver = {
"cxgbe",
@@ -144,7 +144,7 @@ static device_method_t vcxgbe_methods[] = {
DEVMETHOD(device_probe, vcxgbe_probe),
DEVMETHOD(device_attach, vcxgbe_attach),
DEVMETHOD(device_detach, vcxgbe_detach),
- { 0, 0 }
+ DEVMETHOD_END
};
static driver_t vcxgbe_driver = {
"vcxgbe",
@@ -806,7 +806,7 @@ static int validate_mem_range(struct adapter *, uint32_t, uint32_t);
static int fwmtype_to_hwmtype(int);
static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t,
uint32_t *);
-static int fixup_devlog_params(struct adapter *);
+static int fixup_devlog_ncores_params(struct adapter *);
static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *);
static int contact_firmware(struct adapter *);
static int partition_resources(struct adapter *);
@@ -900,6 +900,7 @@ static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
static int sysctl_reset(SYSCTL_HANDLER_ARGS);
+static int sysctl_tcb_cache(SYSCTL_HANDLER_ARGS);
#ifdef TCP_OFFLOAD
static int sysctl_tls(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
@@ -1248,7 +1249,7 @@ t4_calibration(void *arg)
sc = (struct adapter *)arg;
- KASSERT(hw_all_ok(sc), ("!hw_all_ok at t4_calibration"));
+ KASSERT(!hw_off_limits(sc), ("hw_off_limits at t4_calibration"));
hw = t4_read_reg64(sc, A_SGE_TIMESTAMP_LO);
sbt = sbinuptime();
@@ -1424,7 +1425,7 @@ t4_attach(device_t dev)
*/
setup_memwin(sc);
if (t4_init_devlog_ncores_params(sc, 0) == 0)
- fixup_devlog_params(sc);
+ fixup_devlog_ncores_params(sc);
make_dev_args_init(&mda);
mda.mda_devsw = &t4_cdevsw;
mda.mda_uid = UID_ROOT;
@@ -4046,7 +4047,7 @@ t4_map_bar_2(struct adapter *sc)
* request with an implicit doorbell.
*/
- rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
+ rc = pmap_change_attr(__DEVOLATILE(void *, sc->udbs_base),
rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
if (rc == 0) {
clrbit(&sc->doorbells, DOORBELL_UDB);
@@ -4564,11 +4565,15 @@ validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len,
}
static int
-fixup_devlog_params(struct adapter *sc)
+fixup_devlog_ncores_params(struct adapter *sc)
{
struct devlog_params *dparams = &sc->params.devlog;
int rc;
+#ifdef INVARIANTS
+ if (sc->params.ncores > 1)
+ MPASS(chip_id(sc) >= CHELSIO_T7);
+#endif
rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
dparams->size, &dparams->addr);
@@ -5558,7 +5563,7 @@ get_params__pre_init(struct adapter *sc)
/* Read device log parameters. */
rc = -t4_init_devlog_ncores_params(sc, 1);
if (rc == 0)
- fixup_devlog_params(sc);
+ fixup_devlog_ncores_params(sc);
else {
device_printf(sc->dev,
"failed to get devlog parameters: %d.\n", rc);
@@ -5711,8 +5716,6 @@ get_params__post_init(struct adapter *sc)
}
if (sc->params.ncores > 1) {
- MPASS(chip_id(sc) >= CHELSIO_T7);
-
param[0] = FW_PARAM_DEV(TID_QID_SEL_MASK);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
sc->params.tid_qid_sel_mask = rc == 0 ? val[0] : 0;
@@ -8119,6 +8122,12 @@ t4_sysctls(struct adapter *sc)
sysctl_wcwr_stats, "A", "write combined work requests");
}
+ if (chip_id(sc) >= CHELSIO_T7) {
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcb_cache",
+ CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tcb_cache, "I",
+ "1 = enabled (default), 0 = disabled (for debug only)");
+ }
+
#ifdef KERN_TLS
if (is_ktls(sc)) {
/*
@@ -11587,7 +11596,7 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
if (hashen) {
if (x)
sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1);
- sbuf_printf(sb, "%u-%u", y, t->ntids - 1);
+ sbuf_printf(sb, "%u-%u", y, t->tid_base + t->ntids - 1);
} else {
sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base +
t->ntids - 1);
@@ -12204,6 +12213,40 @@ sysctl_reset(SYSCTL_HANDLER_ARGS)
return (0);
}
+static int
+sysctl_tcb_cache(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ u_int val, v;
+ int rc;
+
+ mtx_lock(&sc->reg_lock);
+ if (hw_off_limits(sc)) {
+ rc = ENXIO;
+ goto done;
+ }
+ t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1);
+ mtx_unlock(&sc->reg_lock);
+
+ val = v & F_GLFL ? 0 : 1;
+ rc = sysctl_handle_int(oidp, &val, 0, req);
+ if (rc != 0 || req->newptr == NULL)
+ return (rc);
+ if (val == 0)
+ v |= F_GLFL;
+ else
+ v &= ~F_GLFL;
+
+ mtx_lock(&sc->reg_lock);
+ if (hw_off_limits(sc))
+ rc = ENXIO;
+ else
+ t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1);
+done:
+ mtx_unlock(&sc->reg_lock);
+ return (rc);
+}
+
#ifdef TCP_OFFLOAD
static int
sysctl_tls(SYSCTL_HANDLER_ARGS)
diff --git a/sys/dev/cxgbe/t4_netmap.c b/sys/dev/cxgbe/t4_netmap.c
index 0135bec6e2c1..a858867239c6 100644
--- a/sys/dev/cxgbe/t4_netmap.c
+++ b/sys/dev/cxgbe/t4_netmap.c
@@ -606,10 +606,8 @@ cxgbe_netmap_split_rss(struct adapter *sc, struct vi_info *vi,
(nm_state == NM_OFF && nm_kring_pending_on(kring))) {
MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);
nactive[j]++;
- if (dq[j] == -1) {
+ if (dq[j] == -1)
dq[j] = nm_rxq->iq_abs_id;
- break;
- }
}
}
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index b6d44792dce4..07e4165db4a0 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -828,12 +828,17 @@ t4_tweak_chip_settings(struct adapter *sc)
t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v);
if (sc->debug_flags & DF_DISABLE_TCB_CACHE) {
- m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN |
- V_WRTHRTHRESH(M_WRTHRTHRESH);
t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1);
- v &= ~m;
- v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN |
- V_WRTHRTHRESH(16);
+ if (chip_id(sc) >= CHELSIO_T7) {
+ v |= F_GLFL;
+ } else {
+ m = V_RDTHRESHOLD(M_RDTHRESHOLD) |
+ F_WRTHRTHRESHEN |
+ V_WRTHRTHRESH(M_WRTHRTHRESH);
+ v &= ~m;
+ v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN |
+ V_WRTHRTHRESH(16);
+ }
t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1);
}
}
@@ -4367,18 +4372,17 @@ qsize_to_fthresh(int qsize)
static int
ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq, int idx)
{
- int rc, cntxt_id, core;
+ int rc, cntxt_id;
struct fw_eq_ctrl_cmd c;
int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
- core = sc->params.tid_qid_sel_mask != 0 ? idx % sc->params.ncores : 0;
bzero(&c, sizeof(c));
c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
V_FW_EQ_CTRL_CMD_VFN(0));
c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
- V_FW_EQ_CTRL_CMD_COREGROUP(core) |
+ V_FW_EQ_CTRL_CMD_COREGROUP(idx % sc->params.ncores) |
F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
c.physeqid_pkd = htobe32(0);
@@ -4415,18 +4419,17 @@ ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq, int idx)
static int
eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq, int idx)
{
- int rc, cntxt_id, core;
+ int rc, cntxt_id;
struct fw_eq_eth_cmd c;
int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
- core = sc->params.ncores > 1 ? idx % sc->params.ncores : 0;
bzero(&c, sizeof(c));
c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
V_FW_EQ_ETH_CMD_VFN(0));
c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
- V_FW_EQ_ETH_CMD_COREGROUP(core) |
+ V_FW_EQ_ETH_CMD_COREGROUP(idx % sc->params.ncores) |
F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
@@ -5464,7 +5467,7 @@ csum_to_ctrl(struct adapter *sc, struct mbuf *m)
uint64_t ctrl;
int csum_type, l2hlen, l3hlen;
int x, y;
- static const int csum_types[3][2] = {
+ static const uint8_t csum_types[3][2] = {
{TX_CSUM_TCPIP, TX_CSUM_TCPIP6},
{TX_CSUM_UDPIP, TX_CSUM_UDPIP6},
{TX_CSUM_IP, 0}
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c
index c236ee060bc2..e5f6053e2cb6 100644
--- a/sys/dev/cxgbe/tom/t4_connect.c
+++ b/sys/dev/cxgbe/tom/t4_connect.c
@@ -78,6 +78,7 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
struct toepcb *toep = lookup_atid(sc, atid);
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
@@ -95,7 +96,7 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx];
}
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
/* socket closed by the kernel before hw told us it connected */
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index 5c39ae5fa8f3..6e34d5f54897 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -245,13 +245,13 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
struct cpl_abort_req *req;
int tid = toep->tid;
struct inpcb *inp = toep->inp;
- struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */
+ struct tcpcb *tp = intotcpcb(inp);
INP_WLOCK_ASSERT(inp);
CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
__func__, toep->tid,
- inp->inp_flags & INP_DROPPED ? "inp dropped" :
+ tp->t_flags & TF_DISCONNECTED ? "TCP disconnected" :
tcpstates[tp->t_state],
toep->flags, inp->inp_flags,
toep->flags & TPF_ABORT_SHUTDOWN ?
@@ -273,7 +273,7 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
req = wrtod(wr);
INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
- if (inp->inp_flags & INP_DROPPED)
+ if (tp->t_flags & TF_DISCONNECTED)
req->rsvd0 = htobe32(snd_nxt);
else
req->rsvd0 = htobe32(tp->snd_nxt);
@@ -284,7 +284,7 @@ send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
* XXX: What's the correct way to tell that the inp hasn't been detached
* from its socket? Should I even be flushing the snd buffer here?
*/
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL) /* because I'm not sure. See comment above */
@@ -496,9 +496,6 @@ t4_close_conn(struct adapter *sc, struct toepcb *toep)
#define MIN_ISO_TX_CREDITS (howmany(sizeof(struct cpl_tx_data_iso), 16))
#define MIN_TX_CREDITS(iso) \
(MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
-#define MIN_OFLD_TX_V2_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_v2_wr) + 1, 16))
-#define MIN_TX_V2_CREDITS(iso) \
- (MIN_OFLD_TX_V2_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
_Static_assert(MAX_OFLD_TX_CREDITS <= MAX_OFLD_TX_SDESC_CREDITS,
"MAX_OFLD_TX_SDESC_CREDITS too small");
@@ -546,46 +543,6 @@ max_dsgl_nsegs(int tx_credits, int iso)
return (nseg);
}
-/* Maximum amount of immediate data we could stuff in a WR */
-static inline int
-max_imm_payload_v2(int tx_credits, int iso)
-{
- const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0;
-
- KASSERT(tx_credits >= 0 &&
- tx_credits <= MAX_OFLD_TX_CREDITS,
- ("%s: %d credits", __func__, tx_credits));
-
- if (tx_credits < MIN_TX_V2_CREDITS(iso))
- return (0);
-
- return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_v2_wr) -
- iso_cpl_size);
-}
-
-/* Maximum number of SGL entries we could stuff in a WR */
-static inline int
-max_dsgl_nsegs_v2(int tx_credits, int iso, int imm_payload)
-{
- int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
- int sge_pair_credits = tx_credits - MIN_TX_V2_CREDITS(iso);
-
- KASSERT(tx_credits >= 0 &&
- tx_credits <= MAX_OFLD_TX_CREDITS,
- ("%s: %d credits", __func__, tx_credits));
-
- if (tx_credits < MIN_TX_V2_CREDITS(iso) ||
- sge_pair_credits <= howmany(imm_payload, 16))
- return (0);
- sge_pair_credits -= howmany(imm_payload, 16);
-
- nseg += 2 * (sge_pair_credits * 16 / 24);
- if ((sge_pair_credits * 16) % 24 == 16)
- nseg++;
-
- return (nseg);
-}
-
static inline void
write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
@@ -613,35 +570,6 @@ write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
}
}
-static inline void
-write_tx_v2_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
- unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
- int ulp_submode)
-{
- struct fw_ofld_tx_data_v2_wr *txwr = dst;
- uint32_t flags;
-
- memset(txwr, 0, sizeof(*txwr));
- txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) |
- V_FW_WR_IMMDLEN(immdlen));
- txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
- V_FW_WR_LEN16(credits));
- txwr->plen = htobe32(plen);
- flags = V_TX_ULP_MODE(ULP_MODE_NVMET) | V_TX_ULP_SUBMODE(ulp_submode) |
- V_TX_URG(0) | V_TX_SHOVE(shove);
-
- if (toep->params.tx_align > 0) {
- if (plen < 2 * toep->params.emss)
- flags |= F_FW_OFLD_TX_DATA_WR_LSODISABLE;
- else
- flags |= F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
- (toep->params.nagle == 0 ? 0 :
- F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE);
- }
-
- txwr->lsodisable_to_flags = htobe32(flags);
-}
-
/*
* Generate a DSGL from a starting mbuf. The total number of segments and the
* maximum segments in any one mbuf are provided.
@@ -1300,7 +1228,7 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
{
struct mbuf *m;
const struct nvme_tcp_common_pdu_hdr *hdr;
- struct fw_v2_nvmet_tx_data_wr *txwr;
+ struct fw_ofld_tx_data_wr *txwr;
struct cpl_tx_data_iso *cpl_iso;
void *p;
struct wrqe *wr;
@@ -1330,29 +1258,16 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
return (wr);
}
- /*
- * The first mbuf is the PDU header that is always sent as
- * immediate data.
- */
- imm_data = sndptr->m_len;
-
iso = mbuf_iscsi_iso(sndptr);
- max_imm = max_imm_payload_v2(tx_credits, iso);
-
- /*
- * Not enough credits for the PDU header.
- */
- if (imm_data > max_imm)
- return (NULL);
-
- max_nsegs = max_dsgl_nsegs_v2(tx_credits, iso, imm_data);
+ max_imm = max_imm_payload(tx_credits, iso);
+ max_nsegs = max_dsgl_nsegs(tx_credits, iso);
iso_mss = mbuf_iscsi_iso_mss(sndptr);
- plen = imm_data;
+ plen = 0;
nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
nomap_mbuf_seen = false;
- for (m = sndptr->m_next; m != NULL; m = m->m_next) {
+ for (m = sndptr; m != NULL; m = m->m_next) {
int n;
if (m->m_flags & M_EXTPG)
@@ -1440,13 +1355,13 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
if (iso)
wr_len += sizeof(struct cpl_tx_data_iso);
if (plen <= max_imm && !nomap_mbuf_seen) {
- /* Immediate data tx for full PDU */
+ /* Immediate data tx */
imm_data = plen;
wr_len += plen;
nsegs = 0;
} else {
/* DSGL tx for PDU data */
- wr_len += roundup2(imm_data, 16);
+ imm_data = 0;
wr_len += sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
}
@@ -1460,7 +1375,7 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
credits = howmany(wr->wr_len, 16);
if (iso) {
- write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR,
+ write_tx_wr(txwr, toep, FW_ISCSI_TX_DATA_WR,
imm_data + sizeof(struct cpl_tx_data_iso),
adjusted_plen, credits, shove, ulp_submode | ULP_ISO);
cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1);
@@ -1470,16 +1385,15 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
hdr->pdo);
p = cpl_iso + 1;
} else {
- write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR, imm_data,
+ write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, imm_data,
adjusted_plen, credits, shove, ulp_submode);
p = txwr + 1;
}
- /* PDU header (and immediate data payload). */
- m_copydata(sndptr, 0, imm_data, p);
- if (nsegs != 0) {
- p = roundup2((char *)p + imm_data, 16);
- write_tx_sgl(p, sndptr->m_next, NULL, nsegs, max_nsegs_1mbuf);
+ if (imm_data != 0) {
+ m_copydata(sndptr, 0, plen, p);
+ } else {
+ write_tx_sgl(p, sndptr, NULL, nsegs, max_nsegs_1mbuf);
if (wr_len & 0xf) {
uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len);
*pad = 0;
@@ -1674,8 +1588,8 @@ t4_tod_output(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
t4_push_data(sc, toep, 0);
@@ -1693,8 +1607,8 @@ t4_send_fin(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
toep->flags |= TPF_SEND_FIN;
@@ -1714,8 +1628,8 @@ t4_send_rst(struct toedev *tod, struct tcpcb *tp)
struct toepcb *toep = tp->t_toe;
INP_WLOCK_ASSERT(inp);
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp %p dropped.", __func__, inp));
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: tcpcb %p disconnected", __func__, tp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
/* hmmmm */
@@ -2007,7 +1921,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
}
toep->flags |= TPF_ABORT_SHUTDOWN;
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
@@ -2096,17 +2010,16 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
m_adj(m, sizeof(*cpl));
len = m->m_pkthdr.len;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
}
- tp = intotcpcb(inp);
-
if (__predict_false(ulp_mode(toep) == ULP_MODE_TLS &&
toep->flags & TPF_TLS_RECEIVE)) {
/* Received "raw" data on a TLS socket. */
@@ -2256,6 +2169,7 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
}
inp = toep->inp;
+ tp = intotcpcb(inp);
KASSERT(opcode == CPL_FW4_ACK,
("%s: unexpected opcode 0x%x", __func__, opcode));
@@ -2269,10 +2183,8 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
- KASSERT((inp->inp_flags & INP_DROPPED) == 0,
- ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
-
- tp = intotcpcb(inp);
+ KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
+ ("%s: t_flags 0x%x", __func__, tp->t_flags));
if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
tcp_seq snd_una = be32toh(cpl->snd_una);
@@ -2713,8 +2625,9 @@ sendanother:
/* Inlined tcp_usr_send(). */
inp = toep->inp;
+ tp = intotcpcb(inp);
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
INP_WUNLOCK(inp);
SOCK_IO_SEND_UNLOCK(so);
error = ECONNRESET;
@@ -2728,8 +2641,7 @@ sendanother:
sbappendstream(sb, m, 0);
m = NULL;
- if (!(inp->inp_flags & INP_DROPPED)) {
- tp = intotcpcb(inp);
+ if (!(tp->t_flags & TF_DISCONNECTED)) {
if (moretocome)
tp->t_flags |= TF_MORETOCOME;
error = tcp_output(tp);
diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c
index 35fb1061d867..9d422c2b793e 100644
--- a/sys/dev/cxgbe/tom/t4_ddp.c
+++ b/sys/dev/cxgbe/tom/t4_ddp.c
@@ -641,8 +641,8 @@ handle_ddp_data_aio(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
uint32_t report = be32toh(ddp_report);
unsigned int db_idx;
struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
struct ddp_buffer *db;
- struct tcpcb *tp;
struct socket *so;
struct sockbuf *sb;
struct kaiocb *job;
@@ -664,13 +664,13 @@ handle_ddp_data_aio(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
db = &toep->ddp.db[db_idx];
job = db->job;
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
/*
* This can happen due to an administrative tcpdrop(8).
* Just fail the request with ECONNRESET.
*/
- CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
- __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
+ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, t_flags 0x%x",
+ __func__, toep->tid, be32toh(rcv_nxt), len, tp->t_flags);
if (aio_clear_cancel_function(job))
ddp_complete_one(job, ECONNRESET);
goto completed;
@@ -859,7 +859,7 @@ handle_ddp_data_rcvbuf(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
{
uint32_t report = be32toh(ddp_report);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct sockbuf *sb;
struct ddp_buffer *db;
@@ -881,20 +881,18 @@ handle_ddp_data_rcvbuf(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt,
toep->ddp.active_id, toep->tid));
db = &toep->ddp.db[db_idx];
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
/*
* This can happen due to an administrative tcpdrop(8).
* Just ignore the received data.
*/
- CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
- __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
+ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, t_flags 0x%x",
+ __func__, toep->tid, be32toh(rcv_nxt), len, tp->t_flags);
if (invalidated)
complete_ddp_buffer(toep, db, db_idx);
goto out;
}
- tp = intotcpcb(inp);
-
/*
* For RX_DDP_COMPLETE, len will be zero and rcv_nxt is the
* sequence number of the next byte to receive. The length of
diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c
index b879f6883f25..359267b7db90 100644
--- a/sys/dev/cxgbe/tom/t4_listen.c
+++ b/sys/dev/cxgbe/tom/t4_listen.c
@@ -886,6 +886,7 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
unsigned int status = cpl->status;
struct listen_ctx *lctx = lookup_stid(sc, stid);
struct inpcb *inp = lctx->inp;
+ struct tcpcb *tp = intotcpcb(inp);
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
@@ -911,13 +912,13 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
* If the inp has been dropped (listening socket closed) then
* listen_stop must have run and taken the inp out of the hash.
*/
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
KASSERT(listen_hash_del(sc, inp) == NULL,
("%s: inp %p still in listen hash", __func__, inp));
}
#endif
- if (inp->inp_flags & INP_DROPPED && status != CPL_ERR_NONE) {
+ if (tp->t_flags & TF_DISCONNECTED && status != CPL_ERR_NONE) {
if (release_lctx(sc, lctx) != NULL)
INP_WUNLOCK(inp);
return (status);
@@ -928,7 +929,7 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
* it has started the hardware listener. Stop it; the lctx will be
* released in do_close_server_rpl.
*/
- if (inp->inp_flags & INP_DROPPED) {
+ if (tp->t_flags & TF_DISCONNECTED) {
destroy_server(sc, lctx);
INP_WUNLOCK(inp);
return (status);
@@ -1336,6 +1337,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
unsigned int tid = GET_TID(cpl);
struct listen_ctx *lctx = lookup_stid(sc, stid);
struct inpcb *inp;
+ struct tcpcb *tp;
struct socket *so;
struct in_conninfo inc;
struct tcphdr th;
@@ -1477,10 +1479,11 @@ found:
}
inp = lctx->inp; /* listening socket, not owned by TOE */
+ tp = intotcpcb(inp);
INP_RLOCK(inp);
/* Don't offload if the listening socket has closed */
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
INP_RUNLOCK(inp);
NET_EPOCH_EXIT(et);
REJECT_PASS_ACCEPT_REQ(false);
@@ -1622,6 +1625,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
struct synq_entry *synqe = lookup_tid(sc, tid);
struct listen_ctx *lctx = synqe->lctx;
struct inpcb *inp = lctx->inp, *new_inp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct tcphdr th;
struct tcpopt to;
@@ -1653,7 +1657,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
KASSERT(vi->adapter == sc,
("%s: vi %p, sc %p mismatch", __func__, vi, sc));
- if (__predict_false(inp->inp_flags & INP_DROPPED)) {
+ if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
reset:
send_abort_rpl_synqe(TOEDEV(ifp), synqe, CPL_ABORT_SEND_RST);
INP_WUNLOCK(inp);
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index bbcc1c88c3db..0616279ba15e 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -762,7 +762,7 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
int len;
/* XXX: Should this match do_rx_data instead? */
@@ -781,9 +781,9 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -803,7 +803,6 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
#endif
}
- tp = intotcpcb(inp);
tp->t_rcvtime = ticks;
#ifdef VERBOSE_TRACES
@@ -824,7 +823,7 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
- struct tcpcb *tp;
+ struct tcpcb *tp = intotcpcb(inp);
struct socket *so;
struct sockbuf *sb;
struct mbuf *tls_data;
@@ -851,9 +850,9 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
("%s: payload length mismatch", __func__));
INP_WLOCK(inp);
- if (inp->inp_flags & INP_DROPPED) {
- CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
- __func__, tid, len, inp->inp_flags);
+ if (tp->t_flags & TF_DISCONNECTED) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), t_flags 0x%x",
+ __func__, tid, len, tp->t_flags);
INP_WUNLOCK(inp);
m_freem(m);
return (0);
@@ -862,7 +861,6 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
so = inp_inpcbtosocket(inp);
- tp = intotcpcb(inp);
#ifdef VERBOSE_TRACES
CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 8dfffd465345..950608053be7 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -1830,7 +1830,7 @@ live_tid_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status)
INP_WLOCK(inp);
tp = intotcpcb(inp);
toep->flags |= TPF_ABORT_SHUTDOWN;
- if ((inp->inp_flags & INP_DROPPED) == 0) {
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
@@ -2283,8 +2283,8 @@ find_offload_adapter_cb(struct adapter *sc, void *arg)
struct find_offload_adapter_data *fa = arg;
struct socket *so = fa->so;
struct tom_data *td = sc->tom_softc;
- struct tcpcb *tp;
- struct inpcb *inp;
+ struct inpcb *inp = sotoinpcb(so);
+ struct tcpcb *tp = intotcpcb(inp);
/* Non-TCP were filtered out earlier. */
MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
@@ -2295,10 +2295,8 @@ find_offload_adapter_cb(struct adapter *sc, void *arg)
if (td == NULL)
return; /* TOE not enabled on this adapter. */
- inp = sotoinpcb(so);
INP_WLOCK(inp);
- if ((inp->inp_flags & INP_DROPPED) == 0) {
- tp = intotcpcb(inp);
+ if ((tp->t_flags & TF_DISCONNECTED) == 0) {
if (tp->t_flags & TF_TOE && tp->tod == &td->tod)
fa->sc = sc; /* Found. */
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-28 23:04:00 +0000