cxgbe: Make the TOE TLS stats per-queue instead of per-port.

This avoids some atomics by using counter_u64 for TX and relying on
existing single-threading (single ithread per rxq) for RX.

Reviewed by:	np
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D29383

4 files changed, 34 insertions, 23 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index ed36c1e546d3..88dbbf5d06ac 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -316,10 +316,6 @@ struct port_info {
 	u_int tx_parse_error;
 	int fcs_reg;
 	uint64_t fcs_base;
-	u_long	tx_toe_tls_records;
-	u_long	tx_toe_tls_octets;
-	u_long	rx_toe_tls_records;
-	u_long	rx_toe_tls_octets;
 
 	struct callout tick;
 };
@@ -654,6 +650,8 @@ iq_to_rxq(struct sge_iq *iq)
 struct sge_ofld_rxq {
 	struct sge_iq iq;	/* MUST be first */
 	struct sge_fl fl;	/* MUST follow iq */
+	u_long	rx_toe_tls_records;
+	u_long	rx_toe_tls_octets;
 } __aligned(CACHE_LINE_SIZE);
 
 static inline struct sge_ofld_rxq *
@@ -715,6 +713,8 @@ struct sge_wrq {
 /* ofld_txq: SGE egress queue + miscellaneous items */
 struct sge_ofld_txq {
 	struct sge_wrq wrq;
+	counter_u64_t tx_toe_tls_records;
+	counter_u64_t tx_toe_tls_octets;
 } __aligned(CACHE_LINE_SIZE);
 
 #define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 5d06d4b55d8a..ad622e9999de 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -7218,19 +7218,6 @@ cxgbe_sysctls(struct port_info *pi)
 
 #undef T4_REGSTAT
 #undef T4_PORTSTAT
-
-	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_records",
-	    CTLFLAG_RD, &pi->tx_toe_tls_records,
-	    "# of TOE TLS records transmitted");
-	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_octets",
-	    CTLFLAG_RD, &pi->tx_toe_tls_octets,
-	    "# of payload octets in transmitted TOE TLS records");
-	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_records",
-	    CTLFLAG_RD, &pi->rx_toe_tls_records,
-	    "# of TOE TLS records received");
-	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_octets",
-	    CTLFLAG_RD, &pi->rx_toe_tls_octets,
-	    "# of payload octets in received TOE TLS records");
 }
 
 static int
@@ -10765,6 +10752,8 @@ clear_stats(struct adapter *sc, u_int port_id)
 			for_each_ofld_txq(vi, i, ofld_txq) {
 				ofld_txq->wrq.tx_wrs_direct = 0;
 				ofld_txq->wrq.tx_wrs_copied = 0;
+				counter_u64_zero(ofld_txq->tx_toe_tls_records);
+				counter_u64_zero(ofld_txq->tx_toe_tls_octets);
 			}
 #endif
 #ifdef TCP_OFFLOAD
@@ -10772,6 +10761,8 @@ clear_stats(struct adapter *sc, u_int port_id)
 				ofld_rxq->fl.cl_allocated = 0;
 				ofld_rxq->fl.cl_recycled = 0;
 				ofld_rxq->fl.cl_fast_recycled = 0;
+				ofld_rxq->rx_toe_tls_records = 0;
+				ofld_rxq->rx_toe_tls_octets = 0;
 			}
 #endif
 
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 1f2abcb81078..002b4892e110 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -3958,6 +3958,13 @@ alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq,
 	add_iq_sysctls(&vi->ctx, oid, &ofld_rxq->iq);
 	add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl);
 
+	SYSCTL_ADD_ULONG(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+	    "rx_toe_tls_records", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_records,
+	    "# of TOE TLS records received");
+	SYSCTL_ADD_ULONG(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+	    "rx_toe_tls_octets", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_octets,
+	    "# of payload octets in received TOE TLS records");
+
 	return (rc);
 }
 
@@ -4494,11 +4501,21 @@ alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx,
 	snprintf(name, sizeof(name), "%d", idx);
 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name,
 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue");
+	children = SYSCTL_CHILDREN(oid);
 
 	rc = alloc_wrq(sc, vi, &ofld_txq->wrq, oid);
 	if (rc != 0)
 		return (rc);
 
+	ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK);
+	ofld_txq->tx_toe_tls_octets = counter_u64_alloc(M_WAITOK);
+	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO,
+	    "tx_toe_tls_records", CTLFLAG_RD, &ofld_txq->tx_toe_tls_records,
+	    "# of TOE TLS records transmitted");
+	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO,
+	    "tx_toe_tls_octets", CTLFLAG_RD, &ofld_txq->tx_toe_tls_octets,
+	    "# of payload octets in transmitted TOE TLS records");
+
 	return (rc);
 }
 
@@ -4512,6 +4529,9 @@ free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)
 	if (rc != 0)
 		return (rc);
 
+	counter_u64_free(ofld_txq->tx_toe_tls_records);
+	counter_u64_free(ofld_txq->tx_toe_tls_octets);
+
 	bzero(ofld_txq, sizeof(*ofld_txq));
 	return (0);
 }
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index fff42386fab7..bbd905d8acc3 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -1664,8 +1664,8 @@ t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
 		}
 		toep->txsd_avail--;
 
-		atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1);
-		atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, plen);
+		counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
+		counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, plen);
 
 		t4_l2t_send(sc, wr, toep->l2te);
 	}
@@ -1966,8 +1966,8 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
 		}
 		toep->txsd_avail--;
 
-		atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1);
-		atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, m->m_len);
+		counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
+		counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len);
 
 		t4_l2t_send(sc, wr, toep->l2te);
 	}
@@ -2003,7 +2003,7 @@ do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 	m_adj(m, sizeof(*cpl));
 	len = m->m_pkthdr.len;
 
-	atomic_add_long(&toep->vi->pi->rx_toe_tls_octets, len);
+	toep->ofld_rxq->rx_toe_tls_octets += len;
 
 	KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
 	    ("%s: payload length mismatch", __func__));
@@ -2070,7 +2070,7 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 	m_adj(m, sizeof(*cpl));
 	len = m->m_pkthdr.len;
 
-	atomic_add_long(&toep->vi->pi->rx_toe_tls_records, 1);
+	toep->ofld_rxq->rx_toe_tls_records++;
 
 	KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
 	    ("%s: payload length mismatch", __func__));