cxgbe(4): Allocate a taskqueue per port instead of per channel.

All the channels are not used on all boards and there's no point
allocating taskqueues that will never be used.

MFC after:	1 week
Sponsored by:	Chelsio Communications

3 files changed, 17 insertions, 12 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index b9780809421f..8440d9d94f77 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -471,6 +471,7 @@ struct sge_eq {
 	unsigned int abs_id;	/* absolute SGE id for the eq */
 	uint8_t type;		/* EQ_CTRL/EQ_ETH/EQ_OFLD */
 	uint8_t doorbells;
+	uint8_t port_id;	/* port_id of the port associated with the eq */
 	uint8_t tx_chan;	/* tx channel used by the eq */
 	struct mtx eq_lock;
 
@@ -929,7 +930,7 @@ struct adapter {
 	int rawf_base;
 	int nrawf;
 
-	struct taskqueue *tq[MAX_NCHAN];	/* General purpose taskqueues */
+	struct taskqueue *tq[MAX_NPORTS];	/* General purpose taskqueues */
 	struct port_info *port[MAX_NPORTS];
 	uint8_t chan_map[MAX_NCHAN];		/* channel -> port */
 
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 16b3ac7dd6ff..ca6e35718d21 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -6660,7 +6660,8 @@ adapter_full_init(struct adapter *sc)
 	if (rc != 0)
 		return (rc);
 
-	for (i = 0; i < nitems(sc->tq); i++) {
+	MPASS(sc->params.nports <= nitems(sc->tq));
+	for (i = 0; i < sc->params.nports; i++) {
 		if (sc->tq[i] != NULL)
 			continue;
 		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
@@ -6709,7 +6710,9 @@ adapter_full_uninit(struct adapter *sc)
 
 	t4_teardown_adapter_queues(sc);
 
-	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
+	for (i = 0; i < nitems(sc->tq); i++) {
+		if (sc->tq[i] == NULL)
+			continue;
 		taskqueue_free(sc->tq[i]);
 		sc->tq[i] = NULL;
 	}
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 2ef05b5a9f86..e1705ae063e2 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -3383,13 +3383,14 @@ init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
 
 static inline void
 init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize,
-    uint8_t tx_chan, struct sge_iq *iq, char *name)
+    uint8_t port_id, struct sge_iq *iq, char *name)
 {
 	KASSERT(eqtype >= EQ_CTRL && eqtype <= EQ_OFLD,
 	    ("%s: bad qtype %d", __func__, eqtype));
 
 	eq->type = eqtype;
-	eq->tx_chan = tx_chan;
+	eq->port_id = port_id;
+	eq->tx_chan = sc->port[port_id]->tx_chan;
 	eq->iq = iq;
 	eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
@@ -3830,8 +3831,8 @@ alloc_ctrlq(struct adapter *sc, int idx)
 
 		snprintf(name, sizeof(name), "%s ctrlq%d",
 		    device_get_nameunit(sc->dev), idx);
-		init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE,
-		    sc->port[idx]->tx_chan, &sc->sge.fwq, name);
+		init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, idx,
+		    &sc->sge.fwq, name);
 		rc = alloc_wrq(sc, NULL, ctrlq, &sc->ctx, oid);
 		if (rc != 0) {
 			CH_ERR(sc, "failed to allocate ctrlq%d: %d\n", idx, rc);
@@ -4595,7 +4596,7 @@ alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx)
 		iqidx = vi->first_rxq + (idx % vi->nrxq);
 		snprintf(name, sizeof(name), "%s txq%d",
 		    device_get_nameunit(vi->dev), idx);
-		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan,
+		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->port_id,
 		    &sc->sge.rxq[iqidx].iq, name);
 
 		rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx,
@@ -4812,11 +4813,11 @@ alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)
 		    device_get_nameunit(vi->dev), idx);
 		if (vi->nofldrxq > 0) {
 			iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq);
-			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
+			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->port_id,
 			    &sc->sge.ofld_rxq[iqidx].iq, name);
 		} else {
 			iqidx = vi->first_rxq + (idx % vi->nrxq);
-			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
+			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->port_id,
 			    &sc->sge.rxq[iqidx].iq, name);
 		}
 
@@ -6329,7 +6330,7 @@ handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
 	struct sge_wrq *wrq = (void *)eq;
 
 	atomic_readandclear_int(&eq->equiq);
-	taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
+	taskqueue_enqueue(sc->tq[eq->port_id], &wrq->wrq_tx_task);
 }
 
 static void
@@ -6341,7 +6342,7 @@ handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
 
 	atomic_readandclear_int(&eq->equiq);
 	if (mp_ring_is_idle(txq->r))
-		taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
+		taskqueue_enqueue(sc->tq[eq->port_id], &txq->tx_reclaim_task);
 	else
 		mp_ring_check_drainage(txq->r, 64);
 }