aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--sys/conf/files1
-rw-r--r--sys/dev/bnxt/bnxt_en/bnxt.h28
-rw-r--r--sys/dev/bnxt/bnxt_en/bnxt_hwrm.c54
-rw-r--r--sys/dev/bnxt/bnxt_en/bnxt_mpc.c1075
-rw-r--r--sys/dev/bnxt/bnxt_en/bnxt_mpc.h182
-rw-r--r--sys/dev/bnxt/bnxt_en/bnxt_ulp.c69
-rw-r--r--sys/dev/bnxt/bnxt_en/if_bnxt.c54
-rw-r--r--sys/modules/bnxt/bnxt_en/Makefile1
8 files changed, 1450 insertions, 14 deletions
diff --git a/sys/conf/files b/sys/conf/files
index 73a5d5fa8408..0a6e5874896f 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1333,6 +1333,7 @@ dev/bnxt/bnxt_en/bnxt_log.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_log_data.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci compile-with "${BNXT_C}"
+dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci compile-with "${BNXT_C}"
dev/bwi/bwimac.c optional bwi
dev/bwi/bwiphy.c optional bwi
diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h
index 80cc8905d66b..1de100915f97 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt.h
+++ b/sys/dev/bnxt/bnxt_en/bnxt.h
@@ -608,6 +608,11 @@ struct tx_bd_opaque {
#define TX_RING_MASK(txr) ((txr)->ring_size - 1)
+struct bnxt_sw_mpc_tx_bd {
+ uint8_t inline_bds;
+ unsigned long handle;
+};
+
struct bnxt_ring {
uint64_t paddr;
vm_offset_t doorbell;
@@ -618,6 +623,15 @@ struct bnxt_ring {
uint16_t phys_id;
uint16_t idx;
uint16_t running_bds; /* host TX-completion-coalescing accumulator */
+ uint8_t queue_id; /* CoS queue, or BNXT_MPC_QUEUE_ID below */
+#define BNXT_MPC_QUEUE_ID 0xff
+ uint8_t mpc_chnl_type; /* valid when queue_id == BNXT_MPC_QUEUE_ID */
+ struct iflib_dma_info ring_mem; /* manual DMA alloc for driver-private rings (MPC) */
+ struct bnxt_cp_ring *cp_ring; /* completion ring this TX ring reports to (MPC) */
+ uint16_t prod; /* MPC TX ring producer index */
+ uint16_t cons; /* MPC TX ring consumer index */
+ struct mtx tx_lock; /* MPC TX ring lock */
+ struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
struct bnxt_full_tpa_start *tpa_start;
union {
u64 db_key64;
@@ -649,6 +663,10 @@ struct bnxt_cp_ring {
uint8_t type;
#define Q_TYPE_TX 1
#define Q_TYPE_RX 2
+#define TX_CP_NQ 3 /* MPC-private NQ ring (bnxt_mpc.c) */
+ int msix_vec; /* MPC-private ring: bus_setup_intr vector */
+ struct resource *irq_res; /* MPC-private ring: bus_setup_intr resource */
+ void *irq_cookie; /* MPC-private ring: bus_setup_intr cookie */
};
struct bnxt_full_tpa_start {
@@ -718,6 +736,10 @@ struct bnxt_func_qcfg {
uint16_t alloc_completion_rings;
uint16_t alloc_tx_rings;
uint16_t alloc_rx_rings;
+ /* Firmware-granted counts before the MPC reservation is subtracted;
+ * bnxt_set_dflt_mpc_rings() needs these unreduced values. */
+ uint16_t orig_alloc_completion_rings;
+ uint16_t orig_alloc_tx_rings;
uint16_t alloc_vnics;
uint16_t alloc_rss_ctx;
uint16_t alloc_l2_ctx;
@@ -1435,6 +1457,8 @@ struct bnxt_softc {
struct bnxt_ptp_cfg *ptp_cfg;
bool rx_ts_enabled;
+ struct bnxt_mpc_info *mpc_info;
+
#define BNXT_CAGR_CQCOAL_OFFSET 0xc00
#define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000
#define BNXT_CAGR_TICK_RES_DEFAULT 0x2
@@ -1523,6 +1547,10 @@ int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc *softc,
uint32_t mem_size, uint8_t depth,
struct bnxt_ctx_mem_type *ctxm);
int bnxt_hwrm_ptp_qcfg(struct bnxt_softc *bp);
+int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count);
+void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
+void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
+ uint32_t ring_type);
static inline u32
readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
index 2649100e9b86..fb549bc05c10 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c
@@ -32,6 +32,7 @@
#include "bnxt.h"
#include "bnxt_hwrm.h"
+#include "bnxt_mpc.h"
#include "hsi_struct_def.h"
#include "bnxt_coredump.h"
@@ -1205,6 +1206,7 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
(void *)softc->hwrm_cmd_resp.idi_vaddr;
struct bnxt_func_info *func = &softc->func;
uint32_t flags, flags_ext, flags_ext2;
+ uint8_t mpc_chnls_cap = 0;
bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
req.fid = htole16(0xffff);
@@ -1354,8 +1356,18 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
}
}
+ /*
+ * Only extract the field needed here; the actual alloc/free work
+ * (malloc(M_NOWAIT)/free() in bnxt_alloc_mpc_info()) happens after
+ * BNXT_HWRM_UNLOCK() below instead of under hwrm_lock.
+ */
+ if (BNXT_PF(softc))
+ mpc_chnls_cap = resp->mpc_chnls_cap;
+
fail:
BNXT_HWRM_UNLOCK(softc);
+ if (!rc && BNXT_PF(softc))
+ bnxt_alloc_mpc_info(softc, mpc_chnls_cap);
return rc;
}
@@ -1468,6 +1480,22 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
fn_qcfg->alloc_stat_ctx = le16toh(resp->alloc_stat_ctx);
fn_qcfg->alloc_msix = le16toh(resp->alloc_msix);
+ fn_qcfg->orig_alloc_completion_rings = fn_qcfg->alloc_completion_rings;
+ fn_qcfg->orig_alloc_tx_rings = fn_qcfg->alloc_tx_rings;
+
+ /* Reserve minimum rings for MPC */
+ if (BNXT_PF(softc)) {
+ if (softc->mpc_info) {
+ uint16_t mpc_min = BNXT_MIN_MPC_TCE + BNXT_MIN_MPC_RCE;
+
+ /* Clamp to 0 instead of underflowing these unsigned counts. */
+ fn_qcfg->alloc_completion_rings -=
+ min(fn_qcfg->alloc_completion_rings, mpc_min);
+ fn_qcfg->alloc_tx_rings -=
+ min(fn_qcfg->alloc_tx_rings, mpc_min);
+ }
+ }
+
switch (resp->port_partition_type) {
case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0:
case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2:
@@ -1976,17 +2004,24 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
switch (type) {
case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX:
- cp_ring = &softc->tx_cp_rings[idx];
+ if (ring->queue_id == BNXT_MPC_QUEUE_ID) {
+ cp_ring = ring->cp_ring;
+ req.mpc_chnls_type = ring->mpc_chnl_type;
+ req.enables |= htole32(HWRM_RING_ALLOC_INPUT_ENABLES_MPC_CHNLS_TYPE);
+ } else {
+ cp_ring = &softc->tx_cp_rings[idx];
- req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
- /* queue_id - what CoS queue the TX ring is associated with */
- req.queue_id = htole16(softc->tx_q_info[0].queue_id);
+ /* queue_id - what CoS queue the TX ring is associated with */
+ req.queue_id = htole16(softc->tx_q_info[0].queue_id);
+ req.enables |= htole32(
+ HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
+ }
req.stat_ctx_id = htole32(cp_ring->stats_ctx_id);
- req.enables |= htole32(
- HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
- if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL)
+ if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL &&
+ ring->queue_id != BNXT_MPC_QUEUE_ID)
req.cmpl_coal_cnt = softc->tx_hw_coal_cnt;
+ req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
break;
case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX:
if (!BNXT_CHIP_P5_PLUS(softc))
@@ -2023,7 +2058,10 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
}
req.cq_handle = htole64(ring->id);
- req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
+ if (ring->queue_id == BNXT_MPC_QUEUE_ID)
+ req.nq_ring_id = htole16(softc->mpc_info->mpc_nq_rings[idx].ring.phys_id);
+ else
+ req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
req.enables |= htole32(
HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID);
break;
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.c b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
new file mode 100644
index 000000000000..4e35b3bdca9c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c
@@ -0,0 +1,1075 @@
+/*
+ * Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/time.h>
+#include <machine/atomic.h>
+#include <machine/resource.h>
+#include <dev/pci/pcivar.h>
+
+#include "bnxt.h"
+#include "bnxt_mpc.h"
+#include "bnxt_hwrm.h"
+#include "bnxt_log.h"
+
+/*
+ * bnxt_ktls_mpc_cmp() is defined in bnxt_ktls.c, not yet part of this
+ * branch; forward-declared here so this file builds standalone.
+ */
+void bnxt_ktls_mpc_cmp(struct bnxt_softc *softc, uint32_t client,
+ unsigned long handle, struct bnxt_cmpl_entry *cmpl_entry_arr,
+ uint32_t cmpl_num);
+
+void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap)
+{
+ if (mpc_chnls_cap) {
+ if (!softc->mpc_info)
+ softc->mpc_info = malloc(sizeof(*softc->mpc_info),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ } else {
+ bnxt_free_mpc_info(softc);
+ }
+ if (softc->mpc_info)
+ softc->mpc_info->mpc_chnls_cap = mpc_chnls_cap;
+}
+
+void bnxt_free_mpc_info(struct bnxt_softc *softc)
+{
+ if (softc->mpc_info) {
+ free(softc->mpc_info, M_DEVBUF);
+ softc->mpc_info = NULL;
+ }
+}
+
+static void __bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc,
+ enum bnxt_mpc_type type, int *avail, int avail_cp)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int dflt1, dflt2;
+ int idx1, idx2;
+ int min1, min2;
+ int val1, val2;
+
+ if (type == BNXT_MPC_CRYPTO) {
+ min1 = BNXT_MIN_MPC_TCE;
+ min2 = BNXT_MIN_MPC_RCE;
+ dflt1 = BNXT_DFLT_MPC_TCE;
+ dflt2 = BNXT_DFLT_MPC_RCE;
+ idx1 = BNXT_MPC_TCE_TYPE;
+ idx2 = BNXT_MPC_RCE_TYPE;
+ } else {
+ min1 = BNXT_MIN_MPC_TE_CFA;
+ min2 = BNXT_MIN_MPC_RE_CFA;
+ dflt1 = BNXT_DFLT_MPC_TE_CFA;
+ dflt2 = BNXT_DFLT_MPC_RE_CFA;
+ idx1 = BNXT_MPC_TE_CFA_TYPE;
+ idx2 = BNXT_MPC_RE_CFA_TYPE;
+ }
+ if (*avail < (min1 + min2))
+ return;
+
+ val1 = min(*avail / 2, softc->ntxqsets);
+ val2 = val1;
+
+ val1 = min(val1, dflt1);
+ val2 = min(val2, dflt2);
+
+ if (avail_cp < min1 || avail_cp < min2)
+ return;
+
+ val1 = min(val1, avail_cp);
+ val2 = min(val2, avail_cp);
+
+ mpc->mpc_ring_count[idx1] = val1;
+ mpc->mpc_ring_count[idx2] = val2;
+
+ *avail = *avail - val1 - val2;
+}
+
+void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int avail, mpc_cp, i;
+ int avail_cp;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++)
+ mpc->mpc_ring_count[i] = 0;
+ mpc->mpc_cp_rings_count = 0;
+
+ avail = fn_qcfg->orig_alloc_tx_rings - softc->ntxqsets;
+
+ avail_cp = fn_qcfg->orig_alloc_completion_rings - softc->ntxqsets;
+ avail_cp = min(avail_cp,
+ ((pci_msix_count(softc->dev) - 1) - softc->ntxqsets));
+
+ if (BNXT_MPC_CRYPTO_CAPABLE(softc))
+ __bnxt_set_dflt_mpc_rings(softc, BNXT_MPC_CRYPTO, &avail,
+ avail_cp);
+
+ for (i = 0, mpc_cp = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ if (mpc_cp < mpc->mpc_ring_count[i])
+ mpc_cp = mpc->mpc_ring_count[i];
+ }
+ mpc->mpc_cp_rings_count = mpc_cp;
+}
+
+int bnxt_alloc_mpcs(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i;
+
+ if (!BNXT_MPC_CRYPTO_CAPABLE(softc))
+ return 0;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num;
+ struct bnxt_ring *txr;
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ num = mpc->mpc_ring_count[i];
+ if (!num)
+ continue;
+ txr = malloc(num * sizeof(*txr), M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!txr) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC rings\n",
+ __func__, num);
+ return ENOMEM;
+ }
+ mpc->mpc_rings[i] = txr;
+ }
+
+ mpc->mpc_cp_rings =
+ malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!mpc->mpc_cp_rings) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC completion rings\n",
+ __func__, mpc->mpc_cp_rings_count);
+ return ENOMEM;
+ }
+
+ mpc->mpc_nq_rings =
+ malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!mpc->mpc_nq_rings) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate %d MPC notification rings\n",
+ __func__, mpc->mpc_cp_rings_count);
+ return ENOMEM;
+ }
+ return 0;
+}
+
+void bnxt_free_mpcs(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ if (mpc->mpc_rings[i]) {
+ free(mpc->mpc_rings[i], M_DEVBUF);
+ mpc->mpc_rings[i] = NULL;
+ }
+ }
+
+ if (mpc->mpc_cp_rings) {
+ free(mpc->mpc_cp_rings, M_DEVBUF);
+ mpc->mpc_cp_rings = NULL;
+ }
+
+ if (mpc->mpc_nq_rings) {
+ free(mpc->mpc_nq_rings, M_DEVBUF);
+ mpc->mpc_nq_rings = NULL;
+ }
+}
+
+int bnxt_alloc_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, mpc_idx = 0;
+ int mpc_base_ring_id =
+ (softc->scctx->isc_nrxqsets * 2) + 1 + softc->ntxqsets;
+
+ if (!mpc)
+ return 0;
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+ int rc;
+
+ cpr->ring.queue_id = BNXT_MPC_QUEUE_ID;
+
+ /* Set up the completion ring */
+ cpr->stats_ctx_id = HWRM_NA_SIGNATURE;
+ cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ cpr->ring.softc = softc;
+ cpr->ring.idx = i;
+ cpr->ring.id = mpc_base_ring_id + i;
+ cpr->ring.doorbell = softc->legacy_db_size;
+ cpr->ring.ring_size = softc->scctx->isc_ntxd[0];
+ cpr->ring.db_ring_mask = cpr->ring.ring_size - 1;
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct cmpl_base) * cpr->ring.ring_size,
+ &cpr->ring.ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for completion ring %d (size=%zu), rc=%d\n",
+ __func__, i, sizeof(struct cmpl_base) * cpr->ring.ring_size, rc);
+ return rc;
+ }
+
+ cpr->ring.vaddr = cpr->ring.ring_mem.idi_vaddr;
+ cpr->ring.paddr = cpr->ring.ring_mem.idi_paddr;
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i], rc;
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ /* tx_stats[] is indexed across all MPC types, so use
+ * the flat mpc_idx, not the intra-type counter j. */
+ int stat_idx = mpc_idx;
+
+ txr->queue_id = BNXT_MPC_QUEUE_ID;
+ txr->mpc_chnl_type = i;
+
+ /* alloc mpc TX ring */
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ txr->softc = softc;
+ txr->idx = i << BNXT_RING_MPC_IDX_TYPE_SFT | j;
+ txr->id = mpc_base_ring_id + mpc_idx++;
+ txr->doorbell = softc->legacy_db_size;
+ txr->ring_size = softc->scctx->isc_ntxd[1];
+ txr->db_ring_mask = txr->ring_size - 1;
+
+ /* Init before any early return: bnxt_free_mpc_rings()
+ * unconditionally mtx_destroy()s every txr. */
+ mtx_init(&txr->tx_lock, "MPC TX lock", NULL, MTX_SPIN);
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct tx_bd_short) * txr->ring_size,
+ &txr->ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for TX ring %d (size=%zu), rc=%d\n",
+ __func__, j, sizeof(struct tx_bd_short) * txr->ring_size, rc);
+ return rc;
+ }
+
+ txr->vaddr = txr->ring_mem.idi_vaddr;
+ txr->paddr = txr->ring_mem.idi_paddr;
+
+ txr->tx_mpc_buf_ring =
+ malloc(txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd),
+ M_DEVBUF, M_NOWAIT | M_ZERO);
+ if (!txr->tx_mpc_buf_ring) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate buffer ring for TX ring %d (size=%zu)\n",
+ __func__, j, txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd));
+ return ENOMEM;
+ }
+
+ rc = iflib_dma_alloc(softc->ctx, sizeof(struct ctx_hw_stats),
+ &mpc->tx_stats[stat_idx], 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for TX stats ring %d, rc=%d\n",
+ __func__, j, rc);
+ return rc;
+ }
+ bus_dmamap_sync(mpc->tx_stats[stat_idx].idi_tag,
+ mpc->tx_stats[stat_idx].idi_map, BUS_DMASYNC_PREREAD);
+ }
+ }
+
+ /* Set up the Notification ring (NQ) */
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+ int rc;
+
+ nqr->ring.queue_id = BNXT_MPC_QUEUE_ID;
+
+ nqr->stats_ctx_id = HWRM_NA_SIGNATURE;
+ nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ nqr->ring.softc = softc;
+ nqr->ring.idx = i;
+ nqr->ring.id = softc->ntxqsets + i;
+ nqr->ring.doorbell = softc->legacy_db_size;
+ nqr->ring.ring_size = softc->scctx->isc_ntxd[2];
+ nqr->ring.db_ring_mask = nqr->ring.ring_size - 1;
+
+ rc = iflib_dma_alloc(softc->ctx,
+ sizeof(struct cmpl_base) * nqr->ring.ring_size,
+ &nqr->ring.ring_mem, 0);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate DMA memory for notification ring %d (size=%zu), rc=%d\n",
+ __func__, i, sizeof(struct cmpl_base) * nqr->ring.ring_size, rc);
+ return rc;
+ }
+
+ nqr->ring.vaddr = nqr->ring.ring_mem.idi_vaddr;
+ nqr->ring.paddr = nqr->ring.ring_mem.idi_paddr;
+ nqr->type = TX_CP_NQ;
+ }
+
+ return 0;
+}
+
+void bnxt_free_mpc_rings(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, mpc_idx = 0;
+
+ if (!mpc)
+ return;
+
+ /* A partial alloc failure can leave these NULL; guard each array. */
+ if (mpc->mpc_nq_rings) {
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++)
+ iflib_dma_free(&mpc->mpc_nq_rings[i].ring.ring_mem);
+ }
+
+ if (mpc->mpc_cp_rings) {
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++)
+ iflib_dma_free(&mpc->mpc_cp_rings[i].ring.ring_mem);
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ /*
+ * Same partial-allocation case as above: mpc_ring_count[i] can
+ * be nonzero while mpc_rings[i] itself failed to allocate.
+ */
+ if (!mpc->mpc_rings[i])
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ /* TCE ring vaddr holds cleartext AES-GCM keys; scrub
+ * before freeing the pages. */
+ if (i == BNXT_MPC_TCE_TYPE && txr->ring_mem.idi_vaddr)
+ explicit_bzero(txr->ring_mem.idi_vaddr,
+ txr->ring_mem.idi_size);
+
+ iflib_dma_free(&txr->ring_mem);
+ if (txr->tx_mpc_buf_ring) {
+ free(txr->tx_mpc_buf_ring, M_DEVBUF);
+ txr->tx_mpc_buf_ring = NULL;
+ }
+ /* A ring past a partial alloc failure never had its
+ * tx_lock mtx_init()'d; guard the destroy. */
+ if (mtx_initialized(&txr->tx_lock))
+ mtx_destroy(&txr->tx_lock);
+ /* Must match the flat stat_idx bnxt_alloc_mpc_rings() used. */
+ iflib_dma_free(&mpc->tx_stats[mpc_idx++]);
+ }
+ }
+}
+
+void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j;
+
+ if (!mpc)
+ return;
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ mpc->mpc_nq_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ mpc->mpc_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ mpc->mpc_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE;
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ }
+}
+
+static void bnxt_map_tx_to_cp(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int cp_count = mpc->mpc_cp_rings_count;
+ int i, j;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ int cp_idx = j % cp_count;
+
+ txr->cp_ring = &mpc->mpc_cp_rings[cp_idx];
+ }
+ }
+}
+
+struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc)
+{
+ int idx;
+
+ /* Callers should gate on a non-zero TCE ring count already, but
+ * guard the divide/index here too since this feeds a live TX path. */
+ if (!mpc || !mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE])
+ return NULL;
+
+ idx = PCPU_GET(cpuid) % mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE];
+
+ return &mpc->mpc_rings[BNXT_MPC_TCE_TYPE][idx];
+}
+
+int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, rc;
+
+ if (!mpc)
+ return 0;
+
+ bnxt_map_tx_to_cp(softc);
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+
+ nqr->cons = 0;
+ nqr->v_bit = 1;
+ nqr->raw_cons = 0;
+ nqr->toggle = 0;
+ nqr->last_idx = UINT32_MAX;
+ bnxt_mark_cpr_invalid(nqr);
+
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring);
+
+ bnxt_set_db_mask(softc, &nqr->ring,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM notification ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ softc->db_ops.bnxt_db_nq(nqr, 1);
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+
+ rc = bnxt_hwrm_stat_ctx_alloc(softc, cpr,
+ mpc->tx_stats[i].idi_paddr);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM stats context for completion ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ cpr->v_bit = 1;
+ cpr->cons = 0;
+ cpr->raw_cons = 0;
+ cpr->toggle = 0;
+ bnxt_mark_cpr_invalid(cpr);
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &cpr->ring);
+ bnxt_set_db_mask(softc, &cpr->ring,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM completion ring %d, rc=%d\n",
+ __func__, i, rc);
+ return rc;
+ }
+
+ softc->db_ops.bnxt_db_tx_cq(cpr, 1);
+ }
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+
+ txr->prod = 0;
+ txr->cons = 0;
+ memset(txr->tx_mpc_buf_ring, 0,
+ txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd));
+
+ rc = bnxt_hwrm_ring_alloc(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr);
+ bnxt_set_db_mask(softc, txr,
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_TX);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to allocate HWRM TX ring %d, rc=%d\n",
+ __func__, j, rc);
+ return rc;
+ }
+ softc->db_ops.bnxt_db_tx(txr, 0);
+ }
+ }
+ return 0;
+}
+
+int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int i, j, rc;
+
+ if (!mpc)
+ return 0;
+
+ for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) {
+ int num = mpc->mpc_ring_count[i];
+
+ /* Currently only KTLS TX is supported */
+ if (i != BNXT_MPC_TCE_TYPE)
+ continue;
+
+ for (j = 0; j < num; j++) {
+ struct bnxt_ring *txr = &mpc->mpc_rings[i][j];
+ struct bnxt_cp_ring *cpr = txr->cp_ring;
+
+ if (!cpr)
+ continue;
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr,
+ cpr->ring.phys_id);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM TX ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, j, txr->id, cpr->ring.phys_id, rc);
+ return rc;
+ }
+ txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i];
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
+ &cpr->ring, (uint16_t)HWRM_NA_SIGNATURE);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM completion ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, i, cpr->ring.id, cpr->ring.phys_id, rc);
+ return rc;
+ }
+
+ rc = bnxt_hwrm_stat_ctx_free(softc, cpr);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM stats context id %d for completion ring %d, rc=%d\n",
+ __func__, cpr->stats_ctx_id, i, rc);
+ return rc;
+ }
+
+ cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+
+ for (i = 0; i < mpc->mpc_cp_rings_count; i++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i];
+
+ rc = bnxt_hwrm_ring_free(
+ softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring,
+ (uint16_t)HWRM_NA_SIGNATURE);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to free HWRM notification ring %d (ring_id=%d, phys_id=%d), rc=%d\n",
+ __func__, i, nqr->ring.id, nqr->ring.phys_id, rc);
+ return rc;
+ }
+ nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
+ }
+ return 0;
+}
+
+static void bnxt_mpc_process_cpr(struct bnxt_cp_ring *cpr)
+{
+ struct bnxt_softc *softc = cpr->ring.softc;
+
+ /* Handle completions on the mpc completion ring */
+ uint32_t raw_cons = cpr->raw_cons;
+ uint32_t cons = cpr->cons;
+ struct cmpl_base *cmpl;
+ bool v_bit = cpr->v_bit;
+ bool last_v_bit;
+ uint32_t last_cons;
+ uint16_t type;
+
+ for (;;) {
+ last_cons = cons;
+ last_v_bit = v_bit;
+ cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons];
+
+ if (!CMP_VALID(cmpl, v_bit))
+ break;
+
+ type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK;
+ switch (type) {
+ case CMPL_BASE_TYPE_MID_PATH_SHORT:
+ case CMPL_BASE_TYPE_MID_PATH_LONG: {
+ int rc = bnxt_mpc_cmp(softc, cpr, &raw_cons);
+
+ /* EBUSY: 2nd half of a MID_PATH_LONG completion hasn't
+ * landed yet; stop and retry later. Any other error is
+ * non-transient and already logged/skipped, so fall
+ * through and advance like the success case. */
+ if (rc == EBUSY)
+ goto done;
+ break;
+ }
+ case CMPL_BASE_TYPE_HWRM_DONE:
+ /* ignore this completion */
+ break;
+ default: {
+ /* Rate-limit prints so a stuck ring feeding
+ * unrecognized types can't flood the console.
+ * Statics are unlocked; a best-effort limiter. */
+ static struct timeval last;
+ static int curpps;
+
+ if (ppsratecheck(&last, &curpps, 5))
+ device_printf(softc->dev,
+ "%s: Unknown completion type %u on ring %d (raw_cons=%u, cons=%u)\n",
+ __func__, type, cpr->ring.idx, raw_cons, cons);
+ break;
+ }
+ }
+ raw_cons++;
+ /* Recompute from raw_cons rather than NEXT_CP_CONS_V(): a
+ * MID_PATH_LONG completion can wrap raw_cons inside
+ * bnxt_mpc_cmp(), which would desync v_bit otherwise. */
+ cons = MPC_RING_CMP(cpr, raw_cons);
+ v_bit = !(raw_cons & cpr->ring.ring_size);
+ }
+done:
+ cpr->cons = last_cons;
+ cpr->raw_cons = raw_cons;
+ cpr->v_bit = last_v_bit;
+ softc->db_ops.bnxt_db_tx_cq(cpr, 1);
+}
+
+static void bnxt_mpc_irq(void *arg)
+{
+ struct bnxt_cp_ring *cpr = arg;
+ struct bnxt_softc *softc = cpr->ring.softc;
+ nq_cn_t *cmp = (nq_cn_t *)cpr->ring.vaddr;
+ uint32_t raw_cons = cpr->raw_cons;
+ uint16_t nq_type, nqe_cnt = 0;
+ uint32_t cons = cpr->cons;
+ bool v_bit = cpr->v_bit;
+
+ while (1) {
+ if (!NQ_VALID(&cmp[cons], v_bit))
+ goto done;
+
+ nq_type = NQ_CN_TYPE_MASK & cmp[cons].type;
+
+ if (NQE_CN_TYPE(nq_type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
+ struct bnxt_cp_ring *cpr2 =
+ &softc->mpc_info->mpc_cp_rings[cpr->ring.idx];
+ cpr2->toggle = NQE_CN_TOGGLE(cmp[cons].type);
+ bnxt_mpc_process_cpr(cpr2);
+ }
+
+ NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
+ raw_cons++;
+ nqe_cnt++;
+ }
+done:
+ if (nqe_cnt) {
+ cpr->cons = cons;
+ cpr->raw_cons = raw_cons;
+ cpr->v_bit = v_bit;
+ }
+
+ softc->db_ops.bnxt_db_nq(cpr, 1);
+}
+
+int bnxt_mpc_irq_setup(struct bnxt_softc *softc)
+{
+ int mpc_base_idx = softc->scctx->isc_ntxqsets;
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int mpc_cp_rings_count = 0;
+ int rc, i, j;
+
+ if (mpc)
+ mpc_cp_rings_count = mpc->mpc_cp_rings_count;
+
+ rc = bnxt_populate_irq(softc, mpc_cp_rings_count);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to populate IRQs for %d MPC completion rings, rc=%d\n",
+ __func__, mpc_cp_rings_count, rc);
+ return rc;
+ }
+
+ for (i = mpc_base_idx, j = 0; j < mpc_cp_rings_count; i++, j++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[j];
+ struct resource *irq_res;
+ int rid;
+
+ nqr->msix_vec = softc->irq_tbl[i].vector;
+
+ rid = 1 + i;
+ irq_res = bus_alloc_resource_any(softc->dev, SYS_RES_IRQ, &rid,
+ RF_ACTIVE);
+ if (irq_res == NULL) {
+ /* Release any IRQs already set up for j' < j. */
+ bnxt_mpc_irq_cleanup(softc);
+ return ENOMEM;
+ }
+
+ nqr->irq_res = irq_res;
+
+ rc = bus_setup_intr(softc->dev, irq_res, INTR_TYPE_NET | INTR_MPSAFE,
+ NULL, bnxt_mpc_irq, nqr, &nqr->irq_cookie);
+ if (rc) {
+ device_printf(softc->dev,
+ "%s: Failed to setup IRQ for msix_vec %d, MPC notification ring %d, rc=%d\n",
+ __func__, nqr->msix_vec, j, rc);
+ bus_release_resource(softc->dev, SYS_RES_IRQ, rid, irq_res);
+ nqr->irq_res = NULL;
+ /* Release any IRQs already set up for j' < j. */
+ bnxt_mpc_irq_cleanup(softc);
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+void bnxt_mpc_irq_cleanup(struct bnxt_softc *softc)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ int j;
+
+ if (!mpc)
+ return;
+
+ for (j = 0; j < mpc->mpc_cp_rings_count; j++) {
+ struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[j];
+ int rid;
+
+ if (nqr->irq_res != NULL) {
+ bus_teardown_intr(softc->dev, nqr->irq_res, nqr->irq_cookie);
+ nqr->irq_cookie = NULL;
+ rid = rman_get_rid(nqr->irq_res);
+ bus_release_resource(softc->dev, SYS_RES_IRQ, rid,
+ nqr->irq_res);
+ nqr->irq_res = NULL;
+ }
+ }
+}
+
+static inline uint32_t bnxt_tx_avail(struct bnxt_softc *softc,
+ struct bnxt_ring *txr)
+{
+ uint32_t used, prod, cons;
+
+ prod = atomic_load_acq_16(&txr->prod);
+ cons = atomic_load_acq_16(&txr->cons);
+
+ if (prod >= cons)
+ used = prod - cons;
+ else {
+ used = (txr->ring_size - cons) + prod;
+ }
+
+ return (txr->ring_size - 1 - used);
+}
+
+int bnxt_start_xmit_mpc(struct bnxt_softc *softc, struct bnxt_ring *txr,
+ void *data, uint len, unsigned long handle)
+{
+ uint32_t bds, total_bds, bd_space, free_size;
+ struct bnxt_sw_mpc_tx_bd *tx_buf;
+ struct tx_bd_long *txbd;
+ uint16_t prod;
+
+ /* Caller must hold txr->tx_lock across this sequence; assert rather
+ * than acquire since bnxt_xmit_crypto_cmd() needs it held longer. */
+ mtx_assert(&txr->tx_lock, MA_OWNED);
+
+ bds = howmany(len, sizeof(*txbd));
+ total_bds = bds + 1;
+ free_size = bnxt_tx_avail(softc, txr);
+ if (free_size < total_bds)
+ return EBUSY;
+
+ prod = txr->prod;
+ txbd = &((struct tx_bd_long *)txr->vaddr)[prod];
+
+ tx_buf = &txr->tx_mpc_buf_ring[prod];
+ tx_buf->handle = handle;
+ tx_buf->inline_bds = total_bds;
+
+ txbd->flags_type =
+ htole16(TX_BD_MP_CMD_TYPE_TX_BD_MP_CMD |
+ (total_bds << TX_BD_LONG_FLAGS_BD_CNT_SFT));
+ txbd->len = htole16(len);
+ txbd->opaque = SET_TX_OPAQUE(txr, prod, total_bds);
+
+ prod = NEXT_TX(txr, prod);
+ txbd = &((struct tx_bd_long *)txr->vaddr)[prod];
+
+ bd_space = txr->ring_size - prod;
+ if (bd_space < bds) {
+ uint32_t len0 = bd_space * sizeof(*txbd);
+
+ memcpy(txbd, data, len0);
+ prod = ADVANCE_TX_BY(txr, prod, bd_space);
+ txbd = &((struct tx_bd_long *)txr->vaddr)[prod];
+ bds -= bd_space;
+ len -= len0;
+ data = (uint8_t *)data + len0;
+ }
+ memcpy(txbd, data, len);
+ prod = ADVANCE_TX_BY(txr, prod, bds);
+ /* Release-store pairs with the atomic_load_acq_16(&txr->prod) in
+ * bnxt_tx_avail() and bnxt_adv_mpc_cons(). */
+ atomic_store_rel_16(&txr->prod, prod);
+
+ softc->db_ops.bnxt_db_tx(txr, prod);
+
+ return 0;
+}
+
+static bool bnxt_mpc_unsolicit(struct mpc_cmp *mpcmp)
+{
+ uint32_t client = MPC_CMP_CLIENT_TYPE(mpcmp);
+
+ if (client != MPC_CMP_CLIENT_TCE && client != MPC_CMP_CLIENT_RCE &&
+ client != MPC_CMP_CLIENT_TE_CFA && client != MPC_CMP_CLIENT_RE_CFA)
+ return false;
+ return MPC_CMP_UNSOLICIT_SUBTYPE(mpcmp);
+}
+
+static void bnxt_adv_mpc_cons(struct bnxt_softc *softc, struct bnxt_ring *txr)
+{
+ struct tx_bd_long *bd_base = (struct tx_bd_long *)txr->vaddr;
+ struct bnxt_sw_mpc_tx_bd *mpc_buf;
+ uint16_t tx_cons = txr->cons;
+ uint16_t tail_bds;
+ uint8_t inline_bds;
+
+ mpc_buf = &txr->tx_mpc_buf_ring[tx_cons];
+ do {
+ inline_bds = mpc_buf->inline_bds;
+ /* Scrub the inline-BD region (held CE_ADD command +
+ * AES-GCM key) before the slot is reused. */
+ if (inline_bds > 0) {
+ tail_bds = txr->ring_size - tx_cons;
+ if (tail_bds >= inline_bds) {
+ explicit_bzero(&bd_base[tx_cons],
+ inline_bds * sizeof(*bd_base));
+ } else {
+ explicit_bzero(&bd_base[tx_cons],
+ tail_bds * sizeof(*bd_base));
+ explicit_bzero(&bd_base[0],
+ (inline_bds - tail_bds) * sizeof(*bd_base));
+ }
+ }
+ tx_cons = ADVANCE_TX_BY(txr, tx_cons, mpc_buf->inline_bds);
+ /* Pairs with bnxt_tx_avail()'s atomic_load_acq_16(&txr->cons). */
+ atomic_store_rel_16(&txr->cons, tx_cons);
+ if (tx_cons == atomic_load_acq_16(&txr->prod))
+ break;
+ mpc_buf = &txr->tx_mpc_buf_ring[tx_cons];
+ } while (mpc_buf->handle == BNXT_INV_MPC_HDL);
+}
+
+int bnxt_mpc_cmp(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr,
+ uint32_t *raw_cons)
+{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
+ struct bnxt_cmpl_entry cmpl_entry_arr[2];
+ uint16_t cons = MPC_RING_CMP(cpr, *raw_cons);
+ struct mpc_cmp *mpcmp, *mpcmp1;
+ uint32_t tmp_raw_cons = *raw_cons;
+ unsigned long handle = 0;
+ uint32_t client, cmpl_num;
+ uint8_t type;
+
+ mpcmp = &((struct mpc_cmp *)cpr->ring.vaddr)[cons];
+ type = MPC_CMP_CMP_TYPE(mpcmp);
+ cmpl_entry_arr[0].cmpl = mpcmp;
+ cmpl_entry_arr[0].len = sizeof(*mpcmp);
+ cmpl_num = 1;
+
+ if (type == MPC_CMP_TYPE_MID_PATH_LONG) {
+ tmp_raw_cons = tmp_raw_cons + 1;
+ cons = MPC_RING_CMP(cpr, tmp_raw_cons);
+ mpcmp1 = &((struct mpc_cmp *)cpr->ring.vaddr)[cons];
+ if (!MPC_CMP_VALID(cpr, mpcmp1, tmp_raw_cons)) {
+ /* Expected/transient: caller retries this entry later.
+ * Rate-limit (unlocked statics, best-effort) so a
+ * slow second half can't flood the console. */
+ static struct timeval last;
+ static int curpps;
+
+ if (ppsratecheck(&last, &curpps, 5))
+ device_printf(softc->dev,
+ "%s: Invalid completion entry on ring %d (raw_cons=%u, cons=%u, v_bit=%d)\n",
+ __func__, cpr->ring.idx, tmp_raw_cons, cons, cpr->v_bit);
+ return EBUSY;
+ }
+ /*
+ * The valid test of the entry must be done first before
+ * reading any further.
+ */
+ atomic_thread_fence_acq();
+ if (mpcmp1 == mpcmp + 1) {
+ cmpl_entry_arr[cmpl_num - 1].len += sizeof(*mpcmp1);
+ } else {
+ cmpl_entry_arr[cmpl_num].cmpl = mpcmp1;
+ cmpl_entry_arr[cmpl_num].len = sizeof(*mpcmp1);
+ cmpl_num++;
+ }
+ }
+
+ client = MPC_CMP_CLIENT_TYPE(mpcmp) >> MPC_CMP_CLIENT_SFT;
+
+ if (!bnxt_mpc_unsolicit(mpcmp)) {
+ uint16_t ring_idx, mpc_type;
+ struct bnxt_sw_mpc_tx_bd *mpc_buf;
+ struct bnxt_ring *txr;
+ uint16_t tx_cons, buf_idx;
+ uint32_t opaque;
+
+ opaque = mpcmp->mpc_cmp_opaque;
+ ring_idx = TX_OPAQUE_RING(opaque) & BNXT_RING_MPC_IDX_RING_MASK;
+ mpc_type = TX_OPAQUE_RING(opaque) >> BNXT_RING_MPC_IDX_TYPE_SFT;
+ if (mpc_type != client) {
+ /*
+ * Unlocked statics: see the same note in
+ * bnxt_mpc_process_cpr()'s unknown-completion-type case.
+ */
+ static struct timeval last;
+ static int curpps;
+
+ if (ppsratecheck(&last, &curpps, 5))
+ device_printf(softc->dev,
+ "%s: Wrong opaque 0x%x on completion ring %d, "
+ "expected client type 0x%x, actual type 0x%x (ring_idx=%d)\n",
+ __func__, opaque, cpr->ring.idx, client, mpc_type, ring_idx);
+ /* Corrupt, not pending: advance past both halves so
+ * bnxt_mpc_process_cpr() doesn't spin on it forever. */
+ *raw_cons = tmp_raw_cons;
+ return EINVAL;
+ }
+ if (mpc_type >= BNXT_MPC_TYPE_MAX || mpc->mpc_rings[mpc_type] == NULL) {
+ bnxt_log_live(softc, BNXT_LOGGER_L2,
+ "%s: Invalid mpc_type %d, completion ring %d, opaque=0x%x, client_type=0x%x\n",
+ __func__, mpc_type, cpr->ring.idx, opaque, client);
+ *raw_cons = tmp_raw_cons;
+ return EINVAL;
+ }
+ if (ring_idx >= mpc->mpc_ring_count[mpc_type]) {
+ bnxt_log_live(softc, BNXT_LOGGER_L2,
+ "%s: Invalid ring_idx %d (max=%d), completion ring %d, opaque=0x%x, client_type=0x%x\n",
+ __func__, ring_idx, mpc->mpc_ring_count[mpc_type], cpr->ring.idx, opaque, client);
+ *raw_cons = tmp_raw_cons;
+ return EINVAL;
+ }
+
+ txr = &mpc->mpc_rings[mpc_type][ring_idx];
+ tx_cons = txr->cons;
+ buf_idx = TX_OPAQUE_IDX(opaque);
+ if (buf_idx >= txr->ring_size) {
+ bnxt_log_live(softc, BNXT_LOGGER_L2,
+ "%s: Invalid buf_idx %d (ring_size=%d), completion ring %d, opaque=0x%x\n",
+ __func__, buf_idx, txr->ring_size, cpr->ring.idx, opaque);
+ *raw_cons = tmp_raw_cons;
+ return EINVAL;
+ }
+ mpc_buf = &txr->tx_mpc_buf_ring[buf_idx];
+ handle = mpc_buf->handle;
+ mpc_buf->handle = BNXT_INV_MPC_HDL;
+
+ if (client == BNXT_MPC_TCE_TYPE || client == BNXT_MPC_RCE_TYPE)
+ bnxt_ktls_mpc_cmp(softc, client, handle, cmpl_entry_arr,
+ cmpl_num);
+
+ if (RING_TX(txr, tx_cons) == buf_idx)
+ bnxt_adv_mpc_cons(softc, txr);
+ }
+ *raw_cons = tmp_raw_cons;
+ return 0;
+}
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.h b/sys/dev/bnxt/bnxt_en/bnxt_mpc.h
new file mode 100644
index 000000000000..9e62522afd4d
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.h
@@ -0,0 +1,182 @@
+/*
+ * Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef BNXT_MPC_H
+#define BNXT_MPC_H
+
+#define BNXT_MPC_TCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TCE
+#define BNXT_MPC_RCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RCE
+#define BNXT_MPC_TE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TE_CFA
+#define BNXT_MPC_RE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RE_CFA
+#define BNXT_MPC_TYPE_MAX (BNXT_MPC_RE_CFA_TYPE + 1)
+
+#define BNXT_MAX_MPC 8
+
+#define BNXT_MIN_MPC_TCE 1
+#define BNXT_MIN_MPC_RCE 1
+#define BNXT_DFLT_MPC_TCE BNXT_MAX_MPC
+#define BNXT_DFLT_MPC_RCE BNXT_MAX_MPC
+
+#define BNXT_MIN_MPC_TE_CFA 1
+#define BNXT_MIN_MPC_RE_CFA 1
+#define BNXT_DFLT_MPC_TE_CFA BNXT_MAX_MPC
+#define BNXT_DFLT_MPC_RE_CFA BNXT_MAX_MPC
+
+#define BNXT_MPC_TMO_USECS (1000 * 1000)
+
+struct bnxt_mpc_info {
+ uint8_t mpc_chnls_cap;
+ uint8_t mpc_cp_rings_count;
+ uint8_t mpc_ring_count[BNXT_MPC_TYPE_MAX];
+ struct bnxt_ring *mpc_rings[BNXT_MPC_TYPE_MAX];
+ struct bnxt_cp_ring *mpc_cp_rings;
+ struct bnxt_cp_ring *mpc_nq_rings;
+ struct iflib_dma_info tx_stats[BNXT_MAX_NUM_QUEUES];
+};
+
+#define BNXT_RING_MPC_IDX_TYPE_MASK 0xf0
+#define BNXT_RING_MPC_IDX_TYPE_SFT 4
+#define BNXT_RING_MPC_IDX_RING_MASK 0x0f
+
+enum bnxt_mpc_type {
+ BNXT_MPC_CRYPTO,
+ BNXT_MPC_CFA,
+};
+
+#define BNXT_MPC_CRYPTO_CAP \
+ (HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TCE | \
+ HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RCE)
+
+#define BNXT_MPC_CRYPTO_CAPABLE(softc) \
+ ((softc)->mpc_info ? ((softc)->mpc_info->mpc_chnls_cap & \
+ BNXT_MPC_CRYPTO_CAP) == BNXT_MPC_CRYPTO_CAP \
+ : false)
+
+#define BNXT_MPC_CFA_CAP \
+ (HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TE_CFA | \
+ HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RE_CFA)
+
+#define BNXT_MPC_CFA_CAPABLE(bp) \
+ ((bp)->mpc_info ? ((bp)->mpc_info->mpc_chnls_cap & \
+ BNXT_MPC_CFA_CAP) == BNXT_MPC_CFA_CAP \
+ : false)
+
+#define MPC_CMP_RING_MASK(cpr) ((cpr)->ring.ring_size - 1)
+#define MPC_RING_CMP(cpr, idx) ((idx) & MPC_CMP_RING_MASK(cpr))
+
+#define RING_TX(txr, idx) ((idx) & TX_RING_MASK(txr))
+#define NEXT_TX(txr, idx) ((idx + 1) & TX_RING_MASK(txr))
+#define ADVANCE_TX_BY(txr, idx, count) ((idx + count) & TX_RING_MASK(txr))
+
+#define TX_OPAQUE_IDX_MASK 0x0000ffff
+#define TX_OPAQUE_BDS_MASK 0x00ff0000
+#define TX_OPAQUE_BDS_SHIFT 16
+#define TX_OPAQUE_RING_MASK 0xff000000
+#define TX_OPAQUE_RING_SHIFT 24
+
+#define SET_TX_OPAQUE(txr, prod, bds) \
+ (((txr)->idx << TX_OPAQUE_RING_SHIFT) | \
+ ((bds) << TX_OPAQUE_BDS_SHIFT) | \
+ (((prod) & TX_RING_MASK(txr)) & TX_OPAQUE_IDX_MASK))
+
+#define TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK)
+#define TX_OPAQUE_RING(opq) \
+ (((opq) & TX_OPAQUE_RING_MASK) >> TX_OPAQUE_RING_SHIFT)
+#define TX_OPAQUE_BDS(opq) (((opq) & TX_OPAQUE_BDS_MASK) >> TX_OPAQUE_BDS_SHIFT)
+
+#define BNXT_INV_MPC_HDL (-1UL)
+
+struct bnxt_cmpl_entry {
+ void *cmpl;
+ uint32_t len;
+};
+
+struct mpc_cmp {
+ __le32 mpc_cmp_client_subtype_type;
+#define MPC_CMP_TYPE (0x3f << 0)
+#define MPC_CMP_TYPE_MID_PATH_SHORT 0x1e
+#define MPC_CMP_TYPE_MID_PATH_LONG 0x1f
+
+#define MPC_CMP_SUBTYPE 0xf00
+#define MPC_CMP_SUBTYPE_SFT 8
+#define MPC_CMP_SUBTYPE_SOLICITED (0x0 << 8)
+#define MPC_CMP_SUBTYPE_ERR (0x1 << 8)
+#define MPC_CMP_SUBTYPE_RESYNC (0x2 << 8)
+
+#define MPC_CMP_CLIENT (0xf << 12)
+#define MPC_CMP_CLIENT_SFT 12
+#define MPC_CMP_CLIENT_TCE (0x0 << 12)
+#define MPC_CMP_CLIENT_RCE (0x1 << 12)
+#define MPC_CMP_CLIENT_TE_CFA (0x2 << 12)
+#define MPC_CMP_CLIENT_RE_CFA (0x3 << 12)
+
+ uint32_t mpc_cmp_opaque;
+ __le32 mpc_cmp_v;
+#define MPC_CMP_V (1 << 0)
+ __le32 mpc_cmp_filler;
+};
+
+#define MPC_CMP_CMP_TYPE(mpcmp) \
+ (le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_TYPE)
+
+#define MPC_CMP_CLIENT_TYPE(mpcmp) \
+ (le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_CLIENT)
+
+/*
+ * ERR and RESYNC are both unsolicited (firmware-initiated, no matching TX
+ * descriptor); only SOLICITED completions carry a valid TX opaque to parse.
+ */
+#define MPC_CMP_UNSOLICIT_SUBTYPE(mpcmp) \
+ (((le32toh((mpcmp)->mpc_cmp_client_subtype_type) & \
+ MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_ERR) || \
+ ((le32toh((mpcmp)->mpc_cmp_client_subtype_type) & \
+ MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_RESYNC))
+
+#define MPC_CMP_VALID(cpr, mpcmp, raw_cons) \
+ (!!((mpcmp)->mpc_cmp_v & htole32(MPC_CMP_V)) == \
+ !((raw_cons) & (cpr)->ring.ring_size))
+
+void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap);
+void bnxt_free_mpc_info(struct bnxt_softc *softc);
+void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc);
+int bnxt_alloc_mpcs(struct bnxt_softc *softc);
+void bnxt_free_mpcs(struct bnxt_softc *softc);
+int bnxt_alloc_mpc_rings(struct bnxt_softc *softc);
+void bnxt_free_mpc_rings(struct bnxt_softc *softc);
+int bnxt_mpc_irq_setup(struct bnxt_softc *softc);
+void bnxt_mpc_irq_cleanup(struct bnxt_softc *softc);
+int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc);
+int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc);
+int bnxt_start_xmit_mpc(struct bnxt_softc *softc, struct bnxt_ring *txr,
+ void *data, uint len, unsigned long handle);
+int bnxt_mpc_cmp(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr,
+ uint32_t *raw_cons);
+void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc);
+struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc);
+
+#endif
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
index c38521abd17f..7f25b50bddf4 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c
@@ -426,7 +426,11 @@ void bnxt_destroy_irq(struct bnxt_softc *softc)
kfree(softc->irq_tbl);
}
-static int bnxt_populate_irq(struct bnxt_softc *softc)
+/*
+ * Build the IRQ table from scratch for isc_ntxqsets + irq_count vectors;
+ * usable by any feature needing private IRQs without prior setup.
+ */
+int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count)
{
struct resource_list *rl = NULL;
struct resource_list_entry *rle = NULL;
@@ -434,16 +438,25 @@ static int bnxt_populate_irq(struct bnxt_softc *softc)
struct pci_devinfo *dinfo = NULL;
int i;
- softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT;
+ softc->total_irqs = softc->scctx->isc_ntxqsets + irq_count;
irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL);
if (!irq_tbl) {
device_printf(softc->dev, "Failed to allocate IRQ table\n");
return -1;
}
+
dinfo = device_get_ivars(softc->pdev->dev.bsddev);
rl = &dinfo->resources;
rle = resource_list_find(rl, SYS_RES_IRQ, 1);
+ if (rle == NULL) {
+ device_printf(softc->dev,
+ "No default resources for rid = %d, type = %d\n",
+ 1, SYS_RES_IRQ);
+ kfree(irq_tbl);
+ return -1;
+ }
+
softc->pdev->dev.irq_start = rle->start;
softc->pdev->dev.irq_end = rle->start + softc->total_irqs;
@@ -457,6 +470,56 @@ static int bnxt_populate_irq(struct bnxt_softc *softc)
return 0;
}
+/*
+ * Grow the IRQ table by BNXT_ROCE_IRQ_COUNT on top of existing entries (e.g.
+ * MPC's) instead of reassigning total_irqs, to avoid reclaiming MPC's rids.
+ */
+static int bnxt_populate_irq_roce(struct bnxt_softc *softc)
+{
+ struct resource_list *rl = NULL;
+ struct resource_list_entry *rle = NULL;
+ struct pci_devinfo *dinfo = NULL;
+ struct bnxt_msix_tbl *irq_tbl;
+ int prev_total_irqs = softc->total_irqs;
+ int i;
+
+ softc->total_irqs += BNXT_ROCE_IRQ_COUNT;
+ irq_tbl = krealloc(softc->irq_tbl,
+ softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL);
+
+ if (!irq_tbl) {
+ device_printf(softc->dev, "Failed to allocate IRQ table\n");
+ /* Roll back total_irqs since the table itself wasn't grown. */
+ softc->total_irqs = prev_total_irqs;
+ return -1;
+ }
+
+ dinfo = device_get_ivars(softc->pdev->dev.bsddev);
+ rl = &dinfo->resources;
+ rle = resource_list_find(rl, SYS_RES_IRQ, 1);
+ if (rle == NULL) {
+ device_printf(softc->dev,
+ "No default resources for rid = %d, type = %d\n",
+ 1, SYS_RES_IRQ);
+ /* irq_tbl isn't installed yet; free it and roll back total_irqs. */
+ kfree(irq_tbl);
+ softc->total_irqs = prev_total_irqs;
+ return -1;
+ }
+
+ softc->pdev->dev.irq_start = rle->start;
+ softc->pdev->dev.irq_end = rle->start + softc->total_irqs;
+
+ for (i = prev_total_irqs; i < softc->total_irqs; i++) {
+ irq_tbl[i].entry = i;
+ irq_tbl[i].vector = softc->pdev->dev.irq_start + i;
+ }
+
+ softc->irq_tbl = irq_tbl;
+
+ return 0;
+}
+
static const struct bnxt_en_ops bnxt_en_ops_tbl = {
.bnxt_register_device = bnxt_register_dev,
.bnxt_unregister_device = bnxt_unregister_dev,
@@ -521,7 +584,7 @@ int bnxt_rdma_aux_device_add(struct bnxt_softc *bp)
struct auxiliary_device *aux_dev;
int ret = -1;
- if (bnxt_populate_irq(bp))
+ if (bnxt_populate_irq_roce(bp))
return ret;
device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, "
diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c
index dcfb7e61f0ca..0400b87f5850 100644
--- a/sys/dev/bnxt/bnxt_en/if_bnxt.c
+++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c
@@ -76,6 +76,7 @@
#include "bnxt_log_data.h"
#include "bnxt_coredump.h"
#include "bnxt_ptp.h"
+#include "bnxt_mpc.h"
/*
* PCI Device ID Table
@@ -260,7 +261,7 @@ static int bnxt_handle_isr(void *arg);
static void bnxt_clear_ids(struct bnxt_softc *softc);
static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr);
static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr);
-static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
+void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
static void bnxt_def_cp_task(void *context, int pending);
static void bnxt_handle_async_event(struct bnxt_softc *softc,
struct cmpl_base *cmpl);
@@ -449,6 +450,8 @@ bnxt_init_sctx_variants(uint16_t device_id)
bnxt_sctx_pf_init = bnxt_sctx_template;
bnxt_sctx_pf_init.isc_admin_intrcnt = BNXT_ROCE_IRQ_COUNT;
+ if (device_id == BCM57608)
+ bnxt_sctx_pf_init.isc_admin_intrcnt += BNXT_MAX_MPC;
bnxt_sctx_vf_init = bnxt_sctx_template;
bnxt_sctx_vf_init.isc_flags |= IFLIB_IS_VF;
@@ -553,7 +556,7 @@ bnxt_nq_free(struct bnxt_softc *softc)
}
-static void
+void
bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
u32 ring_type)
{
@@ -668,6 +671,15 @@ bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs,
}
softc->ntxqsets = ntxqsets;
+ bnxt_set_dflt_mpc_rings(softc);
+ rc = bnxt_alloc_mpcs(softc);
+ if (rc == 0)
+ rc = bnxt_alloc_mpc_rings(softc);
+ if (rc) {
+ bnxt_free_mpc_rings(softc);
+ bnxt_free_mpcs(softc);
+ goto dma_alloc_fail;
+ }
return rc;
dma_alloc_fail:
@@ -709,6 +721,9 @@ bnxt_queues_free(if_ctx_t ctx)
free(softc->rx_rings, M_DEVBUF);
free(softc->rx_cp_rings, M_DEVBUF);
bnxt_nq_free(softc);
+
+ bnxt_free_mpc_rings(softc);
+ bnxt_free_mpcs(softc);
}
static int
@@ -1296,12 +1311,21 @@ bnxt_bs_trace_init(struct bnxt_softc *bp, struct bnxt_ctx_mem_type *ctxm)
static int
bnxt_backing_store_cfg_v2(struct bnxt_softc *softc, u32 ena)
{
+ struct bnxt_mpc_info *mpc = softc->mpc_info;
struct bnxt_ctx_mem_info *ctx = softc->ctx_mem;
struct bnxt_ctx_mem_type *ctxm;
u16 last_type = BNXT_CTX_INV;
int rc = 0;
u16 type;
+ if (BNXT_PF(softc) && mpc && mpc->mpc_chnls_cap) {
+ ctxm = &ctx->ctx_arr[BNXT_CTX_MTQM];
+ rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1);
+ if (rc)
+ return rc;
+ last_type = BNXT_CTX_MTQM;
+ }
+
if (BNXT_PF(softc)) {
for (type = BNXT_CTX_SRT_TRACE; type <= BNXT_CTX_ROCE_HWRM_TRACE; type++) {
ctxm = &ctx->ctx_arr[type];
@@ -2971,6 +2995,7 @@ bnxt_attach_pre(if_ctx_t ctx)
failed:
bnxt_free_sysctl_ctx(softc);
+ bnxt_free_mpc_info(softc);
init_sysctl_failed:
bnxt_hwrm_func_drv_unrgtr(softc, false);
if (BNXT_PF(softc))
@@ -3022,6 +3047,10 @@ bnxt_attach_post(if_ctx_t ctx)
softc->rx_buf_size = min(softc->scctx->isc_max_frame_size, BNXT_PAGE_SIZE);
bnxt_dcb_init(softc);
+ rc = bnxt_mpc_irq_setup(softc);
+ if (rc)
+ goto failed;
+
bnxt_rdma_aux_device_init(softc);
/* SR-IOV attach */
@@ -3034,6 +3063,8 @@ bnxt_attach_post(if_ctx_t ctx)
"crash dump init failure rc: %d\n", rc);
failed:
+ /* No cleanup needed: on nonzero return iflib_device_register() calls
+ * IFDI_DETACH and IFDI_QUEUES_FREE, which tear down anything set up so far. */
return rc;
}
@@ -3045,6 +3076,7 @@ bnxt_detach(if_ctx_t ctx)
struct bnxt_vlan_tag *tmp;
int i;
+ bnxt_mpc_irq_cleanup(softc);
bnxt_rdma_aux_device_uninit(softc);
cancel_delayed_work_sync(&softc->fw_reset_task);
cancel_work_sync(&softc->sp_task);
@@ -3057,6 +3089,11 @@ bnxt_detach(if_ctx_t ctx)
bnxt_free_sysctl_ctx(softc);
bnxt_free_crash_dump_mem(softc);
bnxt_hwrm_func_reset(softc);
+ /* Must run before bnxt_free_mpc_info(): these gate on mpc_info being
+ * non-NULL, so freeing it first would leak the MPC rings. */
+ bnxt_free_mpc_rings(softc);
+ bnxt_free_mpcs(softc);
+ bnxt_free_mpc_info(softc);
bnxt_free_ctx_mem(softc);
bnxt_clear_ids(softc);
iflib_irq_free(ctx, &softc->def_cp_ring.irq);
@@ -3133,6 +3170,11 @@ bnxt_hwrm_resource_free(struct bnxt_softc *softc)
if (rc)
goto fail;
}
+
+ rc = bnxt_hwrm_mpc_ring_free(softc);
+ if (rc)
+ goto fail;
+
rc = bnxt_hwrm_free_filter(softc);
if (rc)
goto fail;
@@ -3520,6 +3562,11 @@ skip_def_cp_ring:
softc->db_ops.bnxt_db_tx(&softc->tx_rings[i], 0);
}
+ if (BNXT_PF(softc)) {
+ rc = bnxt_hwrm_mpc_ring_alloc(softc);
+ if (rc)
+ goto fail;
+ }
bnxt_do_enable_intr(&softc->def_cp_ring);
bnxt_get_port_module_status(softc);
bnxt_media_status(softc->ctx, &ifmr);
@@ -5303,6 +5350,7 @@ bnxt_clear_ids(struct bnxt_softc *softc)
softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE;
softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE;
}
+ bnxt_clear_mpc_rings_ids(softc);
softc->vnic_info.filter_id = -1;
softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE;
softc->vnic_info.rss_id = (uint16_t)HWRM_NA_SIGNATURE;
@@ -5310,7 +5358,7 @@ bnxt_clear_ids(struct bnxt_softc *softc)
softc->vnic_info.rss_grp_tbl.idi_size);
}
-static void
+void
bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr)
{
struct cmpl_base *cmp = (void *)cpr->ring.vaddr;
diff --git a/sys/modules/bnxt/bnxt_en/Makefile b/sys/modules/bnxt/bnxt_en/Makefile
index 67bd2646fa4d..e0d371c264de 100644
--- a/sys/modules/bnxt/bnxt_en/Makefile
+++ b/sys/modules/bnxt/bnxt_en/Makefile
@@ -15,6 +15,7 @@ SRCS += bnxt_log.c bnxt_log.h
SRCS += bnxt_log_data.c bnxt_log_data.h
SRCS += bnxt_coredump.c bnxt_coredump.h
SRCS += bnxt_ptp.c bnxt_ptp.h
+SRCS += bnxt_mpc.c bnxt_mpc.h
SRCS += ${LINUXKPI_GENSRCS}
CFLAGS+= -DIFLIB