diff options
| -rw-r--r-- | sys/conf/files | 1 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/bnxt.h | 28 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/bnxt_hwrm.c | 54 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/bnxt_mpc.c | 1075 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/bnxt_mpc.h | 182 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/bnxt_ulp.c | 69 | ||||
| -rw-r--r-- | sys/dev/bnxt/bnxt_en/if_bnxt.c | 54 | ||||
| -rw-r--r-- | sys/modules/bnxt/bnxt_en/Makefile | 1 |
8 files changed, 1450 insertions, 14 deletions
diff --git a/sys/conf/files b/sys/conf/files index 73a5d5fa8408..0a6e5874896f 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1333,6 +1333,7 @@ dev/bnxt/bnxt_en/bnxt_log.c optional bnxt iflib pci compile-with "${BNXT_C}" dev/bnxt/bnxt_en/bnxt_log_data.c optional bnxt iflib pci compile-with "${BNXT_C}" dev/bnxt/bnxt_en/bnxt_coredump.c optional bnxt iflib pci compile-with "${BNXT_C}" dev/bnxt/bnxt_en/bnxt_ptp.c optional bnxt iflib pci compile-with "${BNXT_C}" +dev/bnxt/bnxt_en/bnxt_mpc.c optional bnxt iflib pci compile-with "${BNXT_C}" dev/bnxt/bnxt_en/if_bnxt.c optional bnxt iflib pci compile-with "${BNXT_C}" dev/bwi/bwimac.c optional bwi dev/bwi/bwiphy.c optional bwi diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h index 80cc8905d66b..1de100915f97 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt.h +++ b/sys/dev/bnxt/bnxt_en/bnxt.h @@ -608,6 +608,11 @@ struct tx_bd_opaque { #define TX_RING_MASK(txr) ((txr)->ring_size - 1) +struct bnxt_sw_mpc_tx_bd { + uint8_t inline_bds; + unsigned long handle; +}; + struct bnxt_ring { uint64_t paddr; vm_offset_t doorbell; @@ -618,6 +623,15 @@ struct bnxt_ring { uint16_t phys_id; uint16_t idx; uint16_t running_bds; /* host TX-completion-coalescing accumulator */ + uint8_t queue_id; /* CoS queue, or BNXT_MPC_QUEUE_ID below */ +#define BNXT_MPC_QUEUE_ID 0xff + uint8_t mpc_chnl_type; /* valid when queue_id == BNXT_MPC_QUEUE_ID */ + struct iflib_dma_info ring_mem; /* manual DMA alloc for driver-private rings (MPC) */ + struct bnxt_cp_ring *cp_ring; /* completion ring this TX ring reports to (MPC) */ + uint16_t prod; /* MPC TX ring producer index */ + uint16_t cons; /* MPC TX ring consumer index */ + struct mtx tx_lock; /* MPC TX ring lock */ + struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring; struct bnxt_full_tpa_start *tpa_start; union { u64 db_key64; @@ -649,6 +663,10 @@ struct bnxt_cp_ring { uint8_t type; #define Q_TYPE_TX 1 #define Q_TYPE_RX 2 +#define TX_CP_NQ 3 /* MPC-private NQ ring (bnxt_mpc.c) */ + int msix_vec; /* MPC-private ring: bus_setup_intr vector */ + struct resource *irq_res; /* MPC-private ring: bus_setup_intr resource */ + void *irq_cookie; /* MPC-private ring: bus_setup_intr cookie */ }; struct bnxt_full_tpa_start { @@ -718,6 +736,10 @@ struct bnxt_func_qcfg { uint16_t alloc_completion_rings; uint16_t alloc_tx_rings; uint16_t alloc_rx_rings; + /* Firmware-granted counts before the MPC reservation is subtracted; + * bnxt_set_dflt_mpc_rings() needs these unreduced values. */ + uint16_t orig_alloc_completion_rings; + uint16_t orig_alloc_tx_rings; uint16_t alloc_vnics; uint16_t alloc_rss_ctx; uint16_t alloc_l2_ctx; @@ -1435,6 +1457,8 @@ struct bnxt_softc { struct bnxt_ptp_cfg *ptp_cfg; bool rx_ts_enabled; + struct bnxt_mpc_info *mpc_info; + #define BNXT_CAGR_CQCOAL_OFFSET 0xc00 #define BNXT_CAGR_NQAGG_MAXTIMER 0x5930000 #define BNXT_CAGR_TICK_RES_DEFAULT 0x2 @@ -1523,6 +1547,10 @@ int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc *softc, uint32_t mem_size, uint8_t depth, struct bnxt_ctx_mem_type *ctxm); int bnxt_hwrm_ptp_qcfg(struct bnxt_softc *bp); +int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count); +void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr); +void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db, + uint32_t ring_type); static inline u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c index 2649100e9b86..fb549bc05c10 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c @@ -32,6 +32,7 @@ #include "bnxt.h" #include "bnxt_hwrm.h" +#include "bnxt_mpc.h" #include "hsi_struct_def.h" #include "bnxt_coredump.h" @@ -1205,6 +1206,7 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc) (void *)softc->hwrm_cmd_resp.idi_vaddr; struct bnxt_func_info *func = &softc->func; uint32_t flags, flags_ext, flags_ext2; + uint8_t mpc_chnls_cap = 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS); req.fid = htole16(0xffff); @@ -1354,8 +1356,18 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc) } } + /* + * Only extract the field needed here; the actual alloc/free work + * (malloc(M_NOWAIT)/free() in bnxt_alloc_mpc_info()) happens after + * BNXT_HWRM_UNLOCK() below instead of under hwrm_lock. + */ + if (BNXT_PF(softc)) + mpc_chnls_cap = resp->mpc_chnls_cap; + fail: BNXT_HWRM_UNLOCK(softc); + if (!rc && BNXT_PF(softc)) + bnxt_alloc_mpc_info(softc, mpc_chnls_cap); return rc; } @@ -1468,6 +1480,22 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc) fn_qcfg->alloc_stat_ctx = le16toh(resp->alloc_stat_ctx); fn_qcfg->alloc_msix = le16toh(resp->alloc_msix); + fn_qcfg->orig_alloc_completion_rings = fn_qcfg->alloc_completion_rings; + fn_qcfg->orig_alloc_tx_rings = fn_qcfg->alloc_tx_rings; + + /* Reserve minimum rings for MPC */ + if (BNXT_PF(softc)) { + if (softc->mpc_info) { + uint16_t mpc_min = BNXT_MIN_MPC_TCE + BNXT_MIN_MPC_RCE; + + /* Clamp to 0 instead of underflowing these unsigned counts. */ + fn_qcfg->alloc_completion_rings -= + min(fn_qcfg->alloc_completion_rings, mpc_min); + fn_qcfg->alloc_tx_rings -= + min(fn_qcfg->alloc_tx_rings, mpc_min); + } + } + switch (resp->port_partition_type) { case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0: case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2: @@ -1976,17 +2004,24 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type, switch (type) { case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX: - cp_ring = &softc->tx_cp_rings[idx]; + if (ring->queue_id == BNXT_MPC_QUEUE_ID) { + cp_ring = ring->cp_ring; + req.mpc_chnls_type = ring->mpc_chnl_type; + req.enables |= htole32(HWRM_RING_ALLOC_INPUT_ENABLES_MPC_CHNLS_TYPE); + } else { + cp_ring = &softc->tx_cp_rings[idx]; - req.cmpl_ring_id = htole16(cp_ring->ring.phys_id); - /* queue_id - what CoS queue the TX ring is associated with */ - req.queue_id = htole16(softc->tx_q_info[0].queue_id); + /* queue_id - what CoS queue the TX ring is associated with */ + req.queue_id = htole16(softc->tx_q_info[0].queue_id); + req.enables |= htole32( + HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); + } req.stat_ctx_id = htole32(cp_ring->stats_ctx_id); - req.enables |= htole32( - HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); - if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL) + if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL && + ring->queue_id != BNXT_MPC_QUEUE_ID) req.cmpl_coal_cnt = softc->tx_hw_coal_cnt; + req.cmpl_ring_id = htole16(cp_ring->ring.phys_id); break; case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX: if (!BNXT_CHIP_P5_PLUS(softc)) @@ -2023,7 +2058,10 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type, } req.cq_handle = htole64(ring->id); - req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id); + if (ring->queue_id == BNXT_MPC_QUEUE_ID) + req.nq_ring_id = htole16(softc->mpc_info->mpc_nq_rings[idx].ring.phys_id); + else + req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id); req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID); break; diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.c b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c new file mode 100644 index 000000000000..4e35b3bdca9c --- /dev/null +++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.c @@ -0,0 +1,1075 @@ +/* + * Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2026 Broadcom, All Rights Reserved. + * The term Broadcom refers to Broadcom Limited and/or its subsidiaries + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/endian.h> +#include <sys/bus.h> +#include <sys/rman.h> +#include <sys/pciio.h> +#include <sys/time.h> +#include <machine/atomic.h> +#include <machine/resource.h> +#include <dev/pci/pcivar.h> + +#include "bnxt.h" +#include "bnxt_mpc.h" +#include "bnxt_hwrm.h" +#include "bnxt_log.h" + +/* + * bnxt_ktls_mpc_cmp() is defined in bnxt_ktls.c, not yet part of this + * branch; forward-declared here so this file builds standalone. + */ +void bnxt_ktls_mpc_cmp(struct bnxt_softc *softc, uint32_t client, + unsigned long handle, struct bnxt_cmpl_entry *cmpl_entry_arr, + uint32_t cmpl_num); + +void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap) +{ + if (mpc_chnls_cap) { + if (!softc->mpc_info) + softc->mpc_info = malloc(sizeof(*softc->mpc_info), + M_DEVBUF, M_NOWAIT | M_ZERO); + } else { + bnxt_free_mpc_info(softc); + } + if (softc->mpc_info) + softc->mpc_info->mpc_chnls_cap = mpc_chnls_cap; +} + +void bnxt_free_mpc_info(struct bnxt_softc *softc) +{ + if (softc->mpc_info) { + free(softc->mpc_info, M_DEVBUF); + softc->mpc_info = NULL; + } +} + +static void __bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc, + enum bnxt_mpc_type type, int *avail, int avail_cp) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int dflt1, dflt2; + int idx1, idx2; + int min1, min2; + int val1, val2; + + if (type == BNXT_MPC_CRYPTO) { + min1 = BNXT_MIN_MPC_TCE; + min2 = BNXT_MIN_MPC_RCE; + dflt1 = BNXT_DFLT_MPC_TCE; + dflt2 = BNXT_DFLT_MPC_RCE; + idx1 = BNXT_MPC_TCE_TYPE; + idx2 = BNXT_MPC_RCE_TYPE; + } else { + min1 = BNXT_MIN_MPC_TE_CFA; + min2 = BNXT_MIN_MPC_RE_CFA; + dflt1 = BNXT_DFLT_MPC_TE_CFA; + dflt2 = BNXT_DFLT_MPC_RE_CFA; + idx1 = BNXT_MPC_TE_CFA_TYPE; + idx2 = BNXT_MPC_RE_CFA_TYPE; + } + if (*avail < (min1 + min2)) + return; + + val1 = min(*avail / 2, softc->ntxqsets); + val2 = val1; + + val1 = min(val1, dflt1); + val2 = min(val2, dflt2); + + if (avail_cp < min1 || avail_cp < min2) + return; + + val1 = min(val1, avail_cp); + val2 = min(val2, avail_cp); + + mpc->mpc_ring_count[idx1] = val1; + mpc->mpc_ring_count[idx2] = val2; + + *avail = *avail - val1 - val2; +} + +void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc) +{ + struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg; + struct bnxt_mpc_info *mpc = softc->mpc_info; + int avail, mpc_cp, i; + int avail_cp; + + if (!mpc) + return; + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) + mpc->mpc_ring_count[i] = 0; + mpc->mpc_cp_rings_count = 0; + + avail = fn_qcfg->orig_alloc_tx_rings - softc->ntxqsets; + + avail_cp = fn_qcfg->orig_alloc_completion_rings - softc->ntxqsets; + avail_cp = min(avail_cp, + ((pci_msix_count(softc->dev) - 1) - softc->ntxqsets)); + + if (BNXT_MPC_CRYPTO_CAPABLE(softc)) + __bnxt_set_dflt_mpc_rings(softc, BNXT_MPC_CRYPTO, &avail, + avail_cp); + + for (i = 0, mpc_cp = 0; i < BNXT_MPC_TYPE_MAX; i++) { + if (mpc_cp < mpc->mpc_ring_count[i]) + mpc_cp = mpc->mpc_ring_count[i]; + } + mpc->mpc_cp_rings_count = mpc_cp; +} + +int bnxt_alloc_mpcs(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i; + + if (!BNXT_MPC_CRYPTO_CAPABLE(softc)) + return 0; + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num; + struct bnxt_ring *txr; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + num = mpc->mpc_ring_count[i]; + if (!num) + continue; + txr = malloc(num * sizeof(*txr), M_DEVBUF, M_NOWAIT | M_ZERO); + if (!txr) { + device_printf(softc->dev, + "%s: Failed to allocate %d MPC rings\n", + __func__, num); + return ENOMEM; + } + mpc->mpc_rings[i] = txr; + } + + mpc->mpc_cp_rings = + malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (!mpc->mpc_cp_rings) { + device_printf(softc->dev, + "%s: Failed to allocate %d MPC completion rings\n", + __func__, mpc->mpc_cp_rings_count); + return ENOMEM; + } + + mpc->mpc_nq_rings = + malloc(mpc->mpc_cp_rings_count * sizeof(struct bnxt_cp_ring), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (!mpc->mpc_nq_rings) { + device_printf(softc->dev, + "%s: Failed to allocate %d MPC notification rings\n", + __func__, mpc->mpc_cp_rings_count); + return ENOMEM; + } + return 0; +} + +void bnxt_free_mpcs(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i; + + if (!mpc) + return; + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + if (mpc->mpc_rings[i]) { + free(mpc->mpc_rings[i], M_DEVBUF); + mpc->mpc_rings[i] = NULL; + } + } + + if (mpc->mpc_cp_rings) { + free(mpc->mpc_cp_rings, M_DEVBUF); + mpc->mpc_cp_rings = NULL; + } + + if (mpc->mpc_nq_rings) { + free(mpc->mpc_nq_rings, M_DEVBUF); + mpc->mpc_nq_rings = NULL; + } +} + +int bnxt_alloc_mpc_rings(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i, j, mpc_idx = 0; + int mpc_base_ring_id = + (softc->scctx->isc_nrxqsets * 2) + 1 + softc->ntxqsets; + + if (!mpc) + return 0; + + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i]; + int rc; + + cpr->ring.queue_id = BNXT_MPC_QUEUE_ID; + + /* Set up the completion ring */ + cpr->stats_ctx_id = HWRM_NA_SIGNATURE; + cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + cpr->ring.softc = softc; + cpr->ring.idx = i; + cpr->ring.id = mpc_base_ring_id + i; + cpr->ring.doorbell = softc->legacy_db_size; + cpr->ring.ring_size = softc->scctx->isc_ntxd[0]; + cpr->ring.db_ring_mask = cpr->ring.ring_size - 1; + + rc = iflib_dma_alloc(softc->ctx, + sizeof(struct cmpl_base) * cpr->ring.ring_size, + &cpr->ring.ring_mem, 0); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate DMA memory for completion ring %d (size=%zu), rc=%d\n", + __func__, i, sizeof(struct cmpl_base) * cpr->ring.ring_size, rc); + return rc; + } + + cpr->ring.vaddr = cpr->ring.ring_mem.idi_vaddr; + cpr->ring.paddr = cpr->ring.ring_mem.idi_paddr; + } + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i], rc; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + /* tx_stats[] is indexed across all MPC types, so use + * the flat mpc_idx, not the intra-type counter j. */ + int stat_idx = mpc_idx; + + txr->queue_id = BNXT_MPC_QUEUE_ID; + txr->mpc_chnl_type = i; + + /* alloc mpc TX ring */ + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE; + txr->softc = softc; + txr->idx = i << BNXT_RING_MPC_IDX_TYPE_SFT | j; + txr->id = mpc_base_ring_id + mpc_idx++; + txr->doorbell = softc->legacy_db_size; + txr->ring_size = softc->scctx->isc_ntxd[1]; + txr->db_ring_mask = txr->ring_size - 1; + + /* Init before any early return: bnxt_free_mpc_rings() + * unconditionally mtx_destroy()s every txr. */ + mtx_init(&txr->tx_lock, "MPC TX lock", NULL, MTX_SPIN); + + rc = iflib_dma_alloc(softc->ctx, + sizeof(struct tx_bd_short) * txr->ring_size, + &txr->ring_mem, 0); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate DMA memory for TX ring %d (size=%zu), rc=%d\n", + __func__, j, sizeof(struct tx_bd_short) * txr->ring_size, rc); + return rc; + } + + txr->vaddr = txr->ring_mem.idi_vaddr; + txr->paddr = txr->ring_mem.idi_paddr; + + txr->tx_mpc_buf_ring = + malloc(txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (!txr->tx_mpc_buf_ring) { + device_printf(softc->dev, + "%s: Failed to allocate buffer ring for TX ring %d (size=%zu)\n", + __func__, j, txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd)); + return ENOMEM; + } + + rc = iflib_dma_alloc(softc->ctx, sizeof(struct ctx_hw_stats), + &mpc->tx_stats[stat_idx], 0); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate DMA memory for TX stats ring %d, rc=%d\n", + __func__, j, rc); + return rc; + } + bus_dmamap_sync(mpc->tx_stats[stat_idx].idi_tag, + mpc->tx_stats[stat_idx].idi_map, BUS_DMASYNC_PREREAD); + } + } + + /* Set up the Notification ring (NQ) */ + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i]; + int rc; + + nqr->ring.queue_id = BNXT_MPC_QUEUE_ID; + + nqr->stats_ctx_id = HWRM_NA_SIGNATURE; + nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + nqr->ring.softc = softc; + nqr->ring.idx = i; + nqr->ring.id = softc->ntxqsets + i; + nqr->ring.doorbell = softc->legacy_db_size; + nqr->ring.ring_size = softc->scctx->isc_ntxd[2]; + nqr->ring.db_ring_mask = nqr->ring.ring_size - 1; + + rc = iflib_dma_alloc(softc->ctx, + sizeof(struct cmpl_base) * nqr->ring.ring_size, + &nqr->ring.ring_mem, 0); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate DMA memory for notification ring %d (size=%zu), rc=%d\n", + __func__, i, sizeof(struct cmpl_base) * nqr->ring.ring_size, rc); + return rc; + } + + nqr->ring.vaddr = nqr->ring.ring_mem.idi_vaddr; + nqr->ring.paddr = nqr->ring.ring_mem.idi_paddr; + nqr->type = TX_CP_NQ; + } + + return 0; +} + +void bnxt_free_mpc_rings(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i, j, mpc_idx = 0; + + if (!mpc) + return; + + /* A partial alloc failure can leave these NULL; guard each array. */ + if (mpc->mpc_nq_rings) { + for (i = 0; i < mpc->mpc_cp_rings_count; i++) + iflib_dma_free(&mpc->mpc_nq_rings[i].ring.ring_mem); + } + + if (mpc->mpc_cp_rings) { + for (i = 0; i < mpc->mpc_cp_rings_count; i++) + iflib_dma_free(&mpc->mpc_cp_rings[i].ring.ring_mem); + } + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i]; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + /* + * Same partial-allocation case as above: mpc_ring_count[i] can + * be nonzero while mpc_rings[i] itself failed to allocate. + */ + if (!mpc->mpc_rings[i]) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + + /* TCE ring vaddr holds cleartext AES-GCM keys; scrub + * before freeing the pages. */ + if (i == BNXT_MPC_TCE_TYPE && txr->ring_mem.idi_vaddr) + explicit_bzero(txr->ring_mem.idi_vaddr, + txr->ring_mem.idi_size); + + iflib_dma_free(&txr->ring_mem); + if (txr->tx_mpc_buf_ring) { + free(txr->tx_mpc_buf_ring, M_DEVBUF); + txr->tx_mpc_buf_ring = NULL; + } + /* A ring past a partial alloc failure never had its + * tx_lock mtx_init()'d; guard the destroy. */ + if (mtx_initialized(&txr->tx_lock)) + mtx_destroy(&txr->tx_lock); + /* Must match the flat stat_idx bnxt_alloc_mpc_rings() used. */ + iflib_dma_free(&mpc->tx_stats[mpc_idx++]); + } + } +} + +void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i, j; + + if (!mpc) + return; + + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + mpc->mpc_nq_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + mpc->mpc_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + mpc->mpc_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; + } + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i]; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE; + } + } +} + +static void bnxt_map_tx_to_cp(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int cp_count = mpc->mpc_cp_rings_count; + int i, j; + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i]; + + if (i != BNXT_MPC_TCE_TYPE) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + int cp_idx = j % cp_count; + + txr->cp_ring = &mpc->mpc_cp_rings[cp_idx]; + } + } +} + +struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc) +{ + int idx; + + /* Callers should gate on a non-zero TCE ring count already, but + * guard the divide/index here too since this feeds a live TX path. */ + if (!mpc || !mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE]) + return NULL; + + idx = PCPU_GET(cpuid) % mpc->mpc_ring_count[BNXT_MPC_TCE_TYPE]; + + return &mpc->mpc_rings[BNXT_MPC_TCE_TYPE][idx]; +} + +int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i, j, rc; + + if (!mpc) + return 0; + + bnxt_map_tx_to_cp(softc); + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i]; + + nqr->cons = 0; + nqr->v_bit = 1; + nqr->raw_cons = 0; + nqr->toggle = 0; + nqr->last_idx = UINT32_MAX; + bnxt_mark_cpr_invalid(nqr); + + rc = bnxt_hwrm_ring_alloc( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring); + + bnxt_set_db_mask(softc, &nqr->ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate HWRM notification ring %d, rc=%d\n", + __func__, i, rc); + return rc; + } + + softc->db_ops.bnxt_db_nq(nqr, 1); + } + + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i]; + + rc = bnxt_hwrm_stat_ctx_alloc(softc, cpr, + mpc->tx_stats[i].idi_paddr); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate HWRM stats context for completion ring %d, rc=%d\n", + __func__, i, rc); + return rc; + } + + cpr->v_bit = 1; + cpr->cons = 0; + cpr->raw_cons = 0; + cpr->toggle = 0; + bnxt_mark_cpr_invalid(cpr); + rc = bnxt_hwrm_ring_alloc( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &cpr->ring); + bnxt_set_db_mask(softc, &cpr->ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate HWRM completion ring %d, rc=%d\n", + __func__, i, rc); + return rc; + } + + softc->db_ops.bnxt_db_tx_cq(cpr, 1); + } + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i]; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + + txr->prod = 0; + txr->cons = 0; + memset(txr->tx_mpc_buf_ring, 0, + txr->ring_size * sizeof(struct bnxt_sw_mpc_tx_bd)); + + rc = bnxt_hwrm_ring_alloc( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr); + bnxt_set_db_mask(softc, txr, + HWRM_RING_ALLOC_INPUT_RING_TYPE_TX); + if (rc) { + device_printf(softc->dev, + "%s: Failed to allocate HWRM TX ring %d, rc=%d\n", + __func__, j, rc); + return rc; + } + softc->db_ops.bnxt_db_tx(txr, 0); + } + } + return 0; +} + +int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int i, j, rc; + + if (!mpc) + return 0; + + for (i = 0; i < BNXT_MPC_TYPE_MAX; i++) { + int num = mpc->mpc_ring_count[i]; + + /* Currently only KTLS TX is supported */ + if (i != BNXT_MPC_TCE_TYPE) + continue; + + for (j = 0; j < num; j++) { + struct bnxt_ring *txr = &mpc->mpc_rings[i][j]; + struct bnxt_cp_ring *cpr = txr->cp_ring; + + if (!cpr) + continue; + + rc = bnxt_hwrm_ring_free( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, txr, + cpr->ring.phys_id); + if (rc) { + device_printf(softc->dev, + "%s: Failed to free HWRM TX ring %d (ring_id=%d, phys_id=%d), rc=%d\n", + __func__, j, txr->id, cpr->ring.phys_id, rc); + return rc; + } + txr->phys_id = (uint16_t)HWRM_NA_SIGNATURE; + } + } + + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *cpr = &mpc->mpc_cp_rings[i]; + + rc = bnxt_hwrm_ring_free( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + &cpr->ring, (uint16_t)HWRM_NA_SIGNATURE); + if (rc) { + device_printf(softc->dev, + "%s: Failed to free HWRM completion ring %d (ring_id=%d, phys_id=%d), rc=%d\n", + __func__, i, cpr->ring.id, cpr->ring.phys_id, rc); + return rc; + } + + rc = bnxt_hwrm_stat_ctx_free(softc, cpr); + if (rc) { + device_printf(softc->dev, + "%s: Failed to free HWRM stats context id %d for completion ring %d, rc=%d\n", + __func__, cpr->stats_ctx_id, i, rc); + return rc; + } + + cpr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + } + + for (i = 0; i < mpc->mpc_cp_rings_count; i++) { + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[i]; + + rc = bnxt_hwrm_ring_free( + softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &nqr->ring, + (uint16_t)HWRM_NA_SIGNATURE); + if (rc) { + device_printf(softc->dev, + "%s: Failed to free HWRM notification ring %d (ring_id=%d, phys_id=%d), rc=%d\n", + __func__, i, nqr->ring.id, nqr->ring.phys_id, rc); + return rc; + } + nqr->ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; + } + return 0; +} + +static void bnxt_mpc_process_cpr(struct bnxt_cp_ring *cpr) +{ + struct bnxt_softc *softc = cpr->ring.softc; + + /* Handle completions on the mpc completion ring */ + uint32_t raw_cons = cpr->raw_cons; + uint32_t cons = cpr->cons; + struct cmpl_base *cmpl; + bool v_bit = cpr->v_bit; + bool last_v_bit; + uint32_t last_cons; + uint16_t type; + + for (;;) { + last_cons = cons; + last_v_bit = v_bit; + cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons]; + + if (!CMP_VALID(cmpl, v_bit)) + break; + + type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK; + switch (type) { + case CMPL_BASE_TYPE_MID_PATH_SHORT: + case CMPL_BASE_TYPE_MID_PATH_LONG: { + int rc = bnxt_mpc_cmp(softc, cpr, &raw_cons); + + /* EBUSY: 2nd half of a MID_PATH_LONG completion hasn't + * landed yet; stop and retry later. Any other error is + * non-transient and already logged/skipped, so fall + * through and advance like the success case. */ + if (rc == EBUSY) + goto done; + break; + } + case CMPL_BASE_TYPE_HWRM_DONE: + /* ignore this completion */ + break; + default: { + /* Rate-limit prints so a stuck ring feeding + * unrecognized types can't flood the console. + * Statics are unlocked; a best-effort limiter. */ + static struct timeval last; + static int curpps; + + if (ppsratecheck(&last, &curpps, 5)) + device_printf(softc->dev, + "%s: Unknown completion type %u on ring %d (raw_cons=%u, cons=%u)\n", + __func__, type, cpr->ring.idx, raw_cons, cons); + break; + } + } + raw_cons++; + /* Recompute from raw_cons rather than NEXT_CP_CONS_V(): a + * MID_PATH_LONG completion can wrap raw_cons inside + * bnxt_mpc_cmp(), which would desync v_bit otherwise. */ + cons = MPC_RING_CMP(cpr, raw_cons); + v_bit = !(raw_cons & cpr->ring.ring_size); + } +done: + cpr->cons = last_cons; + cpr->raw_cons = raw_cons; + cpr->v_bit = last_v_bit; + softc->db_ops.bnxt_db_tx_cq(cpr, 1); +} + +static void bnxt_mpc_irq(void *arg) +{ + struct bnxt_cp_ring *cpr = arg; + struct bnxt_softc *softc = cpr->ring.softc; + nq_cn_t *cmp = (nq_cn_t *)cpr->ring.vaddr; + uint32_t raw_cons = cpr->raw_cons; + uint16_t nq_type, nqe_cnt = 0; + uint32_t cons = cpr->cons; + bool v_bit = cpr->v_bit; + + while (1) { + if (!NQ_VALID(&cmp[cons], v_bit)) + goto done; + + nq_type = NQ_CN_TYPE_MASK & cmp[cons].type; + + if (NQE_CN_TYPE(nq_type) == NQ_CN_TYPE_CQ_NOTIFICATION) { + struct bnxt_cp_ring *cpr2 = + &softc->mpc_info->mpc_cp_rings[cpr->ring.idx]; + cpr2->toggle = NQE_CN_TOGGLE(cmp[cons].type); + bnxt_mpc_process_cpr(cpr2); + } + + NEXT_CP_CONS_V(&cpr->ring, cons, v_bit); + raw_cons++; + nqe_cnt++; + } +done: + if (nqe_cnt) { + cpr->cons = cons; + cpr->raw_cons = raw_cons; + cpr->v_bit = v_bit; + } + + softc->db_ops.bnxt_db_nq(cpr, 1); +} + +int bnxt_mpc_irq_setup(struct bnxt_softc *softc) +{ + int mpc_base_idx = softc->scctx->isc_ntxqsets; + struct bnxt_mpc_info *mpc = softc->mpc_info; + int mpc_cp_rings_count = 0; + int rc, i, j; + + if (mpc) + mpc_cp_rings_count = mpc->mpc_cp_rings_count; + + rc = bnxt_populate_irq(softc, mpc_cp_rings_count); + if (rc) { + device_printf(softc->dev, + "%s: Failed to populate IRQs for %d MPC completion rings, rc=%d\n", + __func__, mpc_cp_rings_count, rc); + return rc; + } + + for (i = mpc_base_idx, j = 0; j < mpc_cp_rings_count; i++, j++) { + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[j]; + struct resource *irq_res; + int rid; + + nqr->msix_vec = softc->irq_tbl[i].vector; + + rid = 1 + i; + irq_res = bus_alloc_resource_any(softc->dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (irq_res == NULL) { + /* Release any IRQs already set up for j' < j. */ + bnxt_mpc_irq_cleanup(softc); + return ENOMEM; + } + + nqr->irq_res = irq_res; + + rc = bus_setup_intr(softc->dev, irq_res, INTR_TYPE_NET | INTR_MPSAFE, + NULL, bnxt_mpc_irq, nqr, &nqr->irq_cookie); + if (rc) { + device_printf(softc->dev, + "%s: Failed to setup IRQ for msix_vec %d, MPC notification ring %d, rc=%d\n", + __func__, nqr->msix_vec, j, rc); + bus_release_resource(softc->dev, SYS_RES_IRQ, rid, irq_res); + nqr->irq_res = NULL; + /* Release any IRQs already set up for j' < j. */ + bnxt_mpc_irq_cleanup(softc); + return rc; + } + } + + return 0; +} + +void bnxt_mpc_irq_cleanup(struct bnxt_softc *softc) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + int j; + + if (!mpc) + return; + + for (j = 0; j < mpc->mpc_cp_rings_count; j++) { + struct bnxt_cp_ring *nqr = &mpc->mpc_nq_rings[j]; + int rid; + + if (nqr->irq_res != NULL) { + bus_teardown_intr(softc->dev, nqr->irq_res, nqr->irq_cookie); + nqr->irq_cookie = NULL; + rid = rman_get_rid(nqr->irq_res); + bus_release_resource(softc->dev, SYS_RES_IRQ, rid, + nqr->irq_res); + nqr->irq_res = NULL; + } + } +} + +static inline uint32_t bnxt_tx_avail(struct bnxt_softc *softc, + struct bnxt_ring *txr) +{ + uint32_t used, prod, cons; + + prod = atomic_load_acq_16(&txr->prod); + cons = atomic_load_acq_16(&txr->cons); + + if (prod >= cons) + used = prod - cons; + else { + used = (txr->ring_size - cons) + prod; + } + + return (txr->ring_size - 1 - used); +} + +int bnxt_start_xmit_mpc(struct bnxt_softc *softc, struct bnxt_ring *txr, + void *data, uint len, unsigned long handle) +{ + uint32_t bds, total_bds, bd_space, free_size; + struct bnxt_sw_mpc_tx_bd *tx_buf; + struct tx_bd_long *txbd; + uint16_t prod; + + /* Caller must hold txr->tx_lock across this sequence; assert rather + * than acquire since bnxt_xmit_crypto_cmd() needs it held longer. */ + mtx_assert(&txr->tx_lock, MA_OWNED); + + bds = howmany(len, sizeof(*txbd)); + total_bds = bds + 1; + free_size = bnxt_tx_avail(softc, txr); + if (free_size < total_bds) + return EBUSY; + + prod = txr->prod; + txbd = &((struct tx_bd_long *)txr->vaddr)[prod]; + + tx_buf = &txr->tx_mpc_buf_ring[prod]; + tx_buf->handle = handle; + tx_buf->inline_bds = total_bds; + + txbd->flags_type = + htole16(TX_BD_MP_CMD_TYPE_TX_BD_MP_CMD | + (total_bds << TX_BD_LONG_FLAGS_BD_CNT_SFT)); + txbd->len = htole16(len); + txbd->opaque = SET_TX_OPAQUE(txr, prod, total_bds); + + prod = NEXT_TX(txr, prod); + txbd = &((struct tx_bd_long *)txr->vaddr)[prod]; + + bd_space = txr->ring_size - prod; + if (bd_space < bds) { + uint32_t len0 = bd_space * sizeof(*txbd); + + memcpy(txbd, data, len0); + prod = ADVANCE_TX_BY(txr, prod, bd_space); + txbd = &((struct tx_bd_long *)txr->vaddr)[prod]; + bds -= bd_space; + len -= len0; + data = (uint8_t *)data + len0; + } + memcpy(txbd, data, len); + prod = ADVANCE_TX_BY(txr, prod, bds); + /* Release-store pairs with the atomic_load_acq_16(&txr->prod) in + * bnxt_tx_avail() and bnxt_adv_mpc_cons(). */ + atomic_store_rel_16(&txr->prod, prod); + + softc->db_ops.bnxt_db_tx(txr, prod); + + return 0; +} + +static bool bnxt_mpc_unsolicit(struct mpc_cmp *mpcmp) +{ + uint32_t client = MPC_CMP_CLIENT_TYPE(mpcmp); + + if (client != MPC_CMP_CLIENT_TCE && client != MPC_CMP_CLIENT_RCE && + client != MPC_CMP_CLIENT_TE_CFA && client != MPC_CMP_CLIENT_RE_CFA) + return false; + return MPC_CMP_UNSOLICIT_SUBTYPE(mpcmp); +} + +static void bnxt_adv_mpc_cons(struct bnxt_softc *softc, struct bnxt_ring *txr) +{ + struct tx_bd_long *bd_base = (struct tx_bd_long *)txr->vaddr; + struct bnxt_sw_mpc_tx_bd *mpc_buf; + uint16_t tx_cons = txr->cons; + uint16_t tail_bds; + uint8_t inline_bds; + + mpc_buf = &txr->tx_mpc_buf_ring[tx_cons]; + do { + inline_bds = mpc_buf->inline_bds; + /* Scrub the inline-BD region (held CE_ADD command + + * AES-GCM key) before the slot is reused. */ + if (inline_bds > 0) { + tail_bds = txr->ring_size - tx_cons; + if (tail_bds >= inline_bds) { + explicit_bzero(&bd_base[tx_cons], + inline_bds * sizeof(*bd_base)); + } else { + explicit_bzero(&bd_base[tx_cons], + tail_bds * sizeof(*bd_base)); + explicit_bzero(&bd_base[0], + (inline_bds - tail_bds) * sizeof(*bd_base)); + } + } + tx_cons = ADVANCE_TX_BY(txr, tx_cons, mpc_buf->inline_bds); + /* Pairs with bnxt_tx_avail()'s atomic_load_acq_16(&txr->cons). */ + atomic_store_rel_16(&txr->cons, tx_cons); + if (tx_cons == atomic_load_acq_16(&txr->prod)) + break; + mpc_buf = &txr->tx_mpc_buf_ring[tx_cons]; + } while (mpc_buf->handle == BNXT_INV_MPC_HDL); +} + +int bnxt_mpc_cmp(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr, + uint32_t *raw_cons) +{ + struct bnxt_mpc_info *mpc = softc->mpc_info; + struct bnxt_cmpl_entry cmpl_entry_arr[2]; + uint16_t cons = MPC_RING_CMP(cpr, *raw_cons); + struct mpc_cmp *mpcmp, *mpcmp1; + uint32_t tmp_raw_cons = *raw_cons; + unsigned long handle = 0; + uint32_t client, cmpl_num; + uint8_t type; + + mpcmp = &((struct mpc_cmp *)cpr->ring.vaddr)[cons]; + type = MPC_CMP_CMP_TYPE(mpcmp); + cmpl_entry_arr[0].cmpl = mpcmp; + cmpl_entry_arr[0].len = sizeof(*mpcmp); + cmpl_num = 1; + + if (type == MPC_CMP_TYPE_MID_PATH_LONG) { + tmp_raw_cons = tmp_raw_cons + 1; + cons = MPC_RING_CMP(cpr, tmp_raw_cons); + mpcmp1 = &((struct mpc_cmp *)cpr->ring.vaddr)[cons]; + if (!MPC_CMP_VALID(cpr, mpcmp1, tmp_raw_cons)) { + /* Expected/transient: caller retries this entry later. + * Rate-limit (unlocked statics, best-effort) so a + * slow second half can't flood the console. */ + static struct timeval last; + static int curpps; + + if (ppsratecheck(&last, &curpps, 5)) + device_printf(softc->dev, + "%s: Invalid completion entry on ring %d (raw_cons=%u, cons=%u, v_bit=%d)\n", + __func__, cpr->ring.idx, tmp_raw_cons, cons, cpr->v_bit); + return EBUSY; + } + /* + * The valid test of the entry must be done first before + * reading any further. + */ + atomic_thread_fence_acq(); + if (mpcmp1 == mpcmp + 1) { + cmpl_entry_arr[cmpl_num - 1].len += sizeof(*mpcmp1); + } else { + cmpl_entry_arr[cmpl_num].cmpl = mpcmp1; + cmpl_entry_arr[cmpl_num].len = sizeof(*mpcmp1); + cmpl_num++; + } + } + + client = MPC_CMP_CLIENT_TYPE(mpcmp) >> MPC_CMP_CLIENT_SFT; + + if (!bnxt_mpc_unsolicit(mpcmp)) { + uint16_t ring_idx, mpc_type; + struct bnxt_sw_mpc_tx_bd *mpc_buf; + struct bnxt_ring *txr; + uint16_t tx_cons, buf_idx; + uint32_t opaque; + + opaque = mpcmp->mpc_cmp_opaque; + ring_idx = TX_OPAQUE_RING(opaque) & BNXT_RING_MPC_IDX_RING_MASK; + mpc_type = TX_OPAQUE_RING(opaque) >> BNXT_RING_MPC_IDX_TYPE_SFT; + if (mpc_type != client) { + /* + * Unlocked statics: see the same note in + * bnxt_mpc_process_cpr()'s unknown-completion-type case. + */ + static struct timeval last; + static int curpps; + + if (ppsratecheck(&last, &curpps, 5)) + device_printf(softc->dev, + "%s: Wrong opaque 0x%x on completion ring %d, " + "expected client type 0x%x, actual type 0x%x (ring_idx=%d)\n", + __func__, opaque, cpr->ring.idx, client, mpc_type, ring_idx); + /* Corrupt, not pending: advance past both halves so + * bnxt_mpc_process_cpr() doesn't spin on it forever. */ + *raw_cons = tmp_raw_cons; + return EINVAL; + } + if (mpc_type >= BNXT_MPC_TYPE_MAX || mpc->mpc_rings[mpc_type] == NULL) { + bnxt_log_live(softc, BNXT_LOGGER_L2, + "%s: Invalid mpc_type %d, completion ring %d, opaque=0x%x, client_type=0x%x\n", + __func__, mpc_type, cpr->ring.idx, opaque, client); + *raw_cons = tmp_raw_cons; + return EINVAL; + } + if (ring_idx >= mpc->mpc_ring_count[mpc_type]) { + bnxt_log_live(softc, BNXT_LOGGER_L2, + "%s: Invalid ring_idx %d (max=%d), completion ring %d, opaque=0x%x, client_type=0x%x\n", + __func__, ring_idx, mpc->mpc_ring_count[mpc_type], cpr->ring.idx, opaque, client); + *raw_cons = tmp_raw_cons; + return EINVAL; + } + + txr = &mpc->mpc_rings[mpc_type][ring_idx]; + tx_cons = txr->cons; + buf_idx = TX_OPAQUE_IDX(opaque); + if (buf_idx >= txr->ring_size) { + bnxt_log_live(softc, BNXT_LOGGER_L2, + "%s: Invalid buf_idx %d (ring_size=%d), completion ring %d, opaque=0x%x\n", + __func__, buf_idx, txr->ring_size, cpr->ring.idx, opaque); + *raw_cons = tmp_raw_cons; + return EINVAL; + } + mpc_buf = &txr->tx_mpc_buf_ring[buf_idx]; + handle = mpc_buf->handle; + mpc_buf->handle = BNXT_INV_MPC_HDL; + + if (client == BNXT_MPC_TCE_TYPE || client == BNXT_MPC_RCE_TYPE) + bnxt_ktls_mpc_cmp(softc, client, handle, cmpl_entry_arr, + cmpl_num); + + if (RING_TX(txr, tx_cons) == buf_idx) + bnxt_adv_mpc_cons(softc, txr); + } + *raw_cons = tmp_raw_cons; + return 0; +} diff --git a/sys/dev/bnxt/bnxt_en/bnxt_mpc.h b/sys/dev/bnxt/bnxt_en/bnxt_mpc.h new file mode 100644 index 000000000000..9e62522afd4d --- /dev/null +++ b/sys/dev/bnxt/bnxt_en/bnxt_mpc.h @@ -0,0 +1,182 @@ +/* + * Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2026 Broadcom, All Rights Reserved. + * The term Broadcom refers to Broadcom Limited and/or its subsidiaries + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef BNXT_MPC_H +#define BNXT_MPC_H + +#define BNXT_MPC_TCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TCE +#define BNXT_MPC_RCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RCE +#define BNXT_MPC_TE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TE_CFA +#define BNXT_MPC_RE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RE_CFA +#define BNXT_MPC_TYPE_MAX (BNXT_MPC_RE_CFA_TYPE + 1) + +#define BNXT_MAX_MPC 8 + +#define BNXT_MIN_MPC_TCE 1 +#define BNXT_MIN_MPC_RCE 1 +#define BNXT_DFLT_MPC_TCE BNXT_MAX_MPC +#define BNXT_DFLT_MPC_RCE BNXT_MAX_MPC + +#define BNXT_MIN_MPC_TE_CFA 1 +#define BNXT_MIN_MPC_RE_CFA 1 +#define BNXT_DFLT_MPC_TE_CFA BNXT_MAX_MPC +#define BNXT_DFLT_MPC_RE_CFA BNXT_MAX_MPC + +#define BNXT_MPC_TMO_USECS (1000 * 1000) + +struct bnxt_mpc_info { + uint8_t mpc_chnls_cap; + uint8_t mpc_cp_rings_count; + uint8_t mpc_ring_count[BNXT_MPC_TYPE_MAX]; + struct bnxt_ring *mpc_rings[BNXT_MPC_TYPE_MAX]; + struct bnxt_cp_ring *mpc_cp_rings; + struct bnxt_cp_ring *mpc_nq_rings; + struct iflib_dma_info tx_stats[BNXT_MAX_NUM_QUEUES]; +}; + +#define BNXT_RING_MPC_IDX_TYPE_MASK 0xf0 +#define BNXT_RING_MPC_IDX_TYPE_SFT 4 +#define BNXT_RING_MPC_IDX_RING_MASK 0x0f + +enum bnxt_mpc_type { + BNXT_MPC_CRYPTO, + BNXT_MPC_CFA, +}; + +#define BNXT_MPC_CRYPTO_CAP \ + (HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TCE | \ + HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RCE) + +#define BNXT_MPC_CRYPTO_CAPABLE(softc) \ + ((softc)->mpc_info ? ((softc)->mpc_info->mpc_chnls_cap & \ + BNXT_MPC_CRYPTO_CAP) == BNXT_MPC_CRYPTO_CAP \ + : false) + +#define BNXT_MPC_CFA_CAP \ + (HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TE_CFA | \ + HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RE_CFA) + +#define BNXT_MPC_CFA_CAPABLE(bp) \ + ((bp)->mpc_info ? ((bp)->mpc_info->mpc_chnls_cap & \ + BNXT_MPC_CFA_CAP) == BNXT_MPC_CFA_CAP \ + : false) + +#define MPC_CMP_RING_MASK(cpr) ((cpr)->ring.ring_size - 1) +#define MPC_RING_CMP(cpr, idx) ((idx) & MPC_CMP_RING_MASK(cpr)) + +#define RING_TX(txr, idx) ((idx) & TX_RING_MASK(txr)) +#define NEXT_TX(txr, idx) ((idx + 1) & TX_RING_MASK(txr)) +#define ADVANCE_TX_BY(txr, idx, count) ((idx + count) & TX_RING_MASK(txr)) + +#define TX_OPAQUE_IDX_MASK 0x0000ffff +#define TX_OPAQUE_BDS_MASK 0x00ff0000 +#define TX_OPAQUE_BDS_SHIFT 16 +#define TX_OPAQUE_RING_MASK 0xff000000 +#define TX_OPAQUE_RING_SHIFT 24 + +#define SET_TX_OPAQUE(txr, prod, bds) \ + (((txr)->idx << TX_OPAQUE_RING_SHIFT) | \ + ((bds) << TX_OPAQUE_BDS_SHIFT) | \ + (((prod) & TX_RING_MASK(txr)) & TX_OPAQUE_IDX_MASK)) + +#define TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK) +#define TX_OPAQUE_RING(opq) \ + (((opq) & TX_OPAQUE_RING_MASK) >> TX_OPAQUE_RING_SHIFT) +#define TX_OPAQUE_BDS(opq) (((opq) & TX_OPAQUE_BDS_MASK) >> TX_OPAQUE_BDS_SHIFT) + +#define BNXT_INV_MPC_HDL (-1UL) + +struct bnxt_cmpl_entry { + void *cmpl; + uint32_t len; +}; + +struct mpc_cmp { + __le32 mpc_cmp_client_subtype_type; +#define MPC_CMP_TYPE (0x3f << 0) +#define MPC_CMP_TYPE_MID_PATH_SHORT 0x1e +#define MPC_CMP_TYPE_MID_PATH_LONG 0x1f + +#define MPC_CMP_SUBTYPE 0xf00 +#define MPC_CMP_SUBTYPE_SFT 8 +#define MPC_CMP_SUBTYPE_SOLICITED (0x0 << 8) +#define MPC_CMP_SUBTYPE_ERR (0x1 << 8) +#define MPC_CMP_SUBTYPE_RESYNC (0x2 << 8) + +#define MPC_CMP_CLIENT (0xf << 12) +#define MPC_CMP_CLIENT_SFT 12 +#define MPC_CMP_CLIENT_TCE (0x0 << 12) +#define MPC_CMP_CLIENT_RCE (0x1 << 12) +#define MPC_CMP_CLIENT_TE_CFA (0x2 << 12) +#define MPC_CMP_CLIENT_RE_CFA (0x3 << 12) + + uint32_t mpc_cmp_opaque; + __le32 mpc_cmp_v; +#define MPC_CMP_V (1 << 0) + __le32 mpc_cmp_filler; +}; + +#define MPC_CMP_CMP_TYPE(mpcmp) \ + (le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_TYPE) + +#define MPC_CMP_CLIENT_TYPE(mpcmp) \ + (le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_CLIENT) + +/* + * ERR and RESYNC are both unsolicited (firmware-initiated, no matching TX + * descriptor); only SOLICITED completions carry a valid TX opaque to parse. + */ +#define MPC_CMP_UNSOLICIT_SUBTYPE(mpcmp) \ + (((le32toh((mpcmp)->mpc_cmp_client_subtype_type) & \ + MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_ERR) || \ + ((le32toh((mpcmp)->mpc_cmp_client_subtype_type) & \ + MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_RESYNC)) + +#define MPC_CMP_VALID(cpr, mpcmp, raw_cons) \ + (!!((mpcmp)->mpc_cmp_v & htole32(MPC_CMP_V)) == \ + !((raw_cons) & (cpr)->ring.ring_size)) + +void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap); +void bnxt_free_mpc_info(struct bnxt_softc *softc); +void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc); +int bnxt_alloc_mpcs(struct bnxt_softc *softc); +void bnxt_free_mpcs(struct bnxt_softc *softc); +int bnxt_alloc_mpc_rings(struct bnxt_softc *softc); +void bnxt_free_mpc_rings(struct bnxt_softc *softc); +int bnxt_mpc_irq_setup(struct bnxt_softc *softc); +void bnxt_mpc_irq_cleanup(struct bnxt_softc *softc); +int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc); +int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc); +int bnxt_start_xmit_mpc(struct bnxt_softc *softc, struct bnxt_ring *txr, + void *data, uint len, unsigned long handle); +int bnxt_mpc_cmp(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr, + uint32_t *raw_cons); +void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc); +struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc); + +#endif diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c index c38521abd17f..7f25b50bddf4 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c @@ -426,7 +426,11 @@ void bnxt_destroy_irq(struct bnxt_softc *softc) kfree(softc->irq_tbl); } -static int bnxt_populate_irq(struct bnxt_softc *softc) +/* + * Build the IRQ table from scratch for isc_ntxqsets + irq_count vectors; + * usable by any feature needing private IRQs without prior setup. + */ +int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count) { struct resource_list *rl = NULL; struct resource_list_entry *rle = NULL; @@ -434,16 +438,25 @@ static int bnxt_populate_irq(struct bnxt_softc *softc) struct pci_devinfo *dinfo = NULL; int i; - softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT; + softc->total_irqs = softc->scctx->isc_ntxqsets + irq_count; irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); if (!irq_tbl) { device_printf(softc->dev, "Failed to allocate IRQ table\n"); return -1; } + dinfo = device_get_ivars(softc->pdev->dev.bsddev); rl = &dinfo->resources; rle = resource_list_find(rl, SYS_RES_IRQ, 1); + if (rle == NULL) { + device_printf(softc->dev, + "No default resources for rid = %d, type = %d\n", + 1, SYS_RES_IRQ); + kfree(irq_tbl); + return -1; + } + softc->pdev->dev.irq_start = rle->start; softc->pdev->dev.irq_end = rle->start + softc->total_irqs; @@ -457,6 +470,56 @@ static int bnxt_populate_irq(struct bnxt_softc *softc) return 0; } +/* + * Grow the IRQ table by BNXT_ROCE_IRQ_COUNT on top of existing entries (e.g. + * MPC's) instead of reassigning total_irqs, to avoid reclaiming MPC's rids. + */ +static int bnxt_populate_irq_roce(struct bnxt_softc *softc) +{ + struct resource_list *rl = NULL; + struct resource_list_entry *rle = NULL; + struct pci_devinfo *dinfo = NULL; + struct bnxt_msix_tbl *irq_tbl; + int prev_total_irqs = softc->total_irqs; + int i; + + softc->total_irqs += BNXT_ROCE_IRQ_COUNT; + irq_tbl = krealloc(softc->irq_tbl, + softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); + + if (!irq_tbl) { + device_printf(softc->dev, "Failed to allocate IRQ table\n"); + /* Roll back total_irqs since the table itself wasn't grown. */ + softc->total_irqs = prev_total_irqs; + return -1; + } + + dinfo = device_get_ivars(softc->pdev->dev.bsddev); + rl = &dinfo->resources; + rle = resource_list_find(rl, SYS_RES_IRQ, 1); + if (rle == NULL) { + device_printf(softc->dev, + "No default resources for rid = %d, type = %d\n", + 1, SYS_RES_IRQ); + /* irq_tbl isn't installed yet; free it and roll back total_irqs. */ + kfree(irq_tbl); + softc->total_irqs = prev_total_irqs; + return -1; + } + + softc->pdev->dev.irq_start = rle->start; + softc->pdev->dev.irq_end = rle->start + softc->total_irqs; + + for (i = prev_total_irqs; i < softc->total_irqs; i++) { + irq_tbl[i].entry = i; + irq_tbl[i].vector = softc->pdev->dev.irq_start + i; + } + + softc->irq_tbl = irq_tbl; + + return 0; +} + static const struct bnxt_en_ops bnxt_en_ops_tbl = { .bnxt_register_device = bnxt_register_dev, .bnxt_unregister_device = bnxt_unregister_dev, @@ -521,7 +584,7 @@ int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) struct auxiliary_device *aux_dev; int ret = -1; - if (bnxt_populate_irq(bp)) + if (bnxt_populate_irq_roce(bp)) return ret; device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c index dcfb7e61f0ca..0400b87f5850 100644 --- a/sys/dev/bnxt/bnxt_en/if_bnxt.c +++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c @@ -76,6 +76,7 @@ #include "bnxt_log_data.h" #include "bnxt_coredump.h" #include "bnxt_ptp.h" +#include "bnxt_mpc.h" /* * PCI Device ID Table @@ -260,7 +261,7 @@ static int bnxt_handle_isr(void *arg); static void bnxt_clear_ids(struct bnxt_softc *softc); static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr); static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr); -static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr); +void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr); static void bnxt_def_cp_task(void *context, int pending); static void bnxt_handle_async_event(struct bnxt_softc *softc, struct cmpl_base *cmpl); @@ -449,6 +450,8 @@ bnxt_init_sctx_variants(uint16_t device_id) bnxt_sctx_pf_init = bnxt_sctx_template; bnxt_sctx_pf_init.isc_admin_intrcnt = BNXT_ROCE_IRQ_COUNT; + if (device_id == BCM57608) + bnxt_sctx_pf_init.isc_admin_intrcnt += BNXT_MAX_MPC; bnxt_sctx_vf_init = bnxt_sctx_template; bnxt_sctx_vf_init.isc_flags |= IFLIB_IS_VF; @@ -553,7 +556,7 @@ bnxt_nq_free(struct bnxt_softc *softc) } -static void +void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db, u32 ring_type) { @@ -668,6 +671,15 @@ bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, } softc->ntxqsets = ntxqsets; + bnxt_set_dflt_mpc_rings(softc); + rc = bnxt_alloc_mpcs(softc); + if (rc == 0) + rc = bnxt_alloc_mpc_rings(softc); + if (rc) { + bnxt_free_mpc_rings(softc); + bnxt_free_mpcs(softc); + goto dma_alloc_fail; + } return rc; dma_alloc_fail: @@ -709,6 +721,9 @@ bnxt_queues_free(if_ctx_t ctx) free(softc->rx_rings, M_DEVBUF); free(softc->rx_cp_rings, M_DEVBUF); bnxt_nq_free(softc); + + bnxt_free_mpc_rings(softc); + bnxt_free_mpcs(softc); } static int @@ -1296,12 +1311,21 @@ bnxt_bs_trace_init(struct bnxt_softc *bp, struct bnxt_ctx_mem_type *ctxm) static int bnxt_backing_store_cfg_v2(struct bnxt_softc *softc, u32 ena) { + struct bnxt_mpc_info *mpc = softc->mpc_info; struct bnxt_ctx_mem_info *ctx = softc->ctx_mem; struct bnxt_ctx_mem_type *ctxm; u16 last_type = BNXT_CTX_INV; int rc = 0; u16 type; + if (BNXT_PF(softc) && mpc && mpc->mpc_chnls_cap) { + ctxm = &ctx->ctx_arr[BNXT_CTX_MTQM]; + rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1); + if (rc) + return rc; + last_type = BNXT_CTX_MTQM; + } + if (BNXT_PF(softc)) { for (type = BNXT_CTX_SRT_TRACE; type <= BNXT_CTX_ROCE_HWRM_TRACE; type++) { ctxm = &ctx->ctx_arr[type]; @@ -2971,6 +2995,7 @@ bnxt_attach_pre(if_ctx_t ctx) failed: bnxt_free_sysctl_ctx(softc); + bnxt_free_mpc_info(softc); init_sysctl_failed: bnxt_hwrm_func_drv_unrgtr(softc, false); if (BNXT_PF(softc)) @@ -3022,6 +3047,10 @@ bnxt_attach_post(if_ctx_t ctx) softc->rx_buf_size = min(softc->scctx->isc_max_frame_size, BNXT_PAGE_SIZE); bnxt_dcb_init(softc); + rc = bnxt_mpc_irq_setup(softc); + if (rc) + goto failed; + bnxt_rdma_aux_device_init(softc); /* SR-IOV attach */ @@ -3034,6 +3063,8 @@ bnxt_attach_post(if_ctx_t ctx) "crash dump init failure rc: %d\n", rc); failed: + /* No cleanup needed: on nonzero return iflib_device_register() calls + * IFDI_DETACH and IFDI_QUEUES_FREE, which tear down anything set up so far. */ return rc; } @@ -3045,6 +3076,7 @@ bnxt_detach(if_ctx_t ctx) struct bnxt_vlan_tag *tmp; int i; + bnxt_mpc_irq_cleanup(softc); bnxt_rdma_aux_device_uninit(softc); cancel_delayed_work_sync(&softc->fw_reset_task); cancel_work_sync(&softc->sp_task); @@ -3057,6 +3089,11 @@ bnxt_detach(if_ctx_t ctx) bnxt_free_sysctl_ctx(softc); bnxt_free_crash_dump_mem(softc); bnxt_hwrm_func_reset(softc); + /* Must run before bnxt_free_mpc_info(): these gate on mpc_info being + * non-NULL, so freeing it first would leak the MPC rings. */ + bnxt_free_mpc_rings(softc); + bnxt_free_mpcs(softc); + bnxt_free_mpc_info(softc); bnxt_free_ctx_mem(softc); bnxt_clear_ids(softc); iflib_irq_free(ctx, &softc->def_cp_ring.irq); @@ -3133,6 +3170,11 @@ bnxt_hwrm_resource_free(struct bnxt_softc *softc) if (rc) goto fail; } + + rc = bnxt_hwrm_mpc_ring_free(softc); + if (rc) + goto fail; + rc = bnxt_hwrm_free_filter(softc); if (rc) goto fail; @@ -3520,6 +3562,11 @@ skip_def_cp_ring: softc->db_ops.bnxt_db_tx(&softc->tx_rings[i], 0); } + if (BNXT_PF(softc)) { + rc = bnxt_hwrm_mpc_ring_alloc(softc); + if (rc) + goto fail; + } bnxt_do_enable_intr(&softc->def_cp_ring); bnxt_get_port_module_status(softc); bnxt_media_status(softc->ctx, &ifmr); @@ -5303,6 +5350,7 @@ bnxt_clear_ids(struct bnxt_softc *softc) softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE; } + bnxt_clear_mpc_rings_ids(softc); softc->vnic_info.filter_id = -1; softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.rss_id = (uint16_t)HWRM_NA_SIGNATURE; @@ -5310,7 +5358,7 @@ bnxt_clear_ids(struct bnxt_softc *softc) softc->vnic_info.rss_grp_tbl.idi_size); } -static void +void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr) { struct cmpl_base *cmp = (void *)cpr->ring.vaddr; diff --git a/sys/modules/bnxt/bnxt_en/Makefile b/sys/modules/bnxt/bnxt_en/Makefile index 67bd2646fa4d..e0d371c264de 100644 --- a/sys/modules/bnxt/bnxt_en/Makefile +++ b/sys/modules/bnxt/bnxt_en/Makefile @@ -15,6 +15,7 @@ SRCS += bnxt_log.c bnxt_log.h SRCS += bnxt_log_data.c bnxt_log_data.h SRCS += bnxt_coredump.c bnxt_coredump.h SRCS += bnxt_ptp.c bnxt_ptp.h +SRCS += bnxt_mpc.c bnxt_mpc.h SRCS += ${LINUXKPI_GENSRCS} CFLAGS+= -DIFLIB |
