path: root/sys/dev/ice/ice_iflib_txrx.c

/* SPDX-License-Identifier: BSD-3-Clause */
/*  Copyright (c) 2022, Intel Corporation
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 *   3. Neither the name of the Intel Corporation nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 */
/*$FreeBSD$*/

/**
 * @file ice_iflib_txrx.c
 * @brief iflib Tx/Rx hotpath
 *
 * Main location for the iflib Tx/Rx hotpath implementation.
 *
 * Contains the implementation for the iflib function callbacks and the
 * if_txrx ops structure.
 */

#include "ice_iflib.h"

/* Tx/Rx hotpath utility functions */
#include "ice_common_txrx.h"

/*
 * iflib txrx method declarations
 */
static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi);
static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri);
static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear);
static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget);
static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx);
static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru);
static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi);

/* Macro to help extract the NIC mode flexible Rx descriptor fields from the
 * advanced 32byte Rx descriptors.
 */
#define RX_FLEX_NIC(desc, field) \
	(((struct ice_32b_rx_flex_desc_nic *)desc)->field)

/**
 * @var ice_txrx
 * @brief Tx/Rx operations for the iflib stack
 *
 * Structure defining the Tx and Rx related operations that iflib can request
 * the driver to perform. These are the main entry points for the hot path of
 * the transmit and receive paths in the iflib driver.
 */
struct if_txrx ice_txrx = {
	.ift_txd_encap = ice_ift_txd_encap,
	.ift_txd_flush = ice_ift_txd_flush,
	.ift_txd_credits_update = ice_ift_txd_credits_update,
	.ift_rxd_available = ice_ift_rxd_available,
	.ift_rxd_pkt_get = ice_ift_rxd_pkt_get,
	.ift_rxd_refill = ice_ift_rxd_refill,
	.ift_rxd_flush = ice_ift_rxd_flush,
	.ift_txq_select_v2 = ice_ift_queue_select,
};

/**
 * ice_ift_txd_encap - prepare Tx descriptors for a packet
 * @arg: the iflib softc structure pointer
 * @pi: packet info
 *
 * Prepares and encapsulates the given packet into into Tx descriptors, in
 * preparation for sending to the transmit engine. Sets the necessary context
 * descriptors for TSO and other offloads, and prepares the last descriptor
 * for the writeback status.
 *
 * Return 0 on success, non-zero error code on failure.
 */
static int
ice_ift_txd_encap(void *arg, if_pkt_info_t pi)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx];
	int nsegs = pi->ipi_nsegs;
	bus_dma_segment_t *segs = pi->ipi_segs;
	struct ice_tx_desc *txd = NULL;
	int i, j, mask, pidx_last;
	u32 cmd, off;

	cmd = off = 0;
	i = pi->ipi_pidx;

	/* Set up the TSO/CSUM offload */
	if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) {
		/* Set up the TSO context descriptor if required */
		if (pi->ipi_csum_flags & CSUM_TSO) {
			if (ice_tso_detect_sparse(pi))
				return (EFBIG);
			i = ice_tso_setup(txq, pi);
		}
		ice_tx_setup_offload(txq, pi, &cmd, &off);
	}
	if (pi->ipi_mflags & M_VLANTAG)
		cmd |= ICE_TX_DESC_CMD_IL2TAG1;

	mask = txq->desc_count - 1;
	for (j = 0; j < nsegs; j++) {
		bus_size_t seglen;

		txd = &txq->tx_base[i];
		seglen = segs[j].ds_len;

		txd->buf_addr = htole64(segs[j].ds_addr);
		txd->cmd_type_offset_bsz =
		    htole64(ICE_TX_DESC_DTYPE_DATA
		    | ((u64)cmd  << ICE_TXD_QW1_CMD_S)
		    | ((u64)off << ICE_TXD_QW1_OFFSET_S)
		    | ((u64)seglen  << ICE_TXD_QW1_TX_BUF_SZ_S)
		    | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S));

		txq->stats.tx_bytes += seglen;
		pidx_last = i;
		i = (i+1) & mask;
	}

	/* Set the last descriptor for report */
#define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
	txd->cmd_type_offset_bsz |=
	    htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S));

	/* Add to report status array */
	txq->tx_rsq[txq->tx_rs_pidx] = pidx_last;
	txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask;
	MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx);

	pi->ipi_new_pidx = i;

	++txq->stats.tx_packets;
	return (0);
}

/**
 * ice_ift_txd_flush - Flush Tx descriptors to hardware
 * @arg: device specific softc pointer
 * @txqid: the Tx queue to flush
 * @pidx: descriptor index to advance tail to
 *
 * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that
 * frames are available for transmit.
 */
static void
ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
	struct ice_hw *hw = &sc->hw;

	wr32(hw, txq->tail, pidx);
}

/**
 * ice_ift_txd_credits_update - cleanup Tx descriptors
 * @arg: device private softc
 * @txqid: the Tx queue to update
 * @clear: if false, only report, do not actually clean
 *
 * If clear is false, iflib is asking if we *could* clean up any Tx
 * descriptors.
 *
 * If clear is true, iflib is requesting to cleanup and reclaim used Tx
 * descriptors.
 */
static int
ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];

	qidx_t processed = 0;
	qidx_t cur, prev, ntxd, rs_cidx;
	int32_t delta;
	bool is_done;

	rs_cidx = txq->tx_rs_cidx;
	if (rs_cidx == txq->tx_rs_pidx)
		return (0);
	cur = txq->tx_rsq[rs_cidx];
	MPASS(cur != QIDX_INVALID);
	is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);

	if (!is_done)
		return (0);
	else if (clear == false)
		return (1);

	prev = txq->tx_cidx_processed;
	ntxd = txq->desc_count;
	do {
		MPASS(prev != cur);
		delta = (int32_t)cur - (int32_t)prev;
		if (delta < 0)
			delta += ntxd;
		MPASS(delta > 0);
		processed += delta;
		prev = cur;
		rs_cidx = (rs_cidx + 1) & (ntxd-1);
		if (rs_cidx == txq->tx_rs_pidx)
			break;
		cur = txq->tx_rsq[rs_cidx];
		MPASS(cur != QIDX_INVALID);
		is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
	} while (is_done);

	txq->tx_rs_cidx = rs_cidx;
	txq->tx_cidx_processed = prev;

	return (processed);
}

/**
 * ice_ift_rxd_available - Return number of available Rx packets
 * @arg: device private softc
 * @rxqid: the Rx queue id
 * @pidx: descriptor start point
 * @budget: maximum Rx budget
 *
 * Determines how many Rx packets are available on the queue, up to a maximum
 * of the given budget.
 */
static int
ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
	union ice_32b_rx_flex_desc *rxd;
	uint16_t status0;
	int cnt, i, nrxd;

	nrxd = rxq->desc_count;

	for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) {
		rxd = &rxq->rx_base[i];
		status0 = le16toh(rxd->wb.status_error0);

		if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0)
			break;
		if (++i == nrxd)
			i = 0;
		if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))
			cnt++;
	}

	return (cnt);
}

/**
 * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
 * @arg: device specific softc
 * @ri: receive packet info
 *
 * This function is called by iflib, and executes in ithread context. It is
 * called by iflib to obtain data which has been DMA'ed into host memory.
 * Returns zero on success, and EBADMSG on failure.
 */
static int
ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx];
	union ice_32b_rx_flex_desc *cur;
	u16 status0, plen, ptype;
	bool eop;
	size_t cidx;
	int i;

	cidx = ri->iri_cidx;
	i = 0;
	do {
		/* 5 descriptor receive limit */
		MPASS(i < ICE_MAX_RX_SEGS);

		cur = &rxq->rx_base[cidx];
		status0 = le16toh(cur->wb.status_error0);
		plen = le16toh(cur->wb.pkt_len) &
			ICE_RX_FLX_DESC_PKT_LEN_M;

		/* we should never be called without a valid descriptor */
		MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0);

		ri->iri_len += plen;

		cur->wb.status_error0 = 0;
		eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S));

		ri->iri_frags[i].irf_flid = 0;
		ri->iri_frags[i].irf_idx = cidx;
		ri->iri_frags[i].irf_len = plen;
		if (++cidx == rxq->desc_count)
			cidx = 0;
		i++;
	} while (!eop);

	/* End of Packet reached; cur is eop/last descriptor */

	/* Make sure packets with bad L2 values are discarded.
	 * This bit is only valid in the last descriptor.
	 */
	if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) {
		rxq->stats.desc_errs++;
		return (EBADMSG);
	}

	/* Get VLAN tag information if one is in descriptor */
	if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
		ri->iri_vtag = le16toh(cur->wb.l2tag1);
		ri->iri_flags |= M_VLANTAG;
	}

	/* Capture soft statistics for this Rx queue */
	rxq->stats.rx_packets++;
	rxq->stats.rx_bytes += ri->iri_len;

	/* Get packet type and set checksum flags */
	ptype = le16toh(cur->wb.ptype_flex_flags0) &
		ICE_RX_FLEX_DESC_PTYPE_M;
	if ((iflib_get_ifp(sc->ctx)->if_capenable & IFCAP_RXCSUM) != 0)
		ice_rx_checksum(rxq, &ri->iri_csum_flags,
				&ri->iri_csum_data, status0, ptype);

	/* Set remaining iflib RX descriptor info fields */
	ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash));
	ri->iri_rsstype = ice_ptype_to_hash(ptype);
	ri->iri_nfrags = i;
	return (0);
}

/**
 * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
 * @arg: device specific softc structure
 * @iru: the Rx descriptor update structure
 *
 * Update the Rx descriptor indices for a given queue, assigning new physical
 * addresses to the descriptors, preparing them for re-use by the hardware.
 */
static void
ice_ift_rxd_refill(void *arg, if_rxd_update_t iru)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_rx_queue *rxq;
	uint32_t next_pidx;
	int i;
	uint64_t *paddrs;
	uint32_t pidx;
	uint16_t qsidx, count;

	paddrs = iru->iru_paddrs;
	pidx = iru->iru_pidx;
	qsidx = iru->iru_qsidx;
	count = iru->iru_count;

	rxq = &(sc->pf_vsi.rx_queues[qsidx]);

	for (i = 0, next_pidx = pidx; i < count; i++) {
		rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
		if (++next_pidx == (uint32_t)rxq->desc_count)
			next_pidx = 0;
	}
}

/**
 * ice_ift_rxd_flush - Flush Rx descriptors to hardware
 * @arg: device specific softc pointer
 * @rxqid: the Rx queue to flush
 * @flidx: unused parameter
 * @pidx: descriptor index to advance tail to
 *
 * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that
 * software is done with the descriptor and it can be recycled.
 */
static void
ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused,
		  qidx_t pidx)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
	struct ice_hw *hw = &sc->hw;

	wr32(hw, rxq->tail, pidx);
}

static qidx_t
ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi)
{
	struct ice_softc *sc = (struct ice_softc *)arg;
	struct ice_dcbx_cfg *local_dcbx_cfg;
	struct ice_vsi *vsi = &sc->pf_vsi;
	u16 tc_base_queue, tc_qcount;
	u8 up, tc;

#ifdef ALTQ
	/* Included to match default iflib behavior */
	/* Only go out on default queue if ALTQ is enabled */
	struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx);
	if (ALTQ_IS_ENABLED(&ifp->if_snd))
		return (0);
#endif

	if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) {
		if (M_HASHTYPE_GET(m)) {
			/* Default iflib queue selection method */
			return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues);
		} else
			return (0);
	}

	/* Use default TC unless overridden later */
	tc = 0; /* XXX: Get default TC for traffic if >1 TC? */

	local_dcbx_cfg = &sc->hw.port_info->qos_cfg.local_dcbx_cfg;

#if defined(INET) || defined(INET6)
	if ((local_dcbx_cfg->pfc_mode == ICE_QOS_MODE_DSCP) &&
	    (pi->ipi_flags & (IPI_TX_IPV4 | IPI_TX_IPV6))) {
		u8 dscp_val = pi->ipi_ip_tos >> 2;
		tc = local_dcbx_cfg->dscp_map[dscp_val];
	} else
#endif /* defined(INET) || defined(INET6) */
	if (m->m_flags & M_VLANTAG) { /* ICE_QOS_MODE_VLAN */
		up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag);
		tc = local_dcbx_cfg->etscfg.prio_table[up];
	}

	tc_base_queue = vsi->tc_info[tc].qoffset;
	tc_qcount = vsi->tc_info[tc].qcount_tx;

	if (M_HASHTYPE_GET(m))
		return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue);
	else
		return (tc_base_queue);
}