path: root/sys/dev/aq/aq_hw.c

/*
 * aQuantia Corporation Network Driver
 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   (1) Redistributions of source code must retain the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer.
 *
 *   (2) Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 *   (3)The name of the author may not be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/endian.h>
#include <sys/socket.h>
#include <machine/cpu.h>
#include <net/rss_config.h>

#include "aq_hw.h"
#include "aq2_hw.h"
#include "aq_dbg.h"
#include "aq_hw_llh.h"
#include "aq_fw.h"

#define AQ_HW_FW_SM_RAM        0x2U
#define AQ_CFG_FW_MIN_VER_EXPECTED 0x01050006U

static uint32_t aq_hw_active_tcs(struct aq_hw *hw);

int
aq_hw_err_from_flags(struct aq_hw *hw)
{
	if (atomic_load_acq_long(&hw->flags) & AQ_HW_FLAG_ERR_UNPLUG)
		return (ENXIO);
	return (0);
}

int
aq_hw_invalidate_descriptor_cache(struct aq_hw *hw)
{
	int error;

	/* Atlantic A0 does not implement this stop workaround. */
	if (IS_CHIP_FEATURE(hw, REVISION_A0))
		return (0);

	/*
	 * Cached Rx descriptors retain both descriptor and data addresses.
	 * Toggle the global cache invalidation only after every ring is down.
	 */
	rdm_rx_dma_desc_cache_init_tgl(hw);
	error = aq_hw_err_from_flags(hw);
	if (error != 0)
		return (error);

	error = AQ_HW_WAIT_FOR(rdm_rx_dma_desc_cache_init_done_get(hw) != 0,
	    1000, 10);
	if (error != 0)
		return (error);
	return (aq_hw_err_from_flags(hw));
}

inline uint32_t
aq_hw_read_reg(struct aq_hw *hw, uint32_t reg)
{
	uint32_t val = le32toh(bus_space_read_4(hw->hw_tag, hw->hw_handle, reg));

	if (__predict_false(val == 0xFFFFFFFFU) &&
	    le32toh(bus_space_read_4(hw->hw_tag, hw->hw_handle, 0x10)) ==
	    0xFFFFFFFFU)
		atomic_set_rel_long(&hw->flags, AQ_HW_FLAG_ERR_UNPLUG);

	return (val);
}

static void
aq_hw_chip_features_init(struct aq_hw *hw, uint32_t *p)
{
	uint32_t chip_features = 0U;
	uint32_t val = reg_glb_mif_id_get(hw);
	uint32_t mif_rev = val & 0xFFU;

	if ((0xFU & mif_rev) == 1U) {
		chip_features |= AQ_HW_CHIP_REVISION_A0 | AQ_HW_CHIP_MPI_AQ |
		     AQ_HW_CHIP_MIPS;
	} else if ((0xFU & mif_rev) == 2U) {
		chip_features |= AQ_HW_CHIP_REVISION_B0 | AQ_HW_CHIP_MPI_AQ |
		    AQ_HW_CHIP_MIPS | AQ_HW_CHIP_TPO2 | AQ_HW_CHIP_RPF2;
	} else if ((0xFU & mif_rev) == 0xAU) {
		chip_features |= AQ_HW_CHIP_REVISION_B1 | AQ_HW_CHIP_MPI_AQ |
		    AQ_HW_CHIP_MIPS | AQ_HW_CHIP_TPO2 | AQ_HW_CHIP_RPF2;
	}

	*p = chip_features;
}

int
aq_hw_fw_downld_dwords(struct aq_hw *hw, uint32_t a, uint32_t *p, uint32_t cnt)
{
	int err = 0;

	err = AQ_HW_WAIT_FOR(reg_glb_cpu_sem_get(hw, AQ_HW_FW_SM_RAM) == 1U, 1U,
	     10000U);

	if (err != 0) {
		bool is_locked;

		reg_glb_cpu_sem_set(hw, 1U, AQ_HW_FW_SM_RAM);
		is_locked = reg_glb_cpu_sem_get(hw, AQ_HW_FW_SM_RAM);
		if (!is_locked) {
			err = ETIMEDOUT;
			goto err_exit;
		}
		err = 0;
	}

	mif_mcp_up_mailbox_addr_set(hw, a);

	for (++cnt; --cnt && !err;) {
		mif_mcp_up_mailbox_execute_operation_set(hw, 1);

		if (IS_CHIP_FEATURE(hw, REVISION_B1))
			err = AQ_HW_WAIT_FOR(a != mif_mcp_up_mailbox_addr_get(hw),
			    1U, 1000U);
		else
			err = AQ_HW_WAIT_FOR(!mif_mcp_up_mailbox_busy_get(hw), 1,
			     1000U);

		*(p++) = mif_mcp_up_mailbox_data_get(hw);
		a += 4;
	}

	reg_glb_cpu_sem_set(hw, 1U, AQ_HW_FW_SM_RAM);

err_exit:
	return (err);
}

int
aq_hw_ver_match(const struct aq_hw_fw_version* ver_expected,
    const struct aq_hw_fw_version* ver_actual)
{
	AQ_DBG_ENTER();

	if (ver_actual->major_version != ver_expected->major_version)
		return (ver_actual->major_version > ver_expected->major_version);
	if (ver_actual->minor_version != ver_expected->minor_version)
		return (ver_actual->minor_version > ver_expected->minor_version);
	return (ver_actual->build_number >= ver_expected->build_number);
}

static int
aq_hw_init_ucp(struct aq_hw *hw)
{
	int err = 0;
	AQ_DBG_ENTER();

	hw->fw_version.raw = 0;

	/* Atlantic 2 uses a different reset/firmware handshake. */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2))
		return (aq2_fw_reboot(hw));

	err = aq_fw_reset(hw);
	if (err != 0) {
		device_printf(hw->dev, "aq_hw_init_ucp(): F/W reset failed, err %d\n", err);
		return (err);
	}

	aq_hw_chip_features_init(hw, &hw->chip_features);
	err = aq_fw_ops_init(hw);
	if (err != 0) {
		device_printf(hw->dev, "could not initialize F/W ops, err %d\n", err);
		return (err);
	}

	if (hw->fw_version.major_version == 1) {
		if (!AQ_READ_REG(hw, 0x370)) {
			unsigned int rnd = 0;
			unsigned int ucp_0x370 = 0;

			rnd = arc4random();

			ucp_0x370 = 0x02020202 | (0xFEFEFEFE & rnd);
			AQ_WRITE_REG(hw, AQ_HW_UCP_0X370_REG, ucp_0x370);
		}

		reg_glb_cpu_scratch_scp_set(hw, 0, 25);
	}

	/* check 10 times by 1ms */
	err = AQ_HW_WAIT_FOR((hw->mbox_addr = AQ_READ_REG(hw, 0x360)) != 0, 400U, 20);

	struct aq_hw_fw_version ver_expected = { .raw = AQ_CFG_FW_MIN_VER_EXPECTED };
	if (!aq_hw_ver_match(&ver_expected, &hw->fw_version))
	        device_printf(hw->dev, "aq_hw_init_ucp(), wrong FW version: expected:%x actual:%x\n",
		    AQ_CFG_FW_MIN_VER_EXPECTED, hw->fw_version.raw);

	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_mpi_create(struct aq_hw *hw)
{
	int err = 0;

	AQ_DBG_ENTER();
	err = aq_hw_init_ucp(hw);
	if (err != 0)
		goto err_exit;

err_exit:
	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_mpi_read_stats(struct aq_hw *hw, struct aq_hw_stats *stats)
{
	int err;

	err = hw->fw_ops->get_stats(hw, stats);
	if (err != 0)
		return (err);

	/* No firmware interface reports the RPB drop count; read it directly. */
	stats->dpc = reg_rx_dma_stat_counter7get(hw);

	/* The LRO counter has no A2 counterpart in any reference driver. */
	if (!IS_CHIP_FEATURE(hw, ATLANTIC2))
		stats->cprc = stats_rx_lro_coalesced_pkt_count0_get(hw);

	return (0);
}

static int
aq_hw_mpi_set(struct aq_hw *hw, enum aq_hw_fw_mpi_state state, uint32_t speed)
{
	int err;
	AQ_DBG_ENTERA("speed %d", speed);

	err = hw->fw_ops->set_mode(hw, state, speed);

	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_set_link_speed(struct aq_hw *hw, uint32_t speed)
{
	return aq_hw_mpi_set(hw, MPI_INIT, speed);
}

int
aq_hw_get_link_state(struct aq_hw *hw, uint32_t *link_speed, struct aq_hw_fc_info *fc_neg)
{
	int err = 0;


	enum aq_hw_fw_mpi_state mode;
	enum aq_fw_link_speed speed = aq_fw_none;
	enum aq_fw_link_fc fc;

	*link_speed = 0;
	fc_neg->fc_rx = false;
	fc_neg->fc_tx = false;

	err = hw->fw_ops->get_mode(hw, &mode, &speed, &fc);

	if (err != 0) {
		device_printf(hw->dev, "get_mode() failed, err %d\n", err);
		AQ_DBG_EXIT(err);
		return (err);
	}
	if (mode != MPI_INIT)
		return (0);

	hw->link_speed = speed;	/* remember negotiated rate for interrupt moderation */

	switch (speed) {
	case aq_fw_10G:
		*link_speed = 10000U;
		break;
	case aq_fw_5G:
		*link_speed = 5000U;
		break;
	case aq_fw_2G5:
		*link_speed = 2500U;
		break;
	case aq_fw_1G:
		*link_speed = 1000U;
		break;
	case aq_fw_100M:
		*link_speed = 100U;
		break;
	case aq_fw_10M:
		*link_speed = 10U;
		break;
	default:
		*link_speed = 0U;
		break;
	}

	fc_neg->fc_rx = !!(fc & aq_fw_fc_ENABLE_RX);
	fc_neg->fc_tx = !!(fc & aq_fw_fc_ENABLE_TX);

	return (0);
}

int
aq_hw_get_mac_permanent(struct aq_hw *hw,  uint8_t *mac)
{
	int err;
	AQ_DBG_ENTER();

	err = hw->fw_ops->get_mac_addr(hw, mac);
	if (err != 0) {
		/* A transient mailbox failure must not fail the attach. */
		device_printf(hw->dev, "could not read the MAC address: %d\n",
		    err);
		memset(mac, 0, ETHER_ADDR_LEN);
	}

	/* Couldn't get MAC address from HW. Use auto-generated one. */
	if ((mac[0] & 1) || ((mac[0] | mac[1] | mac[2]) == 0)) {
		uint16_t rnd;
		uint32_t h = 0;
		uint32_t l = 0;

		device_printf(hw->dev,
		    "invalid (multicast/empty) HW MAC %6D; using random\n",
		    mac, ":");

		rnd = arc4random();

		/* chip revision */
		l = 0xE3000000U | (0xFFFFU & rnd) | (0x00 << 16);
		h = 0x8001300EU;

		mac[5] = (uint8_t)(0xFFU & l);
		l >>= 8;
		mac[4] = (uint8_t)(0xFFU & l);
		l >>= 8;
		mac[3] = (uint8_t)(0xFFU & l);
		l >>= 8;
		mac[2] = (uint8_t)(0xFFU & l);
		mac[1] = (uint8_t)(0xFFU & h);
		h >>= 8;
		mac[0] = (uint8_t)(0xFFU & h);

		err = 0;
	}

	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_deinit(struct aq_hw *hw)
{
	AQ_DBG_ENTER();
	aq_hw_mpi_set(hw, MPI_DEINIT, 0);
	AQ_DBG_EXIT(0);
	return (0);
}

int
aq_hw_set_power(struct aq_hw *hw, unsigned int power_state)
{
	AQ_DBG_ENTER();
	aq_hw_mpi_set(hw, MPI_POWER, 0);
	AQ_DBG_EXIT(0);
	return (0);
}


/* HW NIC functions */

int
aq_hw_reset(struct aq_hw *hw, bool reboot)
{
	int err = 0;

	AQ_DBG_ENTER();

	/*
	 * A2 resets by rebooting the MCP; A1 uses the RBL/FLB reset.  At attach
	 * aq_hw_mpi_create() has already done this, so the caller passes
	 * reboot=false to skip the (costly, on A2) redundant MCP reboot.
	 */
	if (reboot) {
		if (IS_CHIP_FEATURE(hw, ATLANTIC2))
			err = aq2_fw_reboot(hw);
		else
			err = aq_fw_reset(hw);
		if (err != 0)
			goto err_exit;
	}

	itr_irq_reg_res_dis_set(hw, 0);
	itr_res_irq_set(hw, 1);

	/* check 10 times by 1ms */
	err = AQ_HW_WAIT_FOR(itr_res_irq_get(hw) == 0, 1000, 10);
	if (err != 0) {
		device_printf(hw->dev, "IRQ reset failed: %d\n", err);
		goto err_exit;
	}

	err = hw->fw_ops->reset(hw);
	if (err != 0)
		goto err_exit;

	err = aq_hw_err_from_flags(hw);

err_exit:
	AQ_DBG_EXIT(err);
	return (err);
}

static int
aq_hw_qos_set(struct aq_hw *hw)
{
	uint32_t tc = 0U;
	uint32_t buff_size = 0U;
	uint32_t n_tcs;
	unsigned int i_priority = 0U;
	unsigned int i;
	int err = 0;

	AQ_DBG_ENTER();

	/* TPS Descriptor rate init */
	tps_tx_pkt_shed_desc_rate_curr_time_res_set(hw, 0x0U);
	tps_tx_pkt_shed_desc_rate_lim_set(hw, 0xA);

	/* TPS VM init */
	tps_tx_pkt_shed_desc_vm_arb_mode_set(hw, 0U);

	/* TPS TC credits init */
	tps_tx_pkt_shed_desc_tc_arb_mode_set(hw, 0U);
	tps_tx_pkt_shed_data_arb_mode_set(hw, 0U);

	/* One TC per active 8-ring group; share the buffer across them. */
	n_tcs = aq_hw_active_tcs(hw);
	buff_size = (IS_CHIP_FEATURE(hw, ATLANTIC2) ?
	    AQ2_HW_TXBUF_MAX : AQ_HW_TXBUF_MAX) / n_tcs;

	for (tc = 0; tc < n_tcs; tc++) {
		if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
			/* Atlantic 2's data-TC credit/weight fields are wider. */
			AQ_WRITE_REG_BIT(hw, TPS_DATA_TCT_REG(tc),
			    TPS2_DATA_TCT_CREDIT_MAX,
			    TPS2_DATA_TCT_CREDIT_MAX_SHIFT, 0xfff0);
			AQ_WRITE_REG_BIT(hw, TPS_DATA_TCT_REG(tc),
			    TPS2_DATA_TCT_WEIGHT,
			    TPS2_DATA_TCT_WEIGHT_SHIFT, 0x640);
		} else {
			tps_tx_pkt_shed_tc_data_max_credit_set(hw, 0xFFF, tc);
			tps_tx_pkt_shed_tc_data_weight_set(hw, 0x64, tc);
		}
		tps_tx_pkt_shed_desc_tc_max_credit_set(hw, 0x50, tc);
		tps_tx_pkt_shed_desc_tc_weight_set(hw, 0x1E, tc);

		tpb_tx_pkt_buff_size_per_tc_set(hw, buff_size, tc);
		tpb_tx_buff_hi_threshold_per_tc_set(hw,
		    AQ_BUF_THRESHOLD(buff_size, 66U), tc);
		tpb_tx_buff_lo_threshold_per_tc_set(hw,
		    AQ_BUF_THRESHOLD(buff_size, 50U), tc);
	}

	/* QoS Rx buf size per TC */
	tc = 0;
	buff_size = IS_CHIP_FEATURE(hw, ATLANTIC2) ?
	    AQ2_HW_RXBUF_MAX : AQ_HW_RXBUF_MAX;

	rpb_rx_pkt_buff_size_per_tc_set(hw, buff_size, tc);
	rpb_rx_buff_hi_threshold_per_tc_set(hw,
	    AQ_BUF_THRESHOLD(buff_size, 66U), tc);
	rpb_rx_buff_lo_threshold_per_tc_set(hw,
	    AQ_BUF_THRESHOLD(buff_size, 50U), tc);

	/* QoS 802.1p priority -> TC mapping */
	for (i_priority = 8U; i_priority--;)
		rpf_rpb_user_priority_tc_map_set(hw, i_priority, 0U);

	/* Atlantic 2 ring -> TC map (TC = ring / HW_ATL_B0_RINGS_PER_TC). */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		AQ_WRITE_REG_BIT(hw, TPB_TX_BUF_REG,
		    TPB_TX_BUF_TC_Q_RAND_MAP_EN,
		    TPB_TX_BUF_TC_Q_RAND_MAP_EN_SHIFT, 1);
		/* TX packs a byte per ring (4/reg); RX a nibble (8/reg). */
		for (i = 0; i < HW_ATL_B0_RINGS_MAX / 4; i++)
			AQ_WRITE_REG(hw, AQ2_TX_Q_TC_MAP_REG(i),
			    (i / 2) * 0x01010101U);
		for (i = 0; i < HW_ATL_B0_RINGS_MAX / 8; i++)
			AQ_WRITE_REG(hw, AQ2_RX_Q_TC_MAP_REG(i),
			    i * 0x11111111U);
	}

	err = aq_hw_err_from_flags(hw);
	AQ_DBG_EXIT(err);
	return (err);
}

/* Tx traffic classes currently provisioned (one per active 8-ring group). */
static uint32_t
aq_hw_active_tcs(struct aq_hw *hw)
{
	uint32_t n = howmany(MAX(hw->tx_rings_count, 1U),
	    HW_ATL_B0_RINGS_PER_TC);

	return (MIN(n, HW_ATL_B0_TCS_MAX));
}

static int
aq_hw_offload_set(struct aq_hw *hw)
{
	int err;

	AQ_DBG_ENTER();
	/* TX checksums offloads*/
	tpo_ipv4header_crc_offload_en_set(hw, 1);
	tpo_tcp_udp_crc_offload_en_set(hw, 1);

	/* RX checksums offloads*/
	rpo_ipv4header_crc_offload_en_set(hw, 1);
	rpo_tcp_udp_crc_offload_en_set(hw, 1);

	/* LSO offloads*/
	tdm_large_send_offload_en_set(hw, 0xFFFFFFFFU);

	/* Outer (S-VLAN) tag parse mode */
	rpo_outer_vlan_tag_mode_set(hw, 1U);

	{
		uint32_t i = 0;
		uint32_t val = (8U < HW_ATL_B0_LRO_RXD_MAX) ? 0x3U :
		    ((4U < HW_ATL_B0_LRO_RXD_MAX) ? 0x2U :
		    ((2U < HW_ATL_B0_LRO_RXD_MAX) ? 0x1U : 0x0));

		for (i = 0; i < HW_ATL_B0_RINGS_MAX; i++)
			rpo_lro_max_num_of_descriptors_set(hw, val, i);

		rpo_lro_time_base_divider_set(hw, 0x61AU);
		rpo_lro_inactive_interval_set(hw, 0);
		/* the LRO timebase divider is 5 uS (0x61a),
		 * to get a maximum coalescing interval of 250 uS,
		 * we need to multiply by 50(0x32) to get
		 * the default value 250 uS
		 */
		rpo_lro_max_coalescing_interval_set(hw, 50);

		rpo_lro_qsessions_lim_set(hw, 1U);

		rpo_lro_total_desc_lim_set(hw, 2U);

		rpo_lro_patch_optimization_en_set(hw, 0U);

		rpo_lro_min_pay_of_first_pkt_set(hw, 10U);

		rpo_lro_pkt_lim_set(hw, 1U);

		rpo_lro_en_set(hw, (hw->lro_enabled ? 0xFFFFFFFFU : 0U));
	}


	err = aq_hw_err_from_flags(hw);

	AQ_DBG_EXIT(err);
	return (err);
}

static int
aq_hw_init_tx_path(struct aq_hw *hw)
{
	int err = 0;

	AQ_DBG_ENTER();

	/* Tx TC/RSS number config */
	tpb_tx_tc_mode_set(hw, 1U);

	thm_lso_tcp_flag_of_first_pkt_set(hw, 0x0FF6U);
	thm_lso_tcp_flag_of_middle_pkt_set(hw, 0x0FF6U);
	thm_lso_tcp_flag_of_last_pkt_set(hw, 0x0F7FU);

	/* Tx interrupts */
	tdm_tx_desc_wr_wb_irq_en_set(hw, 1U);

	/* misc (Atlantic 1 TPO register; absent on Atlantic 2) */
	if (!IS_CHIP_FEATURE(hw, ATLANTIC2))
		AQ_WRITE_REG(hw, 0x00007040U,
		    IS_CHIP_FEATURE(hw, TPO2) ? 0x00010000U : 0x00000000U);
	tdm_tx_dca_en_set(hw, 0U);
	tdm_tx_dca_mode_set(hw, 0U);

	tpb_tx_path_scp_ins_en_set(hw, 1U);

	err = aq_hw_err_from_flags(hw);
	AQ_DBG_EXIT(err);
	return (err);
}

/* Atlantic 2: install one action-resolver-table row under the ART semaphore. */
static int
aq2_art_filter_set(struct aq_hw *hw, uint32_t idx, uint32_t tag,
    uint32_t mask, uint32_t action)
{
	idx += hw->art_filter_base_index;
	if (idx >= AQ2_ART_TABLE_SIZE) {
		device_printf(hw->dev,
		    "ART index %u out of range (firmware base %u)\n", idx,
		    hw->art_filter_base_index);
		return (EINVAL);
	}

	if (AQ_HW_WAIT_FOR(reg_glb_cpu_sem_get(hw, AQ2_ART_SEM_INDEX) == 1U,
	    10U, 1000U) != 0) {
		device_printf(hw->dev, "ART semaphore timeout, idx %u\n", idx);
		return (EBUSY);
	}

	AQ_WRITE_REG(hw, AQ2_RPF_ACT_ART_REQ_TAG_REG(idx), tag);
	AQ_WRITE_REG(hw, AQ2_RPF_ACT_ART_REQ_MASK_REG(idx), mask);
	AQ_WRITE_REG(hw, AQ2_RPF_ACT_ART_REQ_ACTION_REG(idx), action);

	reg_glb_cpu_sem_set(hw, 1U, AQ2_ART_SEM_INDEX);
	return (0);
}

static int
aq_hw_init_rx_path(struct aq_hw *hw)
{
	unsigned int control_reg_val = 0U;
	int i;
	int err = 0;

	AQ_DBG_ENTER();
	/* Rx TC/RSS number config */
	rpb_rpf_rx_traf_class_mode_set(hw, 1U);

	/*
	 * Atlantic 2: program the RSS hash types the kernel honors.  UDP is
	 * excluded by default (see aq_rss_hashconfig()); clearing its per-cast
	 * bits keeps fragmented UDP on a single queue without the A1 L3L4
	 * flow-filter workaround.
	 */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		static const struct {
			u_int		kern;
			uint32_t	hw;
		} ht_map[] = {
			{ RSS_HASHTYPE_RSS_IPV4,	AQ2_RPF_REDIR2_HASHTYPE_IP },
			{ RSS_HASHTYPE_RSS_TCP_IPV4,	AQ2_RPF_REDIR2_HASHTYPE_TCP4 },
			{ RSS_HASHTYPE_RSS_UDP_IPV4,	AQ2_RPF_REDIR2_HASHTYPE_UDP4 },
			{ RSS_HASHTYPE_RSS_IPV6,	AQ2_RPF_REDIR2_HASHTYPE_IP6 },
			{ RSS_HASHTYPE_RSS_TCP_IPV6,	AQ2_RPF_REDIR2_HASHTYPE_TCP6 },
			{ RSS_HASHTYPE_RSS_UDP_IPV6,	AQ2_RPF_REDIR2_HASHTYPE_UDP6 },
			{ RSS_HASHTYPE_RSS_IPV6_EX,	AQ2_RPF_REDIR2_HASHTYPE_IP6EX },
			{ RSS_HASHTYPE_RSS_TCP_IPV6_EX,	AQ2_RPF_REDIR2_HASHTYPE_TCP6EX },
			{ RSS_HASHTYPE_RSS_UDP_IPV6_EX,	AQ2_RPF_REDIR2_HASHTYPE_UDP6EX },
		};
		u_int hc = aq_rss_hashconfig();
		uint32_t ht = 0;

		for (i = 0; i < (int)nitems(ht_map); i++)
			if (hc & ht_map[i].kern)
				ht |= ht_map[i].hw;
		AQ_WRITE_REG_BIT(hw, AQ2_RPF_REDIR2_REG,
		    AQ2_RPF_REDIR2_HASHTYPE, 0, ht);
	}

	/* Rx flow control */
	rpb_rx_flow_ctl_mode_set(hw, 1U);

	/* RSS Ring selection */
	reg_rx_flr_rss_control1set(hw, 0xB3333333U);

	/* Multicast filters */
	for (i = AQ_HW_MAC_MAX; i--;) {
		rpfl2_uc_flr_en_set(hw, (i == 0U) ? 1U : 0U, i);
		rpfl2unicast_flr_act_set(hw, 1U, i);
	}

	reg_rx_flr_mcst_flr_msk_set(hw, 0x00000000U);
	reg_rx_flr_mcst_flr_set(hw, 0x00010FFFU, 0U);

	/* Vlan filters */
	rpf_vlan_outer_etht_set(hw, 0x88A8U);
	rpf_vlan_inner_etht_set(hw, 0x8100U);
	rpf_vlan_accept_untagged_packets_set(hw, true);
	rpf_vlan_untagged_act_set(hw, HW_ATL_RX_HOST);

	rpf_vlan_prom_mode_en_set(hw, 1);

	/* Rx Interrupts */
	rdm_rx_desc_wr_wb_irq_en_set(hw, 1U);

	if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		/* Enable the resolver table and tag L2 unicast/broadcast. */
		AQ_WRITE_REG_BIT(hw, AQ2_RPF_REC_TAB_ENABLE_REG,
		    AQ2_RPF_REC_TAB_ENABLE_MASK, 0, AQ2_RPF_REC_TAB_ENABLE_MASK);
		AQ_WRITE_REG_BIT(hw, RPF_L2UC_MSW_REG(0),
		    RPF_L2UC_MSW_TAG, RPF_L2UC_MSW_TAG_SHIFT, 1);
		AQ_WRITE_REG_BIT(hw, AQ2_RPF_L2BC_TAG_REG,
		    AQ2_RPF_L2BC_TAG_MASK, 0, 1);

		/* Drop rows (promisc lifts them); all PCP -> TC 0. */
		err = aq2_art_filter_set(hw, AQ2_RPF_INDEX_L2_PROMISC_OFF, 0,
		    AQ2_RPF_TAG_UC_MASK | AQ2_RPF_TAG_ALLMC_MASK,
		    AQ2_ART_ACTION_DROP);
		if (err == 0)
			err = aq2_art_filter_set(hw,
			    AQ2_RPF_INDEX_VLAN_PROMISC_OFF, 0,
			    AQ2_RPF_TAG_VLAN_MASK | AQ2_RPF_TAG_UNTAG_MASK,
			    AQ2_ART_ACTION_DROP);
		for (i = 0; err == 0 && i < 8; i++)
			err = aq2_art_filter_set(hw, AQ2_RPF_INDEX_PCP_TO_TC + i,
			    ((uint32_t)i << AQ2_RPF_TAG_PCP_SHIFT),
			    AQ2_RPF_TAG_PCP_MASK,
			    AQ2_ART_ACTION_ASSIGN_TC(0));
	} else {
		/* misc */
		control_reg_val = 0x000F0000U; //RPF2

		/* RSS hash type set for IP/TCP */
		control_reg_val |= 0x1EU;

		AQ_WRITE_REG(hw, 0x00005040U, control_reg_val);
	}

	rpfl2broadcast_en_set(hw, 1U);
	rpfl2broadcast_flr_act_set(hw, 1U);
	rpfl2broadcast_count_threshold_set(hw, 0xFFFFU & (~0U / 256U));

	rdm_rx_dca_en_set(hw, 0U);
	rdm_rx_dca_mode_set(hw, 0U);

	if (err == 0)
		err = aq_hw_err_from_flags(hw);
	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_mac_addr_set(struct aq_hw *hw, uint8_t *mac_addr, uint8_t index)
{
	int err = 0;
	unsigned int h = 0U;
	unsigned int l = 0U;

	AQ_DBG_ENTER();
	if (!mac_addr) {
		err = EINVAL;
		goto err_exit;
	}
	if (index >= AQ_HW_MAC_MAX) {
		err = EINVAL;
		goto err_exit;
	}
	h = (mac_addr[0] << 8) | (mac_addr[1]);
	l = (mac_addr[2] << 24) | (mac_addr[3] << 16) | (mac_addr[4] << 8) |
	    mac_addr[5];

	rpfl2_uc_flr_en_set(hw, 0U, index);
	rpfl2unicast_dest_addresslsw_set(hw, l, index);
	rpfl2unicast_dest_addressmsw_set(hw, h, index);
	/* Atlantic 2: classify this address into the resolver table. */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2))
		AQ_WRITE_REG_BIT(hw, RPF_L2UC_MSW_REG(index),
		    RPF_L2UC_MSW_TAG, RPF_L2UC_MSW_TAG_SHIFT, 1);
	rpfl2_uc_flr_en_set(hw, 1U, index);

	err = aq_hw_err_from_flags(hw);

err_exit:
	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_init(struct aq_hw *hw, uint8_t *mac_addr, uint8_t adm_irq, bool msix)
{

	int err = 0;
	uint32_t val = 0;

	AQ_DBG_ENTER();

	/* Atlantic 1 only: clamp MRRS and TX DMA total request limit. */
	if (!IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		/* Force limit MRRS on RDM/TDM to 2K */
		val = AQ_READ_REG(hw, AQ_HW_PCI_REG_CONTROL_6_ADR);
		AQ_WRITE_REG(hw, AQ_HW_PCI_REG_CONTROL_6_ADR,
		    (val & ~0x707) | 0x404);

		/* TX DMA total request limit. B0 hardware is not capable to
		* handle more than (8K-MRRS) incoming DMA data.
		* Value 24 in 256byte units
		*/
		AQ_WRITE_REG(hw, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_ADR, 24);
	} else {
		/* Atlantic 2: launch-time clock ratio per FPGA version. */
		uint32_t fpgaver = AQ_READ_REG(hw, AQ2_HW_FPGA_VERSION_REG);
		uint32_t ratio;

		if (fpgaver < 0x01000000U)
			ratio = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_FULL;
		else if (fpgaver >= 0x01008502U)
			ratio = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_HALF;
		else
			ratio = AQ2_LAUNCHTIME_CTRL_RATIO_SPEED_QUARTER;
		AQ_WRITE_REG_BIT(hw, AQ2_LAUNCHTIME_CTRL_REG,
		    AQ2_LAUNCHTIME_CTRL_RATIO, 8, ratio);
	}

	err = aq_hw_init_tx_path(hw);
	if (err != 0)
		goto err_exit;
	err = aq_hw_init_rx_path(hw);
	if (err != 0)
		goto err_exit;

	aq_hw_mac_addr_set(hw, mac_addr, AQ_HW_MAC);

	/* A lost ack must not skip the setup that follows. */
	err = aq_hw_mpi_set(hw, MPI_INIT, hw->link_rate);
	if (err != 0)
		device_printf(hw->dev, "could not set F/W link mode: %d\n", err);

	aq_hw_qos_set(hw);

	/* Atlantic 2: turn on the new RPF / action-resolver engine. */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2))
		AQ_WRITE_REG_BIT(hw, AQ2_RPF_NEW_CTRL_REG,
		    AQ2_RPF_NEW_CTRL_ENABLE, 11, 1);

	err = aq_hw_err_from_flags(hw);
	if (err != 0)
		goto err_exit;

	/* Interrupts */
	//Enable interrupt
	itr_irq_status_cor_en_set(hw, 0); //Disable clear-on-read for status
	itr_irq_auto_mask_clr_en_set(hw, 1); // Enable auto-mask clear.
	if (msix)
		itr_irq_mode_set(hw, 0x6); //MSIX + multi vector
	else
		itr_irq_mode_set(hw, 0x5); //MSI + multi vector

	/* Route both hardware error causes (map reg 0) to the admin vector. */
	reg_gen_irq_map_set(hw,
	    ((0x80U | adm_irq) << 24) | ((0x80U | adm_irq) << 16), 0);
	reg_gen_irq_map_set(hw, 0x80 | adm_irq, 3);

	err = aq_hw_offload_set(hw);

err_exit:
	AQ_DBG_EXIT(err);
	return (err);
}


int
aq_hw_start(struct aq_hw *hw)
{
	int err;

	AQ_DBG_ENTER();
	tpb_tx_buff_en_set(hw, 1U);
	rpb_rx_buff_en_set(hw, 1U);
	err = aq_hw_err_from_flags(hw);
	AQ_DBG_EXIT(err);
	return (err);
}


int
aq_hw_interrupt_moderation_set(struct aq_hw *hw)
{
	/* Rows in enum aq_fw_link_speed bit order (10M=0 .. 10G=5); index = ffs(speed)-1. */
	static unsigned int aq_itr_timers_rx[][2] = {
	    {0x4U, 0x50U},	/* 10Mbit */
	    {0x4U, 0x50U},	/* 100Mbit */
	    {0x30U, 0x80U},	/* 1Gbit */
	    {0x18U, 0xE0U},	/* 2.5Gbit */
	    {0xCU, 0x70U},	/* 5Gbit */
	    {80, 120},		/* 10Gbit */
	};
	static unsigned int aq_itr_timers_tx[][2] = {
	    {0x4fU, 0xffU},	/* 10Mbit */
	    {0x4fU, 0xffU},	/* 100Mbit */
	    {0x4fU, 0xffU},	/* 1Gbit */
	    {0x4fU, 0xffU},	/* 2.5Gbit */
	    {0x4fU, 0xffU},	/* 5Gbit */
	    {0x4fU, 0x1ff},	/* 10Gbit */
	};

	uint32_t speed_index = ffs(hw->link_speed ? hw->link_speed : aq_fw_10G) - 1;
	uint32_t itr_rx = 2U;
	uint32_t itr_tx = 2U;
	int custom_itr = hw->itr;
	int active = custom_itr != 0;
	int err;


	AQ_DBG_ENTER();

	if (custom_itr == -1) {
		/* set min timer value */
		itr_rx |= aq_itr_timers_rx[speed_index][0] << 0x8U;
		/* set max timer value */
		itr_rx |= aq_itr_timers_rx[speed_index][1] << 0x10U;

		/* set min timer value */
		itr_tx |= aq_itr_timers_tx[speed_index][0] << 0x8U;
		/* set max timer value */
		itr_tx |= aq_itr_timers_tx[speed_index][1] << 0x10U;
	} else {
		if (custom_itr > 0x1FF)
			custom_itr = 0x1FF;

		itr_rx |= (custom_itr/2) << 0x8U; /* set min timer value */
		itr_rx |= custom_itr << 0x10U; /* set max timer value */

		itr_tx |= (custom_itr/2) << 0x8U; /* set min timer value */
		itr_tx |= custom_itr << 0x10U; /* set max timer value */
	}

	tdm_tx_desc_wr_wb_irq_en_set(hw, !active);
	tdm_tdm_intr_moder_en_set(hw, active);
	rdm_rx_desc_wr_wb_irq_en_set(hw, !active);
	rdm_rdm_intr_moder_en_set(hw, active);

	for (int i = HW_ATL_B0_RINGS_MAX; i--;) {
		/* A2 Tx moderation register moved; same layout.  Rx is shared. */
		if (IS_CHIP_FEATURE(hw, ATLANTIC2))
			AQ_WRITE_REG(hw, AQ2_TX_INTR_MODERATION_CTL_REG(i),
			    itr_tx);
		else
			reg_tx_intr_moder_ctrl_set(hw, itr_tx, i);
		reg_rx_intr_moder_ctrl_set(hw, itr_rx, i);
	}

	err = aq_hw_err_from_flags(hw);
	AQ_DBG_EXIT(err);
	return (err);
}

/**
 * @brief Set VLAN filter table
 * @details Configure VLAN filter table to accept (and assign the queue) traffic
 *  for the particular vlan ids.
 * Note: use this function under vlan promisc mode not to lost the traffic
 *
 * @param aq_hw
 * @param aq_rx_filter_vlan VLAN filter configuration
 * @return 0 - OK, <0 - error
 */
int
hw_atl_b0_hw_vlan_set(struct aq_hw *hw, struct aq_rx_filter_vlan *aq_vlans)
{
	int i;

	for (i = 0; i < AQ_HW_VLAN_MAX_FILTERS; i++) {
		hw_atl_rpf_vlan_flr_en_set(hw, 0U, i);
		hw_atl_rpf_vlan_rxq_en_flr_set(hw, 0U, i);
		if (aq_vlans[i].enable) {
			hw_atl_rpf_vlan_id_flr_set(hw,
						   aq_vlans[i].vlan_id,
						   i);
			hw_atl_rpf_vlan_flr_act_set(hw, 1U, i);
			/* A2: tag the filter so the VLAN drop row passes it. */
			if (IS_CHIP_FEATURE(hw, ATLANTIC2))
				AQ_WRITE_REG_BIT(hw,
				    AQ2_RPF_VLAN_FLR_TAG_REG(i),
				    AQ2_RPF_VLAN_FLR_TAG,
				    AQ2_RPF_VLAN_FLR_TAG_SHIFT, 1U);
			hw_atl_rpf_vlan_flr_en_set(hw, 1U, i);
			if (aq_vlans[i].queue != 0xFF) {
				hw_atl_rpf_vlan_rxq_flr_set(hw,
							    aq_vlans[i].queue,
							    i);
				hw_atl_rpf_vlan_rxq_en_flr_set(hw, 1U, i);
			}
		}
	}

	return aq_hw_err_from_flags(hw);
}

int
hw_atl_b0_hw_vlan_promisc_set(struct aq_hw *hw, bool promisc)
{
	int err;

	/* A2: fallible ART write first; a timeout leaves the legacy bit as-is. */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		err = aq2_art_filter_set(hw, AQ2_RPF_INDEX_VLAN_PROMISC_OFF,
		    0, AQ2_RPF_TAG_VLAN_MASK | AQ2_RPF_TAG_UNTAG_MASK,
		    promisc ? AQ2_ART_ACTION_DISABLE : AQ2_ART_ACTION_DROP);
		if (err != 0)
			return (err);
	}
	hw_atl_rpf_vlan_prom_mode_en_set(hw, promisc);
	return aq_hw_err_from_flags(hw);
}


int
aq_hw_set_promisc(struct aq_hw *hw, bool l2_promisc, bool vlan_promisc,
    bool mc_promisc)
{
	int err = 0;

	AQ_DBG_ENTERA("promisc %d, vlan_promisc %d, allmulti %d", l2_promisc,
	    vlan_promisc, mc_promisc);

	rpfl2promiscuous_mode_en_set(hw, l2_promisc);

	err = hw_atl_b0_hw_vlan_promisc_set(hw, l2_promisc | vlan_promisc);

	/* A2: promisc toggles the unicast drop row. */
	if (err == 0 && IS_CHIP_FEATURE(hw, ATLANTIC2))
		err = aq2_art_filter_set(hw, AQ2_RPF_INDEX_L2_PROMISC_OFF, 0,
		    AQ2_RPF_TAG_UC_MASK | AQ2_RPF_TAG_ALLMC_MASK,
		    l2_promisc ? AQ2_ART_ACTION_DISABLE : AQ2_ART_ACTION_DROP);

	rpfl2_accept_all_mc_packets_set(hw, mc_promisc);
	rpfl2multicast_flr_en_set(hw, mc_promisc, 0);

	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_rss_hash_set(struct aq_hw *hw,
    uint8_t rss_key[HW_ATL_RSS_HASHKEY_SIZE])
{
	uint32_t rss_key_dw[HW_ATL_RSS_HASHKEY_SIZE / 4];
	uint32_t addr = 0U;
	uint32_t i = 0U;
	int err = 0;

	AQ_DBG_ENTER();

	memcpy(rss_key_dw, rss_key, HW_ATL_RSS_HASHKEY_SIZE);

	for (i = 10, addr = 0U; i--; ++addr) {
		uint32_t key_data = bswap32(rss_key_dw[i]);
		rpf_rss_key_wr_data_set(hw, key_data);
		rpf_rss_key_addr_set(hw, addr);
		rpf_rss_key_wr_en_set(hw, 1U);
		err = AQ_HW_WAIT_FOR(rpf_rss_key_wr_en_get(hw) == 0,
			       1000U, 10U);
		if (err != 0)
			goto err_exit;
	}

	err = aq_hw_err_from_flags(hw);

err_exit:
	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_rss_hash_get(struct aq_hw *hw,
    uint8_t rss_key[HW_ATL_RSS_HASHKEY_SIZE])
{
	uint32_t rss_key_dw[HW_ATL_RSS_HASHKEY_SIZE / 4];
	uint32_t addr = 0U;
	uint32_t i = 0U;
	int err = 0;

	AQ_DBG_ENTER();

	for (i = 10, addr = 0U; i--; ++addr) {
		rpf_rss_key_addr_set(hw, addr);
		rss_key_dw[i] = bswap32(rpf_rss_key_rd_data_get(hw));
	}
	memcpy(rss_key, rss_key_dw, HW_ATL_RSS_HASHKEY_SIZE);

	err = aq_hw_err_from_flags(hw);

	AQ_DBG_EXIT(err);
	return (err);
}

int
aq_hw_rss_set(struct aq_hw *hw,
    uint8_t rss_table[HW_ATL_RSS_INDIRECTION_TABLE_MAX])
{
	uint16_t bitary[(HW_ATL_RSS_INDIRECTION_TABLE_MAX *
					3 / 16U)];
	int err = 0;
	uint32_t i = 0U;

	memset(bitary, 0, sizeof(bitary));

	/* A2 per-TC redirection table; flat RX rings, same queue per TC. */
	if (IS_CHIP_FEATURE(hw, ATLANTIC2)) {
		uint32_t n_tcs = aq_hw_active_tcs(hw);
		int tc, slot, q;
		uint32_t word;

		AQ_WRITE_REG_BIT(hw, AQ2_RPF_REDIR2_REG, AQ2_RPF_REDIR2_INDEX,
		    12, 0);
		for (slot = 0; slot < AQ2_RPF_RSS_REDIR_MAX; slot++) {
			q = rss_table[slot] &
			    (HW_ATL_RSS_INDIRECTION_QUEUES_MAX - 1U);
			word = 0U;
			for (tc = 0; (uint32_t)tc < n_tcs; tc++)
				word |= (uint32_t)q << (5 * tc);
			AQ_WRITE_REG(hw, AQ2_RPF_RSS_REDIR_REG(0, slot), word);
		}
		/* A2 uses this table only; skip the legacy indirection writes. */
		return (aq_hw_err_from_flags(hw));
	}

	for (i = HW_ATL_RSS_INDIRECTION_TABLE_MAX; i--;) {
		uint32_t bit_pos = i * HW_ATL_RSS_INDIRECTION_ENTRY_BITS;
		uint32_t word = bit_pos / 16U;
		uint32_t shift = bit_pos % 16U;
		uint32_t field = (uint32_t)(rss_table[i] &
		    (HW_ATL_RSS_INDIRECTION_QUEUES_MAX - 1U)) << shift;

		bitary[word] |= (uint16_t)field;
		if (shift + HW_ATL_RSS_INDIRECTION_ENTRY_BITS > 16U &&
		    word + 1U < nitems(bitary))
			bitary[word + 1U] |= (uint16_t)(field >> 16);
	}

	for (i = nitems(bitary); i--;) {
		rpf_rss_redir_tbl_wr_data_set(hw, bitary[i]);
		rpf_rss_redir_tbl_addr_set(hw, i);
		rpf_rss_redir_wr_en_set(hw, 1U);
		err = AQ_HW_WAIT_FOR(rpf_rss_redir_wr_en_get(hw) == 0,
			       1000U, 10U);
		if (err != 0)
			goto err_exit;
	}

	err = aq_hw_err_from_flags(hw);

err_exit:
	return (err);
}

int
aq_hw_udp_rss_enable(struct aq_hw *hw, bool enable)
{
	int err = 0;
	if (!enable) {
		/* HW bug workaround:
		 * Disable RSS for UDP using rx flow filter 0.
		 * HW does not track RSS stream for fragmenged UDP,
		 * 0x5040 control reg does not work.
		 */
		hw_atl_rpf_l3_l4_enf_set(hw, true, 0);
		hw_atl_rpf_l4_protf_en_set(hw, true, 0);
		hw_atl_rpf_l3_l4_rxqf_en_set(hw, true, 0);
		hw_atl_rpf_l3_l4_actf_set(hw, L2_FILTER_ACTION_HOST, 0);
		hw_atl_rpf_l3_l4_rxqf_set(hw, 0, 0);
		hw_atl_rpf_l4_protf_set(hw, HW_ATL_RX_UDP, 0);
	} else {
		hw_atl_rpf_l3_l4_enf_set(hw, false, 0);
	}

	err = aq_hw_err_from_flags(hw);
	return (err);

}