/* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright (c) 2025, Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /** * @file ice_iov.c * @brief Virtualization support functions * * Contains functions for enabling and managing PCIe virtual function devices, * including enabling new VFs, and managing VFs over the virtchnl interface. */ #include "ice_iov.h" #include "ice_fault.h" #include /* Version 1 driver.ice extension schema; documented in ice(4). */ #define ICE_VF_STATUS_NAMESPACE "driver.ice" #define ICE_VF_STATUS_VERSION 1 #define ICE_VF_STATUS_MIRROR_CONFIGURED "mirror-configured" #define ICE_VF_STATUS_MIRROR_SOURCE_VSI "mirror-source-vsi" #define ICE_VF_STATUS_MIRROR_INGRESS_ACTIVE "mirror-ingress-active" #define ICE_VF_STATUS_MIRROR_EGRESS_ACTIVE "mirror-egress-active" #define ICE_VF_STATUS_MDD_BLOCKED "mdd-blocked" #define ICE_VF_STATUS_MDD_TX_EVENTS "mdd-tx-events" #define ICE_VF_STATUS_MDD_RX_EVENTS "mdd-rx-events" #define ICE_VF_STATUS_MBX_BLOCKED "mailbox-blocked" #define ICE_VF_STATUS_MBX_OVERFLOW_EVENTS "mailbox-overflow-events" #define ICE_VF_STATUS_MAC_FILTER_COUNT "mac-filter-count" #define ICE_VF_STATUS_MAC_FILTER_LIMIT "mac-filter-limit" #define ICE_VF_STATUS_RESET_FAILED "reset-failed" #define ICE_VF_STATUS_REBUILD_REQUIRED "rebuild-required" /* Optional fields are compacted when absent; values define schema order. */ enum ice_vf_status_field { ICE_VF_STATUS_FIELD_MIRROR_CONFIGURED, ICE_VF_STATUS_FIELD_MIRROR_SOURCE_VSI, ICE_VF_STATUS_FIELD_MIRROR_INGRESS_ACTIVE, ICE_VF_STATUS_FIELD_MIRROR_EGRESS_ACTIVE, ICE_VF_STATUS_FIELD_MDD_BLOCKED, ICE_VF_STATUS_FIELD_MDD_TX_EVENTS, ICE_VF_STATUS_FIELD_MDD_RX_EVENTS, ICE_VF_STATUS_FIELD_MBX_BLOCKED, ICE_VF_STATUS_FIELD_MBX_OVERFLOW_EVENTS, ICE_VF_STATUS_FIELD_MAC_FILTER_COUNT, ICE_VF_STATUS_FIELD_MAC_FILTER_LIMIT, ICE_VF_STATUS_FIELD_RESET_FAILED, ICE_VF_STATUS_FIELD_REBUILD_REQUIRED, ICE_VF_STATUS_NUM_FIELDS, }; #define ICE_VC_MAX_RX_BUFFER \ ((16 * 1024) - BIT(ICE_RLAN_CTX_DBUF_S)) #define ICE_VIRTCHNL_QUEUE_MAP_SIZE 16 #ifdef DRIVER_FAILPOINTS static SYSCTL_NODE(_debug_fail_point_ice, OID_AUTO, iov, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ice SR-IOV fail points"); static int ice_iov_fail_vf = -1; SYSCTL_INT(_debug_fail_point_ice_iov, OID_AUTO, vf, CTLFLAG_RW | CTLFLAG_MPSAFE, &ice_iov_fail_vf, 0, "VF eligible for ice SR-IOV fail points (-1 selects every VF)"); #endif /* DRIVER_FAILPOINTS */ static struct ice_vf *ice_iov_get_vf(struct ice_softc *sc, int vf_num); static int ice_iov_configure_mac_anti_spoof(struct ice_softc *sc, struct ice_vf *vf); static int ice_iov_restore_vf_host_config(struct ice_softc *sc, struct ice_vf *vf); static void ice_iov_clear_vf_queue_state(struct ice_vf *vf); static void ice_iov_clear_vf_mbx(struct ice_softc *sc, struct ice_vf *vf); static void ice_iov_complete_vf_reset(struct ice_softc *sc, struct ice_vf *vf, bool restore_mapping); static void ice_iov_ready_vf(struct ice_softc *sc, struct ice_vf *vf); static int ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf, bool trigger_reset, bool release_vf); static void ice_iov_setup_intr_mapping(struct ice_softc *sc, struct ice_vf *vf); static void ice_vc_version_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_get_vf_res_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_add_eth_addr_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_del_eth_addr_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static bool ice_vc_isvalid_ring_len(u32 ring_len); static void ice_vc_cfg_vsi_qs_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_cfg_rss_key_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_set_rss_hena_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_enable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_notify_vf_link_state(struct ice_softc *sc, struct ice_vf *vf); static void ice_vc_disable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static int ice_vc_disable_queues(struct ice_softc *sc, struct ice_vf *vf, u32 tx_queues, u32 rx_queues); static void ice_vc_cfg_irq_map_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_get_stats_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_eth_stats_to_virtchnl_eth_stats(struct ice_eth_stats *istats, struct virtchnl_eth_stats *vstats); static void ice_vc_cfg_rss_lut_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_cfg_promisc_mode_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_add_vlan_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static void ice_vc_del_vlan_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf); static int ice_vc_select_vlans(struct ice_vf *vf, u16 *vids, u16 count, bool add, u16 *selected_count); static enum virtchnl_status_code ice_iov_err_to_virt_err(int ice_err); static int ice_vf_mac_filter_index(struct ice_vf *vf, const uint8_t *addr); static int ice_vf_validate_mac(struct ice_vf *vf, const uint8_t *addr); #ifdef DRIVER_FAILPOINTS static bool ice_iov_fail_vf_matches(uint16_t vfnum) { return (ice_iov_fail_vf == -1 || ice_iov_fail_vf == vfnum); } #endif #define ICE_IOV_FAIL_POINT(_sc, _vfnum, _name, _error, _label) do { \ ICE_FAIL_POINT_CODE_COND(_sc, _debug_fail_point_ice_iov, _name, \ ice_iov_fail_vf_matches((_vfnum)), \ FAIL_POINT_NONSLEEPABLE, { \ (_error) = RETURN_VALUE; \ if ((_error) <= 0) \ (_error) = EIO; \ device_printf((_sc)->dev, \ "injecting VF %u failure at %s: %d\n", \ (unsigned int)(_vfnum), #_name, (_error)); \ goto _label; \ }); \ } while (0) /** * ice_iov_attach - Initialize SR-IOV PF host support * @sc: device softc structure * * Initialize SR-IOV PF host support at the end of the driver attach process. * * @pre Must be called from sleepable context (calls malloc() w/ M_WAITOK) * * @returns 0 if successful, or * - ENOMEM if there is no memory for the PF/VF schemas or iov device * - ENXIO if the device isn't PCI-E or doesn't support the same SR-IOV * version as the kernel * - ENOENT if the device doesn't have the SR-IOV capability */ int ice_iov_attach(struct ice_softc *sc) { device_t dev = sc->dev; nvlist_t *pf_schema, *vf_schema; int error; pf_schema = pci_iov_schema_alloc_node(); vf_schema = pci_iov_schema_alloc_node(); pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", IOV_SCHEMA_HASDEFAULT, TRUE); pci_iov_schema_add_bool(vf_schema, "allow-set-mac", IOV_SCHEMA_HASDEFAULT, FALSE); pci_iov_schema_add_bool(vf_schema, "allow-promisc", IOV_SCHEMA_HASDEFAULT, FALSE); pci_iov_schema_add_uint16(vf_schema, "num-queues", IOV_SCHEMA_HASDEFAULT, ICE_DEFAULT_VF_QUEUES); pci_iov_schema_add_uint16(vf_schema, "mirror-src-vsi", IOV_SCHEMA_HASDEFAULT, ICE_INVALID_MIRROR_VSI); pci_iov_schema_add_uint16(vf_schema, "max-vlan-allowed", IOV_SCHEMA_HASDEFAULT, ICE_DEFAULT_VF_VLAN_LIMIT); pci_iov_schema_add_uint16(vf_schema, "max-mac-filters", IOV_SCHEMA_HASDEFAULT, ICE_DEFAULT_VF_FILTER_LIMIT); error = pci_iov_attach(dev, pf_schema, vf_schema); if (error != 0) { device_printf(dev, "pci_iov_attach failed (error=%s)\n", ice_err_str(error)); ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); } else { ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_en); if (ice_is_e830(&sc->hw)) ice_iov_reconfigure_mbx(sc); else ice_mbx_init_snapshot(&sc->hw); } return (error); } /** * ice_iov_reconfigure_mbx - Restore hardware mailbox flood protection * @sc: device softc structure * * E830 limits each VF's outstanding messages in hardware. The threshold * register is reset by a core reset and must be restored during rebuild. * Older devices use the software snapshot detector instead. */ void ice_iov_reconfigure_mbx(struct ice_softc *sc) { struct ice_hw *hw = &sc->hw; if (!ice_is_e830(hw)) return; wr32(hw, E830_MBX_PF_IN_FLIGHT_VF_MSGS_THRESH, ICE_MBX_OVERFLOW_WATERMARK); ice_flush(hw); } /** * ice_iov_detach - Teardown SR-IOV PF host support * @sc: device softc structure * * Teardown SR-IOV PF host support at the start of the driver detach process. * * @returns 0 if successful or IOV support hasn't been setup, or * - EBUSY if VFs still exist */ int ice_iov_detach(struct ice_softc *sc) { device_t dev = sc->dev; int error; error = pci_iov_detach(dev); if (error != 0) { device_printf(dev, "pci_iov_detach failed (error=%s)\n", ice_err_str(error)); } return (error); } /** * ice_iov_init - Called by the OS before the first VF is created. * @sc: device softc structure * @num_vfs: number of VFs to setup resources for * @params: configuration parameters for the PF * * @returns 0 if successful or an error code on failure */ int ice_iov_init(struct ice_softc *sc, uint16_t num_vfs, const nvlist_t *params __unused) { /* Allocate array of VFs, for tracking */ sc->vfs = (struct ice_vf *)malloc(sizeof(struct ice_vf) * num_vfs, M_ICE, M_NOWAIT | M_ZERO); if (sc->vfs == NULL) return (ENOMEM); /* Initialize each VF with basic information */ for (int i = 0; i < num_vfs; i++) { sc->vfs[i].vf_num = i; if (ice_is_e830(&sc->hw)) ice_mbx_vf_clear_cnt_e830(&sc->hw, i); else ice_mbx_init_vf_info(&sc->hw, &sc->vfs[i].mbx_info); } /* Save off number of configured VFs */ sc->num_vfs = num_vfs; return (0); } /** * ice_iov_get_vf - Get pointer to VF at given index * @sc: device softc structure * @vf_num: Index of VF to retrieve * * @remark will throw an assertion if vf_num is not in the * range of allocated VFs * * @returns a pointer to the VF structure at the given index */ static struct ice_vf * ice_iov_get_vf(struct ice_softc *sc, int vf_num) { MPASS(vf_num < sc->num_vfs); return &sc->vfs[vf_num]; } /** * ice_iov_configure_mac_anti_spoof - Apply a VF's source-MAC policy * @sc: device softc structure * @vf: VF whose VSI security policy should be configured * * PF and device resets discard the hardware VSI context, so callers must * replay this policy after creating or rebuilding the VF's VSI. Also reapply * the PF-owned policy defensively before releasing a VF after VFR. */ static int ice_iov_configure_mac_anti_spoof(struct ice_softc *sc, struct ice_vf *vf) { struct ice_vsi_ctx ctx = { 0 }; struct ice_vsi *vsi = vf->vsi; struct ice_hw *hw = &sc->hw; bool enable; #ifdef DRIVER_FAILPOINTS int error; #endif int status; enable = (atomic_load_acq_32(&vf->vf_flags) & VF_FLAG_MAC_ANTI_SPOOF) != 0; ctx.info.sec_flags = vsi->info.sec_flags; ctx.info.valid_sections = CPU_TO_LE16(ICE_AQ_VSI_PROP_SECURITY_VALID); if (enable) ctx.info.sec_flags |= ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF; else ctx.info.sec_flags &= ~ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF; ICE_IOV_FAIL_POINT(sc, vf->vf_num, mac_anti_spoof_update, error, fail); status = ice_update_vsi(hw, vsi->idx, &ctx, NULL); if (status != 0) { device_printf(sc->dev, "Unable to configure VF %u MAC anti-spoof %s, " "err %s aq_err %s\n", vf->vf_num, enable ? "on" : "off", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); return (EIO); } vsi->info.sec_flags = ctx.info.sec_flags; return (0); #ifdef DRIVER_FAILPOINTS fail: return (error); #endif } /** * ice_iov_restore_vf_host_config - Restore PF-owned policy after a VF reset * @sc: device softc structure * @vf: VF whose host configuration should be restored * * A VF reset discards the guest's filter configuration. Remove the matching * software switch state as well so that replayed guest requests reach * firmware instead of being mistaken for filters which still exist. Restore * the PF-owned source-MAC policy and base filters before releasing the VF. */ static int ice_iov_restore_vf_host_config(struct ice_softc *sc, struct ice_vf *vf) { struct ice_vsi *vsi = vf->vsi; int error; ice_remove_vsi_fltr(&sc->hw, vsi->idx); vf->mac_filter_cnt = 0; vf->vlan_cnt = 0; bzero(vf->vlans_map, sizeof(vf->vlans_map)); error = ice_iov_configure_mac_anti_spoof(sc, vf); if (error != 0) return (error); error = ice_add_vsi_mac_filter(vsi, broadcastaddr); if (error != 0) return (error); if (!ETHER_IS_ZERO(vf->mac)) { error = ice_add_vsi_mac_filter(vsi, vf->mac); if (error != 0) return (error); } return (0); } /** * ice_iov_add_vf - Called by the OS for each VF to create * @sc: device softc structure * @vfnum: index of VF to configure * @params: configuration parameters for the VF * * @returns 0 if successful or an error code on failure */ int ice_iov_add_vf(struct ice_softc *sc, uint16_t vfnum, const nvlist_t *params) { struct ice_tx_queue *txq; struct ice_rx_queue *rxq; device_t dev = sc->dev; struct ice_vsi *vsi; struct ice_vf *vf; int vf_num_queues; const void *mac; size_t size; int error; int i; vf = ice_iov_get_vf(sc, vfnum); vf->vf_flags = 0; /* This VF needs at least one VSI */ vsi = ice_alloc_vsi(sc, ICE_VSI_VF); if (vsi == NULL) return (ENOMEM); vf->vsi = vsi; vsi->vf_num = vfnum; ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_alloc, error, release_vsi); vf_num_queues = nvlist_get_number(params, "num-queues"); /* Validate and clamp value if invalid */ if (vf_num_queues < 1 || vf_num_queues > ICE_MAX_SCATTERED_QUEUES) device_printf(dev, "Invalid num-queues (%d) for VF %d\n", vf_num_queues, vf->vf_num); if (vf_num_queues < 1) { device_printf(dev, "Setting VF %d num-queues to 1\n", vf->vf_num); vf_num_queues = 1; } else if (vf_num_queues > ICE_MAX_SCATTERED_QUEUES) { device_printf(dev, "Setting VF %d num-queues to %d\n", vf->vf_num, ICE_MAX_SCATTERED_QUEUES); vf_num_queues = ICE_MAX_SCATTERED_QUEUES; } vsi->qmap_type = ICE_RESMGR_ALLOC_SCATTERED; /* Reserve VF queue allocation from PF queues */ ice_alloc_vsi_qmap(vsi, vf_num_queues, vf_num_queues); vsi->num_tx_queues = vsi->num_rx_queues = vf_num_queues; ICE_IOV_FAIL_POINT(sc, vfnum, add_after_queue_maps, error, release_vsi); /* Assign Tx queues from PF space */ error = ice_resmgr_assign_scattered(&sc->tx_qmgr, vsi->tx_qmap, vsi->num_tx_queues); if (error) { device_printf(sc->dev, "Unable to assign VF Tx queues: %s\n", ice_err_str(error)); goto release_vsi; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_reservation, error, release_vsi); /* Assign Rx queues from PF space */ error = ice_resmgr_assign_scattered(&sc->rx_qmgr, vsi->rx_qmap, vsi->num_rx_queues); if (error) { device_printf(sc->dev, "Unable to assign VF Rx queues: %s\n", ice_err_str(error)); goto release_vsi; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_reservation, error, release_vsi); vsi->max_frame_size = ICE_MAX_FRAME_SIZE; /* Allocate queue structure memory */ vsi->tx_queues = (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * vsi->num_tx_queues, M_ICE, M_NOWAIT | M_ZERO); if (!vsi->tx_queues) { device_printf(sc->dev, "VF-%d: Unable to allocate Tx queue memory\n", vfnum); error = ENOMEM; goto release_vsi; } for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { txq->me = i; txq->vsi = vsi; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_queue_memory, error, free_txqs); /* Allocate queue structure memory */ vsi->rx_queues = (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * vsi->num_rx_queues, M_ICE, M_NOWAIT | M_ZERO); if (!vsi->rx_queues) { device_printf(sc->dev, "VF-%d: Unable to allocate Rx queue memory\n", vfnum); error = ENOMEM; goto free_txqs; } for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) { rxq->me = i; rxq->vsi = vsi; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_queue_memory, error, free_rxqs); /* Allocate space to store the IRQ vector data */ vf->num_irq_vectors = vf_num_queues + 1; vf->tx_irqvs = (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (vf->num_irq_vectors), M_ICE, M_NOWAIT); if (!vf->tx_irqvs) { device_printf(sc->dev, "Unable to allocate TX irqv memory for VF-%d's %d vectors\n", vfnum, vf->num_irq_vectors); error = ENOMEM; goto free_rxqs; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_tx_irq_memory, error, free_txirqvs); vf->rx_irqvs = (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (vf->num_irq_vectors), M_ICE, M_NOWAIT); if (!vf->rx_irqvs) { device_printf(sc->dev, "Unable to allocate RX irqv memory for VF-%d's %d vectors\n", vfnum, vf->num_irq_vectors); error = ENOMEM; goto free_txirqvs; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_rx_irq_memory, error, free_rxirqvs); /* Assign VF interrupts from PF space */ if (!(vf->vf_imap = (u16 *)malloc(sizeof(u16) * vf->num_irq_vectors, M_ICE, M_NOWAIT))) { device_printf(dev, "Unable to allocate VF-%d imap memory\n", vfnum); error = ENOMEM; goto free_rxirqvs; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_memory, error, free_imap); error = ice_resmgr_assign_contiguous(&sc->dev_imgr, vf->vf_imap, vf->num_irq_vectors); if (error) { device_printf(dev, "Unable to assign VF-%d interrupt mapping: %s\n", vfnum, ice_err_str(error)); goto free_imap; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_imap_reservation, error, release_imap); if (nvlist_exists_binary(params, "mac-addr")) { mac = nvlist_get_binary(params, "mac-addr", &size); memcpy(vf->mac, mac, ETHER_ADDR_LEN); if (nvlist_get_bool(params, "allow-set-mac")) vf->vf_flags |= VF_FLAG_SET_MAC_CAP; } else /* * If the administrator has not specified a MAC address then * we must allow the VF to choose one. */ vf->vf_flags |= VF_FLAG_SET_MAC_CAP; if (nvlist_get_bool(params, "mac-anti-spoof")) vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; if (nvlist_get_bool(params, "allow-promisc")) vf->vf_flags |= VF_FLAG_PROMISC_CAP; vsi->mirror_src_vsi = nvlist_get_number(params, "mirror-src-vsi"); vf->vlan_limit = nvlist_get_number(params, "max-vlan-allowed"); vf->mac_filter_limit = nvlist_get_number(params, "max-mac-filters"); if (vf->mac_filter_limit != 0) { vf->mac_filters = mallocarray(vf->mac_filter_limit, sizeof(*vf->mac_filters), M_ICE, M_NOWAIT | M_ZERO); if (vf->mac_filters == NULL) { device_printf(sc->dev, "Unable to allocate VF-%d MAC filter memory\n", vfnum); error = ENOMEM; goto release_imap; } } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_mac_filter_memory, error, free_mac_filters); vf->vf_flags |= VF_FLAG_VLAN_CAP; /* Create and setup VSI in HW */ error = ice_initialize_vsi(vsi); if (error) { device_printf(sc->dev, "Unable to initialize VF %d VSI: %s\n", vfnum, ice_err_str(error)); goto free_mac_filters; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_vsi_init, error, free_mac_filters); error = ice_iov_configure_mac_anti_spoof(sc, vf); if (error != 0) goto free_mac_filters; /* Add the broadcast address */ error = ice_add_vsi_mac_filter(vsi, broadcastaddr); if (error) { device_printf(sc->dev, "Unable to add broadcast filter VF %d VSI: %s\n", vfnum, ice_err_str(error)); goto free_mac_filters; } ICE_IOV_FAIL_POINT(sc, vfnum, add_after_broadcast_filter, error, free_mac_filters); atomic_set_32(&vf->vf_flags, VF_FLAG_ENABLED); ice_iov_ready_vf(sc, vf); return (0); free_mac_filters: free(vf->mac_filters, M_ICE); vf->mac_filters = NULL; vf->mac_filter_cnt = 0; release_imap: ice_resmgr_release_map(&sc->dev_imgr, vf->vf_imap, vf->num_irq_vectors); free_imap: free(vf->vf_imap, M_ICE); vf->vf_imap = NULL; free_rxirqvs: free(vf->rx_irqvs, M_ICE); vf->rx_irqvs = NULL; free_txirqvs: free(vf->tx_irqvs, M_ICE); vf->tx_irqvs = NULL; free_rxqs: free(vsi->rx_queues, M_ICE); vsi->rx_queues = NULL; free_txqs: free(vsi->tx_queues, M_ICE); vsi->tx_queues = NULL; release_vsi: if (vsi->hw_vsi_created) ice_release_vsi(vsi); else ice_release_vsi_resources(vsi); vf->vsi = NULL; atomic_store_rel_32(&vf->vf_flags, 0); return (error); } /** * ice_iov_vf_status - report configured VF state * @sc: device private structure * @statusp: returned status snapshot * * The iflib context lock protects VF state and VSI lifetime while this * method constructs the report. */ int ice_iov_vf_status(struct ice_softc *sc, struct if_vf_status **statusp) { struct ice_vf *vf; struct ice_vsi *vsi; struct if_vf_extension *extension; struct if_vf_info *info; struct if_vf_status *status; u32 vf_flags; bool mirror_configured, software_mbx_limit; uint32_t field, num_fields; int i; if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SRIOV)) return (EOPNOTSUPP); status = if_vf_status_alloc(sc->num_vfs); if (status == NULL) return (ENOMEM); for (i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; vsi = vf->vsi; vf_flags = atomic_load_acq_32(&vf->vf_flags); info = &status->vfs[i]; info->fields = IFVF_F_CONFIGURED | IFVF_F_INITIALIZED | IFVF_F_TRAFFIC_ALLOWED | IFVF_F_FAULT_BLOCKED | IFVF_F_LINK_STATE_POLICY | IFVF_F_VLAN_MODE | IFVF_F_VLAN_COUNT | IFVF_F_ALLOW_SET_MAC | IFVF_F_ALLOW_SET_VLAN | IFVF_F_MAC_ANTI_SPOOF | IFVF_F_ALLOW_PROMISC; info->index = i; info->configured = (vf_flags & VF_FLAG_ENABLED) != 0 && vsi != NULL; info->initialized = info->configured && (vf_flags & VF_FLAG_INITIALIZED) != 0; info->traffic_allowed = info->configured && (vf_flags & (VF_FLAG_MDD_BLOCKED | VF_FLAG_MBX_BLOCKED)) == 0; info->fault_blocked = (vf_flags & (VF_FLAG_MDD_BLOCKED | VF_FLAG_MBX_BLOCKED)) != 0; info->link_state_policy = IFVF_LINK_AUTO; if (info->initialized) { snprintf(info->api_version, sizeof(info->api_version), "%u.%u", vf->version.major, vf->version.minor); info->fields |= IFVF_F_API_VERSION; } if (!ETHER_IS_ZERO(vf->mac)) { memcpy(info->mac, vf->mac, sizeof(info->mac)); info->fields |= IFVF_F_MAC; } /* The ICE IOV schema exposes only VF-managed trunk membership. */ info->vlan_mode = IFVF_VLAN_TRUNK; info->vlan_count = vf->vlan_cnt; if (info->configured) { info->vlan_limit = vf->vlan_limit; info->fields |= IFVF_F_VLAN_LIMIT; } if (vsi != NULL) { info->tx_queue_count = vsi->num_tx_queues; info->rx_queue_count = vsi->num_rx_queues; info->fields |= IFVF_F_NUM_TX_QUEUES | IFVF_F_NUM_RX_QUEUES; } mirror_configured = vsi != NULL && vsi->mirror_src_vsi != ICE_INVALID_MIRROR_VSI; software_mbx_limit = !ice_is_e830(&sc->hw); num_fields = ICE_VF_STATUS_NUM_FIELDS - (mirror_configured ? 0 : 1) - (software_mbx_limit ? 0 : 2) - (info->configured ? 0 : 1); extension = if_vf_status_add_extension(info, ICE_VF_STATUS_NAMESPACE, ICE_VF_STATUS_VERSION, num_fields); if (extension == NULL) { if_vf_status_free(status); return (ENOMEM); } field = ICE_VF_STATUS_FIELD_MIRROR_CONFIGURED; if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_MIRROR_CONFIGURED, mirror_configured); if (mirror_configured) if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MIRROR_SOURCE_VSI, vsi->mirror_src_vsi); if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_MIRROR_INGRESS_ACTIVE, vsi != NULL && vsi->rule_mir_ingress != ICE_INVAL_MIRROR_RULE_ID); if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_MIRROR_EGRESS_ACTIVE, vsi != NULL && vsi->rule_mir_egress != ICE_INVAL_MIRROR_RULE_ID); if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_MDD_BLOCKED, (vf_flags & VF_FLAG_MDD_BLOCKED) != 0); if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MDD_TX_EVENTS, vf->mdd_tx_events); if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MDD_RX_EVENTS, vf->mdd_rx_events); if (software_mbx_limit) { if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_MBX_BLOCKED, (vf_flags & VF_FLAG_MBX_BLOCKED) != 0); if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MBX_OVERFLOW_EVENTS, vf->mbx_overflow_events); } if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MAC_FILTER_COUNT, vf->mac_filter_cnt); if (info->configured) if_vf_extension_set_number(extension, field++, ICE_VF_STATUS_MAC_FILTER_LIMIT, vf->mac_filter_limit); if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_RESET_FAILED, (vf_flags & VF_FLAG_RESET_FAILED) != 0); if_vf_extension_set_bool(extension, field++, ICE_VF_STATUS_REBUILD_REQUIRED, (vf_flags & VF_FLAG_REBUILD_REQUIRED) != 0); KASSERT(field == num_fields, ("ICE VF status field count %u != %u", field, num_fields)); info->allow_set_mac = (vf_flags & VF_FLAG_SET_MAC_CAP) != 0; info->allow_set_vlan = (vf_flags & VF_FLAG_VLAN_CAP) != 0; info->mac_anti_spoof = (vf_flags & VF_FLAG_MAC_ANTI_SPOOF) != 0; info->allow_promisc = (vf_flags & VF_FLAG_PROMISC_CAP) != 0; } *statusp = status; return (0); } /** * ice_iov_uninit - Called by the OS when VFs are destroyed * @sc: device softc structure */ void ice_iov_uninit(struct ice_softc *sc) { struct ice_vf *vf; struct ice_vsi *vsi; /* Release per-VF resources */ for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; if (!ice_is_e830(&sc->hw)) LIST_DEL(&vf->mbx_info.list_entry); atomic_store_rel_32(&vf->vf_flags, 0); vsi = vf->vsi; free(vf->mac_filters, M_ICE); vf->mac_filters = NULL; vf->mac_filter_cnt = 0; /* Free VF interrupt reservation */ if (vf->vf_imap) { ice_resmgr_release_map(&sc->dev_imgr, vf->vf_imap, vf->num_irq_vectors); free(vf->vf_imap, M_ICE); vf->vf_imap = NULL; } /* Free queue interrupt mapping trackers */ if (vf->tx_irqvs) { free(vf->tx_irqvs, M_ICE); vf->tx_irqvs = NULL; } if (vf->rx_irqvs) { free(vf->rx_irqvs, M_ICE); vf->rx_irqvs = NULL; } if (!vsi) continue; /* Free VSI queues */ if (vsi->tx_queues) { free(vsi->tx_queues, M_ICE); vsi->tx_queues = NULL; } if (vsi->rx_queues) { free(vsi->rx_queues, M_ICE); vsi->rx_queues = NULL; } if (vsi->hw_vsi_created) ice_release_vsi(vsi); else ice_release_vsi_resources(vsi); vf->vsi = NULL; } /* Release memory used for VF tracking */ if (sc->vfs) { free(sc->vfs, M_ICE); sc->vfs = NULL; } sc->num_vfs = 0; } /** * ice_iov_handle_vflr - Process VFLR event * @sc: device softc structure * * Identifys which VFs have been reset and re-configure * them. */ void ice_iov_handle_vflr(struct ice_softc *sc) { struct ice_hw *hw = &sc->hw; struct ice_vf *vf; u32 reg, reg_idx, bit_idx, vf_flags; for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; reg_idx = (hw->func_caps.vf_base_id + vf->vf_num) / 32; bit_idx = (hw->func_caps.vf_base_id + vf->vf_num) % 32; reg = rd32(hw, GLGEN_VFLRSTAT(reg_idx)); if ((reg & BIT(bit_idx)) == 0) continue; vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) != 0 && vf->vsi != NULL) { if ((vf_flags & VF_FLAG_REBUILD_REQUIRED) != 0) { /* Consume the event but leave the invalid VF held. */ wr32(hw, GLGEN_VFLRSTAT(reg_idx), BIT(bit_idx)); ice_flush(hw); continue; } ice_reset_vf(sc, vf, false, true); continue; } /* Consume reset events for inactive or incompletely added VFs. */ wr32(hw, GLGEN_VFLRSTAT(reg_idx), BIT(bit_idx)); ice_flush(hw); } } /** * ice_iov_handle_mdd - Attribute malicious-driver events to VFs * @sc: device softc structure * * Consume every per-VF MDD latch. Block further virtchnl requests and reset a * newly blocked VF without restoring its queues, so even event classes which * only drop the offending packet cannot continue traffic. An optional policy * reconstructs and releases the VF immediately instead. * * @returns a mask of enum ice_mdd_source_bits attributed to configured or * unconfigured VFs of this PF. */ u32 ice_iov_handle_mdd(struct ice_softc *sc) { static const struct timeval log_interval = { 2, 0 }; struct ice_hw *hw = &sc->hw; struct virtchnl_pf_event event = {}; struct ice_vf *vf; u32 reg, sources, vf_sources, tx_events, rx_events, vf_flags; bool newly_blocked; int error; event.event = VIRTCHNL_EVENT_RESET_IMPENDING; event.severity = PF_EVENT_SEVERITY_CERTAIN_DOOM; vf_sources = 0; for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; sources = 0; tx_events = 0; rx_events = 0; reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_num)); if ((reg & VP_MDET_TX_PQM_VALID_M) != 0) { wr32(hw, VP_MDET_TX_PQM(vf->vf_num), 0xffff); sources |= ICE_MDD_TX_PQM; tx_events++; } reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_num)); if ((reg & VP_MDET_TX_TCLAN_VALID_M) != 0) { wr32(hw, VP_MDET_TX_TCLAN(vf->vf_num), 0xffff); sources |= ICE_MDD_TX_TCLAN; tx_events++; } reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_num)); if ((reg & VP_MDET_TX_TDPU_VALID_M) != 0) { wr32(hw, VP_MDET_TX_TDPU(vf->vf_num), 0xffff); sources |= ICE_MDD_TX_TDPU; tx_events++; } reg = rd32(hw, VP_MDET_RX(vf->vf_num)); if ((reg & VP_MDET_RX_VALID_M) != 0) { wr32(hw, VP_MDET_RX(vf->vf_num), 0xffff); sources |= ICE_MDD_RX; rx_events++; } if (tx_events == 0 && rx_events == 0) continue; vf_sources |= sources; vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) == 0 || vf->vsi == NULL) continue; vf->mdd_tx_events += tx_events; vf->mdd_rx_events += rx_events; newly_blocked = (vf_flags & VF_FLAG_MDD_BLOCKED) == 0; atomic_set_32(&vf->vf_flags, VF_FLAG_MDD_BLOCKED); if (ratecheck(&vf->last_mdd_log, &log_interval)) { device_printf(sc->dev, "malicious-driver event from VF-%d " "(tx %ju, rx %ju); %s\n", vf->vf_num, (uintmax_t)vf->mdd_tx_events, (uintmax_t)vf->mdd_rx_events, sc->mdd_auto_reset_vf && newly_blocked ? "resetting VF" : "VF remains blocked"); } if (!newly_blocked) continue; /* Ignore notification failure; reset does not require VF help. */ if (sc->mdd_auto_reset_vf && (vf_flags & VF_FLAG_INITIALIZED) != 0 && ice_check_sq_alive(hw, &hw->mailboxq)) { (void)ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_EVENT, VIRTCHNL_STATUS_SUCCESS, (u8 *)&event, sizeof(event), NULL); } /* * TDPU MDD drops only the offending packet. Reset the entire VF so * the software blocked state always means that traffic is actually * fenced. The opt-in policy reconstructs its queues immediately. */ error = ice_reset_vf(sc, vf, true, sc->mdd_auto_reset_vf); if (error != 0) { device_printf(sc->dev, "failed to quiesce MDD-blocked VF-%d: %d\n", vf->vf_num, error); } else if (!sc->mdd_auto_reset_vf) { /* * Complete VFR without restoring queues. This leaves the VF * inactive and DMA-fenced, but permits a later physical FLR to * create a new reset edge and recover it. */ ice_iov_complete_vf_reset(sc, vf, false); } } ice_flush(hw); return (vf_sources); } /** * ice_iov_notify_vfs_reset - Notify initialized VFs of an impending reset * @sc: device softc structure * * Give VF drivers advance notice while the mailbox control queue is still * alive. Ignore individual send failures so one VF cannot prevent the PF from * notifying its siblings or proceeding with the reset. */ void ice_iov_notify_vfs_reset(struct ice_softc *sc) { struct virtchnl_pf_event event = {}; struct ice_hw *hw = &sc->hw; struct ice_vf *vf; if (!ice_check_sq_alive(hw, &hw->mailboxq)) return; event.event = VIRTCHNL_EVENT_RESET_IMPENDING; event.severity = PF_EVENT_SEVERITY_CERTAIN_DOOM; for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; if ((atomic_load_acq_32(&vf->vf_flags) & VF_FLAG_INITIALIZED) == 0) continue; (void)ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_EVENT, VIRTCHNL_STATUS_SUCCESS, (u8 *)&event, sizeof(event), NULL); } } /** * ice_iov_clear_vf_queue_state - Clear tracked VF queue state * @vf: driver's VF structure for the VF to update */ static void ice_iov_clear_vf_queue_state(struct ice_vf *vf) { vf->txq_configured = 0; vf->rxq_configured = 0; vf->rxq_enabled = 0; } /** * ice_iov_clear_vf_mdd - Clear hardware MDD latches for a reset VF * @sc: device softc structure * @vf: driver's VF structure for the VF to update * * Function reset can generate a spurious anti-spoof MDD indication. Consume * all per-VF latches before releasing reset so it cannot re-block a VF which * has just been reconstructed successfully. */ static void ice_iov_clear_vf_mdd(struct ice_softc *sc, struct ice_vf *vf) { struct ice_hw *hw = &sc->hw; wr32(hw, VP_MDET_TX_PQM(vf->vf_num), 0xffff); wr32(hw, VP_MDET_TX_TCLAN(vf->vf_num), 0xffff); wr32(hw, VP_MDET_TX_TDPU(vf->vf_num), 0xffff); wr32(hw, VP_MDET_RX(vf->vf_num), 0xffff); ice_flush(hw); } /** * ice_iov_complete_vf_reset - Complete a VF reset * @sc: device softc structure * @vf: driver's VF structure for the VF to update * @restore_mapping: restore the VF queue and interrupt mappings * * Clear VFSWR after the hardware drain, optionally restore the VF mappings, * and then publish VFACTIVE. The mapping registers do not retain writes made * while VFSWR remains asserted. Callers may instead leave a software-blocked * VF with no queue or interrupt mappings. */ static void ice_iov_complete_vf_reset(struct ice_softc *sc, struct ice_vf *vf, bool restore_mapping) { struct ice_hw *hw = &sc->hw; u32 reg; reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_num)); reg &= ~VPGEN_VFRTRIG_VFSWR_M; wr32(hw, VPGEN_VFRTRIG(vf->vf_num), reg); if (restore_mapping) ice_iov_setup_intr_mapping(sc, vf); wr32(hw, VFGEN_RSTAT(vf->vf_num), VIRTCHNL_VFR_VFACTIVE); ice_flush(hw); } /** * ice_iov_ready_vf - Setup VF interrupts and mark it as ready * @sc: device softc structure * @vf: driver's VF structure for the VF to update * * Clears VF reset triggering bit, sets up the PF<->VF interrupt * mapping and marks the VF as active in the HW so that the VF * driver can use it. */ static void ice_iov_ready_vf(struct ice_softc *sc, struct ice_vf *vf) { /* A VF or PF reset discards all queue configuration and state. */ ice_iov_clear_vf_queue_state(vf); ice_iov_clear_vf_mdd(sc, vf); atomic_clear_32(&vf->vf_flags, VF_FLAG_MDD_BLOCKED); ice_iov_clear_vf_mbx(sc, vf); ice_iov_complete_vf_reset(sc, vf, true); } /** * ice_iov_clear_vf_mbx - Release mailbox isolation after a completed reset * @sc: device softc structure * @vf: VF whose mailbox state should be cleared */ static void ice_iov_clear_vf_mbx(struct ice_softc *sc, struct ice_vf *vf) { if (ice_is_e830(&sc->hw)) ice_mbx_vf_clear_cnt_e830(&sc->hw, vf->vf_num); else ice_mbx_clear_malvf(&vf->mbx_info); atomic_clear_32(&vf->vf_flags, VF_FLAG_MBX_BLOCKED); } /** * ice_iov_rebuild_vf - Rebuild a VF VSI after a PF or device reset * @sc: device softc structure * @vsi: VF VSI to rebuild * * PF and device resets discard the hardware VSI and interrupt state for every * VF. Re-add the VSI and replay its configuration before reporting the VF as * active. A failed rebuild leaves the VF inactive while allowing the PF and * other VFs to recover. */ int ice_iov_rebuild_vf(struct ice_softc *sc, struct ice_vsi *vsi) { struct ice_eth_stats accumulated_stats; struct ice_hw *hw = &sc->hw; struct ice_vf *vf; int error, status; MPASS(vsi->type == ICE_VSI_VF); vf = ice_iov_get_vf(sc, vsi->vf_num); atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED); atomic_set_32(&vf->vf_flags, VF_FLAG_REBUILD_REQUIRED); ice_iov_clear_vf_queue_state(vf); ICE_IOV_FAIL_POINT(sc, vf->vf_num, rebuild_before_initialize, error, fail); /* A new hardware VSI starts a new raw statistics epoch. */ accumulated_stats = vsi->hw_stats.cur; error = ice_initialize_vsi(vsi); if (error != 0) { device_printf(sc->dev, "Unable to re-initialize VF %d VSI, err %s\n", vf->vf_num, ice_err_str(error)); return (error); } vsi->hw_stats.cur = accumulated_stats; error = ice_iov_configure_mac_anti_spoof(sc, vf); if (error != 0) return (error); status = ice_replay_vsi(hw, vsi->idx); if (status != 0) { device_printf(sc->dev, "Failed to replay VF %d VSI, err %s aq_err %s\n", vf->vf_num, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); return (EIO); } atomic_clear_32(&vf->vf_flags, VF_FLAG_REBUILD_REQUIRED | VF_FLAG_RESET_FAILED); ice_iov_ready_vf(sc, vf); return (0); #ifdef DRIVER_FAILPOINTS fail: return (error); #endif /* DRIVER_FAILPOINTS */ } /** * ice_reset_vf - Perform a hardware reset (VFR) on a VF * @sc: device softc structure * @vf: driver's VF structure for VF to be reset * @trigger_reset: trigger a reset or only handle an already executed reset * @release_vf: publish VFACTIVE after reset; otherwise leave the VF held * * Performs a VFR for the given VF. This function busy waits until the reset * completes in the HW and publishes VFACTIVE only after every mandatory * reset stage succeeds. In quiesce mode, it returns with VFSWR asserted and * without restoring interrupt mappings or publishing VFACTIVE. * * @remark Release mode also sets up the PF<->VF interrupt mapping and * allocations in the hardware after the hardware reset is finished, via * ice_iov_setup_intr_mapping() */ static int ice_reset_vf(struct ice_softc *sc, struct ice_vf *vf, bool trigger_reset, bool release_vf) { u16 global_vf_num, reg_idx, bit_idx; struct ice_hw *hw = &sc->hw; bool reset_done; int error, status; u32 reg; int bit, i; /* A VFR cannot recover PF-owned VSI state lost during PF rebuild. */ if (release_vf && (atomic_load_acq_32(&vf->vf_flags) & VF_FLAG_REBUILD_REQUIRED) != 0) return (EIO); global_vf_num = vf->vf_num + hw->func_caps.vf_base_id; atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED); error = 0; if (trigger_reset) { reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_num)); reg |= VPGEN_VFRTRIG_VFSWR_M; wr32(hw, VPGEN_VFRTRIG(vf->vf_num), reg); ice_flush(hw); } /* * Remove the tracked queue leaves from the software scheduler before * issuing the reset-only AQ command. That command drains hardware but * does not update the shared scheduler database. Retain unresolved queue * state if cleanup fails so a later reset can retry it. */ status = ice_vc_disable_queues(sc, vf, vf->txq_configured, vf->rxq_enabled); if (status == 0) ice_iov_clear_vf_queue_state(vf); else if (error == 0) error = status; /* This zero-queue command is required to complete every VF reset. */ status = ice_dis_vsi_txq(hw->port_info, vf->vsi->idx, 0, 0, NULL, NULL, NULL, ICE_VF_RESET, vf->vf_num, NULL); ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, vf_reset_tx_disable, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { status = ICE_ERR_AQ_ERROR; }); if (status) { device_printf(sc->dev, "%s: Failed to disable LAN Tx queues: err %s aq_err %s\n", __func__, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); if (error == 0) error = EIO; } /* Then check for the VF reset to finish in HW. */ reset_done = false; for (i = 0; i < ICE_VPGEN_VFRSTAT_WAIT_COUNT; i++) { reg = rd32(hw, VPGEN_VFRSTAT(vf->vf_num)); if ((reg & VPGEN_VFRSTAT_VFRD_M)) { reset_done = true; break; } DELAY(ICE_VPGEN_VFRSTAT_WAIT_DELAY_US); } ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, vf_reset_vfr_timeout, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { reset_done = false; }); if (!reset_done) { device_printf(sc->dev, "VF-%d Reset is stuck\n", vf->vf_num); if (error == 0) error = ETIMEDOUT; } else { /* VFLR status is W1C only after the hardware drain completes. */ reg_idx = global_vf_num / 32; bit_idx = global_vf_num % 32; wr32(hw, GLGEN_VFLRSTAT(reg_idx), BIT(bit_idx)); ice_flush(hw); /* Hardware resets Tx queues; the PF must disable every Rx. */ for (bit = 0; bit < vf->vsi->num_rx_queues; bit++) { status = ice_control_rx_queue(vf->vsi, bit, false); ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, vf_reset_rx_disable, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { status = EIO; }); if (status != 0) { device_printf(sc->dev, "Unable to disable VF-%d Rx queue %d: %s\n", vf->vf_num, bit, ice_err_str(status)); if (error == 0) error = status; } } } /* Verify that post-drain cleanup left no outstanding DMA. */ wr32(hw, PF_PCI_CIAA, ICE_PCIE_DEV_STATUS | (global_vf_num << PF_PCI_CIAA_VF_NUM_S)); for (i = 0; i < ICE_PCI_CIAD_WAIT_COUNT; i++) { reg = rd32(hw, PF_PCI_CIAD); if (!(reg & PCIEM_STA_TRANSACTION_PND)) break; DELAY(ICE_PCI_CIAD_WAIT_DELAY_US); } ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, vf_reset_pcie_pending, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { i = ICE_PCI_CIAD_WAIT_COUNT; }); if (i == ICE_PCI_CIAD_WAIT_COUNT) { device_printf(sc->dev, "VF-%d PCI transactions remain after reset\n", vf->vf_num); if (error == 0) error = ETIMEDOUT; } if (error != 0) { atomic_set_32(&vf->vf_flags, VF_FLAG_RESET_FAILED); return (error); } if (!release_vf) { /* Discard any anti-spoof MDD indication caused by the reset. */ ice_iov_clear_vf_mdd(sc, vf); atomic_clear_32(&vf->vf_flags, VF_FLAG_RESET_FAILED); return (0); } error = ice_iov_restore_vf_host_config(sc, vf); if (error != 0) { atomic_set_32(&vf->vf_flags, VF_FLAG_RESET_FAILED); return (error); } atomic_clear_32(&vf->vf_flags, VF_FLAG_RESET_FAILED); ice_iov_ready_vf(sc, vf); return (0); } /** * ice_iov_quiesce_vfs_for_reset - Hold configured VFs before device reset * @sc: device softc structure * * Gate VF master accesses, drain each VF data path, and leave VFSWR asserted. * Process VFs serially to remain below the E810 limit of four concurrent * VM/VF reset flows. A successful VSI rebuild releases each VF individually. */ int ice_iov_quiesce_vfs_for_reset(struct ice_softc *sc) { struct virtchnl_pf_event event = {}; struct ice_hw *hw = &sc->hw; struct ice_vf *vf; int error, first_error; u32 reg, vf_flags; bool notify; notify = ice_check_sq_alive(hw, &hw->mailboxq); event.event = VIRTCHNL_EVENT_RESET_IMPENDING; event.severity = PF_EVENT_SEVERITY_CERTAIN_DOOM; /* Notify and then gate each VF before it can release its buffers. */ for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) == 0 || vf->vsi == NULL) continue; if (notify && (vf_flags & VF_FLAG_INITIALIZED) != 0) ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_EVENT, VIRTCHNL_STATUS_SUCCESS, (u8 *)&event, sizeof(event), NULL); /* Block mailbox reconfiguration before asserting reset. */ atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED); atomic_set_32(&vf->vf_flags, VF_FLAG_REBUILD_REQUIRED); reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_num)); reg |= VPGEN_VFRTRIG_VFSWR_M; wr32(hw, VPGEN_VFRTRIG(vf->vf_num), reg); } ice_flush(hw); /* Firmware data-path drains remain serialized below its limit. */ first_error = 0; for (int i = 0; i < sc->num_vfs; i++) { vf = &sc->vfs[i]; vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) == 0 || vf->vsi == NULL) continue; error = ice_reset_vf(sc, vf, false, false); if (error != 0) { device_printf(sc->dev, "Failed to quiesce VF-%d for device reset: %d\n", vf->vf_num, error); if (first_error == 0) first_error = error; } } return (first_error); } /** * ice_vc_get_vf_res_msg - Handle VIRTCHNL_OP_GET_VF_RESOURCES msg from VF * @sc: device private structure * @vf: VF tracking structure * @msg_buf: raw message buffer from the VF * * Receives a message from the VF listing its supported capabilities, and * replies to the VF with information about what resources the PF has * allocated for the VF. * * @remark This always replies to the VF with a success status; it does not * fail. It's up to the VF driver to reject or complain about the PF's response. */ static void ice_vc_get_vf_res_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_vf_resource *vf_res; struct virtchnl_vsi_resource *vsi_res; u16 vf_res_len; u32 vf_caps; int status; /* XXX: Only support one VSI per VF, so this size doesn't need adjusting */ vf_res_len = sizeof(struct virtchnl_vf_resource); vf_res = (struct virtchnl_vf_resource *)malloc(vf_res_len, M_ICE, M_WAITOK | M_ZERO); vf_res->num_vsis = 1; vf_res->num_queue_pairs = vf->vsi->num_tx_queues; vf_res->max_vectors = vf_res->num_queue_pairs + 1; vf_res->rss_key_size = ICE_GET_SET_RSS_KEY_EXTEND_KEY_SIZE; vf_res->rss_lut_size = ICE_VSIQF_HLUT_ARRAY_SIZE; vf_res->max_mtu = ICE_MAX_FRAME_SIZE; vf_res->vf_cap_flags = VF_BASE_MODE_OFFLOADS; if (msg_buf != NULL) { vf_caps = *((u32 *)(msg_buf)); if (vf_caps & VIRTCHNL_VF_CAP_ADV_LINK_SPEED) vf_res->vf_cap_flags |= VIRTCHNL_VF_CAP_ADV_LINK_SPEED; if (vf_caps & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) vf_res->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_WB_ON_ITR; } vsi_res = &vf_res->vsi_res[0]; vsi_res->vsi_id = vf->vsi->idx; vsi_res->num_queue_pairs = vf->vsi->num_tx_queues; vsi_res->vsi_type = VIRTCHNL_VSI_SRIOV; vsi_res->qset_handle = 0; if (!ETHER_IS_ZERO(vf->mac)) memcpy(vsi_res->default_mac_addr, vf->mac, ETHER_ADDR_LEN); status = ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_GET_VF_RESOURCES, VIRTCHNL_STATUS_SUCCESS, (u8 *)vf_res, vf_res_len, NULL); if (status == 0) atomic_set_32(&vf->vf_flags, VF_FLAG_INITIALIZED); else device_printf(sc->dev, "Unable to send VF-%u resource response, err %s\n", vf->vf_num, ice_status_str(status)); free(vf_res, M_ICE); } /** * ice_vc_version_msg - Handle VIRTCHNL_OP_VERSION msg from VF * @sc: device private structure * @vf: VF tracking structure * @msg_buf: raw message buffer from the VF * * Receives a version message from the VF, and responds to the VF with * the version number that the PF will use. * * @remark This always replies to the VF with a success status; it does not * fail. */ static void ice_vc_version_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct virtchnl_version_info *recv_vf_version; struct ice_hw *hw = &sc->hw; device_t dev = sc->dev; recv_vf_version = (struct virtchnl_version_info *)msg_buf; /* VFs running the 1.0 API expect to get 1.0 back */ if (VF_IS_V10(recv_vf_version)) { vf->version.major = 1; vf->version.minor = VIRTCHNL_VERSION_MINOR_NO_VF_CAPS; } else { vf->version.major = VIRTCHNL_VERSION_MAJOR; vf->version.minor = VIRTCHNL_VERSION_MINOR; if ((recv_vf_version->major != VIRTCHNL_VERSION_MAJOR) || (recv_vf_version->minor != VIRTCHNL_VERSION_MINOR)) device_printf(dev, "%s: VF-%d requested version (%d.%d) differs from PF version (%d.%d)\n", __func__, vf->vf_num, recv_vf_version->major, recv_vf_version->minor, VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR); } ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_VERSION, VIRTCHNL_STATUS_SUCCESS, (u8 *)&vf->version, sizeof(vf->version), NULL); } /** * ice_vf_validate_mac - Validate MAC address before adding it * @vf: VF tracking structure * @addr: MAC address to validate * * Validate a MAC address before adding it to a VF during the handling * of a VIRTCHNL_OP_ADD_ETH_ADDR operation. Notably, this also checks if * the VF is allowed to set its own arbitrary MAC addresses. * * Returns 0 if MAC address is valid for the given vf */ static int ice_vf_validate_mac(struct ice_vf *vf, const uint8_t *addr) { if (ETHER_IS_ZERO(addr) || ETHER_IS_BROADCAST(addr)) return (EINVAL); /* * If the VF is not allowed to change its MAC address, don't let it * set a MAC filter for an address that is not a multicast address and * is not its assigned MAC. */ if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && !(ETHER_IS_MULTICAST(addr) || !bcmp(addr, vf->mac, ETHER_ADDR_LEN))) return (EPERM); return (0); } /** * ice_vf_mac_filter_index - Find a VF-owned MAC filter * @vf: VF tracking structure * @addr: MAC address to find * * The administrator-assigned address does not consume the configurable VF * filter quota and is therefore not stored in this array. */ static int ice_vf_mac_filter_index(struct ice_vf *vf, const uint8_t *addr) { for (u16 i = 0; i < vf->mac_filter_cnt; i++) { if (memcmp(vf->mac_filters[i].addr, addr, ETHER_ADDR_LEN) == 0) return (i); } return (-1); } /** * ice_vc_add_eth_addr_msg - Handle VIRTCHNL_OP_ADD_ETH_ADDR msg from VF * @sc: device private structure * @vf: VF tracking structure * @msg_buf: raw message buffer from the VF * * Receives a list of MAC addresses from the VF and adds those addresses * to the VSI's filter list. */ static void ice_vc_add_eth_addr_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct virtchnl_ether_addr_list *addr_list; struct ice_hw *hw = &sc->hw; u16 new_filters; int error = 0; addr_list = (struct virtchnl_ether_addr_list *)msg_buf; /* Validate the entire batch and charge only unique, absent filters. */ new_filters = 0; for (int i = 0; i < addr_list->num_elements; i++) { u8 *addr = addr_list->list[i].addr; int j; error = ice_vf_validate_mac(vf, addr); if (error != 0) { device_printf(sc->dev, "%s: VF-%d: invalid or unauthorized MAC for VSI %d\n", __func__, vf->vf_num, vf->vsi->idx); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } for (j = 0; j < i; j++) { if (memcmp(addr_list->list[j].addr, addr, ETHER_ADDR_LEN) == 0) break; } if (j != i || memcmp(addr, vf->mac, ETHER_ADDR_LEN) == 0 || ice_vf_mac_filter_index(vf, addr) >= 0) continue; new_filters++; } if ((u32)vf->mac_filter_cnt + new_filters > vf->mac_filter_limit) { v_status = VIRTCHNL_STATUS_ERR_NO_MEMORY; goto done; } for (int i = 0; i < addr_list->num_elements; i++) { u8 *addr = addr_list->list[i].addr; bool assigned; /* The type flag is currently ignored; every MAC address is * treated as the LEGACY type */ assigned = memcmp(addr, vf->mac, ETHER_ADDR_LEN) == 0; if (!assigned && ice_vf_mac_filter_index(vf, addr) >= 0) continue; error = ice_add_vsi_mac_filter(vf->vsi, addr); if (error) { device_printf(sc->dev, "%s: VF-%d: Error adding MAC addr for VSI %d\n", __func__, vf->vf_num, vf->vsi->idx); v_status = VIRTCHNL_STATUS_ERR_PARAM; continue; } if (!assigned) { MPASS(vf->mac_filter_cnt < vf->mac_filter_limit); memcpy(vf->mac_filters[vf->mac_filter_cnt].addr, addr, ETHER_ADDR_LEN); vf->mac_filter_cnt++; } } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_ADD_ETH_ADDR, v_status, NULL, 0, NULL); } /** * ice_vc_del_eth_addr_msg - Handle VIRTCHNL_OP_DEL_ETH_ADDR msg from VF * @sc: device private structure * @vf: VF tracking structure * @msg_buf: raw message buffer from the VF * * Receives a list of MAC addresses from the VF and removes those addresses * from the VSI's filter list. */ static void ice_vc_del_eth_addr_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct virtchnl_ether_addr_list *addr_list; struct ice_hw *hw = &sc->hw; int error = 0; addr_list = (struct virtchnl_ether_addr_list *)msg_buf; for (int i = 0; i < addr_list->num_elements; i++) { u8 *addr = addr_list->list[i].addr; bool assigned; int index; error = ice_vf_validate_mac(vf, addr); if (error != 0) { v_status = VIRTCHNL_STATUS_ERR_PARAM; continue; } assigned = memcmp(addr, vf->mac, ETHER_ADDR_LEN) == 0; if (assigned && (vf->vf_flags & VF_FLAG_SET_MAC_CAP) == 0) continue; index = assigned ? -1 : ice_vf_mac_filter_index(vf, addr); if (!assigned && index < 0) continue; error = ice_remove_vsi_mac_filter(vf->vsi, addr); if (error) { device_printf(sc->dev, "%s: VF-%d: Error removing MAC addr for VSI %d\n", __func__, vf->vf_num, vf->vsi->idx); v_status = VIRTCHNL_STATUS_ERR_PARAM; continue; } if (!assigned) { if (index + 1 < vf->mac_filter_cnt) { memmove(&vf->mac_filters[index], &vf->mac_filters[index + 1], (vf->mac_filter_cnt - index - 1) * sizeof(*vf->mac_filters)); } vf->mac_filter_cnt--; } } ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_DEL_ETH_ADDR, v_status, NULL, 0, NULL); } /** * ice_vc_select_vlans - Compact a VF VLAN request in place * @vf: VF tracking structure * @vids: VLAN IDs supplied by the VF * @count: number of VLAN IDs in the request * @add: select absent VLANs for add, or present VLANs for delete * @selected_count: returned number of VLAN IDs requiring a hardware change * * A VF may replay its entire VLAN configuration after a reset or retry a * request whose reply was lost. Select only unique IDs whose membership * actually changes so those requests remain idempotent and filter accounting * continues to enforce the configured limit. */ static int ice_vc_select_vlans(struct ice_vf *vf, u16 *vids, u16 count, bool add, u16 *selected_count) { bitstr_t bit_decl(seen, ICE_VF_VLAN_MAP_LEN); u16 selected, vid; bzero(seen, sizeof(seen)); selected = 0; for (u16 i = 0; i < count; i++) { vid = vids[i]; if (vid > EVL_VLID_MASK) return (EINVAL); if (bit_test(seen, vid)) continue; bit_set(seen, vid); if (bit_test(vf->vlans_map, vid) == add) continue; vids[selected++] = vid; } *selected_count = selected; return (0); } /** * ice_vc_add_vlan_msg - Handle VIRTCHNL_OP_ADD_VLAN msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Adds the VLANs in msg_buf to the VF's VLAN filter list. */ static void ice_vc_add_vlan_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_vlan_filter_list *vlan_list; u16 selected; int status = 0; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; vlan_list = (struct virtchnl_vlan_filter_list *)msg_buf; if (vlan_list->vsi_id != vsi->idx) { device_printf(sc->dev, "VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", vf->vf_num, vsi->idx, vlan_list->vsi_id); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } status = ice_vc_select_vlans(vf, vlan_list->vlan_id, vlan_list->num_elements, true, &selected); if (status != 0) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } if ((u32)vf->vlan_cnt + selected > vf->vlan_limit) { v_status = VIRTCHNL_STATUS_ERR_NO_MEMORY; goto done; } if (selected == 0) goto done; for (u16 i = 0; i < selected; i++) { status = ice_add_vlan_hw_filter(vsi, vlan_list->vlan_id[i]); if (status != 0 && status != ICE_ERR_ALREADY_EXISTS) { device_printf(sc->dev, "VF-%d: Failure adding VLAN %d to VSI %d, err %s aq_err %s\n", vf->vf_num, vlan_list->vlan_id[i], vsi->idx, ice_status_str(status), ice_aq_str(sc->hw.adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } bit_set(vf->vlans_map, vlan_list->vlan_id[i]); vf->vlan_cnt++; } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_ADD_VLAN, v_status, NULL, 0, NULL); } /** * ice_vc_del_vlan_msg - Handle VIRTCHNL_OP_DEL_VLAN msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Removes the VLANs in msg_buf from the VF's VLAN filter list. */ static void ice_vc_del_vlan_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_vlan_filter_list *vlan_list; u16 selected; int status = 0; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; vlan_list = (struct virtchnl_vlan_filter_list *)msg_buf; if (vlan_list->vsi_id != vsi->idx) { device_printf(sc->dev, "VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", vf->vf_num, vsi->idx, vlan_list->vsi_id); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } status = ice_vc_select_vlans(vf, vlan_list->vlan_id, vlan_list->num_elements, false, &selected); if (status != 0) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } if (selected == 0) goto done; for (u16 i = 0; i < selected; i++) { status = ice_remove_vlan_hw_filter(vsi, vlan_list->vlan_id[i]); if (status != 0 && status != ICE_ERR_DOES_NOT_EXIST) { device_printf(sc->dev, "VF-%d: Failure deleting VLAN %d from VSI %d, err %s aq_err %s\n", vf->vf_num, vlan_list->vlan_id[i], vsi->idx, ice_status_str(status), ice_aq_str(sc->hw.adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } bit_clear(vf->vlans_map, vlan_list->vlan_id[i]); MPASS(vf->vlan_cnt > 0); vf->vlan_cnt--; } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_DEL_VLAN, v_status, NULL, 0, NULL); } /** * ice_vc_validate_queue_select - Validate a VF queue selection * @sc: PF's softc structure * @vf: VF tracking structure * @vqs: queue selection from the VF * * Return true when the VSI ID and both queue masks are valid for the VF. */ static bool ice_vc_validate_queue_select(struct ice_softc *sc, struct ice_vf *vf, const struct virtchnl_queue_select *vqs) { struct ice_vsi *vsi = vf->vsi; int bit; if (vqs->vsi_id != vsi->idx) { device_printf(sc->dev, "%s: VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", __func__, vf->vf_num, vsi->idx, vqs->vsi_id); return (false); } if (vqs->rx_queues == 0 && vqs->tx_queues == 0) { device_printf(sc->dev, "%s: VF-%d: message queue masks are empty\n", __func__, vf->vf_num); return (false); } bit = fls(vqs->rx_queues); if (bit > vsi->num_rx_queues) { device_printf(sc->dev, "%s: VF-%d: message's Rx queue map (0x%08x) has invalid bit set (%d)\n", __func__, vf->vf_num, vqs->rx_queues, bit); return (false); } bit = fls(vqs->tx_queues); if (bit > vsi->num_tx_queues) { device_printf(sc->dev, "%s: VF-%d: message's Tx queue map (0x%08x) has invalid bit set (%d)\n", __func__, vf->vf_num, vqs->tx_queues, bit); return (false); } return (true); } /** * ice_vc_disable_tx_queue - Disable one configured VF Tx queue * @sc: PF's softc structure * @vf: VF tracking structure * @qid: VF-relative queue ID */ static int ice_vc_disable_tx_queue(struct ice_softc *sc, struct ice_vf *vf, u16 qid) { struct ice_vsi *vsi = vf->vsi; struct ice_tx_queue *txq = &vsi->tx_queues[qid]; struct ice_hw *hw = &sc->hw; u16 q_handle, q_id; u32 q_teid; int status; q_handle = txq->q_handle; q_id = vsi->tx_qmap[qid]; q_teid = txq->q_teid; status = ice_dis_vsi_txq(hw->port_info, vsi->idx, txq->tc, 1, &q_handle, &q_id, &q_teid, ICE_NO_RESET, 0, NULL); if (status != ICE_SUCCESS && status != ICE_ERR_DOES_NOT_EXIST && status != ICE_ERR_RESET_ONGOING) { device_printf(sc->dev, "Failed to disable VF-%d Tx queue %u, err %s aq_err %s\n", vf->vf_num, qid, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); return (EIO); } txq->q_handle = 0; txq->q_teid = 0; return (0); } /** * ice_vc_disable_queues - Disable selected configured VF queues * @sc: PF's softc structure * @vf: VF tracking structure * @tx_queues: VF-relative Tx queue bitmap * @rx_queues: VF-relative Rx queue bitmap * * Queue disable is idempotent. Queues which are not configured or enabled * have no hardware work to perform and are treated as successfully disabled. * Since CONFIG_VSI_QUEUES also enables Tx, disabling a Tx queue removes its * tracked configuration and the VF must configure it before enabling it again. */ static int ice_vc_disable_queues(struct ice_softc *sc, struct ice_vf *vf, u32 tx_queues, u32 rx_queues) { struct ice_vsi *vsi = vf->vsi; u32 queues; int bit, error; queues = rx_queues & vf->rxq_enabled; while (queues != 0) { bit = ffs(queues) - 1; error = ice_control_rx_queue(vsi, bit, false); if (error != 0) { device_printf(sc->dev, "Unable to disable VF-%d Rx queue %d: %s\n", vf->vf_num, bit, ice_err_str(error)); return (error); } vf->rxq_enabled &= ~BIT(bit); queues &= ~BIT(bit); } queues = tx_queues & vf->txq_configured; if (queues == vf->txq_configured && queues != 0) { error = ice_vsi_disable_tx(vsi); if (error != 0) return (error); vf->txq_configured = 0; return (0); } while (queues != 0) { bit = ffs(queues) - 1; error = ice_vc_disable_tx_queue(sc, vf, bit); if (error != 0) return (error); vf->txq_configured &= ~BIT(bit); queues &= ~BIT(bit); } return (0); } /** * ice_vc_validate_ring_len - Check to see if a descriptor ring length is valid * @ring_len: length of ring * * Check whether a ring size value is valid. * * @returns true if given ring size is valid */ static bool ice_vc_isvalid_ring_len(u32 ring_len) { return (ring_len >= ICE_MIN_DESC_COUNT && ring_len <= ICE_MAX_DESC_COUNT && !(ring_len % ICE_DESC_COUNT_INCR)); } /** * ice_vc_isvalid_txq - Validate a VF transmit queue description * @txq: VF-supplied transmit queue description * * Queue base addresses are encoded in the hardware context in 128-byte * units. Reject values which would be truncated while building the context. */ static bool ice_vc_isvalid_txq(const struct virtchnl_txq_info *txq) { u64 align; align = BIT_ULL(ICE_TLAN_CTX_BASE_S); return (ice_vc_isvalid_ring_len(txq->ring_len) && txq->dma_ring_addr != 0 && (txq->dma_ring_addr & (align - 1)) == 0 && txq->headwb_enabled == 0); } /** * ice_vc_isvalid_rxq - Validate a VF receive queue description * @rxq: VF-supplied receive queue description * * The receive queue context stores its ring base and data buffer size in * 128-byte units. It can represent data buffers from 128 through 16256 * bytes. The current driver supports neither header splitting nor retaining * the Ethernet CRC for VFs. Some older iavf drivers request the maximum PF * frame size with a buffer too small to hold it in five segments. Accept that * advisory mismatch; ice_setup_rx_ctx() safely limits the hardware RXMAX to * five data buffers. */ static bool ice_vc_isvalid_rxq(const struct virtchnl_rxq_info *rxq) { u64 ring_align; u32 buffer_align; ring_align = BIT_ULL(ICE_RLAN_BASE_S); buffer_align = BIT(ICE_RLAN_CTX_DBUF_S); return (ice_vc_isvalid_ring_len(rxq->ring_len) && rxq->dma_ring_addr != 0 && (rxq->dma_ring_addr & (ring_align - 1)) == 0 && rxq->databuffer_size >= buffer_align && rxq->databuffer_size <= ICE_VC_MAX_RX_BUFFER && (rxq->databuffer_size & (buffer_align - 1)) == 0 && rxq->max_pkt_size >= ETHER_MIN_LEN && rxq->max_pkt_size <= ICE_MAX_FRAME_SIZE && rxq->splithdr_enabled == 0 && rxq->crc_disable == 0); } /** * ice_vc_isvalid_itr_idx - Validate a virtchnl interrupt throttle index * @itr_idx: VF-supplied ITR index */ static bool ice_vc_isvalid_itr_idx(u16 itr_idx) { return (itr_idx == VIRTCHNL_ITR_IDX_0 || itr_idx == VIRTCHNL_ITR_IDX_1 || itr_idx == VIRTCHNL_ITR_IDX_NO_ITR); } /** * ice_vc_cfg_vsi_qs_msg - Handle VIRTCHNL_OP_CONFIG_VSI_QUEUES msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF */ static void ice_vc_cfg_vsi_qs_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { device_t dev = sc->dev; struct ice_hw *hw = &sc->hw; struct virtchnl_vsi_queue_config_info *vqci; struct virtchnl_queue_pair_info *vqpi; enum virtchnl_status_code status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; struct ice_tx_queue *txq; struct ice_rx_queue *rxq; u32 expected_map, max_pkt_size, queue_map, rx_buffer_size; int i, error = 0; vqci = (struct virtchnl_vsi_queue_config_info *)msg_buf; ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, malformed_queues, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { switch (RETURN_VALUE) { case 1: vqci->qpair[0].txq.dma_ring_addr |= 1; break; case 2: vqci->qpair[0].rxq.dma_ring_addr |= 1; break; case 3: vqci->qpair[0].rxq.databuffer_size++; break; case 4: vqci->qpair[0].rxq.max_pkt_size = 0; break; case 5: if (vqci->num_queue_pairs > 1) { vqci->qpair[1].txq.queue_id = vqci->qpair[0].txq.queue_id; vqci->qpair[1].rxq.queue_id = vqci->qpair[0].rxq.queue_id; } else { vqci->qpair[0].txq.queue_id++; } break; case 6: vqci->qpair[0].rxq.databuffer_size = ICE_VC_MAX_RX_BUFFER + BIT(ICE_RLAN_CTX_DBUF_S); break; default: vqci->vsi_id++; break; } }); if (vqci->vsi_id != vsi->idx || vqci->num_queue_pairs == 0 || vqci->num_queue_pairs > sizeof(queue_map) * NBBY || vqci->num_queue_pairs > vsi->num_tx_queues || vqci->num_queue_pairs > vsi->num_rx_queues) { status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } queue_map = 0; rx_buffer_size = 0; max_pkt_size = 0; vqpi = vqci->qpair; for (i = 0; i < vqci->num_queue_pairs; i++, vqpi++) { if (vqpi->txq.vsi_id != vsi->idx || vqpi->rxq.vsi_id != vsi->idx || vqpi->txq.queue_id != vqpi->rxq.queue_id || vqpi->txq.queue_id >= vsi->num_tx_queues || vqpi->rxq.queue_id >= vsi->num_rx_queues || (queue_map & BIT(vqpi->txq.queue_id)) != 0 || !ice_vc_isvalid_txq(&vqpi->txq) || !ice_vc_isvalid_rxq(&vqpi->rxq)) { status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } if (i == 0) { rx_buffer_size = vqpi->rxq.databuffer_size; max_pkt_size = vqpi->rxq.max_pkt_size; } else if (vqpi->rxq.databuffer_size != rx_buffer_size || vqpi->rxq.max_pkt_size != max_pkt_size) { status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } queue_map |= BIT(vqpi->txq.queue_id); } if (vqci->num_queue_pairs == sizeof(queue_map) * NBBY) expected_map = ~0U; else expected_map = BIT(vqci->num_queue_pairs) - 1; if (queue_map != expected_map) { status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } error = ice_vc_disable_queues(sc, vf, vf->txq_configured, vf->rxq_enabled); if (error != 0) { status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; goto done; } vf->txq_configured = 0; vf->rxq_configured = 0; vf->rxq_enabled = 0; /* * Clear TX and RX queues config in case VF * requests different number of queues. */ for (i = 0; i < vsi->num_tx_queues; i++) { txq = &vsi->tx_queues[i]; txq->desc_count = 0; txq->tx_paddr = 0; txq->q_teid = 0; txq->q_handle = 0; txq->tc = 0; } for (i = 0; i < vsi->num_rx_queues; i++) { rxq = &vsi->rx_queues[i]; rxq->desc_count = 0; rxq->rx_paddr = 0; } vqpi = vqci->qpair; for (i = 0; i < vqci->num_queue_pairs; i++, vqpi++) { /* Copy parameters into VF's queue/VSI structs */ txq = &vsi->tx_queues[vqpi->txq.queue_id]; txq->desc_count = vqpi->txq.ring_len; txq->tx_paddr = vqpi->txq.dma_ring_addr; txq->q_handle = vqpi->txq.queue_id; txq->tc = 0; rxq = &vsi->rx_queues[vqpi->rxq.queue_id]; rxq->desc_count = vqpi->rxq.ring_len; rxq->rx_paddr = vqpi->rxq.dma_ring_addr; } vsi->mbuf_sz = rx_buffer_size; vsi->max_frame_size = max_pkt_size; /* Configure TX queues in HW */ /* * Record the intended map before programming hardware so a partial * firmware failure remains discoverable and can be cleaned up by the * next configuration attempt. */ vf->txq_configured = queue_map; error = ice_cfg_vsi_for_tx(vsi); if (error) { device_printf(dev, "VF-%d: Unable to configure VSI for Tx: %s\n", vf->vf_num, ice_err_str(error)); status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; if (ice_vsi_disable_tx(vsi) == 0) vf->txq_configured = 0; goto done; } /* Configure RX queues in HW */ error = ice_cfg_vsi_for_rx(vsi); if (error) { device_printf(dev, "VF-%d: Unable to configure VSI for Rx: %s\n", vf->vf_num, ice_err_str(error)); status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; (void)ice_vc_disable_queues(sc, vf, vf->txq_configured, 0); goto done; } vf->rxq_configured = queue_map; done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_CONFIG_VSI_QUEUES, status, NULL, 0, NULL); } /** * ice_vc_cfg_rss_key_msg - Handle VIRTCHNL_OP_CONFIG_RSS_KEY msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Sets the RSS key for the given VF, using the contents of msg_buf. */ static void ice_vc_cfg_rss_key_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_aqc_get_set_rss_keys keydata = { .standard_rss_key = {0}, .extended_hash_key = {0} }; struct ice_hw *hw = &sc->hw; struct virtchnl_rss_key *vrk; int status = 0; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; vrk = (struct virtchnl_rss_key *)msg_buf; ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, malformed_rss_key, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { vrk->key_len--; }); if (vrk->vsi_id != vsi->idx) { device_printf(sc->dev, "VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", vf->vf_num, vsi->idx, vrk->vsi_id); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } /* The VF must use the exact key size advertised by this PF. */ if (vrk->key_len != ICE_GET_SET_RSS_KEY_EXTEND_KEY_SIZE) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } memcpy(&keydata, vrk->key, vrk->key_len); status = ice_aq_set_rss_key(hw, vsi->idx, &keydata); if (status) { device_printf(sc->dev, "ice_aq_set_rss_key status %s, error %s\n", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_CONFIG_RSS_KEY, v_status, NULL, 0, NULL); } /** * ice_vc_cfg_rss_lut_msg - Handle VIRTCHNL_OP_CONFIG_RSS_LUT msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Adds the LUT from the VF in msg_buf to the PF via an admin queue call. */ static void ice_vc_cfg_rss_lut_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_rss_lut *vrl; int i, status = 0; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_aq_get_set_rss_lut_params lut_params = {}; struct ice_vsi *vsi = vf->vsi; vrl = (struct virtchnl_rss_lut *)msg_buf; ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, malformed_rss_lut, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { if (RETURN_VALUE == 1) vrl->lut_entries--; else vrl->lut[0] = vsi->num_rx_queues; }); if (vrl->vsi_id != vsi->idx) { device_printf(sc->dev, "VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", vf->vf_num, vsi->idx, vrl->vsi_id); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } /* The VF must use the exact LUT size advertised by this PF. */ if (vrl->lut_entries != vsi->rss_table_size) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } for (i = 0; i < vrl->lut_entries; i++) { if (vrl->lut[i] >= vsi->num_rx_queues) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } } lut_params.vsi_handle = vsi->idx; lut_params.lut_size = vrl->lut_entries; lut_params.lut_type = vsi->rss_lut_type; lut_params.lut = vrl->lut; lut_params.global_lut_id = 0; status = ice_aq_set_rss_lut(hw, &lut_params); if (status) { device_printf(sc->dev, "VF-%d: Cannot set RSS lut, err %s aq_err %s\n", vf->vf_num, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_CONFIG_RSS_LUT, v_status, NULL, 0, NULL); } /** * ice_vc_set_rss_hena_msg - Handle VIRTCHNL_OP_SET_RSS_HENA msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Adds the VF's hena (hash enable) bits as flow types to the PF's RSS flow * type list. */ static void ice_vc_set_rss_hena_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_rss_hena *vrh; int status = 0; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; MPASS(vsi != NULL); vrh = (struct virtchnl_rss_hena *)msg_buf; /* * Remove existing configuration to make sure only requested * config is applied and allow VFs to disable RSS completly. */ status = ice_rem_vsi_rss_cfg(hw, vsi->idx); if (vrh->hena) { /* * Problem with removing config is not fatal, when new one * is requested. Warn about it but try to apply new config * anyway. */ if (status) device_printf(sc->dev, "ice_rem_vsi_rss_cfg status %s, error %s\n", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); status = ice_add_avf_rss_cfg(hw, vsi->idx, vrh->hena); if (status) device_printf(sc->dev, "ice_add_avf_rss_cfg status %s, error %s\n", ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); } v_status = ice_iov_err_to_virt_err(status); ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_SET_RSS_HENA, v_status, NULL, 0, NULL); } /** * ice_vc_enable_queues_msg - Handle VIRTCHNL_OP_ENABLE_QUEUES msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Enables VF queues selected in msg_buf for Tx/Rx traffic. * * @remark Only actually operates on Rx queues; Tx queues are enabled in * CONFIG_VSI_QUEUES message handler. */ static void ice_vc_enable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_queue_select *vqs; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; u32 queues; int bit, error; vqs = (struct virtchnl_queue_select *)msg_buf; if (!ice_vc_validate_queue_select(sc, vf, vqs)) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } if ((vqs->tx_queues & ~vf->txq_configured) != 0 || (vqs->rx_queues & ~vf->rxq_configured) != 0) { device_printf(sc->dev, "%s: VF-%d: cannot enable unconfigured queues " "(Tx 0x%08x, Rx 0x%08x)\n", __func__, vf->vf_num, vqs->tx_queues, vqs->rx_queues); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } queues = vqs->rx_queues & ~vf->rxq_enabled; while (queues != 0) { bit = ffs(queues) - 1; error = ice_control_rx_queue(vsi, bit, true); if (error) { device_printf(sc->dev, "Unable to enable VF-%d Rx queue %d: %s\n", vf->vf_num, bit, ice_err_str(error)); v_status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; goto done; } vf->rxq_enabled |= BIT(bit); queues &= ~BIT(bit); } /* Tx queues were enabled when their contexts were configured. */ done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_ENABLE_QUEUES, v_status, NULL, 0, NULL); } /** * ice_vc_disable_queues_msg - Handle VIRTCHNL_OP_DISABLE_QUEUES msg * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Disables the selected VF Tx and Rx queues. Repeated requests for queues * which are already disabled complete successfully without touching hardware. */ static void ice_vc_disable_queues_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_queue_select *vqs; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; int error; vqs = (struct virtchnl_queue_select *)msg_buf; if (!ice_vc_validate_queue_select(sc, vf, vqs)) { v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } error = ice_vc_disable_queues(sc, vf, vqs->tx_queues, vqs->rx_queues); if (error != 0) v_status = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_DISABLE_QUEUES, v_status, NULL, 0, NULL); } /** * ice_vc_cfg_irq_map_msg - Handle VIRTCHNL_OP_CFG_IRQ_MAP msg from VF * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Configures the interrupt vectors described in the message in msg_buf. The * VF needs to send this message during init, so that queues can be allowed * to generate interrupts. */ static void ice_vc_cfg_irq_map_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_irq_map_info *vimi; struct virtchnl_vector_map *vvm; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; struct ice_vsi *vsi = vf->vsi; u32 vectors_seen; u16 rxqs_seen, txqs_seen, valid_rxqs, valid_txqs, vector; vimi = (struct virtchnl_irq_map_info *)msg_buf; ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, malformed_irq_map, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { switch (RETURN_VALUE) { case 1: vimi->vecmap[0].rxitr_idx = VIRTCHNL_ITR_IDX_NO_ITR + 1; break; case 2: vimi->vecmap[0].vector_id = 0; vimi->vecmap[0].rxq_map = 1; break; case 3: if (vimi->num_vectors > 1) { vimi->vecmap[1].vector_id = vimi->vecmap[0].vector_id; } else { vimi->vecmap[0].vsi_id++; } break; default: vimi->vecmap[0].vsi_id++; break; } }); if (vimi->num_vectors == 0 || vimi->num_vectors > vf->num_irq_vectors || vimi->num_vectors > sizeof(vectors_seen) * NBBY || vsi->num_tx_queues < 1 || vsi->num_tx_queues > ICE_VIRTCHNL_QUEUE_MAP_SIZE || vsi->num_rx_queues < 1 || vsi->num_rx_queues > ICE_VIRTCHNL_QUEUE_MAP_SIZE) { device_printf(sc->dev, "%s: VF-%d: invalid vector count %d (VF has %d)\n", __func__, vf->vf_num, vimi->num_vectors, vf->num_irq_vectors); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } valid_txqs = vsi->num_tx_queues == ICE_VIRTCHNL_QUEUE_MAP_SIZE ? (u16)~0U : (u16)(BIT(vsi->num_tx_queues) - 1); valid_rxqs = vsi->num_rx_queues == ICE_VIRTCHNL_QUEUE_MAP_SIZE ? (u16)~0U : (u16)(BIT(vsi->num_rx_queues) - 1); vectors_seen = 0; txqs_seen = 0; rxqs_seen = 0; /* Validate the complete request before changing any queue state. */ vvm = vimi->vecmap; for (int i = 0; i < vimi->num_vectors; i++, vvm++) { /* vvm->vector_id is relative to VF space */ vector = vvm->vector_id; if (vvm->vsi_id != vsi->idx || vector >= vf->num_irq_vectors || vector >= sizeof(vectors_seen) * NBBY || (vectors_seen & BIT(vector)) != 0 || !ice_vc_isvalid_itr_idx(vvm->txitr_idx) || !ice_vc_isvalid_itr_idx(vvm->rxitr_idx) || (vvm->txq_map & ~valid_txqs) != 0 || (vvm->rxq_map & ~valid_rxqs) != 0 || (txqs_seen & vvm->txq_map) != 0 || (rxqs_seen & vvm->rxq_map) != 0 || (vector == 0 && (vvm->txq_map != 0 || vvm->rxq_map != 0))) { device_printf(sc->dev, "%s: VF-%d: invalid queue mapping for vector %u\n", __func__, vf->vf_num, vector); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } vectors_seen |= BIT(vector); txqs_seen |= vvm->txq_map; rxqs_seen |= vvm->rxq_map; } /* Save the validated queue-to-vector mappings. */ vvm = vimi->vecmap; for (int i = 0; i < vimi->num_vectors; i++, vvm++) { struct ice_tx_queue *txq; struct ice_rx_queue *rxq; int bit; vector = vvm->vector_id; /* The Misc/Admin Queue vector doesn't need mapping */ if (vector == 0) continue; for (bit = 0; bit < ICE_VIRTCHNL_QUEUE_MAP_SIZE; bit++) { if ((vvm->txq_map & BIT(bit)) == 0) continue; vf->tx_irqvs[vector].me = vector; txq = &vsi->tx_queues[bit]; txq->irqv = &vf->tx_irqvs[vector]; txq->itr_idx = vvm->txitr_idx; } for (bit = 0; bit < ICE_VIRTCHNL_QUEUE_MAP_SIZE; bit++) { if ((vvm->rxq_map & BIT(bit)) == 0) continue; vf->rx_irqvs[vector].me = vector; rxq = &vsi->rx_queues[bit]; rxq->irqv = &vf->rx_irqvs[vector]; rxq->itr_idx = vvm->rxitr_idx; } } /* Write to T/RQCTL registers to actually map vectors to queues */ for (int i = 0; i < vf->vsi->num_rx_queues; i++) if (vsi->rx_queues[i].irqv != NULL) ice_configure_rxq_interrupt(hw, vsi->rx_qmap[i], vsi->rx_queues[i].irqv->me, vsi->rx_queues[i].itr_idx); for (int i = 0; i < vf->vsi->num_tx_queues; i++) if (vsi->tx_queues[i].irqv != NULL) ice_configure_txq_interrupt(hw, vsi->tx_qmap[i], vsi->tx_queues[i].irqv->me, vsi->tx_queues[i].itr_idx); ice_flush(hw); done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_CONFIG_IRQ_MAP, v_status, NULL, 0, NULL); } /** * ice_eth_stats_to_virtchnl_eth_stats - Convert stats for virtchnl * @istats: VSI stats from HW to convert * @vstats: stats struct to copy to * * This function copies all known stats in struct virtchnl_eth_stats from the * input struct ice_eth_stats to an output struct virtchnl_eth_stats. * * @remark These two structure types currently have the same definition up to * the size of struct virtchnl_eth_stats (on FreeBSD), but that could change * in the future. */ static void ice_eth_stats_to_virtchnl_eth_stats(struct ice_eth_stats *istats, struct virtchnl_eth_stats *vstats) { vstats->rx_bytes = istats->rx_bytes; vstats->rx_unicast = istats->rx_unicast; vstats->rx_multicast = istats->rx_multicast; vstats->rx_broadcast = istats->rx_broadcast; vstats->rx_discards = istats->rx_discards; vstats->rx_unknown_protocol = istats->rx_unknown_protocol; vstats->tx_bytes = istats->tx_bytes; vstats->tx_unicast = istats->tx_unicast; vstats->tx_multicast = istats->tx_multicast; vstats->tx_broadcast = istats->tx_broadcast; vstats->tx_discards = istats->tx_discards; vstats->tx_errors = istats->tx_errors; } /** * ice_vc_get_stats_msg - Handle VIRTCHNL_OP_GET_STATS msg * @sc: device private structure * @vf: VF tracking structure * @msg_buf: raw message buffer from the VF * * Updates the VF's VSI stats and sends those stats back to the VF. */ static void ice_vc_get_stats_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct virtchnl_queue_select *vqs; struct virtchnl_eth_stats stats; struct ice_vsi *vsi = vf->vsi; struct ice_hw *hw = &sc->hw; vqs = (struct virtchnl_queue_select *)msg_buf; ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, get_stats_bad_vsi, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { vqs->vsi_id = vsi->idx + 1; device_printf(sc->dev, "injecting invalid GET_STATS VSI ID for VF %u\n", (unsigned int)vf->vf_num); }); if (vqs->vsi_id != vsi->idx) { device_printf(sc->dev, "%s: VF-%d: message has invalid VSI ID %d (VF has VSI ID %d)\n", __func__, vf->vf_num, vqs->vsi_id, vsi->idx); ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_GET_STATS, VIRTCHNL_STATUS_ERR_PARAM, NULL, 0, NULL); return; } ice_update_vsi_hw_stats(vf->vsi); ice_eth_stats_to_virtchnl_eth_stats(&vsi->hw_stats.cur, &stats); ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_GET_STATS, VIRTCHNL_STATUS_SUCCESS, (u8 *)&stats, sizeof(struct virtchnl_eth_stats), NULL); } /** * ice_vc_cfg_promisc_mode_msg - Handle VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE * @sc: PF's softc structure * @vf: VF tracking structure * @msg_buf: message buffer from VF * * Configures the promiscuous modes for the given VSI in msg_buf. */ static void ice_vc_cfg_promisc_mode_msg(struct ice_softc *sc, struct ice_vf *vf, u8 *msg_buf) { struct ice_hw *hw = &sc->hw; struct virtchnl_promisc_info *vpi; enum virtchnl_status_code v_status = VIRTCHNL_STATUS_SUCCESS; int status = 0; struct ice_vsi *vsi = vf->vsi; ice_declare_bitmap(old_promisc_mask, ICE_PROMISC_MAX); ice_declare_bitmap(req_promisc_mask, ICE_PROMISC_MAX); ice_declare_bitmap(clear_promisc_mask, ICE_PROMISC_MAX); ice_declare_bitmap(set_promisc_mask, ICE_PROMISC_MAX); ice_declare_bitmap(old_req_xor_mask, ICE_PROMISC_MAX); u16 vid; vpi = (struct virtchnl_promisc_info *)msg_buf; /* Check to see if VF has permission to configure promiscuous mode */ if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { device_printf(sc->dev, "VF-%d: attempted to configure promiscuous mode\n", vf->vf_num); /* Don't reply to VF with an error */ goto done; } if (vpi->vsi_id != vsi->idx) { device_printf(sc->dev, "VF-%d: Message has invalid VSI ID (expected %d, got %d)\n", vf->vf_num, vsi->idx, vpi->vsi_id); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } if (vpi->flags & ~ICE_VIRTCHNL_VALID_PROMISC_FLAGS) { device_printf(sc->dev, "VF-%d: Message has invalid promiscuous flags set (valid 0x%02x, got 0x%02x)\n", vf->vf_num, ICE_VIRTCHNL_VALID_PROMISC_FLAGS, vpi->flags); v_status = VIRTCHNL_STATUS_ERR_PARAM; goto done; } ice_zero_bitmap(req_promisc_mask, ICE_PROMISC_MAX); /* Convert virtchnl flags to ice AQ promiscuous mode flags */ if (vpi->flags & FLAG_VF_UNICAST_PROMISC) { ice_set_bit(ICE_PROMISC_UCAST_TX, req_promisc_mask); ice_set_bit(ICE_PROMISC_UCAST_RX, req_promisc_mask); } if (vpi->flags & FLAG_VF_MULTICAST_PROMISC) { ice_set_bit(ICE_PROMISC_MCAST_TX, req_promisc_mask); ice_set_bit(ICE_PROMISC_MCAST_RX, req_promisc_mask); } status = ice_get_vsi_promisc(hw, vsi->idx, old_promisc_mask, &vid); if (status) { device_printf(sc->dev, "VF-%d: Failed to get promiscuous mode mask for VSI %d, err %s aq_err %s\n", vf->vf_num, vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } /* Figure out what got added and what got removed */ ice_zero_bitmap(old_req_xor_mask, ICE_PROMISC_MAX); ice_xor_bitmap(old_req_xor_mask, old_promisc_mask, req_promisc_mask, ICE_PROMISC_MAX); ice_and_bitmap(clear_promisc_mask, old_req_xor_mask, old_promisc_mask, ICE_PROMISC_MAX); ice_and_bitmap(set_promisc_mask, old_req_xor_mask, req_promisc_mask, ICE_PROMISC_MAX); if (ice_is_any_bit_set(clear_promisc_mask, ICE_PROMISC_MAX)) { status = ice_clear_vsi_promisc(hw, vsi->idx, clear_promisc_mask, 0); if (status) { device_printf(sc->dev, "VF-%d: Failed to clear promiscuous mode for VSI %d, err %s aq_err %s\n", vf->vf_num, vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } } if (ice_is_any_bit_set(set_promisc_mask, ICE_PROMISC_MAX)) { status = ice_set_vsi_promisc(hw, vsi->idx, set_promisc_mask, 0); if (status) { device_printf(sc->dev, "VF-%d: Failed to set promiscuous mode for VSI %d, err %s aq_err %s\n", vf->vf_num, vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status)); v_status = ice_iov_err_to_virt_err(status); goto done; } } done: ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, v_status, NULL, 0, NULL); } /** * ice_vc_notify_all_vfs_link_state - Notify all VFs of PF link state * @sc: device private structure * * Sends a message to all VFs about the status of the PF's link * state. For more details, @see ice_vc_notify_vf_link_state. */ void ice_vc_notify_all_vfs_link_state(struct ice_softc *sc) { for (int i = 0; i < sc->num_vfs; i++) ice_vc_notify_vf_link_state(sc, &sc->vfs[i]); } /** * ice_vc_notify_vf_link_state - Notify VF of PF link state * @sc: device private structure * @vf: VF tracking structure * * Sends an event message to the specified VF with information about * the current link state from the PF's port. This includes whether * link is up or down, and the link speed in 100Mbps units. */ static void ice_vc_notify_vf_link_state(struct ice_softc *sc, struct ice_vf *vf) { struct virtchnl_pf_event event = {}; struct ice_hw *hw = &sc->hw; event.event = VIRTCHNL_EVENT_LINK_CHANGE; event.severity = PF_EVENT_SEVERITY_INFO; event.event_data.link_event_adv.link_status = sc->link_up; event.event_data.link_event_adv.link_speed = (u32)ice_conv_link_speed_to_virtchnl(true, hw->port_info->phy.link_info.link_speed); ice_aq_send_msg_to_vf(hw, vf->vf_num, VIRTCHNL_OP_EVENT, VIRTCHNL_STATUS_SUCCESS, (u8 *)&event, sizeof(event), NULL); } /** * ice_iov_mbx_overflow - Detect and isolate a VF flooding the PF mailbox * @sc: device private structure * @vf: VF which sent the current message * @mbx_data: software mailbox snapshot data, or NULL on E830 * * E830 enforces the per-VF watermark in hardware. On older devices, reset * and block a VF after the Intel snapshot detector first attributes an * overflow. A later external VF reset, PF reset, or SR-IOV recreation * releases it. * * @returns true if the current message must be discarded. */ static bool ice_iov_mbx_overflow(struct ice_softc *sc, struct ice_vf *vf, struct ice_mbx_data *mbx_data) { struct ice_hw *hw = &sc->hw; bool report_malvf; u32 reg, vf_flags; int error, status; if (mbx_data == NULL) return (false); /* Every message advances the snapshot, including a blocked VF's. */ report_malvf = false; status = ice_mbx_vf_state_handler(hw, mbx_data, &vf->mbx_info, &report_malvf); if ((atomic_load_acq_32(&vf->vf_flags) & VF_FLAG_MBX_BLOCKED) != 0) return (true); ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov, mailbox_overflow, ice_iov_fail_vf_matches(vf->vf_num), FAIL_POINT_NONSLEEPABLE, { status = 0; vf->mbx_info.malicious = 1; report_malvf = true; }); if (status != 0) { device_printf(sc->dev, "Unable to check VF %u mailbox overflow, err %s\n", vf->vf_num, ice_status_str(status)); return (false); } if (!report_malvf) return (vf->mbx_info.malicious != 0); vf->mbx_overflow_events++; atomic_set_32(&vf->vf_flags, VF_FLAG_MBX_BLOCKED); device_printf(sc->dev, "VF %u exceeded the mailbox message limit; resetting and blocking it\n", vf->vf_num); vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) != 0 && vf->vsi != NULL) { error = ice_reset_vf(sc, vf, true, false); if (error != 0) { device_printf(sc->dev, "Unable to isolate VF %u after mailbox overflow: %s\n", vf->vf_num, ice_err_str(error)); } else { /* * Leave queues and mailbox requests blocked, but complete VFR * so a later physical FLR can create a new reset edge and * recover the VF. */ ice_iov_complete_vf_reset(sc, vf, false); } } else { /* An incompletely configured VF has no queues to drain. */ reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_num)); reg |= VPGEN_VFRTRIG_VFSWR_M; wr32(hw, VPGEN_VFRTRIG(vf->vf_num), reg); ice_flush(hw); } return (true); } /** * ice_vc_handle_vf_msg - Handle a message from a VF * @sc: device private structure * @event: event received from the HW MBX queue * @mbx_data: software overflow-detection data, or NULL on E830 * * Called whenever an event is received from a VF on the HW mailbox queue. * Responsible for handling these messages as well as responding to the * VF afterwards, depending on the received message type. */ void ice_vc_handle_vf_msg(struct ice_softc *sc, struct ice_rq_event_info *event, struct ice_mbx_data *mbx_data) { struct ice_hw *hw = &sc->hw; device_t dev = sc->dev; struct ice_vf *vf; int err = 0; u32 vf_flags; u32 v_opcode = event->desc.cookie_high; u16 v_id = event->desc.retval; u8 *msg = event->msg_buf; u16 msglen = event->msg_len; if (v_id >= sc->num_vfs) { device_printf(dev, "%s: Received msg from invalid VF-%d: opcode %d, len %d\n", __func__, v_id, v_opcode, msglen); return; } vf = &sc->vfs[v_id]; if (ice_iov_mbx_overflow(sc, vf, mbx_data)) return; /* Perform basic checks on the msg */ err = virtchnl_vc_validate_vf_msg(&vf->version, v_opcode, msg, msglen); if (err) { device_printf(dev, "%s: Received invalid msg from VF-%d: opcode %d, len %d, error %d\n", __func__, vf->vf_num, v_opcode, msglen, err); ice_aq_send_msg_to_vf(hw, v_id, v_opcode, VIRTCHNL_STATUS_ERR_PARAM, NULL, 0, NULL); return; } vf_flags = atomic_load_acq_32(&vf->vf_flags); if ((vf_flags & VF_FLAG_ENABLED) == 0 || vf->vsi == NULL) return; /* Only a reset outside this dispatcher may release an isolated VF. */ if ((vf_flags & (VF_FLAG_MDD_BLOCKED | VF_FLAG_MBX_BLOCKED)) != 0) return; /* * Permit only reset negotiation while VF hardware state is unsafe. * A VFR can retry RESET_FAILED; REBUILD_REQUIRED needs a PF rebuild. */ if ((vf_flags & (VF_FLAG_REBUILD_REQUIRED | VF_FLAG_RESET_FAILED)) != 0 && v_opcode != VIRTCHNL_OP_VERSION && v_opcode != VIRTCHNL_OP_RESET_VF) { ice_aq_send_msg_to_vf(hw, v_id, v_opcode, VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR, NULL, 0, NULL); return; } switch (v_opcode) { case VIRTCHNL_OP_VERSION: ice_vc_version_msg(sc, vf, msg); break; case VIRTCHNL_OP_RESET_VF: ice_reset_vf(sc, vf, true, true); break; case VIRTCHNL_OP_GET_VF_RESOURCES: ice_vc_get_vf_res_msg(sc, vf, msg); break; case VIRTCHNL_OP_ADD_ETH_ADDR: ice_vc_add_eth_addr_msg(sc, vf, msg); break; case VIRTCHNL_OP_DEL_ETH_ADDR: ice_vc_del_eth_addr_msg(sc, vf, msg); break; case VIRTCHNL_OP_ADD_VLAN: ice_vc_add_vlan_msg(sc, vf, msg); break; case VIRTCHNL_OP_DEL_VLAN: ice_vc_del_vlan_msg(sc, vf, msg); break; case VIRTCHNL_OP_CONFIG_VSI_QUEUES: ice_vc_cfg_vsi_qs_msg(sc, vf, msg); break; case VIRTCHNL_OP_CONFIG_RSS_KEY: ice_vc_cfg_rss_key_msg(sc, vf, msg); break; case VIRTCHNL_OP_CONFIG_RSS_LUT: ice_vc_cfg_rss_lut_msg(sc, vf, msg); break; case VIRTCHNL_OP_SET_RSS_HENA: ice_vc_set_rss_hena_msg(sc, vf, msg); break; case VIRTCHNL_OP_ENABLE_QUEUES: ice_vc_enable_queues_msg(sc, vf, msg); ice_vc_notify_vf_link_state(sc, vf); break; case VIRTCHNL_OP_DISABLE_QUEUES: ice_vc_disable_queues_msg(sc, vf, msg); break; case VIRTCHNL_OP_CONFIG_IRQ_MAP: ice_vc_cfg_irq_map_msg(sc, vf, msg); break; case VIRTCHNL_OP_GET_STATS: ice_vc_get_stats_msg(sc, vf, msg); break; case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: ice_vc_cfg_promisc_mode_msg(sc, vf, msg); break; default: device_printf(dev, "%s: Received unknown msg from VF-%d: opcode %d, len %d\n", __func__, vf->vf_num, v_opcode, msglen); ice_aq_send_msg_to_vf(hw, v_id, v_opcode, VIRTCHNL_STATUS_ERR_NOT_SUPPORTED, NULL, 0, NULL); break; } } /** * ice_iov_setup_intr_mapping - Setup interrupt config for a VF * @sc: device softc structure * @vf: driver's VF structure for VF to be configured * * Before a VF can be used, and after a VF reset, the PF must configure * the VF's interrupt allocation registers. This includes allocating * interrupts from the PF's interrupt pool to the VF using the * VPINT_ALLOC(_PCI) registers, and setting up a mapping from PF vectors * to VF vectors in GLINT_VECT2FUNC. * * As well, this sets up queue allocation registers and maps the mailbox * interrupt for the VF. */ static void ice_iov_setup_intr_mapping(struct ice_softc *sc, struct ice_vf *vf) { struct ice_hw *hw = &sc->hw; struct ice_vsi *vsi = vf->vsi; u16 v; /* Calculate indices for register ops below */ u16 vf_first_irq_idx = vf->vf_imap[0]; u16 vf_last_irq_idx = (vf_first_irq_idx + vf->num_irq_vectors) - 1; u16 abs_vf_first_irq_idx = hw->func_caps.common_cap.msix_vector_first_id + vf_first_irq_idx; u16 abs_vf_last_irq_idx = (abs_vf_first_irq_idx + vf->num_irq_vectors) - 1; u16 abs_vf_num = vf->vf_num + hw->func_caps.vf_base_id; /* Map out VF interrupt allocation in global device space. Both * VPINT_ALLOC and VPINT_ALLOC_PCI use the same values. */ wr32(hw, VPINT_ALLOC(vf->vf_num), (((abs_vf_first_irq_idx << VPINT_ALLOC_FIRST_S) & VPINT_ALLOC_FIRST_M) | ((abs_vf_last_irq_idx << VPINT_ALLOC_LAST_S) & VPINT_ALLOC_LAST_M) | VPINT_ALLOC_VALID_M)); wr32(hw, VPINT_ALLOC_PCI(vf->vf_num), (((abs_vf_first_irq_idx << VPINT_ALLOC_PCI_FIRST_S) & VPINT_ALLOC_PCI_FIRST_M) | ((abs_vf_last_irq_idx << VPINT_ALLOC_PCI_LAST_S) & VPINT_ALLOC_PCI_LAST_M) | VPINT_ALLOC_PCI_VALID_M)); /* Create inverse mapping of vectors to PF/VF combinations */ for (v = vf_first_irq_idx; v <= vf_last_irq_idx; v++) { wr32(hw, GLINT_VECT2FUNC(v), (((abs_vf_num << GLINT_VECT2FUNC_VF_NUM_S) & GLINT_VECT2FUNC_VF_NUM_M) | ((hw->pf_id << GLINT_VECT2FUNC_PF_NUM_S) & GLINT_VECT2FUNC_PF_NUM_M))); } /* Map mailbox interrupt to MSI-X index 0. Disable ITR for it, too. */ wr32(hw, VPINT_MBX_CTL(abs_vf_num), ((0 << VPINT_MBX_CTL_MSIX_INDX_S) & VPINT_MBX_CTL_MSIX_INDX_M) | ((0x3 << VPINT_MBX_CTL_ITR_INDX_S) & VPINT_MBX_CTL_ITR_INDX_M) | VPINT_MBX_CTL_CAUSE_ENA_M); /* Mark the TX queue mapping registers as valid */ wr32(hw, VPLAN_TXQ_MAPENA(vf->vf_num), VPLAN_TXQ_MAPENA_TX_ENA_M); /* Indicate to HW that VF has scattered queue allocation */ wr32(hw, VPLAN_TX_QBASE(vf->vf_num), VPLAN_TX_QBASE_VFQTABLE_ENA_M); for (int i = 0; i < vsi->num_tx_queues; i++) { wr32(hw, VPLAN_TX_QTABLE(i, vf->vf_num), (vsi->tx_qmap[i] << VPLAN_TX_QTABLE_QINDEX_S) & VPLAN_TX_QTABLE_QINDEX_M); } /* Mark the RX queue mapping registers as valid */ wr32(hw, VPLAN_RXQ_MAPENA(vf->vf_num), VPLAN_RXQ_MAPENA_RX_ENA_M); wr32(hw, VPLAN_RX_QBASE(vf->vf_num), VPLAN_RX_QBASE_VFQTABLE_ENA_M); for (int i = 0; i < vsi->num_rx_queues; i++) { wr32(hw, VPLAN_RX_QTABLE(i, vf->vf_num), (vsi->rx_qmap[i] << VPLAN_RX_QTABLE_QINDEX_S) & VPLAN_RX_QTABLE_QINDEX_M); } } /** * ice_err_to_virt err - translate ice errors into virtchnl errors * @ice_err: status returned from ice function */ static enum virtchnl_status_code ice_iov_err_to_virt_err(int ice_err) { switch (ice_err) { case 0: return VIRTCHNL_STATUS_SUCCESS; case ICE_ERR_BAD_PTR: case ICE_ERR_INVAL_SIZE: case ICE_ERR_DEVICE_NOT_SUPPORTED: case ICE_ERR_PARAM: case ICE_ERR_CFG: return VIRTCHNL_STATUS_ERR_PARAM; case ICE_ERR_NO_MEMORY: return VIRTCHNL_STATUS_ERR_NO_MEMORY; case ICE_ERR_NOT_READY: case ICE_ERR_RESET_FAILED: case ICE_ERR_FW_API_VER: case ICE_ERR_AQ_ERROR: case ICE_ERR_AQ_TIMEOUT: case ICE_ERR_AQ_FULL: case ICE_ERR_AQ_NO_WORK: case ICE_ERR_AQ_EMPTY: return VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; default: return VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; } }