/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2001-2024, Intel Corporation * Copyright (c) 2026 Kevin Bowling * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "if_em.h" #include #define IGBV_82576_QUEUES 2 #define IGBV_I350_QUEUES 1 #define IGBV_MAX_MAC_FILTERS 3 #define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10 #define IGBV_QUEUE_DISABLE_DELAY_US 10 #define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US) #define IGBV_QUEUE_DISABLE_RETRIES 20 #define IGBV_QUEUE_SANITIZE_ATTEMPTS 3 #define IGBV_VLAN_RETRY_BATCH 4 #define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S) /* Reading these six configuration bytes performs the Hyper-V VF handshake. */ #define IGBV_HV_RESET_OFFSET 0x201 static const struct timeval igbv_queue_log_interval = { 2, 0 }; static const struct timeval igbv_mbx_log_interval = { 60, 0 }; static const sbintime_t igbv_queue_retry_delay[] = { 100 * SBT_1MS, 500 * SBT_1MS, }; static const sbintime_t igbv_mbx_retry_delay[] = { 250 * SBT_1MS, 1 * SBT_1S, 4 * SBT_1S, 8 * SBT_1S, }; _Static_assert(nitems(igbv_queue_retry_delay) + 1 == IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay"); struct igb_vf_uc_addr_list { struct e1000_softc *sc; u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN]; }; static bool igbv_tx_pending(struct e1000_softc *); static bool igbv_vlan_retry_pending(const struct e1000_softc *); static void igbv_vlan_retry_tick(struct e1000_softc *); static s32 igbv_hv_read_mac_addr(struct e1000_hw *hw) { if (!em_is_valid_ether_addr(hw->mac.perm_addr)) return (-E1000_ERR_MAC_INIT); memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); return (E1000_SUCCESS); } static s32 igbv_hv_reset_hw(struct e1000_hw *hw) { struct e1000_osdep *osdep; u8 addr[ETHER_ADDR_LEN]; u32 ctrl; int i; osdep = hw->back; hw->mac.get_link_status = true; memset(hw->mac.perm_addr, 0, ETHER_ADDR_LEN); /* Hyper-V does not service the native VF posted-message protocol. */ hw->mbx.timeout = 0; ctrl = E1000_READ_REG(hw, E1000_CTRL); if (ctrl == 0xffffffff) return (-E1000_ERR_RESET); /* Reset VF-local state before the Hyper-V host reset/MAC exchange. */ E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST); E1000_WRITE_FLUSH(hw); for (i = 0; i < E1000_VF_INIT_TIMEOUT; i++) { if (hw->mbx.ops.check_for_rst(hw, 0) != E1000_SUCCESS) break; DELAY(5); } /* * The PF handles 0x201..0x206 as a reset/MAC exchange, including * partial reads. Do not use these bytes for routine status polling. * The host-assigned address also identifies the matching hn interface. */ for (i = 0; i < ETHER_ADDR_LEN; i++) addr[i] = pci_read_config(osdep->dev, IGBV_HV_RESET_OFFSET + i, 1); if (!em_is_valid_ether_addr(addr)) return (-E1000_ERR_MAC_INIT); memcpy(hw->mac.perm_addr, addr, ETHER_ADDR_LEN); return (E1000_SUCCESS); } static s32 igbv_hv_check_for_link(struct e1000_hw *hw) { u32 status; /* Leave reset indications for the statistics counter-epoch check. */ status = E1000_READ_REG(hw, E1000_STATUS); hw->mac.get_link_status = true; if (status == 0xffffffff) return (-E1000_ERR_MAC_INIT); /* Poll even after link-up so a host port link-down cannot stay cached. */ hw->mac.get_link_status = (status & E1000_STATUS_LU) == 0; return (E1000_SUCCESS); } static int igbv_hv_rar_set(struct e1000_hw *hw, u8 *addr, u32 index) { int error; error = index == 0 && memcmp(addr, hw->mac.perm_addr, ETHER_ADDR_LEN) == 0 ? E1000_SUCCESS : -E1000_ERR_MAC_INIT; if (igbv_hv_read_mac_addr(hw) != E1000_SUCCESS) return (-E1000_ERR_MAC_INIT); return (error); } static void igbv_hv_update_mc_addr_list(struct e1000_hw *hw __unused, u8 *addrs __unused, u32 count __unused) { /* The host owns receive-filter policy for the synthetic/VF pair. */ } void igbv_init_hv_ops(struct e1000_hw *hw) { /* Install after the ordinary VF MAC and mailbox parameter setup. */ hw->mac.ops.reset_hw = igbv_hv_reset_hw; /* Native init_hw calls rar_set_vf directly, bypassing the RAR op. */ hw->mac.ops.init_hw = igbv_hv_read_mac_addr; hw->mac.ops.read_mac_addr = igbv_hv_read_mac_addr; hw->mac.ops.rar_set = igbv_hv_rar_set; hw->mac.ops.check_for_link = igbv_hv_check_for_link; hw->mac.ops.update_mc_addr_list = igbv_hv_update_mc_addr_list; } static void igbv_queue_retry_callout(void *arg) { struct e1000_softc *sc; if_t ifp; sc = arg; if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0) return; ifp = iflib_get_ifp(sc->ctx); if ((if_getflags(ifp) & IFF_UP) == 0) { atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); return; } iflib_request_reset_if_up(sc->ctx); iflib_admin_intr_deferred(sc->ctx); } /* * A missing PF can make the posted reset handshake wait for a full mailbox * timeout. Keep that work out of stopped status paths. An administratively * up VF retries complete initialization with an exponential delay capped at * eight seconds, so it recovers without creating a tight mailbox poller. */ static void igbv_mbx_retry_callout(void *arg) { struct e1000_softc *sc; if_t ifp; sc = arg; if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 || atomic_load_acq_32(&sc->vf_mbx_ready) != 0 || iflib_in_detach(sc->ctx)) return; ifp = iflib_get_ifp(sc->ctx); if ((if_getflags(ifp) & IFF_UP) == 0) return; iflib_request_reset_if_up(sc->ctx); iflib_admin_intr_deferred(sc->ctx); } static const char * igbv_reset_error_desc(s32 error) { switch (error) { case -E1000_ERR_RESET: return ("PF reset acknowledgement timed out"); case -E1000_ERR_MAC_INIT: return ("PF returned an invalid VF reset response"); case -E1000_ERR_MBX: return ("PF mailbox reset exchange failed"); default: return ("VF reset handshake failed"); } } void igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching) { /* Report each backoff stage, then limit the steady eight-second retry. */ if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 && !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval)) return; device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error), error, attaching ? "; continuing attach" : ""); } void igbv_mbx_retry_detach(struct e1000_softc *sc) { if (!sc->vf_mbx_retry_initialized) return; atomic_readandclear_32(&sc->vf_mbx_retry_pending); callout_drain(&sc->vf_mbx_retry); sc->vf_mbx_retry_initialized = false; } void igbv_mbx_retry_prepare(struct e1000_softc *sc) { if (!sc->vf_mbx_retry_initialized) return; atomic_readandclear_32(&sc->vf_mbx_retry_pending); callout_drain(&sc->vf_mbx_retry); } void igbv_mbx_retry_stop(struct e1000_softc *sc) { if_t ifp; if (!sc->vf_mbx_retry_initialized) return; atomic_readandclear_32(&sc->vf_mbx_retry_pending); callout_drain(&sc->vf_mbx_retry); ifp = iflib_get_ifp(sc->ctx); if ((if_getflags(ifp) & IFF_UP) == 0) sc->vf_mbx_retry_stage = 0; } void igbv_mbx_retry_failed(if_ctx_t ctx) { struct e1000_softc *sc; if_t ifp; sbintime_t delay; u_int stage; sc = iflib_get_softc(ctx); atomic_store_rel_32(&sc->vf_mbx_ready, 0); sc->link_speed = 0; sc->link_duplex = 0; if (sc->link_state != EM_LINK_STATE_DOWN) { sc->link_state = EM_LINK_STATE_DOWN; iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); } iflib_init_failed(ctx); ifp = iflib_get_ifp(ctx); if (!sc->vf_mbx_retry_initialized || (if_getflags(ifp) & IFF_UP) == 0) return; stage = sc->vf_mbx_retry_stage; if (stage >= nitems(igbv_mbx_retry_delay)) stage = nitems(igbv_mbx_retry_delay) - 1; delay = igbv_mbx_retry_delay[stage]; if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay)) sc->vf_mbx_retry_stage++; atomic_set_32(&sc->vf_mbx_retry_pending, 1); callout_reset_sbt(&sc->vf_mbx_retry, delay, 0, igbv_mbx_retry_callout, sc, C_PREL(1)); } static void igbv_mbx_retry_succeeded(struct e1000_softc *sc) { atomic_store_rel_32(&sc->vf_mbx_ready, 1); atomic_readandclear_32(&sc->vf_mbx_retry_pending); if (sc->vf_mbx_retry_initialized) callout_stop(&sc->vf_mbx_retry); sc->vf_mbx_retry_stage = 0; sc->vf_last_mbx_log.tv_sec = 0; sc->vf_last_mbx_log.tv_usec = 0; } void igbv_queue_retry_detach(struct e1000_softc *sc) { if (!sc->vf_queue_retry_initialized) return; atomic_readandclear_32(&sc->vf_queue_retry_pending); callout_drain(&sc->vf_queue_retry); sc->vf_queue_retry_initialized = false; } void igbv_queue_retry_stop(struct e1000_softc *sc) { if (!sc->vf_queue_retry_initialized) return; if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0) atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); callout_stop(&sc->vf_queue_retry); } void igbv_queue_retry_prepare(struct e1000_softc *sc) { bool new_epoch; new_epoch = atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0; if (!sc->vf_queue_gave_up && !new_epoch) return; sc->vf_queue_failures = 0; sc->vf_queue_gave_up = false; } static void igbv_queue_retry_succeeded(struct e1000_softc *sc) { atomic_readandclear_32(&sc->vf_queue_retry_pending); atomic_readandclear_32(&sc->vf_queue_retry_new_epoch); if (sc->vf_queue_retry_initialized) callout_stop(&sc->vf_queue_retry); sc->vf_queue_failures = 0; sc->vf_queue_gave_up = false; } void igbv_queue_retry_failed(if_ctx_t ctx) { struct e1000_softc *sc; sbintime_t delay; sc = iflib_get_softc(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) sc->vf_queue_failures++; if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) { delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1]; atomic_set_32(&sc->vf_queue_retry_pending, 1); callout_reset_sbt(&sc->vf_queue_retry, delay, 0, igbv_queue_retry_callout, sc, C_PREL(1)); } else if (!sc->vf_queue_gave_up) { atomic_readandclear_32(&sc->vf_queue_retry_pending); callout_stop(&sc->vf_queue_retry); sc->vf_queue_gave_up = true; device_printf(sc->dev, "retained VF queues remained active after %u attempts; " "interface left down; toggle it down/up to retry\n", sc->vf_queue_failures); } iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); iflib_init_failed(ctx); } void igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid) { bool pending; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); pending = igbv_vlan_retry_pending(sc); sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); /* Bound the whole batch from its first failure, not each new VID. */ if (!pending) sc->vf_vlan_retry_deadline = getsbinuptime() + IGBV_VLAN_RETRY_WINDOW; } void igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid) { KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f)); } static bool igbv_vlan_retry_pending(const struct e1000_softc *sc) { int i; for (i = 0; i < EM_VFTA_SIZE; i++) if (sc->vf_vfta_retry[i] != 0) return (true); return (false); } static void igbv_vlan_retry_tick(struct e1000_softc *sc) { u32 bit; u16 vid; int attempts, i, remaining; if (!igbv_vlan_retry_pending(sc)) { sc->vf_vlan_retry_deadline = 0; return; } if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { remaining = 0; for (i = 0; i < EM_VFTA_SIZE; i++) remaining += bitcount32(sc->vf_vfta_retry[i]); memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); sc->vf_vlan_retry_deadline = 0; device_printf(sc->dev, "VF VLAN restore retries exhausted for %d VIDs\n", remaining); return; } /* * The mailbox NACK does not distinguish a transient PF rate limit * from permanent VLVF exhaustion. Retry at the PF's sustained * allowance, but bound the entire recovery window so ENOSPC cannot * create a permanent mailbox poller. */ for (attempts = 0, i = 0; attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) { vid = sc->vf_vlan_retry_cursor; sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; bit = 1U << (vid & 0x1f); if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) continue; attempts++; if ((sc->shadow_vfta[vid >> 5] & bit) == 0 || e1000_vfta_set_vf(&sc->hw, vid, true) == E1000_SUCCESS) sc->vf_vfta_retry[vid >> 5] &= ~bit; } if (!igbv_vlan_retry_pending(sc)) sc->vf_vlan_retry_deadline = 0; } int igbv_if_attach_pre(if_ctx_t ctx) { struct e1000_softc *sc; device_t dev; int error; dev = iflib_get_dev(ctx); if (pci_msix_count(dev) < 2) { device_printf(dev, "VF operation requires two MSI-X vectors\n"); return (ENXIO); } error = em_if_attach_pre(ctx); if (error != 0) return (error); sc = iflib_get_softc(ctx); callout_init(&sc->vf_queue_retry, 1); sc->vf_queue_retry_initialized = true; callout_init(&sc->vf_mbx_retry, 1); sc->vf_mbx_retry_initialized = true; KASSERT(sc->vf_ifp && (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0, ("%s: igbv attached without VF policy", __func__)); return (0); } int igbv_if_attach_post(if_ctx_t ctx) { struct e1000_softc *sc; int error; sc = iflib_get_softc(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (sc->intr_type != IFLIB_INTR_MSIX) { device_printf(sc->dev, "VF operation requires MSI-X\n"); return (ENXIO); } error = em_if_attach_post(ctx); if (error != 0) return (error); /* * Attach failures can leave the device sysctl tree registered when * hw.bus.disable_failed_devices is set. Do not publish handlers with * softc arguments until iflib has successfully allocated MSI-X. */ em_add_device_sysctls(sc); return (0); } int igbv_if_media_change(if_ctx_t ctx __unused) { return (EOPNOTSUPP); } void igbv_if_update_admin_status(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; device_t dev; bool link_check, timer_tick; sc = iflib_get_softc(ctx); hw = &sc->hw; dev = iflib_get_dev(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (!iflib_is_running(ctx) || !sc->vf_queues_sanitized || atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { if (sc->link_state != EM_LINK_STATE_DOWN) { sc->link_speed = 0; sc->link_duplex = 0; sc->link_state = EM_LINK_STATE_DOWN; iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); } return; } if (igbv_is_hyperv(sc)) { u32 status, txdctl; if (sc->vf_reset_pending) return; txdctl = E1000_READ_REG(hw, E1000_TXDCTL(0)); if (txdctl != 0xffffffff && (txdctl & E1000_TXDCTL_QUEUE_ENABLE) == 0) { /* Carrier can remain up after the PF disables queue DMA. */ sc->vf_stats_valid = false; sc->link_speed = sc->link_duplex = 0; if (sc->link_state != EM_LINK_STATE_DOWN) { sc->link_state = EM_LINK_STATE_DOWN; iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); } /* * Do not reset while carrier is down. Limit requests * if the host keeps the queue disabled with carrier up. */ status = E1000_READ_REG(hw, E1000_STATUS); if (status != 0xffffffff && (status & E1000_STATUS_LU) != 0 && ratecheck(&sc->vf_last_queue_log, &igbv_queue_log_interval)) { device_printf(dev, "Hyper-V VF queue disabled; requesting recovery\n"); sc->vf_reset_pending = true; iflib_request_reset_if_up(ctx); iflib_admin_intr_deferred(ctx); } return; } } if (!sc->vf_reset_pending && atomic_readandclear_32(&sc->promisc_pending) != 0) (void)em_if_set_promisc_impl(ctx, if_getflags(iflib_get_ifp(ctx))); if (e1000_check_for_link(hw) != E1000_SUCCESS && !sc->vf_reset_pending) { sc->vf_reset_pending = true; iflib_request_reset(ctx); iflib_admin_intr_deferred(ctx); } link_check = !hw->mac.get_link_status; if (link_check && (sc->link_state == EM_LINK_STATE_DOWN || sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { e1000_get_speed_and_duplex(hw, &sc->link_speed, &sc->link_duplex); if (bootverbose) device_printf(dev, "Link is up %d Mbps %s\n", sc->link_speed, sc->link_duplex == FULL_DUPLEX ? "Full Duplex" : "Half Duplex"); sc->link_state = EM_LINK_STATE_UP; iflib_link_state_change(ctx, LINK_STATE_UP, IF_Mbps(sc->link_speed)); } else if (!link_check && (sc->link_state == EM_LINK_STATE_UP || sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { sc->link_speed = 0; sc->link_duplex = 0; sc->link_state = EM_LINK_STATE_DOWN; iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); } /* * A VF stops transmit DMA when its PF reports link down. Reset if * descriptors remain queued so they cannot be sent stale when carrier * returns, matching the periodic check in Linux igbvf. */ if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) { sc->vf_reset_pending = true; iflib_request_reset(ctx); iflib_admin_intr_deferred(ctx); } /* em_if_init() establishes a new counter baseline after the reset. */ timer_tick = !sc->vf_reset_pending && atomic_readandclear_32(&sc->stats_pending) != 0; if (timer_tick) { em_update_stats_counters(sc); /* iflib closes admission before stop; do not replay after reset. */ if (iflib_is_running(ctx)) igbv_vlan_retry_tick(sc); } } static bool igbv_tx_pending(struct e1000_softc *sc) { struct tx_ring *txr; u32 head, tail; for (int i = 0; i < sc->tx_num_queues; i++) { txr = &sc->tx_queues[i].txr; head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)); tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me)); if (head != tail) return (true); } return (false); } bool igbv_sanitize_queues(struct e1000_softc *sc) { struct e1000_hw *hw; u32 rxdctl, txdctl; int i, nqueues, retry; hw = &sc->hw; switch (hw->mac.type) { case e1000_vfadapt: nqueues = IGBV_82576_QUEUES; break; case e1000_vfadapt_i350: nqueues = IGBV_I350_QUEUES; break; default: return (true); } /* * The 82576 and I350 specification updates, Software Clarification 3, * note that VFLR leaves this queue configuration intact. Clear it * before programming the new rings so igbv does not depend on its PF * to sanitize state left by a previous VF owner. igbv uses only queue * zero, but must also clear the unused second 82576 queue. * * Disable every queue first and wait for outstanding DMA activity to * stop before programming TDWBAL/H. Spin only for the normal fast * transition, then sleep until the bounded deadline. */ for (i = 0; i < nqueues; i++) { E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0); E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0); } E1000_WRITE_FLUSH(hw); for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) { for (i = 0; i < nqueues; i++) { rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 || (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0) break; } if (i == nqueues) break; if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) { if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES) DELAY(IGBV_QUEUE_DISABLE_DELAY_US); else pause_sbt("igbvqds", IGBV_QUEUE_DISABLE_PAUSE, 0, C_PREL(1)); } } if (retry == IGBV_QUEUE_DISABLE_RETRIES) { if (ratecheck(&sc->vf_last_queue_log, &igbv_queue_log_interval)) device_printf(sc->dev, "could not disable retained VF queues; " "reset deferred\n"); return (false); } for (i = 0; i < nqueues; i++) { E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0); E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0); E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0); E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0); E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0); } E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0); E1000_WRITE_FLUSH(hw); return (true); } bool igbv_reset(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; s32 error; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); /* * Receive-buffer allocation and flow control are port resources owned * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size. */ sc->pba = 0; hw->fc = (struct e1000_fc_info){ .current_mode = e1000_fc_none, .requested_mode = e1000_fc_none, }; error = e1000_reset_hw(hw); atomic_store_rel_32(&sc->vf_mbx_ready, 0); sc->vf_queues_sanitized = igbv_sanitize_queues(sc); if (!sc->vf_queues_sanitized) { return (false); } igbv_queue_retry_succeeded(sc); if (error != E1000_SUCCESS) { igbv_log_reset_failure(sc, error, false); return (false); } memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale)); memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); sc->vf_vlan_retry_deadline = 0; sc->vf_vlan_retry_cursor = 0; if (e1000_init_hw(hw) < 0) { device_printf(sc->dev, "Hardware Initialization Failed\n"); return (false); } e1000_check_for_link(hw); igbv_mbx_retry_succeeded(sc); return (true); } void igbv_initialize_transmit_unit(if_ctx_t ctx) { KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, ("%s called for a PF", __func__)); em_initialize_transmit_rings(ctx); } void igbv_initialize_receive_unit(if_ctx_t ctx) { KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, ("%s called for a PF", __func__)); igb_initialize_receive_rings(ctx, true); } void igbv_if_intr_enable(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; u32 mask; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (!sc->vf_queues_sanitized || atomic_load_acq_32(&sc->vf_mbx_ready) == 0) return; mask = sc->que_mask | sc->link_mask; E1000_WRITE_REG(hw, E1000_EIAC, mask); E1000_WRITE_REG(hw, E1000_EIAM, mask); E1000_WRITE_REG(hw, E1000_EIMS, mask); E1000_WRITE_FLUSH(hw); } void igbv_if_intr_disable(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff); E1000_WRITE_REG(hw, E1000_EIAC, 0); E1000_WRITE_FLUSH(hw); } int igbv_get_regs(SYSCTL_HANDLER_ARGS) { struct e1000_softc *sc; struct e1000_hw *hw; struct sbuf *sb; int error; sc = (struct e1000_softc *)arg1; hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); if (sb == NULL) return (ENOMEM); /* * Limited VF register set: * Don't read EICR here because it is clear-on-read. The VF register * file exposes its queue pair at index zero, so this diagnostic does * not depend on the narrower lifetime of iflib's queue arrays. */ sbuf_printf(sb, "VF Registers\n"); sbuf_printf(sb, "\tVTCTRL\t %08x\n", E1000_READ_REG(hw, E1000_CTRL)); sbuf_printf(sb, "\tSTATUS\t %08x\n", E1000_READ_REG(hw, E1000_STATUS)); sbuf_printf(sb, "\tRDLEN\t %08x\n", E1000_READ_REG(hw, E1000_RDLEN(0))); sbuf_printf(sb, "\tRDH\t %08x\n", E1000_READ_REG(hw, E1000_RDH(0))); sbuf_printf(sb, "\tRDT\t %08x\n", E1000_READ_REG(hw, E1000_RDT(0))); sbuf_printf(sb, "\tTDLEN\t %08x\n", E1000_READ_REG(hw, E1000_TDLEN(0))); sbuf_printf(sb, "\tTDH\t %08x\n", E1000_READ_REG(hw, E1000_TDH(0))); sbuf_printf(sb, "\tTDT\t %08x\n", E1000_READ_REG(hw, E1000_TDT(0))); error = sbuf_finish(sb); sbuf_delete(sb); return (error); } static u_int igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx) { struct igb_vf_uc_addr_list *list; const u8 *addr; list = arg; addr = (const u8 *)LLADDR(sdl); if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) return (0); if (idx < IGBV_MAX_MAC_FILTERS) memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN); return (1); } void igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp) { struct igb_vf_uc_addr_list list = { .sc = sc, }; u_int count; if (igbv_is_hyperv(sc)) return; count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list); if (count > IGBV_MAX_MAC_FILTERS) { device_printf(sc->dev, "too many secondary unicast addresses; maximum is %u\n", IGBV_MAX_MAC_FILTERS); } if (count == 0 && !sc->vf_uc_filters_set) return; /* * Linux igb PFs validate the address field before dispatching the CLR * subcommand. Supply the primary address rather than the zero payload * used by igbvf so those PFs actually remove the old filters. FreeBSD * PFs dispatch CLR before inspecting the otherwise-ignored address. */ if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR, sc->hw.mac.addr) != E1000_SUCCESS) { device_printf(sc->dev, "VF secondary unicast filter clear request failed\n"); return; } sc->vf_uc_filters_set = false; if (count > IGBV_MAX_MAC_FILTERS) return; for (u_int i = 0; i < count; i++) { if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD, list.addrs[i]) != E1000_SUCCESS) { device_printf(sc->dev, "VF secondary unicast filter add request failed " "for %6D\n", list.addrs[i], ":"); } else sc->vf_uc_filters_set = true; usec_delay(200); } } void igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp) { u8 *lladdr; if (!em_is_valid_ether_addr(sc->hw.mac.addr)) return; lladdr = (u8 *)if_getlladdr(ifp); if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) return; device_printf(sc->dev, "PF rejected or replaced the requested MAC; using %6D\n", sc->hw.mac.addr, ":"); /* * if_setlladdr() would re-enter the driver's address-change path. * Initialization already holds the context lock, so update the * storage directly and issue the notification it would have sent. */ memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN); CURVNET_SET_QUIET(if_getvnet(ifp)); EVENTHANDLER_INVOKE(iflladdr_event, ifp); CURVNET_RESTORE(); }