aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--UPDATING13
-rw-r--r--share/man/man4/Makefile2
-rw-r--r--share/man/man4/em.4212
-rw-r--r--sys/conf/files4
-rw-r--r--sys/dev/e1000/e1000_82575.h3
-rw-r--r--sys/dev/e1000/e1000_hw.h3
-rw-r--r--sys/dev/e1000/e1000_mbx.c107
-rw-r--r--sys/dev/e1000/e1000_mbx.h5
-rw-r--r--sys/dev/e1000/e1000_osdep.h114
-rw-r--r--sys/dev/e1000/e1000_regs.h1
-rw-r--r--sys/dev/e1000/e1000_vf.c107
-rw-r--r--sys/dev/e1000/e1000_vf.h6
-rw-r--r--sys/dev/e1000/em_txrx.c5
-rw-r--r--sys/dev/e1000/if_em.c1585
-rw-r--r--sys/dev/e1000/if_em.h106
-rw-r--r--sys/dev/e1000/if_igb_iov.c2268
-rw-r--r--sys/dev/e1000/if_igb_iov.h61
-rw-r--r--sys/dev/e1000/if_igbv.c973
-rw-r--r--sys/dev/e1000/igb_txrx.c55
-rw-r--r--sys/modules/em/Makefile7
20 files changed, 5191 insertions, 446 deletions
diff --git a/UPDATING b/UPDATING
index 9116e396bb8f..777efc49b8c4 100644
--- a/UPDATING
+++ b/UPDATING
@@ -12,6 +12,19 @@ Items affecting the ports and packages system can be found in
/usr/ports/UPDATING. Please read that file before updating system packages
and/or ports.
+20261031:
+ The Intel 82576 and I350 Virtual Function devices now use the igbv(4)
+ driver. VF interfaces previously named igbN are now named igbvN.
+ Before booting the new kernel, update interface references in rc.conf,
+ firewall rules, and other network configuration, and change VF device
+ tunables from dev.igb.N to dev.igbv.N. Systems with both PFs and VFs
+ should also check PF unit numbers after the driver split.
+
+ The drivers remain in if_em.ko; existing if_em_load and if_igb_load
+ settings still load them. if_igbv_load is also supported.
+ The igb(4) PF driver now supports SR-IOV on these controllers; see
+ igb(4) for the PCI_IOV, MSI-X, and single-queue requirements.
+
20261029:
struct if_rxd_info now carries a receive timestamp. iflib drivers
must be rebuilt. __FreeBSD_version has been bumped to 1501506.
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 66c4d6c8537f..a9d56d7496ae 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -716,6 +716,8 @@ MLINKS+=edsc.4 if_edsc.4
MLINKS+=em.4 if_em.4 \
em.4 igb.4 \
em.4 if_igb.4 \
+ em.4 igbv.4 \
+ em.4 if_igbv.4 \
em.4 lem.4 \
em.4 if_lem.4
MLINKS+=ena.4 if_ena.4
diff --git a/share/man/man4/em.4 b/share/man/man4/em.4
index 96d68eb4fcca..133f0d3bbac8 100644
--- a/share/man/man4/em.4
+++ b/share/man/man4/em.4
@@ -32,13 +32,14 @@
.\"
.\" * Other names and brands may be claimed as the property of others.
.\"
-.Dd August 30, 2026
+.Dd September 16, 2026
.Dt EM 4
.Os
.Sh NAME
.Nm em ,
.Nm lem ,
-.Nm igb
+.Nm igb ,
+.Nm igbv
.Nd "Intel(R) PRO/1000 Gigabit Ethernet adapter driver"
.Sh SYNOPSIS
To compile this driver into the kernel,
@@ -49,6 +50,11 @@ kernel configuration file:
.Cd "device em"
.Ed
.Pp
+PF SR-IOV support additionally requires:
+.Bd -ragged -offset indent
+.Cd "options PCI_IOV"
+.Ed
+.Pp
Alternatively, to load the driver as a
module at boot time, place the following line in
.Xr loader.conf 5 :
@@ -78,12 +84,43 @@ The
.Nm
driver provides support for PCI Express Gigabit Ethernet adapters
based on the Intel 82575, 82576, 82580, i210, i211, and i35x.
-These appear as
+Physical functions appear as
.Nm igb
interfaces to maintain compatibility with existing infrastructure.
+Virtual functions provided by 82576 and I350 physical functions appear as
+.Nm igbv
+interfaces.
.Pp
-The driver supports Transmit/Receive checksum offload and Jumbo Frames
-on all but 82542-based adapters.
+On Hyper-V,
+.Nm igbv
+supports 82576 and I350 virtual functions using the host-assigned MAC address
+and a single transmit/receive queue pair.
+The VF can be used by
+.Xr hn 4
+as the accelerated data path for its synthetic interface.
+The host controls MAC, multicast, promiscuous-mode, and VLAN-filter policy;
+changing these on the VF does not reconfigure the host.
+Hyper-V VFs currently support an MTU of at most 1500 bytes.
+.Pp
+Only host-assigned access VLANs with untagged guest traffic are supported
+on Hyper-V; guest VLAN registration is ignored.
+With Windows I350 PF driver version 14.1.5.0, tagged VF transmissions can
+trigger host malicious driver detection and
+disable the VF's queues even when the Hyper-V port is configured for trunk mode.
+This limitation is also reproducible with Windows VF drivers.
+For guest VLAN trunks, disable SR-IOV for the virtual network adapter on
+the host and use the
+.Xr hn 4
+synthetic data path.
+.Pp
+If the host unexpectedly disables a running Hyper-V VF transmit queue,
+the driver reports link down.
+If the hardware reports carrier up, the driver requests recovery through
+the normal stop/init path.
+These recovery requests are rate-limited if the host keeps the queue disabled.
+.Pp
+The driver supports Transmit/Receive checksum offload.
+Jumbo Frames are supported except on 82542-based adapters and Hyper-V VFs.
.Pp
Physical functions on supported boards and ports advertise one or more of the
.Cm wol_magic ,
@@ -96,7 +133,9 @@ The available modes depend on the controller and board.
The NVM APME setting normally selects whether magic-packet wake is enabled by
default without hiding modes that the hardware and board support.
Wake support can be restricted to particular ports on multi-port adapters and
-is not available on virtual functions.
+is not available on
+.Nm igbv
+virtual functions.
Waking the system from an ACPI sleep state also requires enabling the
corresponding device wake source when it is exposed, for example:
.Bd -literal -offset indent
@@ -186,6 +225,161 @@ is not specified, the driver defaults to
.Pp
For more information on configuring this device, see
.Xr ifconfig 8 .
+.Sh SR-IOV
+When the kernel is built with
+.Cd PCI_IOV ,
+the
+.Nm igb
+PF driver supports SR-IOV on 82576 and I350 controllers.
+Up to seven VFs can be created per PF.
+PF SR-IOV is not featured on 82575, 82580, I210, I211, or I354 controllers.
+.Pp
+Passing a VF through to a guest requires an IOMMU to isolate its DMA.
+The packet filters described below do not constrain descriptor DMA;
+.Xr bhyve 8
+refuses to assign a passthrough device when it cannot create an IOMMU domain.
+.Pp
+The supported SR-IOV configuration uses one PF transmit and receive queue and
+requires MSI-X.
+I350 requires this layout; 82576 hardware can combine VMDq with per-pool RSS,
+but that mode is not implemented by this driver.
+The PF may be administratively down when VFs are created and remains down
+afterward.
+VF mailbox and datapath service begin when the PF is brought up, and the PF
+must remain up to provide those services.
+Creating or destroying VFs temporarily stops and restarts a running PF.
+Set the following per-device
+.Xr iflib 4
+loader tunables before the PF attaches:
+.Bd -literal -offset indent
+dev.igb.N.iflib.override_ntxqs=1
+dev.igb.N.iflib.override_nrxqs=1
+.Ed
+.Pp
+The following VF parameters are accepted in
+.Xr iovctl.conf 5 :
+.Bl -tag -width "mac-anti-spoof"
+.It Va mac-addr
+Assign a unicast MAC address to the VF.
+If omitted, the PF uses the stable address generator described by
+.Xr ether_gen_addr 9
+with a distinct name for each VF.
+.It Va mac-anti-spoof
+Enable source MAC anti-spoofing.
+The default is true.
+The check binds transmissions to MAC addresses currently assigned to that VF.
+It prevents impersonating the PF or a sibling VF, but a VF permitted to change
+its MAC can select an address belonging to an off-host station.
+.It Va allow-set-mac
+Permit the VF to replace its primary MAC address, whether assigned by
+.Va mac-addr
+or generated by the PF, and to install up to three auxiliary unicast
+addresses.
+Each auxiliary address expands both the VF's receive filters and the source
+addresses permitted by transmit anti-spoofing.
+The default is false.
+.It Va allow-promisc
+Permit explicit VF mailbox requests for unicast or multicast promiscuous mode
+within its VLAN membership.
+The default is false.
+VF promiscuous-mode changes are applied asynchronously.
+If the PF refuses a request, the VF logs the failure after
+.Xr ifconfig 8
+has returned.
+.It Va vlan
+Assign the VF an access VLAN from 1 through 4095.
+The PF inserts this tag on VF transmissions, admits only this VLAN to the VF,
+and strips the tag on reception so the VF operates on untagged traffic.
+VF mailbox requests to change VLAN membership are rejected while an access
+VLAN is assigned.
+The default value,
+.Dq trunk ,
+instead permits the VF to manage its VLAN memberships through the mailbox.
+VLAN 0 cannot be assigned as an access VLAN.
+.El
+.Pp
+I350 hides an assigned access VLAN in hardware.
+On 82576, transparent delivery depends on the VF driver ignoring stripped
+VLAN metadata for VLANs it did not request; the in-tree
+.Nm igbv
+VF driver does so.
+DPDK guests and
+.Fx
+guests with an older
+.Nm igb
+driver do not apply this workaround and can expose the assigned VLAN tag
+instead.
+.Pp
+The VF mailbox carries at most 30 multicast hashes.
+If the VF reports more, the PF automatically enables reception of all
+multicast within that VF's VLAN membership.
+This overflow fallback is independent of
+.Va allow-promisc .
+The multicast hash table is shared by the port, so hash collisions can also
+deliver matching multicast traffic to another multicast-enabled pool in the
+same VLAN.
+.Pp
+A VF with
+.Va allow-set-mac
+enabled can install at most three auxiliary unicast receive addresses.
+These filters come from a controller-wide pool; exhaustion makes later
+requests fail without replacing existing filters.
+.Pp
+The hardware provides 32 VLAN-to-pool filter entries for VLANs assigned to
+VFs, including PF-assigned access VLANs, and rejects a request that would
+exceed 32 distinct VF VLANs.
+A trunk VF's implicit VID-0 membership consumes one shared entry, leaving
+31 entries for nonzero VF VLANs while any trunk VF is active.
+.Pp
+Each VF can burst 64 VLAN-add requests and then receives eight additional
+additions per second.
+This permits two complete 31-VLAN replays while bounding repeated global
+filter-table rebuilds.
+VLAN removals and requests that do not change membership are not limited.
+A PF-wide reinitialization replenishes the burst; VF-initiated resets do not.
+The
+.Nm igbv
+VF driver retries failed VLAN additions at the PF's sustained rate for up to
+eight seconds.
+This heals transient rate-limit failures without polling indefinitely when
+the shared VLAN table is full.
+.Pp
+On 82576, a PF-only VLAN uses the default PF pool without consuming one of
+these entries; an entry shared by the PF and VFs includes all member pools.
+I350 can additionally place the PF in VLAN-promiscuous mode when its PF
+memberships do not fit.
+Because the entries are shared, VF allocations can force this PF fallback.
+DMA coalescing is disabled while I350 SR-IOV is active.
+.Pp
+The PF enables malicious-driver detection.
+Hardware stops a VF transmit queue that submits a malformed descriptor, such as
+a legacy descriptor in a virtualized mode, or a descriptor requesting VLAN
+insertion while an access VLAN is assigned.
+The PF logs
+.Ql malicious-driver event ,
+clears the offending VF's queue enables and PF-owned filter state, and sends a
+control message directing the guest to complete a new reset handshake.
+Recovery is unconditional; there is no tunable to disable it.
+A VF that submits further malformed descriptors is stopped and recovered again.
+.Pp
+Transmissions failing the anti-spoof checks are dropped by hardware without
+stopping the queue or resetting the VF.
+The PF logs
+.Ql spoof event detected
+for an offending VF at most once every two seconds.
+.Pp
+If a VF retains mailbox ownership while the PF is reading a request, the PF
+retries with a bounded backoff, then logs
+.Ql mailbox remained busy
+and revokes that VF's clear-to-send state.
+Requests from that VF are ignored until a hardware reset begins a new mailbox
+epoch.
+A guest that resets through the VF control register or through a function level
+reset recovers automatically; a guest that only sends the reset mailbox message
+does not.
+.Pp
+PF and VF statistics are sampled once per interface watchdog interval, so
+exported counters can trail the hardware by up to 500 milliseconds.
.Sh HARDWARE
The
.Nm
@@ -350,13 +544,17 @@ issue to
.Sh SEE ALSO
.Xr altq 4 ,
.Xr arp 4 ,
+.Xr hn 4 ,
.Xr iflib 4 ,
.Xr led 4 ,
.Xr netintro 4 ,
.Xr ng_ether 4 ,
.Xr polling 4 ,
.Xr vlan 4 ,
-.Xr ifconfig 8
+.Xr iovctl.conf 5 ,
+.Xr ifconfig 8 ,
+.Xr iovctl 8 ,
+.Xr ether_gen_addr 9
.Sh HISTORY
The
.Nm
diff --git a/sys/conf/files b/sys/conf/files
index 72c6439d9bba..8551f999a2a7 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1633,6 +1633,10 @@ dev/efidev/efirt.c optional efirt
dev/efidev/efirtc.c optional efirt efirtc
dev/e1000/if_em.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
+dev/e1000/if_igbv.c optional em \
+ compile-with "${NORMAL_C} -I$S/dev/e1000"
+dev/e1000/if_igb_iov.c optional em \
+ compile-with "${NORMAL_C} -I$S/dev/e1000"
dev/e1000/em_txrx.c optional em \
compile-with "${NORMAL_C} -I$S/dev/e1000"
dev/e1000/igb_txrx.c optional em \
diff --git a/sys/dev/e1000/e1000_82575.h b/sys/dev/e1000/e1000_82575.h
index 69c195cc199d..430eed4202e3 100644
--- a/sys/dev/e1000/e1000_82575.h
+++ b/sys/dev/e1000/e1000_82575.h
@@ -403,9 +403,10 @@ enum e1000_promisc_type {
e1000_num_promisc_types
};
-void e1000_vfta_set_vf(struct e1000_hw *, u16, bool);
+s32 e1000_vfta_set_vf(struct e1000_hw *, u16, bool);
void e1000_rlpml_set_vf(struct e1000_hw *, u16);
s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type type);
+s32 e1000_set_uc_addr_vf(struct e1000_hw *, u32, u8 *);
void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value);
u16 e1000_rxpbs_adjust_82580(u32 data);
s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data);
diff --git a/sys/dev/e1000/e1000_hw.h b/sys/dev/e1000/e1000_hw.h
index 43f41869a6f5..980aaedf845b 100644
--- a/sys/dev/e1000/e1000_hw.h
+++ b/sys/dev/e1000/e1000_hw.h
@@ -944,13 +944,14 @@ struct e1000_fc_info {
struct e1000_mbx_operations {
s32 (*init_params)(struct e1000_hw *hw);
- s32 (*read)(struct e1000_hw *, u32 *, u16, u16);
+ s32 (*read)(struct e1000_hw *, u32 *, u16, u16, bool);
s32 (*write)(struct e1000_hw *, u32 *, u16, u16);
s32 (*read_posted)(struct e1000_hw *, u32 *, u16, u16);
s32 (*write_posted)(struct e1000_hw *, u32 *, u16, u16);
s32 (*check_for_msg)(struct e1000_hw *, u16);
s32 (*check_for_ack)(struct e1000_hw *, u16);
s32 (*check_for_rst)(struct e1000_hw *, u16);
+ s32 (*unlock)(struct e1000_hw *, u16);
};
struct e1000_mbx_stats {
diff --git a/sys/dev/e1000/e1000_mbx.c b/sys/dev/e1000/e1000_mbx.c
index 97097fd777a1..985d315d527e 100644
--- a/sys/dev/e1000/e1000_mbx.c
+++ b/sys/dev/e1000/e1000_mbx.c
@@ -64,6 +64,17 @@ static s32 e1000_null_mbx_transact(struct e1000_hw E1000_UNUSEDARG *hw,
return E1000_SUCCESS;
}
+static s32 e1000_null_mbx_read(struct e1000_hw E1000_UNUSEDARG *hw,
+ u32 E1000_UNUSEDARG *msg,
+ u16 E1000_UNUSEDARG size,
+ u16 E1000_UNUSEDARG mbx_id,
+ bool E1000_UNUSEDARG unlock)
+{
+ DEBUGFUNC("e1000_null_mbx_read");
+
+ return E1000_SUCCESS;
+}
+
/**
* e1000_read_mbx - Reads a message from the mailbox
* @hw: pointer to the HW structure
@@ -73,7 +84,8 @@ static s32 e1000_null_mbx_transact(struct e1000_hw E1000_UNUSEDARG *hw,
*
* returns SUCCESS if it successfully read message from buffer
**/
-s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id)
+s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id,
+ bool unlock)
{
struct e1000_mbx_info *mbx = &hw->mbx;
s32 ret_val = -E1000_ERR_MBX;
@@ -85,7 +97,7 @@ s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id)
size = mbx->size;
if (mbx->ops.read)
- ret_val = mbx->ops.read(hw, msg, size, mbx_id);
+ ret_val = mbx->ops.read(hw, msg, size, mbx_id, unlock);
return ret_val;
}
@@ -176,6 +188,24 @@ s32 e1000_check_for_rst(struct e1000_hw *hw, u16 mbx_id)
}
/**
+ * e1000_unlock_mbx - release mailbox ownership
+ * @hw: pointer to the HW structure
+ * @mbx_id: id of mailbox to unlock
+ **/
+s32 e1000_unlock_mbx(struct e1000_hw *hw, u16 mbx_id)
+{
+ struct e1000_mbx_info *mbx = &hw->mbx;
+ s32 ret_val = -E1000_ERR_MBX;
+
+ DEBUGFUNC("e1000_unlock_mbx");
+
+ if (mbx->ops.unlock)
+ ret_val = mbx->ops.unlock(hw, mbx_id);
+
+ return (ret_val);
+}
+
+/**
* e1000_poll_for_msg - Wait for message notification
* @hw: pointer to the HW structure
* @mbx_id: id of mailbox to write
@@ -261,7 +291,7 @@ s32 e1000_read_posted_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id)
/* if ack received read message, otherwise we timed out */
if (!ret_val)
- ret_val = mbx->ops.read(hw, msg, size, mbx_id);
+ ret_val = mbx->ops.read(hw, msg, size, mbx_id, true);
out:
return ret_val;
}
@@ -307,11 +337,12 @@ void e1000_init_mbx_ops_generic(struct e1000_hw *hw)
{
struct e1000_mbx_info *mbx = &hw->mbx;
mbx->ops.init_params = e1000_null_ops_generic;
- mbx->ops.read = e1000_null_mbx_transact;
+ mbx->ops.read = e1000_null_mbx_read;
mbx->ops.write = e1000_null_mbx_transact;
mbx->ops.check_for_msg = e1000_null_mbx_check_for_flag;
mbx->ops.check_for_ack = e1000_null_mbx_check_for_flag;
mbx->ops.check_for_rst = e1000_null_mbx_check_for_flag;
+ mbx->ops.unlock = e1000_null_mbx_check_for_flag;
mbx->ops.read_posted = e1000_read_posted_mbx;
mbx->ops.write_posted = e1000_write_posted_mbx;
}
@@ -500,7 +531,8 @@ out_no_write:
* returns SUCCESS if it successfully read message from buffer
**/
static s32 e1000_read_mbx_vf(struct e1000_hw *hw, u32 *msg, u16 size,
- u16 E1000_UNUSEDARG mbx_id)
+ u16 E1000_UNUSEDARG mbx_id,
+ bool E1000_UNUSEDARG unlock)
{
s32 ret_val = E1000_SUCCESS;
u16 i;
@@ -649,28 +681,37 @@ static s32 e1000_obtain_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number)
{
s32 ret_val = -E1000_ERR_MBX;
u32 p2v_mailbox;
- int count = 10;
DEBUGFUNC("e1000_obtain_mbx_lock_pf");
- do {
- /* Take ownership of the buffer */
- E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number),
- E1000_P2VMAILBOX_PFU);
-
- /* reserve mailbox for pf use */
- p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number));
- if (p2v_mailbox & E1000_P2VMAILBOX_PFU) {
- ret_val = E1000_SUCCESS;
- break;
- }
- usec_delay(1000);
- } while (count-- > 0);
+ /*
+ * A VF request releases VFU as it raises REQ. If the VF still owns
+ * the buffer, leave the request for a later admin pass rather than
+ * sleeping under the PF's context lock and delaying every other VF.
+ */
+ E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number),
+ E1000_P2VMAILBOX_PFU);
+ p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number));
+ if (p2v_mailbox & E1000_P2VMAILBOX_PFU)
+ ret_val = E1000_SUCCESS;
return ret_val;
}
+static s32
+e1000_release_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number)
+{
+ u32 p2v_mailbox;
+
+ p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number));
+ if (p2v_mailbox & E1000_P2VMAILBOX_PFU)
+ E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number),
+ p2v_mailbox & ~E1000_P2VMAILBOX_PFU);
+
+ return (E1000_SUCCESS);
+}
+
/**
* e1000_write_mbx_pf - Places a message in the mailbox
* @hw: pointer to the HW structure
@@ -693,6 +734,17 @@ static s32 e1000_write_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size,
if (ret_val)
goto out_no_write;
+ /*
+ * A VF request wins over an asynchronous PF message. Do not clear
+ * VFREQ here: the PF mailbox handler still needs to consume it.
+ */
+ if (E1000_READ_REG(hw, E1000_MBVFICR) &
+ (E1000_MBVFICR_VFREQ_VF1 << vf_number)) {
+ e1000_release_mbx_lock_pf(hw, vf_number);
+ ret_val = -E1000_ERR_MBX;
+ goto out_no_write;
+ }
+
/* flush msg and acks as we are overwriting the message buffer */
e1000_check_for_msg_pf(hw, vf_number);
e1000_check_for_ack_pf(hw, vf_number);
@@ -724,7 +776,7 @@ out_no_write:
* a message due to a VF request so no polling for message is needed.
**/
static s32 e1000_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size,
- u16 vf_number)
+ u16 vf_number, bool unlock)
{
s32 ret_val;
u16 i;
@@ -736,12 +788,21 @@ static s32 e1000_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size,
if (ret_val)
goto out_no_read;
+ /*
+ * A second VF request can arrive while the PF retries ownership. PFU
+ * now keeps VMBMEM stable, so consume any reasserted VFREQ before
+ * reading the request that it describes.
+ */
+ (void)e1000_check_for_msg_pf(hw, vf_number);
+
/* copy the message to the mailbox memory buffer */
for (i = 0; i < size; i++)
msg[i] = E1000_READ_REG_ARRAY(hw, E1000_VMBMEM(vf_number), i);
- /* Acknowledge the message and release buffer */
- E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_ACK);
+ /* Acknowledge the message and optionally retain PF ownership. */
+ E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number),
+ E1000_P2VMAILBOX_ACK |
+ (unlock ? 0 : E1000_P2VMAILBOX_PFU));
/* update stats */
hw->mbx.stats.msgs_rx++;
@@ -776,6 +837,7 @@ s32 e1000_init_mbx_params_pf(struct e1000_hw *hw)
mbx->ops.check_for_msg = e1000_check_for_msg_pf;
mbx->ops.check_for_ack = e1000_check_for_ack_pf;
mbx->ops.check_for_rst = e1000_check_for_rst_pf;
+ mbx->ops.unlock = e1000_release_mbx_lock_pf;
mbx->stats.msgs_tx = 0;
mbx->stats.msgs_rx = 0;
@@ -787,4 +849,3 @@ s32 e1000_init_mbx_params_pf(struct e1000_hw *hw)
return E1000_SUCCESS;
}
}
-
diff --git a/sys/dev/e1000/e1000_mbx.h b/sys/dev/e1000/e1000_mbx.h
index 61f8d5cbe265..edd74a49f00b 100644
--- a/sys/dev/e1000/e1000_mbx.h
+++ b/sys/dev/e1000/e1000_mbx.h
@@ -77,6 +77,8 @@
#define E1000_VF_RESET 0x01 /* VF requests reset */
#define E1000_VF_SET_MAC_ADDR 0x02 /* VF requests to set MAC addr */
+#define E1000_VF_MAC_FILTER_CLR (0x01 << E1000_VT_MSGINFO_SHIFT)
+#define E1000_VF_MAC_FILTER_ADD (0x02 << E1000_VT_MSGINFO_SHIFT)
#define E1000_VF_SET_MULTICAST 0x03 /* VF requests to set MC addr */
#define E1000_VF_SET_MULTICAST_COUNT_MASK (0x1F << E1000_VT_MSGINFO_SHIFT)
#define E1000_VF_SET_MULTICAST_OVERFLOW (0x80 << E1000_VT_MSGINFO_SHIFT)
@@ -92,13 +94,14 @@
#define E1000_VF_MBX_INIT_TIMEOUT 2000 /* number of retries on mailbox */
#define E1000_VF_MBX_INIT_DELAY 500 /* microseconds between retries */
-s32 e1000_read_mbx(struct e1000_hw *, u32 *, u16, u16);
+s32 e1000_read_mbx(struct e1000_hw *, u32 *, u16, u16, bool);
s32 e1000_write_mbx(struct e1000_hw *, u32 *, u16, u16);
s32 e1000_read_posted_mbx(struct e1000_hw *, u32 *, u16, u16);
s32 e1000_write_posted_mbx(struct e1000_hw *, u32 *, u16, u16);
s32 e1000_check_for_msg(struct e1000_hw *, u16);
s32 e1000_check_for_ack(struct e1000_hw *, u16);
s32 e1000_check_for_rst(struct e1000_hw *, u16);
+s32 e1000_unlock_mbx(struct e1000_hw *, u16);
void e1000_init_mbx_ops_generic(struct e1000_hw *hw);
s32 e1000_init_mbx_params_vf(struct e1000_hw *);
s32 e1000_init_mbx_params_pf(struct e1000_hw *);
diff --git a/sys/dev/e1000/e1000_osdep.h b/sys/dev/e1000/e1000_osdep.h
index bac71d34de45..b3b16b102edb 100644
--- a/sys/dev/e1000/e1000_osdep.h
+++ b/sys/dev/e1000/e1000_osdep.h
@@ -162,10 +162,113 @@ struct e1000_osdep
device_t dev;
if_ctx_t ctx;
bool pcim2pci_arbiter_wa;
+ bool vf;
+ bool vf_82576;
};
void e1000_pcim2pci_write(struct e1000_osdep *, uint32_t, uint32_t);
+#ifdef INVARIANTS
+/*
+ * 82576 and I350 VFs expose a sparse register file. Keep this list local to
+ * the OS accessors so a PF-only register that leaks into the shared VF path
+ * fails at its first access instead of returning reserved-register garbage.
+ *
+ * The driver intentionally uses only queue pair zero. It also clears the
+ * retained configuration of the unused second 82576 queue pair after VFLR.
+ * Expanding VF data-path queue support must extend this predicate from the
+ * applicable device CSR map.
+ */
+static __inline bool
+e1000_vf_reg_valid(uint32_t reg, bool write, bool vf_82576)
+{
+ /* VF mailbox memory: 16 dwords beginning at 0x800. */
+ if (reg >= 0x00800 && reg <= 0x0083c && (reg & 3) == 0)
+ return (true);
+
+ /* The three VF MSI-X interrupt-throttling registers. */
+ if (reg >= 0x01680 && reg <= 0x01688 && (reg & 3) == 0)
+ return (true);
+
+ /* Receive queue zero. */
+ switch (reg) {
+ case 0x02800: /* RDBAL */
+ case 0x02804: /* RDBAH */
+ case 0x02808: /* RDLEN */
+ case 0x0280c: /* SRRCTL */
+ case 0x02810: /* RDH */
+ case 0x02814: /* RXCTL */
+ case 0x02818: /* RDT */
+ case 0x02828: /* RXDCTL */
+ return (true);
+ }
+
+ /* Transmit queue zero. */
+ switch (reg) {
+ case 0x03800: /* TDBAL */
+ case 0x03804: /* TDBAH */
+ case 0x03808: /* TDLEN */
+ case 0x03810: /* TDH */
+ case 0x03814: /* TXCTL */
+ case 0x03818: /* TDT */
+ case 0x03828: /* TXDCTL */
+ case 0x03838: /* TDWBAL */
+ case 0x0383c: /* TDWBAH */
+ return (true);
+ }
+
+ /* Retained configuration from 82576 virtual queue one. */
+ if (vf_82576) {
+ switch (reg) {
+ case 0x0290c: /* SRRCTL(1) */
+ case 0x02914: /* RXCTL(1) */
+ case 0x02928: /* RXDCTL(1) */
+ case 0x03914: /* TXCTL(1) */
+ case 0x03928: /* TXDCTL(1) */
+ case 0x03938: /* TDWBAL(1) */
+ case 0x0393c: /* TDWBAH(1) */
+ return (true);
+ }
+ }
+
+ /*
+ * 82576 exposes VFMPRC at 0xf3c. I350 erratum 31 makes
+ * its corrected 0xf38 address inaccessible to a VF.
+ */
+ if (vf_82576 && reg == 0x00f3c)
+ return (!write);
+
+ switch (reg) {
+ case 0x00000: /* CTRL */
+ case 0x000c4: /* Legacy ITR, listed but unused by igb VFs */
+ case 0x00c40: /* V2PMAILBOX(0) */
+ case 0x00f0c: /* VFPSRTYPE */
+ case 0x01524: /* EIMS */
+ case 0x0152c: /* EIAC */
+ case 0x01530: /* EIAM */
+ case 0x01700: /* IVAR0 */
+ case 0x01740: /* IVAR_MISC */
+ return (true);
+ case 0x01520: /* EICS */
+ case 0x01528: /* EIMC */
+ return (write);
+ case 0x00008: /* STATUS */
+ case 0x00f10: /* VFGPRC */
+ case 0x00f14: /* VFGPTC */
+ case 0x00f18: /* VFGORC */
+ case 0x00f34: /* VFGOTC */
+ case 0x00f40: /* VFGPRLBC */
+ case 0x00f44: /* VFGPTLBC */
+ case 0x00f48: /* VFGORLBC */
+ case 0x00f50: /* VFGOTLBC */
+ case 0x01580: /* EICR */
+ return (!write);
+ default:
+ return (false);
+ }
+}
+#endif
+
#define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \
? reg : e1000_translate_register_82542(reg))
@@ -183,6 +286,11 @@ e1000_rd32(struct e1000_osdep *osdep, uint32_t reg)
KASSERT(reg < osdep->mem_bus_space_size,
("e1000: register offset %#jx too large (max is %#jx)",
(uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size));
+#ifdef INVARIANTS
+ KASSERT(!osdep->vf ||
+ e1000_vf_reg_valid(reg, false, osdep->vf_82576),
+ ("e1000: invalid VF register read at %#x", reg));
+#endif
return (bus_space_read_4(osdep->mem_bus_space_tag,
osdep->mem_bus_space_handle, reg));
@@ -196,6 +304,11 @@ e1000_wr32(struct e1000_osdep *osdep, uint32_t reg, uint32_t value)
KASSERT(reg < osdep->mem_bus_space_size,
("e1000: register offset %#jx too large (max is %#jx)",
(uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size));
+#ifdef INVARIANTS
+ KASSERT(!osdep->vf ||
+ e1000_vf_reg_valid(reg, true, osdep->vf_82576),
+ ("e1000: invalid VF register write at %#x", reg));
+#endif
if (__predict_true(!osdep->pcim2pci_arbiter_wa)) {
bus_space_write_4(osdep->mem_bus_space_tag,
@@ -283,4 +396,3 @@ e1000_wr32(struct e1000_osdep *osdep, uint32_t reg, uint32_t value)
#endif
#endif /* _FREEBSD_OS_H_ */
-
diff --git a/sys/dev/e1000/e1000_regs.h b/sys/dev/e1000/e1000_regs.h
index 3cb2d62a5a4c..b70fc4aa1255 100644
--- a/sys/dev/e1000/e1000_regs.h
+++ b/sys/dev/e1000/e1000_regs.h
@@ -299,6 +299,7 @@
#define E1000_TXDMAC 0x03000 /* Tx DMA Control - RW */
#define E1000_KABGTXD 0x03004 /* AFE Band Gap Transmit Ref Data */
#define E1000_PSRTYPE(_i) (0x05480 + ((_i) * 4))
+#define E1000_VFPSRTYPE 0x00F0C
#define E1000_RAL(_i) (((_i) <= 15) ? (0x05400 + ((_i) * 8)) : \
(0x054E0 + ((_i - 16) * 8)))
#define E1000_RAH(_i) (((_i) <= 15) ? (0x05404 + ((_i) * 8)) : \
diff --git a/sys/dev/e1000/e1000_vf.c b/sys/dev/e1000/e1000_vf.c
index 5b19375e6034..9528ab6009cb 100644
--- a/sys/dev/e1000/e1000_vf.c
+++ b/sys/dev/e1000/e1000_vf.c
@@ -260,7 +260,7 @@ static s32 e1000_reset_hw_vf(struct e1000_hw *hw)
struct e1000_mbx_info *mbx = &hw->mbx;
u32 timeout = E1000_VF_INIT_TIMEOUT;
s32 ret_val = -E1000_ERR_MAC_INIT;
- u32 ctrl, msgbuf[3];
+ u32 ctrl, msgbuf[3] = { E1000_VF_RESET, ~0U, ~0U };
u8 *addr = (u8 *)(&msgbuf[1]);
DEBUGFUNC("e1000_reset_hw_vf");
@@ -275,23 +275,40 @@ static s32 e1000_reset_hw_vf(struct e1000_hw *hw)
usec_delay(5);
}
- if (timeout) {
- /* mailbox timeout can now become active */
- mbx->timeout = E1000_VF_MBX_INIT_TIMEOUT;
+ if (!timeout)
+ return -E1000_ERR_RESET;
- msgbuf[0] = E1000_VF_RESET;
- mbx->ops.write_posted(hw, msgbuf, 1, 0);
+ /* mailbox timeout can now become active */
+ mbx->timeout = E1000_VF_MBX_INIT_TIMEOUT;
- msec_delay(10);
+ /*
+ * Linux PFs return a three-dword, zero-filled NACK when the reset
+ * completed without an assigned MAC address. FreeBSD PFs also use a
+ * one-dword NACK while retained queues are still being sanitized. Seed
+ * the unused request payload so the two responses remain distinguishable.
+ */
+ ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0);
+ if (ret_val)
+ return ret_val;
- /* set our "perm_addr" based on info provided by PF */
- ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0);
- if (!ret_val) {
- if (msgbuf[0] == (E1000_VF_RESET |
- E1000_VT_MSGTYPE_ACK))
- memcpy(hw->mac.perm_addr, addr, 6);
+ msec_delay(10);
+
+ /* set our "perm_addr" based on info provided by PF */
+ ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0);
+ if (!ret_val) {
+ switch (msgbuf[0]) {
+ case E1000_VF_RESET | E1000_VT_MSGTYPE_ACK:
+ memcpy(hw->mac.perm_addr, addr, ETHER_ADDR_LEN);
+ break;
+ case E1000_VF_RESET | E1000_VT_MSGTYPE_NACK:
+ if (msgbuf[1] == 0 && msgbuf[2] == 0)
+ memset(hw->mac.perm_addr, 0, ETHER_ADDR_LEN);
else
ret_val = -E1000_ERR_MAC_INIT;
+ break;
+ default:
+ ret_val = -E1000_ERR_MAC_INIT;
+ break;
}
}
@@ -391,6 +408,41 @@ static void e1000_write_msg_read_ack(struct e1000_hw *hw,
}
/**
+ * e1000_set_uc_addr_vf - Add or clear secondary unicast addresses
+ * @hw: pointer to the HW structure
+ * @sub_cmd: E1000_VF_MAC_FILTER_ADD or E1000_VF_MAC_FILTER_CLR
+ * @addr: address to add, or a valid compatibility address when clearing
+ *
+ * Uses the secondary-MAC mailbox subprotocol implemented by Linux igbvf.
+ * Linux igb PFs validate this field before dispatching the clear subcommand,
+ * even though they do not otherwise use it for a clear request.
+ **/
+s32
+e1000_set_uc_addr_vf(struct e1000_hw *hw, u32 sub_cmd, u8 *addr)
+{
+ struct e1000_mbx_info *mbx = &hw->mbx;
+ u32 msgbuf[3] = {};
+ u32 request;
+ s32 ret_val;
+
+ msgbuf[0] = E1000_VF_SET_MAC_ADDR | sub_cmd;
+ request = msgbuf[0];
+ if (addr != NULL)
+ memcpy(&msgbuf[1], addr, ETHER_ADDR_LEN);
+
+ ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0);
+ if (ret_val == E1000_SUCCESS)
+ ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0);
+
+ msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
+ if (ret_val == E1000_SUCCESS &&
+ msgbuf[0] == (request | E1000_VT_MSGTYPE_NACK))
+ ret_val = -E1000_ERR_NO_SPACE;
+
+ return (ret_val);
+}
+
+/**
* e1000_update_mc_addr_list_vf - Update Multicast addresses
* @hw: pointer to the HW structure
* @mc_addr_list: array of multicast addresses to program
@@ -402,7 +454,7 @@ static void e1000_write_msg_read_ack(struct e1000_hw *hw,
void e1000_update_mc_addr_list_vf(struct e1000_hw *hw,
u8 *mc_addr_list, u32 mc_addr_count)
{
- u32 msgbuf[E1000_VFMAILBOX_SIZE];
+ u32 msgbuf[E1000_VFMAILBOX_SIZE] = {};
u16 *hash_list = (u16 *)&msgbuf[1];
u32 hash_value;
u32 i;
@@ -444,10 +496,14 @@ void e1000_update_mc_addr_list_vf(struct e1000_hw *hw,
* @hw: pointer to the HW structure
* @vid: determines the vfta register and bit to set/unset
* @set: if true then set bit, else clear bit
+ *
+ * Returns success if the PF accepted the request, or an error otherwise.
**/
-void e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set)
+s32 e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set)
{
+ struct e1000_mbx_info *mbx = &hw->mbx;
u32 msgbuf[2];
+ s32 ret_val;
msgbuf[0] = E1000_VF_SET_VLAN;
msgbuf[1] = vid;
@@ -455,7 +511,15 @@ void e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set)
if (set)
msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
- e1000_write_msg_read_ack(hw, msgbuf, 2);
+ ret_val = mbx->ops.write_posted(hw, msgbuf, 2, 0);
+ if (!ret_val)
+ ret_val = mbx->ops.read_posted(hw, msgbuf, 1, 0);
+ if (!ret_val &&
+ ((msgbuf[0] & 0xffff) != E1000_VF_SET_VLAN ||
+ !(msgbuf[0] & E1000_VT_MSGTYPE_ACK)))
+ ret_val = -E1000_ERR_MAC_INIT;
+
+ return (ret_val);
}
/** e1000_rlpml_set_vf - Set the maximum receive packet length
@@ -561,13 +625,17 @@ static s32 e1000_check_for_link_vf(struct e1000_hw *hw)
/* if the read failed it could just be a mailbox collision, best wait
* until we are called again and don't report an error */
- if (mbx->ops.read(hw, &in_msg, 1, 0))
+ if (mbx->ops.read(hw, &in_msg, 1, 0, true))
goto out;
/* if incoming message isn't clear to send we are waiting on response */
if (!(in_msg & E1000_VT_MSGTYPE_CTS)) {
- /* message is not CTS and is NACK we have lost CTS status */
- if (in_msg & E1000_VT_MSGTYPE_NACK)
+ /*
+ * A NACK or a PF control message without CTS means that the PF
+ * discarded our state and requires a new VF reset handshake.
+ */
+ if ((in_msg & E1000_VT_MSGTYPE_NACK) != 0 ||
+ (in_msg & 0xffff) == E1000_PF_CONTROL_MSG)
ret_val = -E1000_ERR_MAC_INIT;
goto out;
}
@@ -587,4 +655,3 @@ static s32 e1000_check_for_link_vf(struct e1000_hw *hw)
out:
return ret_val;
}
-
diff --git a/sys/dev/e1000/e1000_vf.h b/sys/dev/e1000/e1000_vf.h
index aace8e78ed6a..17116a86b084 100644
--- a/sys/dev/e1000/e1000_vf.h
+++ b/sys/dev/e1000/e1000_vf.h
@@ -227,13 +227,14 @@ struct e1000_mac_info {
struct e1000_mbx_operations {
s32 (*init_params)(struct e1000_hw *hw);
- s32 (*read)(struct e1000_hw *, u32 *, u16, u16);
+ s32 (*read)(struct e1000_hw *, u32 *, u16, u16, bool);
s32 (*write)(struct e1000_hw *, u32 *, u16, u16);
s32 (*read_posted)(struct e1000_hw *, u32 *, u16, u16);
s32 (*write_posted)(struct e1000_hw *, u32 *, u16, u16);
s32 (*check_for_msg)(struct e1000_hw *, u16);
s32 (*check_for_ack)(struct e1000_hw *, u16);
s32 (*check_for_rst)(struct e1000_hw *, u16);
+ s32 (*unlock)(struct e1000_hw *, u16);
};
struct e1000_mbx_stats {
@@ -290,7 +291,8 @@ enum e1000_promisc_type {
/* These functions must be implemented by drivers */
s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value);
-void e1000_vfta_set_vf(struct e1000_hw *, u16, bool);
+s32 e1000_vfta_set_vf(struct e1000_hw *, u16, bool);
void e1000_rlpml_set_vf(struct e1000_hw *, u16);
s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type);
+s32 e1000_set_uc_addr_vf(struct e1000_hw *, u32, u8 *);
#endif /* _E1000_VF_H_ */
diff --git a/sys/dev/e1000/em_txrx.c b/sys/dev/e1000/em_txrx.c
index 927dd9df8cb9..df952c8f2ef1 100644
--- a/sys/dev/e1000/em_txrx.c
+++ b/sys/dev/e1000/em_txrx.c
@@ -96,6 +96,11 @@ em_dump_rs(struct e1000_softc *sc)
int16_t rs_cidx;
uint8_t status;
+ if (sc->tx_queues == NULL) {
+ device_printf(sc->dev, "queue state is unavailable\n");
+ return;
+ }
+
printf("\n");
ntxd = scctx->isc_ntxd[0];
for (qid = 0; qid < sc->tx_num_queues; qid++) {
diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index 696fed6e8e25..5a44956f2a88 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -28,6 +28,7 @@
*/
#include "if_em.h"
+#include "if_igb_iov.h"
#include <sys/sbuf.h>
#include <machine/_inttypes.h>
@@ -346,8 +347,6 @@ static const pci_vendor_info_t igb_vendor_info_array[] =
"Intel(R) PRO/1000 ET 82576 (Quad Copper)"),
PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2,
"Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"),
- PVID(0x8086, E1000_DEV_ID_82576_VF,
- "Intel(R) PRO/1000 82576 Virtual Function"),
PVID(0x8086, E1000_DEV_ID_82580_COPPER,
"Intel(R) I340 82580 (Copper)"),
PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"),
@@ -369,7 +368,6 @@ static const pci_vendor_info_t igb_vendor_info_array[] =
PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"),
PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"),
PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"),
- PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) I350 Virtual Function"),
PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"),
PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT,
"Intel(R) I210 IT (Copper)"),
@@ -393,13 +391,29 @@ static const pci_vendor_info_t igb_vendor_info_array[] =
PVID_END
};
+static const pci_vendor_info_t igbv_vendor_info_array[] = {
+ PVID(0x8086, E1000_DEV_ID_82576_VF,
+ "Intel(R) PRO/1000 82576 Virtual Function"),
+ PVID(0x8086, E1000_DEV_ID_82576_VF_HV,
+ "Intel(R) PRO/1000 82576 Hyper-V Virtual Function"),
+ PVID(0x8086, E1000_DEV_ID_I350_VF,
+ "Intel(R) I350 Virtual Function"),
+ PVID(0x8086, E1000_DEV_ID_I350_VF_HV,
+ "Intel(R) I350 Hyper-V Virtual Function"),
+ PVID_END
+};
+
/*********************************************************************
* Function prototypes
*********************************************************************/
static void *em_register(device_t);
static void *igb_register(device_t);
-static int em_if_attach_pre(if_ctx_t);
-static int em_if_attach_post(if_ctx_t);
+static void *igbv_register(device_t);
+static int igb_device_attach(device_t);
+#ifdef PCI_IOV
+static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *);
+static void igb_device_iov_uninit(device_t);
+#endif
static int em_if_detach(if_ctx_t);
static int em_if_shutdown(if_ctx_t);
static int em_if_suspend(if_ctx_t);
@@ -415,6 +429,8 @@ static uint64_t em_if_get_vf_counter(if_ctx_t, ift_counter);
static uint64_t em_if_get_counter(if_ctx_t, ift_counter);
static void em_if_init(if_ctx_t);
static void em_if_stop(if_ctx_t);
+static void em_fence_pci_busmaster(struct e1000_softc *);
+static int em_enable_pci_busmaster(struct e1000_softc *);
static void em_if_media_status(if_ctx_t, struct ifmediareq *);
static int em_if_media_change(if_ctx_t);
static int em_if_mtu_set(if_ctx_t, uint32_t);
@@ -458,11 +474,13 @@ static void em_update_82580_ecc_stats(struct e1000_softc *, u32, u32,
static void em_if_multi_set(if_ctx_t);
static void em_if_update_admin_status(if_ctx_t);
static void em_if_debug(if_ctx_t);
+static void em_initialize_vf_stats(struct e1000_softc *);
+static void em_rebase_vf_stats(struct e1000_softc *);
static void em_update_vf_stats_counters(struct e1000_softc *);
-static void em_update_stats_counters(struct e1000_softc *);
static void em_add_hw_stats(struct e1000_softc *);
static bool em_mac_has_eee(enum e1000_mac_type);
static int em_if_set_promisc(if_ctx_t, int);
+static bool em_if_defer_promisc(struct e1000_softc *);
static bool em_if_vlan_filter_capable(if_ctx_t);
static bool em_if_vlan_filter_used(if_ctx_t);
static void em_if_vlan_filter_enable(struct e1000_softc *);
@@ -478,7 +496,6 @@ static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS);
static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
static int em_get_rs(SYSCTL_HANDLER_ARGS);
static void em_print_debug_info(struct e1000_softc *);
-static int em_is_valid_ether_addr(u8 *);
static void em_newitr(struct e1000_softc *, struct em_rx_queue *,
struct rx_ring *);
static bool em_automask_tso(if_ctx_t);
@@ -534,7 +551,6 @@ static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
static void em_if_led_func(if_ctx_t, int);
static int em_get_regs(SYSCTL_HANDLER_ARGS);
-
static void lem_smartspeed(struct e1000_softc *);
static void igb_configure_queues(struct e1000_softc *);
static void igb_initialize_interrupt_rate(struct e1000_softc *);
@@ -560,6 +576,23 @@ static device_method_t igb_methods[] = {
/* Device interface */
DEVMETHOD(device_register, igb_register),
DEVMETHOD(device_probe, iflib_device_probe),
+ DEVMETHOD(device_attach, igb_device_attach),
+ DEVMETHOD(device_detach, iflib_device_detach),
+ DEVMETHOD(device_shutdown, iflib_device_shutdown),
+ DEVMETHOD(device_suspend, iflib_device_suspend),
+ DEVMETHOD(device_resume, iflib_device_resume),
+#ifdef PCI_IOV
+ DEVMETHOD(pci_iov_init, igb_device_iov_init),
+ DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit),
+ DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf),
+#endif
+ DEVMETHOD_END
+};
+
+static device_method_t igbv_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_register, igbv_register),
+ DEVMETHOD(device_probe, iflib_device_probe),
DEVMETHOD(device_attach, iflib_device_attach),
DEVMETHOD(device_detach, iflib_device_detach),
DEVMETHOD(device_shutdown, iflib_device_shutdown),
@@ -593,6 +626,18 @@ MODULE_DEPEND(igb, iflib, 1, 1, 1);
IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array);
+static driver_t igbv_driver = {
+ "igbv", igbv_methods, sizeof(struct e1000_softc),
+};
+
+DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0);
+
+MODULE_DEPEND(igbv, pci, 1, 1, 1);
+MODULE_DEPEND(igbv, ether, 1, 1, 1);
+MODULE_DEPEND(igbv, iflib, 1, 1, 1);
+
+IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array);
+
static device_method_t em_if_methods[] = {
DEVMETHOD(ifdi_attach_pre, em_if_attach_pre),
DEVMETHOD(ifdi_attach_post, em_if_attach_post),
@@ -660,6 +705,12 @@ static device_method_t igb_if_methods[] = {
DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable),
DEVMETHOD(ifdi_debug, em_if_debug),
DEVMETHOD(ifdi_needs_restart, em_if_needs_restart),
+#ifdef PCI_IOV
+ DEVMETHOD(ifdi_iov_init, igb_if_iov_init),
+ DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit),
+ DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add),
+ DEVMETHOD(ifdi_vf_status, igb_if_vf_status),
+#endif
DEVMETHOD_END
};
@@ -667,6 +718,42 @@ static driver_t igb_if_driver = {
"igb_if", igb_if_methods, sizeof(struct e1000_softc)
};
+static device_method_t igbv_if_methods[] = {
+ DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre),
+ DEVMETHOD(ifdi_attach_post, igbv_if_attach_post),
+ DEVMETHOD(ifdi_detach, em_if_detach),
+ DEVMETHOD(ifdi_shutdown, em_if_shutdown),
+ DEVMETHOD(ifdi_suspend, em_if_suspend),
+ DEVMETHOD(ifdi_resume, em_if_resume),
+ DEVMETHOD(ifdi_init, em_if_init),
+ DEVMETHOD(ifdi_stop, em_if_stop),
+ DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign),
+ DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable),
+ DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable),
+ DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc),
+ DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc),
+ DEVMETHOD(ifdi_queues_free, em_if_queues_free),
+ DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status),
+ DEVMETHOD(ifdi_multi_set, em_if_multi_set),
+ DEVMETHOD(ifdi_media_status, em_if_media_status),
+ DEVMETHOD(ifdi_media_change, igbv_if_media_change),
+ DEVMETHOD(ifdi_mtu_set, em_if_mtu_set),
+ DEVMETHOD(ifdi_promisc_set, em_if_set_promisc),
+ DEVMETHOD(ifdi_timer, em_if_timer),
+ DEVMETHOD(ifdi_vlan_register, em_if_vlan_register),
+ DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister),
+ DEVMETHOD(ifdi_get_counter, em_if_get_counter),
+ DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable),
+ DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable),
+ DEVMETHOD(ifdi_debug, em_if_debug),
+ DEVMETHOD(ifdi_needs_restart, em_if_needs_restart),
+ DEVMETHOD_END
+};
+
+static driver_t igbv_if_driver = {
+ "igbv_if", igbv_if_methods, sizeof(struct e1000_softc)
+};
+
/*********************************************************************
* Tunable default values.
*********************************************************************/
@@ -800,6 +887,40 @@ static struct if_shared_ctx igb_sctx_init = {
.isc_ntxd_default = {EM_DEFAULT_TXD},
};
+/*
+ * igb PFs and igbv VFs share the common datapath implementation. Keep a
+ * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or
+ * interrupt modes.
+ */
+static struct if_shared_ctx igbv_sctx_init = {
+ .isc_magic = IFLIB_MAGIC,
+ .isc_q_align = PAGE_SIZE,
+ .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
+ .isc_tx_maxsegsize = PAGE_SIZE,
+ .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
+ .isc_tso_maxsegsize = EM_TSO_SEG_SIZE,
+ .isc_rx_maxsize = MJUM9BYTES,
+ .isc_rx_nsegments = 1,
+ .isc_rx_maxsegsize = MJUM9BYTES,
+ .isc_nfl = 1,
+ .isc_nrxqs = 1,
+ .isc_ntxqs = 1,
+ .isc_admin_intrcnt = 1,
+ .isc_vendor_info = igbv_vendor_info_array,
+ .isc_driver_version = igb_driver_version,
+ .isc_driver = &igbv_if_driver,
+ .isc_flags =
+ IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM |
+ IFLIB_IS_VF,
+
+ .isc_nrxd_min = {EM_MIN_RXD},
+ .isc_ntxd_min = {EM_MIN_TXD},
+ .isc_nrxd_max = {IGB_MAX_RXD},
+ .isc_ntxd_max = {IGB_MAX_TXD},
+ .isc_nrxd_default = {EM_DEFAULT_RXD},
+ .isc_ntxd_default = {EM_DEFAULT_TXD},
+};
+
/*****************************************************************
*
* Dump Registers
@@ -814,6 +935,7 @@ static int em_get_regs(SYSCTL_HANDLER_ARGS)
struct sbuf *sb;
u32 *regs_buff;
int rc;
+ uint32_t rxqid, txqid;
/*
* This sysctl is registered before iflib allocates the queue arrays,
@@ -824,6 +946,8 @@ static int em_get_regs(SYSCTL_HANDLER_ARGS)
regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK);
memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32));
+ rxqid = sc->rx_queues[0].rxr.me;
+ txqid = sc->tx_queues[0].txr.me;
rc = sysctl_wire_old_buffer(req, 0);
MPASS(rc == 0);
@@ -845,19 +969,19 @@ static int em_get_regs(SYSCTL_HANDLER_ARGS)
regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT);
regs_buff[3] = E1000_READ_REG(hw, E1000_ICR);
regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL);
- regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0));
- regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0));
- regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0));
- regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0));
- regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0));
- regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0));
+ regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid));
+ regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid));
+ regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid));
+ regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid));
+ regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid));
+ regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid));
regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL);
- regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0));
- regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0));
- regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0));
- regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0));
- regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0));
- regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0));
+ regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid));
+ regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid));
+ regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid));
+ regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid));
+ regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid));
+ regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid));
regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH);
regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT);
regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS);
@@ -944,6 +1068,65 @@ igb_register(device_t dev)
return (&igb_sctx_init);
}
+static void *
+igbv_register(device_t dev)
+{
+ return (&igbv_sctx_init);
+}
+
+static int
+igb_device_attach(device_t dev)
+{
+ struct e1000_softc *sc;
+ if_ctx_t ctx;
+ int error;
+
+ error = iflib_device_attach(dev);
+ if (error != 0)
+ return (error);
+
+ ctx = device_get_softc(dev);
+ sc = iflib_get_softc(ctx);
+ (void)igb_iov_attach(sc);
+ return (0);
+}
+
+#ifdef PCI_IOV
+static int
+igb_device_iov_init(device_t dev, uint16_t num_vfs,
+ const nvlist_t *params)
+{
+ struct e1000_softc *sc;
+ if_ctx_t ctx;
+ int error;
+
+ ctx = device_get_softc(dev);
+ sc = iflib_get_softc(ctx);
+ error = igb_iov_validate(sc, num_vfs);
+ if (error != 0)
+ return (error);
+ return (iflib_device_iov_init_restart(dev, num_vfs, params));
+}
+
+static void
+igb_device_iov_uninit(device_t dev)
+{
+ struct e1000_softc *sc;
+ if_ctx_t ctx;
+
+ ctx = device_get_softc(dev);
+ sc = iflib_get_softc(ctx);
+ /*
+ * pci_iov(4) has already detached the VF devices. Tell the stop
+ * half of iflib's restart transaction not to wait for acknowledgements
+ * from VFs which can no longer service their mailbox vectors.
+ */
+ atomic_store_rel_32(&sc->iov_teardown, 1);
+ iflib_device_iov_uninit_restart(dev);
+}
+
+#endif
+
static int
em_set_num_queues(if_ctx_t ctx)
{
@@ -966,6 +1149,11 @@ em_set_num_queues(if_ctx_t ctx)
case e1000_82574:
maxqueues = 2;
break;
+ case e1000_vfadapt:
+ /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */
+ case e1000_vfadapt_i350:
+ maxqueues = 1;
+ break;
default:
maxqueues = 1;
break;
@@ -974,98 +1162,76 @@ em_set_num_queues(if_ctx_t ctx)
return (maxqueues);
}
-#define LEM_CAPS \
+#define LEM_CAPS ( \
IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \
IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \
- IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6
+ IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6)
-#define EM_CAPS \
+#define EM_CAPS ( \
IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \
IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \
IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \
- IFCAP_TSO6
+ IFCAP_TSO6)
-#define IGB_CAPS \
+#define IGB_CAPS ( \
IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \
IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \
IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \
- IFCAP_TSO6
+ IFCAP_TSO6)
-/*********************************************************************
- * Device initialization routine
- *
- * The attach entry point is called when the driver is being loaded.
- * This routine identifies the type of hardware, allocates all resources
- * and initializes the hardware.
- *
- * return 0 on success, positive on failure
- *********************************************************************/
-static int
-em_if_attach_pre(if_ctx_t ctx)
+/*
+ * VLAN filtering is an effective VF capability, but its policy is owned by
+ * the PF and cannot be disabled from the VF. vlan(4) registration callbacks
+ * are independent of this capability bit.
+ */
+#define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL)
+
+void
+em_add_device_sysctls(struct e1000_softc *sc)
{
- struct e1000_softc *sc;
- if_softc_ctx_t scctx;
- device_t dev;
struct e1000_hw *hw;
struct sysctl_oid_list *child;
struct sysctl_ctx_list *ctx_list;
- int error = 0;
- INIT_DEBUGOUT("em_if_attach_pre: begin");
- dev = iflib_get_dev(ctx);
- sc = iflib_get_softc(ctx);
+ hw = &sc->hw;
+ ctx_list = device_get_sysctl_ctx(sc->dev);
+ child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
- if (em_max_interrupt_rate <= 0) {
- device_printf(dev,
- "Invalid max_interrupt_rate %d; using default %d\n",
- em_max_interrupt_rate, EM_INTS_DEFAULT);
- em_max_interrupt_rate = EM_INTS_DEFAULT;
- }
+ sc->enable_aim = em_enable_aim;
+ SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim",
+ CTLFLAG_RW, &sc->enable_aim, 0,
+ "Interrupt Moderation (1=normal, 2=lowlatency)");
- sc->ctx = sc->osdep.ctx = ctx;
- sc->dev = sc->osdep.dev = dev;
- scctx = sc->shared = iflib_get_softc_ctx(ctx);
- sc->media = iflib_get_media(ctx);
- hw = &sc->hw;
+ SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug",
+ CTLTYPE_INT | CTLFLAG_RW, sc, 0,
+ em_sysctl_debug_info, "I", "Debug Information");
- /* Determine hardware and mac info */
- em_identify_hardware(ctx);
+ SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump",
+ CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
+ em_get_rs, "I", "Dump RS indexes");
- /* SYSCTL stuff */
- ctx_list = device_get_sysctl_ctx(dev);
- child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
+ if (sc->vf_ifp) {
+ SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump",
+ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
+ igbv_get_regs, "A", "Dump VF registers");
+ return;
+ }
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm",
CTLTYPE_INT | CTLFLAG_RW, sc, 0,
em_sysctl_nvm_info, "I", "NVM Information");
-
- sc->enable_aim = em_enable_aim;
- SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim",
- CTLFLAG_RW, &sc->enable_aim, 0,
- "Interrupt Moderation (1=normal, 2=lowlatency)");
-
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
em_sysctl_print_fw_version, "A",
"Prints FW/NVM Versions");
-
- SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug",
- CTLTYPE_INT | CTLFLAG_RW, sc, 0,
- em_sysctl_debug_info, "I", "Debug Information");
-
sc->fc = e1000_fc_full;
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc",
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
em_set_flowcntl, "I", "Flow Control");
-
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump",
CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
em_get_regs, "A", "Dump Registers");
- SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump",
- CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
- em_get_rs, "I", "Dump RS indexes");
-
if (hw->mac.type >= e1000_i350 && hw->mac.type != e1000_i211) {
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac",
CTLTYPE_INT | CTLFLAG_RW, sc, 0,
@@ -1077,18 +1243,67 @@ em_if_attach_pre(if_ctx_t ctx)
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
sc, 0, em_sysctl_tso_tcp_flags_mask, "IU",
"TSO TCP flags mask for first segment");
-
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO,
"tso_tcp_flags_mask_middle_segment",
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
sc, 1, em_sysctl_tso_tcp_flags_mask, "IU",
"TSO TCP flags mask for middle segment");
-
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO,
"tso_tcp_flags_mask_last_segment",
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
sc, 2, em_sysctl_tso_tcp_flags_mask, "IU",
"TSO TCP flags mask for last segment");
+}
+
+/*********************************************************************
+ * Device initialization routine
+ *
+ * The attach entry point is called when the driver is being loaded.
+ * This routine identifies the type of hardware, allocates all resources
+ * and initializes the hardware.
+ *
+ * return 0 on success, positive on failure
+ *********************************************************************/
+int
+em_if_attach_pre(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ if_softc_ctx_t scctx;
+ device_t dev;
+ struct e1000_hw *hw;
+ struct sysctl_oid_list *child;
+ struct sysctl_ctx_list *ctx_list;
+ int error = 0;
+
+ INIT_DEBUGOUT("em_if_attach_pre: begin");
+ dev = iflib_get_dev(ctx);
+ sc = iflib_get_softc(ctx);
+
+ if (em_max_interrupt_rate <= 0) {
+ device_printf(dev,
+ "Invalid max_interrupt_rate %d; using default %d\n",
+ em_max_interrupt_rate, EM_INTS_DEFAULT);
+ em_max_interrupt_rate = EM_INTS_DEFAULT;
+ }
+
+ sc->ctx = sc->osdep.ctx = ctx;
+ sc->dev = sc->osdep.dev = dev;
+ scctx = sc->shared = iflib_get_softc_ctx(ctx);
+ sc->media = iflib_get_media(ctx);
+ hw = &sc->hw;
+ sc->vf_ifp =
+ (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0;
+ sc->osdep.vf = sc->vf_ifp;
+
+ /* Determine hardware and mac info */
+ em_identify_hardware(ctx);
+ sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt;
+
+ /* VF sysctls are deferred until attach-post confirms MSI-X. */
+ ctx_list = device_get_sysctl_ctx(dev);
+ child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
+ if (!sc->vf_ifp)
+ em_add_device_sysctls(sc);
scctx->isc_tx_nsegments = EM_MAX_SCATTER;
scctx->isc_nrxqsets_max =
@@ -1108,7 +1323,14 @@ em_if_attach_pre(if_ctx_t ctx)
scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER;
scctx->isc_tx_tso_size_max = EM_TSO_SIZE;
scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE;
- scctx->isc_capabilities = scctx->isc_capenable = IGB_CAPS;
+ scctx->isc_capabilities = scctx->isc_capenable =
+ sc->vf_ifp ? IGBV_CAPS : IGB_CAPS;
+ if (igbv_is_hyperv(sc)) {
+ /* The host owns VLAN filters and the receive-frame limit. */
+ scctx->isc_capabilities &=
+ ~(IFCAP_VLAN_HWFILTER | IFCAP_JUMBO_MTU);
+ scctx->isc_capenable = scctx->isc_capabilities;
+ }
scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO |
CSUM_IP6_TCP | CSUM_IP6_UDP;
if (hw->mac.type != e1000_82575)
@@ -1222,6 +1444,13 @@ em_if_attach_pre(if_ctx_t ctx)
goto err_pci;
}
/*
+ * A VF can retain queue enable bits and DMA addresses across VFLR.
+ * Fence bus mastering before the first mailbox reset so state left by
+ * a previous owner cannot issue DMA while the driver attaches.
+ */
+ if (sc->vf_ifp)
+ em_fence_pci_busmaster(sc);
+ /*
* 82579 can lose a host CSR write while the Management Engine owns
* the PCIm2PCI arbiter. Enable the OS register write interlock before
* shared code initialization performs any MAC writes.
@@ -1279,6 +1508,9 @@ em_if_attach_pre(if_ctx_t ctx)
goto err_pci;
}
+ if (igbv_is_hyperv(sc))
+ igbv_init_hv_ops(hw);
+
em_setup_msix(ctx);
e1000_get_bus_info(hw);
@@ -1364,19 +1596,21 @@ em_if_attach_pre(if_ctx_t ctx)
/* Clear the IFCAP_TSO auto mask */
sc->tso_automasked = 0;
- /* Check SOL/IDER usage */
- if (e1000_check_reset_block(hw))
+ /* Check SOL/IDER usage on physical functions. */
+ if (!sc->vf_ifp && e1000_check_reset_block(hw))
device_printf(dev,
"PHY reset is blocked due to SOL/IDER session.\n");
/* Sysctl for setting Energy Efficient Ethernet */
- if (hw->mac.type < igb_mac_min)
- hw->dev_spec.ich8lan.eee_disable = eee_setting;
- else
- hw->dev_spec._82575.eee_disable = eee_setting;
- SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control",
- CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
- em_sysctl_eee, "I", "Disable Energy Efficient Ethernet");
+ if (!sc->vf_ifp) {
+ if (hw->mac.type < igb_mac_min)
+ hw->dev_spec.ich8lan.eee_disable = eee_setting;
+ else
+ hw->dev_spec._82575.eee_disable = eee_setting;
+ SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control",
+ CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
+ em_sysctl_eee, "I", "Disable Energy Efficient Ethernet");
+ }
/*
** Start from a known state, this is
@@ -1384,14 +1618,20 @@ em_if_attach_pre(if_ctx_t ctx)
** mac from that.
*/
error = e1000_reset_hw(hw);
- if (!sc->vf_ifp && error != E1000_SUCCESS) {
+ if (sc->vf_ifp) {
+ atomic_store_rel_32(&sc->vf_mbx_ready,
+ error == E1000_SUCCESS);
+ if (error != E1000_SUCCESS)
+ igbv_log_reset_failure(sc, error, true);
+ sc->vf_queues_sanitized = igbv_sanitize_queues(sc);
+ } else if (error != E1000_SUCCESS) {
device_printf(dev, "Hardware reset failed: %d\n", error);
error = EIO;
goto err_late;
}
- /* Make sure we have a good EEPROM before we read from it */
- if (e1000_validate_nvm_checksum(hw) < 0) {
+ /* Make sure a PF has a good EEPROM before we read from it. */
+ if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) {
/*
** Some PCI-E parts fail the first check due to
** the link being in sleep state, call it again,
@@ -1408,13 +1648,16 @@ em_if_attach_pre(if_ctx_t ctx)
/* Copy the permanent MAC address out of the EEPROM */
if (e1000_read_mac_addr(hw) < 0) {
device_printf(dev,
- "EEPROM read error while reading MAC address\n");
+ "Unable to read MAC address\n");
error = EIO;
goto err_late;
}
if (!em_is_valid_ether_addr(hw->mac.addr)) {
if (sc->vf_ifp) {
+ device_printf(dev,
+ "PF did not assign a MAC address; using a "
+ "locally generated address\n");
ether_gen_addr(iflib_get_ifp(ctx),
(struct ether_addr *)hw->mac.addr);
} else {
@@ -1424,18 +1667,16 @@ em_if_attach_pre(if_ctx_t ctx)
}
}
- /* Save the EEPROM/NVM versions, must be done under IFLIB_CTX_LOCK */
- em_fw_version_locked(ctx);
-
- em_print_fw_version(sc);
+ if (!sc->vf_ifp) {
+ /* Save NVM versions while holding the IFLIB context lock. */
+ em_fw_version_locked(ctx);
+ em_print_fw_version(sc);
+ }
/*
* Get Wake-on-Lan and Management info for later use
*/
- if (sc->vf_ifp) {
- scctx->isc_capabilities &= ~IFCAP_WOL;
- scctx->isc_capenable &= ~IFCAP_WOL;
- } else {
+ if (!sc->vf_ifp) {
em_get_wakeup(ctx);
}
@@ -1448,11 +1689,12 @@ err_late:
err_pci:
em_free_pci_resources(ctx);
free(sc->mta, M_DEVBUF);
+ sc->mta = NULL;
return (error);
}
-static int
+int
em_if_attach_post(if_ctx_t ctx)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
@@ -1467,7 +1709,7 @@ em_if_attach_post(if_ctx_t ctx)
}
if (sc->vf_ifp) {
- (void)em_reset(ctx);
+ (void)igbv_reset(ctx);
} else if (em_reset(ctx) != E1000_SUCCESS) {
error = EIO;
goto err_late;
@@ -1475,14 +1717,17 @@ em_if_attach_post(if_ctx_t ctx)
/* Initialize statistics */
if (sc->vf_ifp)
- sc->ustats.vf_stats = (struct e1000_vf_stats){};
+ em_initialize_vf_stats(sc);
else
sc->ustats.stats = (struct e1000_hw_stats){};
em_update_stats_counters(sc);
atomic_readandclear_32(&sc->stats_pending);
hw->mac.get_link_status = 1;
- em_if_update_admin_status(ctx);
+ if (sc->vf_ifp)
+ igbv_if_update_admin_status(ctx);
+ else
+ em_if_update_admin_status(ctx);
em_add_hw_stats(sc);
/* Non-AMT based hardware can now take control from firmware */
@@ -1517,7 +1762,13 @@ em_if_detach(if_ctx_t ctx)
INIT_DEBUGOUT("em_if_detach: begin");
- e1000_phy_hw_reset(&sc->hw);
+ igb_iov_detach(sc);
+ if (sc->vf_ifp) {
+ igbv_queue_retry_detach(sc);
+ igbv_mbx_retry_detach(sc);
+ } else {
+ e1000_phy_hw_reset(&sc->hw);
+ }
em_release_manageability(sc);
em_release_hw_control(sc);
@@ -1555,6 +1806,10 @@ em_if_suspend(if_ctx_t ctx)
struct e1000_softc *sc = iflib_get_softc(ctx);
int error;
+ if (sc->vf_ifp) {
+ igbv_queue_retry_stop(sc);
+ igbv_mbx_retry_stop(sc);
+ }
error = em_enable_wakeup(ctx);
em_release_manageability(sc);
em_release_hw_control(sc);
@@ -1611,6 +1866,10 @@ em_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
+ /* No native SET_LPE exchange is available with a Hyper-V PF. */
+ if (igbv_is_hyperv(sc) && mtu > ETHERMTU)
+ return (EINVAL);
+
switch (sc->hw.mac.type) {
case e1000_82571:
case e1000_82572:
@@ -1641,7 +1900,7 @@ em_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
break;
default:
if (sc->hw.mac.type >= igb_mac_min)
- max_frame_size = 9234;
+ max_frame_size = IGB_MAX_FRAME_SIZE;
else /* lem */
max_frame_size = MAX_JUMBO_FRAME_SIZE;
}
@@ -1670,18 +1929,26 @@ em_if_init(if_ctx_t ctx)
if_softc_ctx_t scctx = sc->shared;
if_t ifp = iflib_get_ifp(ctx);
struct em_tx_queue *tx_que;
- u32 icr;
int i;
INIT_DEBUGOUT("em_if_init: begin");
+ if (sc->vf_ifp) {
+ igbv_queue_retry_prepare(sc);
+ igbv_mbx_retry_prepare(sc);
+ sc->vf_reset_pending = true;
+ }
if (sc->suspend_link_powered_down)
em_power_up_wakeup_link(sc);
/* Get the latest mac address, User can use a LAA */
bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN);
- /* Put the address into the Receive Address Array */
- e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0);
+ /*
+ * A VF restores its address only after its reset handshake establishes
+ * its PF-assigned state. The PF path programs RAR[0] directly here.
+ */
+ if (!sc->vf_ifp)
+ e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0);
/*
* With the 82571 adapter, RAR[0] may be overwritten
@@ -1696,17 +1963,51 @@ em_if_init(if_ctx_t ctx)
}
/* Initialize the hardware */
+ igb_iov_reset_prepare(sc);
if (sc->vf_ifp) {
- (void)em_reset(ctx);
+ (void)igbv_reset(ctx);
+ em_rebase_vf_stats(sc);
} else if (em_reset(ctx) != E1000_SUCCESS) {
+ em_fence_pci_busmaster(sc);
iflib_init_failed(ctx);
return;
}
+ if (sc->vf_ifp && !sc->vf_queues_sanitized) {
+ /*
+ * Do not program or enable rings while retained queue state
+ * might still contain a previous VF owner's DMA address. A
+ * bounded callout retries initialization after iflib leaves the
+ * failed initialization stopped.
+ */
+ em_fence_pci_busmaster(sc);
+ igbv_queue_retry_failed(ctx);
+ return;
+ }
+ if (sc->vf_ifp &&
+ atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
+ igbv_mbx_retry_failed(ctx);
+ return;
+ }
+ /*
+ * Keep a fail-closed device fenced until reset and VF queue
+ * sanitization have removed every stale DMA address.
+ */
+ if (em_enable_pci_busmaster(sc) != 0) {
+ device_printf(sc->dev,
+ "Unable to enable PCI bus mastering\n");
+ iflib_init_failed(ctx);
+ return;
+ }
+ if (sc->vf_ifp)
+ igbv_reconcile_mac(sc, ifp);
/* Re-arm a link-up transition deferred for this reset. */
if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING ||
sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)
sc->link_state = EM_LINK_STATE_DOWN;
- em_if_update_admin_status(ctx);
+ if (sc->vf_ifp)
+ igbv_if_update_admin_status(ctx);
+ else
+ em_if_update_admin_status(ctx);
for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues;
i++, tx_que++) {
@@ -1722,8 +2023,9 @@ em_if_init(if_ctx_t ctx)
txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
}
- /* Setup VLAN support, basic and offload if available */
- E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN);
+ /* The VF VLAN EtherType is fixed and has no VET register. */
+ if (!sc->vf_ifp)
+ E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN);
/* Clear bad data from Rx FIFOs */
if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp)
@@ -1733,19 +2035,33 @@ em_if_init(if_ctx_t ctx)
em_init_manageability(sc);
/* Prepare transmit descriptors and buffers */
- em_initialize_transmit_unit(ctx);
+ if (sc->vf_ifp)
+ igbv_initialize_transmit_unit(ctx);
+ else
+ em_initialize_transmit_unit(ctx);
- /* Setup Multicast table */
+ /*
+ * A failed VF reset has no CTS channel on which to restore mailbox
+ * state. The reset detector schedules another complete init, which
+ * replays these interface-owned lists after the handshake succeeds.
+ */
em_if_multi_set(ctx);
sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
- em_initialize_receive_unit(ctx);
+ if (sc->vf_ifp)
+ igbv_initialize_receive_unit(ctx);
+ else
+ em_initialize_receive_unit(ctx);
- /* Set up VLAN support and filter */
+ /* Set up VLAN support and filter. */
em_setup_vlan_hw_support(ctx);
- /* Don't lose promiscuous settings */
- em_if_set_promisc(ctx, if_getflags(ifp));
+ /* Don't lose promiscuous settings. */
+ em_if_set_promisc_impl(ctx, if_getflags(ifp));
+ atomic_readandclear_32(&sc->promisc_pending);
+
+ /* Restore PF/VF pool configuration after the global reset. */
+ igb_iov_initialize(sc);
if (sc->hw.mac.ops.clear_hw_cntrs != NULL)
sc->hw.mac.ops.clear_hw_cntrs(&sc->hw);
@@ -1780,18 +2096,24 @@ em_if_init(if_ctx_t ctx)
em_configure_peind_memory_errors(sc);
em_configure_82575_memory_errors(sc);
em_configure_82580_memory_errors(sc);
+ if (sc->vf_ifp) {
+ sc->vf_reset_pending = false;
+ } else {
+ u32 icr;
- /*
- * Drain stale causes only after register reconstruction is complete.
- * DRSTA and DEV_RST_SET together close the window in which another
- * device reset can arrive while interrupts are masked.
- */
- icr = E1000_READ_REG(&sc->hw, E1000_ICR);
- if (igb_finish_device_reset(sc, icr)) {
- iflib_init_failed(ctx);
- return;
+ /*
+ * Drain stale causes only after register reconstruction is
+ * complete. DRSTA and DEV_RST_SET together close the window in
+ * which another device reset can arrive while interrupts are
+ * masked.
+ */
+ icr = E1000_READ_REG(&sc->hw, E1000_ICR);
+ if (igb_finish_device_reset(sc, icr)) {
+ iflib_init_failed(ctx);
+ return;
+ }
+ E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC);
}
- E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC);
}
/*
@@ -1930,7 +2252,8 @@ em_newitr(struct e1000_softc *sc, struct em_rx_queue *que,
} else if (sc->link_speed < SPEED_1000) {
/* Use half default (4K) ITR if sub-gig */
newitr = EM_INTS_4K;
- } else if (sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) {
+ } else if (!sc->vf_ifp &&
+ sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) {
/* Want at least enough packet buffer for two frames to AIM */
newitr = em_max_interrupt_rate;
} else {
@@ -2669,8 +2992,7 @@ static u32
igb_device_reset_intr_mask(struct e1000_softc *sc)
{
- return (!sc->vf_ifp && sc->hw.mac.type >= e1000_82580 ?
- E1000_IMS_DRSTA : 0);
+ return (sc->hw.mac.type >= e1000_82580 ? E1000_IMS_DRSTA : 0);
}
/* Keep interrupt-side work quiesced until device-reset recovery completes. */
@@ -3167,11 +3489,27 @@ em_msix_link(void *arg)
++sc->link_irq;
MPASS(sc->hw.back != NULL);
+ /*
+ * The VF's admin vector represents mailbox and link activity. It has
+ * no PF ICR at E1000_ICR, so process every admin-vector interrupt,
+ * matching the igbvf misc-vector model.
+ */
+ if (sc->vf_ifp) {
+ sc->hw.mac.get_link_status = true;
+ iflib_admin_intr_deferred(sc->ctx);
+ E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask);
+ return (FILTER_HANDLED);
+ }
+
reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR);
if (igb_device_reset_pending(sc))
return (FILTER_HANDLED);
- /* Ignore transient all-ones MMIO reads as interrupt causes. */
+ /*
+ * Enabling or disabling SR-IOV can briefly make PF MMIO reads return
+ * all ones. This is not an interrupt cause; in particular, do not
+ * turn it into a malicious-driver event.
+ */
if (__predict_false(reg_icr == 0xffffffff))
goto rearm;
if (igb_handle_device_reset(sc, reg_icr))
@@ -3182,6 +3520,10 @@ em_msix_link(void *arg)
if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
em_handle_link(sc->ctx);
+ if (reg_icr & E1000_ICR_MDDET)
+ igb_iov_mdd_event(sc);
+ if (reg_icr & E1000_ICR_VMMB)
+ iflib_admin_intr_deferred(sc->ctx);
em_handle_fatal_error_intr(sc, reg_icr);
rearm:
@@ -3189,6 +3531,7 @@ rearm:
if (sc->hw.mac.type >= igb_mac_min) {
E1000_WRITE_REG(&sc->hw, E1000_IMS,
E1000_IMS_LSC | igb_device_reset_intr_mask(sc) |
+ igb_iov_intr_mask(sc) |
em_fatal_error_intr_mask(sc));
E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask);
} else if (sc->hw.mac.type == e1000_82574) {
@@ -3326,11 +3669,68 @@ em_if_media_change(if_ctx_t ctx)
static int
em_if_set_promisc(if_ctx_t ctx, int flags)
{
+ struct e1000_softc *sc;
+
+ sc = iflib_get_softc(ctx);
+ if (em_if_defer_promisc(sc))
+ return (0);
+ return (em_if_set_promisc_impl(ctx, flags));
+}
+
+static bool
+em_if_defer_promisc(struct e1000_softc *sc)
+{
+ switch (sc->hw.mac.type) {
+ case e1000_82576:
+ case e1000_i350:
+ case e1000_vfadapt:
+ case e1000_vfadapt_i350:
+ break;
+ default:
+ return (false);
+ }
+
+ /*
+ * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox
+ * and IOV register operations later from the locked admin task.
+ * A deferred VF mailbox rejection cannot be returned to ifconfig; the
+ * admin task logs it instead.
+ */
+ atomic_set_32(&sc->promisc_pending, 1);
+ iflib_admin_intr_deferred(sc->ctx);
+ return (true);
+}
+
+int
+em_if_set_promisc_impl(if_ctx_t ctx, int flags)
+{
struct e1000_softc *sc = iflib_get_softc(ctx);
if_t ifp = iflib_get_ifp(ctx);
+ enum e1000_promisc_type type;
+ s32 error;
u32 reg_rctl;
int mcnt = 0;
+ /* Hyper-V receive-mode policy is configured through the host. */
+ if (igbv_is_hyperv(sc))
+ return (0);
+
+ if (sc->vf_ifp) {
+ if (flags & IFF_PROMISC)
+ type = e1000_promisc_enabled;
+ else if (flags & IFF_ALLMULTI)
+ type = e1000_promisc_multicast;
+ else
+ type = e1000_promisc_disabled;
+ error = e1000_promisc_set_vf(&sc->hw, type);
+ if (error != E1000_SUCCESS) {
+ device_printf(sc->dev,
+ "VF promiscuous-mode request failed\n");
+ return (EPERM);
+ }
+ return (0);
+ }
+
reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL);
reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE);
if (flags & IFF_ALLMULTI)
@@ -3349,16 +3749,21 @@ em_if_set_promisc(if_ctx_t ctx, int flags)
if (em_debug_sbp)
reg_rctl |= E1000_RCTL_SBP;
E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl);
- em_if_vlan_filter_disable(sc);
+ if (igb_iov_enabled(sc))
+ em_if_vlan_filter_enable(sc);
+ else
+ em_if_vlan_filter_disable(sc);
} else {
if (flags & IFF_ALLMULTI) {
reg_rctl |= E1000_RCTL_MPE;
reg_rctl &= ~E1000_RCTL_UPE;
E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl);
}
- if (em_if_vlan_filter_used(ctx))
+ if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx))
em_if_vlan_filter_enable(sc);
}
+ igb_iov_update_pf_vmolr(sc);
+ igb_iov_rebuild_vlan(sc);
return (0);
}
@@ -3420,7 +3825,14 @@ em_if_multi_set(if_ctx_t ctx)
mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta);
- if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
+ if (sc->vf_ifp) {
+ e1000_update_mc_addr_list(&sc->hw, mta, mcnt);
+ igbv_update_uc_addr_list(sc, ifp);
+ return;
+ }
+
+ if (mcnt < MAX_NUM_MULTICAST_ADDRESSES &&
+ !igb_iov_enabled(sc))
e1000_update_mc_addr_list(&sc->hw, mta, mcnt);
reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL);
@@ -3445,6 +3857,8 @@ em_if_multi_set(if_ctx_t ctx)
if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
e1000_pci_set_mwi(&sc->hw);
}
+ igb_iov_rebuild_mta(sc);
+ igb_iov_update_pf_vmolr(sc);
}
/*********************************************************************
@@ -3477,9 +3891,10 @@ em_if_update_admin_status(if_ctx_t ctx)
u32 link_check, thstat, ctrl;
bool reset_requested = false;
+ KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__));
if (em_handle_fatal_error_admin(sc))
return;
- /* A sibling-port reset invalidated the port registers. */
+ /* A sibling-port reset invalidated the registers and VF mailboxes. */
if (atomic_cmpset_acq_32(&sc->device_reset_state,
IGB_DEVICE_RESET_DETECTED, IGB_DEVICE_RESET_REQUESTED)) {
if (sc->link_state == EM_LINK_STATE_UP)
@@ -3497,6 +3912,12 @@ em_if_update_admin_status(if_ctx_t ctx)
IGB_DEVICE_RESET_NONE)
return;
+ if (atomic_readandclear_32(&sc->promisc_pending) != 0)
+ (void)em_if_set_promisc_impl(ctx,
+ if_getflags(iflib_get_ifp(ctx)));
+ igb_iov_handle_mdd(sc);
+ igb_iov_handle_mbx(sc);
+
link_check = thstat = ctrl = 0;
/* Get the cached link value or read phy for real */
switch (hw->phy.media_type) {
@@ -3522,11 +3943,6 @@ em_if_update_admin_status(if_ctx_t ctx)
e1000_check_for_link(hw);
link_check = hw->mac.serdes_has_link;
break;
- /* VF device is type_unknown */
- case e1000_media_type_unknown:
- e1000_check_for_link(hw);
- link_check = !hw->mac.get_link_status;
- /* FALLTHROUGH */
default:
break;
}
@@ -3563,7 +3979,8 @@ em_if_update_admin_status(if_ctx_t ctx)
"Full Duplex" : "Half Duplex"));
sc->link_state = EM_LINK_STATE_UP;
sc->smartspeed = 0;
- if ((ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) ==
+ if (hw->mac.type == e1000_i350 &&
+ (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) ==
E1000_CTRL_EXT_LINK_MODE_GMII &&
(thstat & E1000_THSTAT_LINK_THROTTLE))
device_printf(dev, "Link: thermal downshift\n");
@@ -3595,6 +4012,7 @@ em_if_update_admin_status(if_ctx_t ctx)
iflib_link_state_change(ctx, LINK_STATE_UP,
IF_Mbps(sc->link_speed));
}
+ igb_iov_ping_all_vfs(sc);
} else if (!link_check &&
(sc->link_state == EM_LINK_STATE_UP ||
sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) {
@@ -3610,6 +4028,7 @@ em_if_update_admin_status(if_ctx_t ctx)
EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN;
if (link_was_published)
iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ igb_iov_ping_all_vfs(sc);
}
/*
* Mailbox, link, and timer events share this admin task. The PF
@@ -3651,6 +4070,65 @@ em_if_update_admin_status(if_ctx_t ctx)
lem_smartspeed(sc);
}
+/*
+ * Last-resort DMA fence. iflib releases DMA mappings after the driver's
+ * stop callback, so continuing with bus mastering still enabled would turn
+ * a recoverable NIC failure into memory corruption. Treat failure of the
+ * PCI command bit as a fail-stop invariant violation.
+ */
+static void
+em_fence_pci_busmaster(struct e1000_softc *sc)
+{
+ device_t dev;
+ u_int timeout;
+ u16 command;
+ int error;
+
+ dev = sc->dev;
+ error = pci_disable_busmaster(dev);
+ command = pci_read_config(dev, PCIR_COMMAND, 2);
+ if (command != 0xffff && (command & PCIM_CMD_BUSMASTEREN) != 0)
+ panic("%s: unable to fence device DMA (error %d)",
+ device_get_nameunit(dev), error);
+ if (error != 0 && command != 0xffff)
+ device_printf(dev,
+ "PCI bus-master disable returned %d; readback is disabled\n",
+ error);
+
+ timeout = max(pcie_get_max_completion_timeout(dev) / 1000, 10);
+ if (command != 0xffff &&
+ !pcie_wait_for_pending_transactions(dev, timeout)) {
+ /* A function removed during the wait can no longer issue DMA. */
+ command = pci_read_config(dev, PCIR_COMMAND, 2);
+ if (command != 0xffff)
+ panic("%s: DMA transactions remain pending after fencing",
+ device_get_nameunit(dev));
+ }
+}
+
+static int
+em_enable_pci_busmaster(struct e1000_softc *sc)
+{
+ device_t dev;
+ u16 command;
+ int error;
+
+ dev = sc->dev;
+ command = pci_read_config(dev, PCIR_COMMAND, 2);
+ if (command == 0xffff)
+ return (ENXIO);
+ if ((command & PCIM_CMD_BUSMASTEREN) != 0)
+ return (0);
+
+ error = pci_enable_busmaster(dev);
+ command = pci_read_config(dev, PCIR_COMMAND, 2);
+ if (command == 0xffff)
+ return (ENXIO);
+ if ((command & PCIM_CMD_BUSMASTEREN) == 0)
+ return (error != 0 ? error : EIO);
+ return (0);
+}
+
/*********************************************************************
*
* This routine disables all traffic on the adapter by issuing a
@@ -3665,22 +4143,48 @@ em_if_stop(if_ctx_t ctx)
INIT_DEBUGOUT("em_if_stop: begin");
+ if (sc->vf_ifp) {
+ igbv_queue_retry_stop(sc);
+ igbv_mbx_retry_stop(sc);
+ }
+
/* I219 needs special flushing to avoid hangs */
if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min)
em_flush_desc_rings(sc);
- em_prepare_fatal_error_reset(sc);
- error = e1000_reset_hw(&sc->hw);
- if (!sc->vf_ifp && error != E1000_SUCCESS) {
- device_printf(sc->dev, "Hardware reset failed while "
- "stopping: %d\n", error);
- return;
+ igb_iov_reset_prepare(sc);
+ if (!sc->vf_ifp ||
+ (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 &&
+ (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) {
+ em_prepare_fatal_error_reset(sc);
+ error = e1000_reset_hw(&sc->hw);
+ if (!sc->vf_ifp && error != E1000_SUCCESS) {
+ device_printf(sc->dev, "Hardware reset failed while "
+ "stopping: %d\n", error);
+ em_fence_pci_busmaster(sc);
+ return;
+ }
+ }
+ if (sc->vf_ifp) {
+ sc->vf_queues_sanitized = igbv_sanitize_queues(sc);
+ atomic_store_rel_32(&sc->vf_mbx_ready, 0);
+ if (!sc->vf_queues_sanitized)
+ em_fence_pci_busmaster(sc);
}
if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp)
E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0);
- e1000_led_off(&sc->hw);
- e1000_cleanup_led(&sc->hw);
+ if (!sc->vf_ifp) {
+ e1000_led_off(&sc->hw);
+ e1000_cleanup_led(&sc->hw);
+ } else {
+ sc->link_speed = 0;
+ sc->link_duplex = 0;
+ if (sc->link_state != EM_LINK_STATE_DOWN) {
+ sc->link_state = EM_LINK_STATE_DOWN;
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ }
+ }
}
/*********************************************************************
@@ -3710,12 +4214,15 @@ em_identify_hardware(if_ctx_t ctx)
return;
}
- /* Are we a VF device? */
- if ((sc->hw.mac.type == e1000_vfadapt) ||
- (sc->hw.mac.type == e1000_vfadapt_i350))
- sc->vf_ifp = 1;
- else
- sc->vf_ifp = 0;
+ /*
+ * Function type comes from the selected iflib shared context, not from
+ * enum ordering. Keep the detected MAC type as an independent check
+ * that the igb/igbv probe tables selected the right policy.
+ */
+ KASSERT(sc->vf_ifp ==
+ (sc->hw.mac.type == e1000_vfadapt ||
+ sc->hw.mac.type == e1000_vfadapt_i350),
+ ("%s: iflib function type and MAC type disagree", __func__));
}
static int
@@ -3885,8 +4392,16 @@ igb_configure_queues(struct e1000_softc *sc)
struct em_tx_queue *tx_que;
u32 tmp, ivar = 0;
- /* First turn on RSS capability */
- if (hw->mac.type != e1000_82575)
+ /*
+ * Queue ownership can change when SR-IOV is enabled or disabled.
+ * Rebuild the interrupt mask for the current layout instead of
+ * retaining vectors from a previous initialization.
+ */
+ sc->que_mask = 0;
+ sc->link_mask = 0;
+
+ /* GPIE controls the PF interrupt block and is not in the VF BAR. */
+ if (!sc->vf_ifp && hw->mac.type != e1000_82575)
E1000_WRITE_REG(hw, E1000_GPIE,
E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
E1000_GPIE_PBA | E1000_GPIE_NSICR);
@@ -3902,10 +4417,13 @@ igb_configure_queues(struct e1000_softc *sc)
case e1000_vfadapt_i350:
/* RX entries */
for (int i = 0; i < sc->rx_num_queues; i++) {
- u32 index = i >> 1;
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ uint32_t index, qid;
+
rx_que = &sc->rx_queues[i];
- if (i & 1) {
+ qid = rx_que->rxr.me;
+ index = qid >> 1;
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ if (qid & 1) {
ivar &= 0xFF00FFFF;
ivar |= (rx_que->msix | E1000_IVAR_VALID) <<
16;
@@ -3914,13 +4432,17 @@ igb_configure_queues(struct e1000_softc *sc)
ivar |= rx_que->msix | E1000_IVAR_VALID;
}
E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
+ sc->que_mask |= rx_que->eims;
}
/* TX entries */
for (int i = 0; i < sc->tx_num_queues; i++) {
- u32 index = i >> 1;
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ uint32_t index, qid;
+
tx_que = &sc->tx_queues[i];
- if (i & 1) {
+ qid = tx_que->txr.me;
+ index = qid >> 1;
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ if (qid & 1) {
ivar &= 0x00FFFFFF;
ivar |= (tx_que->msix | E1000_IVAR_VALID) <<
24;
@@ -3934,17 +4456,27 @@ igb_configure_queues(struct e1000_softc *sc)
}
/* And for the link interrupt */
- ivar = (sc->linkvec | E1000_IVAR_VALID) << 8;
+ if (sc->vf_ifp) {
+ /*
+ * VTIVAR_MISC maps the VF mailbox in bits 7:0.
+ * The PF IVAR_MISC maps other causes in bits 15:8.
+ */
+ ivar = sc->linkvec | E1000_IVAR_VALID;
+ } else
+ ivar = (sc->linkvec | E1000_IVAR_VALID) << 8;
sc->link_mask = 1 << sc->linkvec;
E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
break;
case e1000_82576:
/* RX entries */
for (int i = 0; i < sc->rx_num_queues; i++) {
- u32 index = i & 0x7; /* Each IVAR has two entries */
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ uint32_t index, qid;
+
rx_que = &sc->rx_queues[i];
- if (i < 8) {
+ qid = rx_que->rxr.me;
+ index = qid & 0x7; /* Each IVAR has two entries */
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ if (qid < 8) {
ivar &= 0xFFFFFF00;
ivar |= rx_que->msix | E1000_IVAR_VALID;
} else {
@@ -3957,10 +4489,13 @@ igb_configure_queues(struct e1000_softc *sc)
}
/* TX entries */
for (int i = 0; i < sc->tx_num_queues; i++) {
- u32 index = i & 0x7; /* Each IVAR has two entries */
- ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ uint32_t index, qid;
+
tx_que = &sc->tx_queues[i];
- if (i < 8) {
+ qid = tx_que->txr.me;
+ index = qid & 0x7; /* Each IVAR has two entries */
+ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
+ if (qid < 8) {
ivar &= 0xFFFF00FF;
ivar |= (tx_que->msix | E1000_IVAR_VALID) <<
8;
@@ -4029,6 +4564,8 @@ igb_initialize_interrupt_rate(struct e1000_softc *sc)
E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix),
rx_que->itr_setting);
}
+ if (sc->intr_type == IFLIB_INTR_MSIX)
+ E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr);
}
static void
@@ -4167,9 +4704,22 @@ igb_init_dmac(struct e1000_softc *sc, u32 pba)
u16 hwm;
u16 max_frame_size;
+ KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF",
+ __func__));
+
if (hw->mac.type == e1000_i211)
return;
+ /*
+ * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve
+ * the configured value so it can be restored after IOV is disabled.
+ */
+ if (igb_iov_enabled(sc)) {
+ if (hw->mac.type > e1000_82580)
+ igb_disable_dmac(hw);
+ return;
+ }
+
max_frame_size = sc->shared->isc_max_frame_size;
if (hw->mac.type > e1000_82580) {
@@ -4400,6 +4950,8 @@ em_reset(if_ctx_t ctx)
s32 error;
INIT_DEBUGOUT("em_reset: begin");
+ KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__));
+
/* Let the firmware know the OS is in control */
em_get_hw_control(sc);
@@ -4473,14 +5025,12 @@ em_reset(if_ctx_t ctx)
pba = E1000_PBA_32K;
break;
case e1000_82576:
- case e1000_vfadapt:
pba = E1000_READ_REG(hw, E1000_RXPBS);
pba &= E1000_RXPBS_SIZE_MASK_82576;
break;
case e1000_82580:
case e1000_i350:
case e1000_i354:
- case e1000_vfadapt_i350:
pba = E1000_READ_REG(hw, E1000_RXPBS);
pba = e1000_rxpbs_adjust_82580(pba);
break;
@@ -4525,7 +5075,7 @@ em_reset(if_ctx_t ctx)
if (hw->mac.type < igb_mac_min)
E1000_WRITE_REG(hw, E1000_PBA, pba);
- INIT_DEBUGOUT1("em_reset: pba=%dK",pba);
+ INIT_DEBUGOUT1("em_reset: pba=%dK", pba);
/*
* These parameters control the automatic generation (Tx) and
@@ -4604,8 +5154,6 @@ em_reset(if_ctx_t ctx)
case e1000_i354:
case e1000_i210:
case e1000_i211:
- case e1000_vfadapt:
- case e1000_vfadapt_i350:
/* 16-byte granularity */
hw->fc.low_water = hw->fc.high_water - 16;
break;
@@ -4631,17 +5179,14 @@ em_reset(if_ctx_t ctx)
em_prepare_fatal_error_reset(sc);
error = e1000_reset_hw(hw);
if (error != E1000_SUCCESS) {
- if (!sc->vf_ifp)
- device_printf(dev, "Hardware reset failed: %d\n", error);
+ device_printf(dev, "Hardware reset failed: %d\n", error);
return (error);
}
- if (!sc->vf_ifp) {
- if (hw->mac.type >= igb_mac_min) {
- E1000_WRITE_REG(hw, E1000_WUC, 0);
- } else {
- E1000_WRITE_REG(hw, E1000_WUFC, 0);
- em_disable_aspm(sc);
- }
+ if (hw->mac.type >= igb_mac_min) {
+ E1000_WRITE_REG(hw, E1000_WUC, 0);
+ } else {
+ E1000_WRITE_REG(hw, E1000_WUFC, 0);
+ em_disable_aspm(sc);
}
if (sc->flags & IGB_MEDIA_RESET) {
e1000_setup_init_funcs(hw, true);
@@ -4660,7 +5205,7 @@ em_reset(if_ctx_t ctx)
if (hw->mac.type >= igb_mac_min)
igb_init_dmac(sc, pba);
- /* Save the final PBA off if it needs to be used elsewhere i.e. AIM */
+ /* Save the receive packet-buffer allocation for AIM. */
sc->pba = pba;
E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
@@ -4835,6 +5380,14 @@ em_setup_interface(if_ctx_t ctx)
* Specify the media types supported by this adapter and register
* callbacks to update media and link information
*/
+ if (sc->vf_ifp) {
+ ifmedia_add(sc->media,
+ IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
+ ifmedia_set(sc->media,
+ IFM_ETHER | IFM_1000_T | IFM_FDX);
+ return (0);
+ }
+
if (sc->hw.phy.media_type == e1000_media_type_fiber ||
sc->hw.phy.media_type == e1000_media_type_internal_serdes) {
u_char fiber_type = IFM_1000_SX; /* default type */
@@ -5098,24 +5651,24 @@ igb_txdctl(struct e1000_hw *hw)
* Enable transmit unit.
*
**********************************************************************/
-static void
-em_initialize_transmit_unit(if_ctx_t ctx)
+void
+em_initialize_transmit_rings(if_ctx_t ctx)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
if_softc_ctx_t scctx = sc->shared;
struct em_tx_queue *que;
struct tx_ring *txr;
struct e1000_hw *hw = &sc->hw;
- u32 tctl, txdctl = 0, tarc, tipg = 0;
-
- INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
+ u32 txdctl;
- for (int i = 0; i < sc->tx_num_queues; i++, txr++) {
+ for (int i = 0; i < sc->tx_num_queues; i++) {
u64 bus_addr;
caddr_t offp, endp;
+ uint32_t qid;
que = &sc->tx_queues[i];
txr = &que->txr;
+ qid = txr->me;
bus_addr = txr->tx_paddr;
/* Clear checksum offload context. */
@@ -5124,32 +5677,45 @@ em_initialize_transmit_unit(if_ctx_t ctx)
memset(offp, 0, endp - offp);
if (hw->mac.type >= igb_mac_min) {
- txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
- E1000_WRITE_REG(hw, E1000_TXDCTL(i),
+ txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid));
+ E1000_WRITE_REG(hw, E1000_TXDCTL(qid),
txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
E1000_WRITE_FLUSH(hw);
}
/* Base and Len of TX Ring */
- E1000_WRITE_REG(hw, E1000_TDLEN(i),
+ E1000_WRITE_REG(hw, E1000_TDLEN(qid),
scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc));
- E1000_WRITE_REG(hw, E1000_TDBAH(i), (u32)(bus_addr >> 32));
- E1000_WRITE_REG(hw, E1000_TDBAL(i), (u32)bus_addr);
+ E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32));
+ E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr);
/* Init the HEAD/TAIL indices */
- E1000_WRITE_REG(hw, E1000_TDT(i), 0);
- E1000_WRITE_REG(hw, E1000_TDH(i), 0);
+ E1000_WRITE_REG(hw, E1000_TDT(qid), 0);
+ E1000_WRITE_REG(hw, E1000_TDH(qid), 0);
HW_DEBUGOUT2("Base = %x, Length = %x\n",
- E1000_READ_REG(hw, E1000_TDBAL(i)),
- E1000_READ_REG(hw, E1000_TDLEN(i)));
+ E1000_READ_REG(hw, E1000_TDBAL(qid)),
+ E1000_READ_REG(hw, E1000_TDLEN(qid)));
if (hw->mac.type < igb_mac_min)
txdctl = em_legacy_txdctl(hw);
else
txdctl = igb_txdctl(hw);
- E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
+ E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl);
}
+}
+
+static void
+em_initialize_transmit_unit(if_ctx_t ctx)
+{
+ struct e1000_softc *sc = iflib_get_softc(ctx);
+ struct e1000_hw *hw = &sc->hw;
+ u32 tctl, tarc, tipg = 0;
+
+ INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
+ KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__));
+
+ em_initialize_transmit_rings(ctx);
/* Set the default values for the Tx Inter Packet Gap timer */
switch (hw->mac.type) {
@@ -5290,6 +5856,51 @@ igb_rxdctl(struct e1000_softc *sc, u32 rxdctl)
return (rxdctl);
}
+void
+igb_initialize_receive_rings(if_ctx_t ctx, bool drop)
+{
+ struct e1000_softc *sc = iflib_get_softc(ctx);
+ if_softc_ctx_t scctx = sc->shared;
+ struct e1000_hw *hw = &sc->hw;
+ struct em_rx_queue *que;
+ u32 srrctl;
+
+ srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
+ E1000_SRRCTL_BSIZEPKT_SHIFT;
+ srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
+ if (drop)
+ srrctl |= E1000_SRRCTL_DROP_EN;
+
+ for (int i = 0; i < sc->rx_num_queues; i++) {
+ struct rx_ring *rxr;
+ u64 bus_addr;
+ u32 rxdctl;
+ uint32_t qid;
+
+ que = &sc->rx_queues[i];
+ rxr = &que->rxr;
+ bus_addr = rxr->rx_paddr;
+ qid = rxr->me;
+
+ rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid));
+ E1000_WRITE_REG(hw, E1000_RXDCTL(qid),
+ rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
+ E1000_WRITE_FLUSH(hw);
+
+ E1000_WRITE_REG(hw, E1000_RDLEN(qid),
+ scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc));
+ E1000_WRITE_REG(hw, E1000_RDBAH(qid),
+ (uint32_t)(bus_addr >> 32));
+ E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr);
+ E1000_WRITE_REG(hw, E1000_RDH(qid), 0);
+ E1000_WRITE_REG(hw, E1000_RDT(qid), 0);
+ E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl);
+
+ rxdctl = igb_rxdctl(sc, rxdctl);
+ E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl);
+ }
+}
+
static bool
em_integrated_jumbo_rx(struct e1000_hw *hw)
{
@@ -5324,32 +5935,29 @@ em_initialize_receive_unit(if_ctx_t ctx)
uint32_t rctl, rxcsum;
INIT_DEBUGOUT("em_initialize_receive_units: begin");
+ KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__));
/*
- * Make sure receives are disabled while setting
- * up the descriptor ring
+ * Make sure receives are disabled while setting up the descriptor
+ * ring.
*/
rctl = E1000_READ_REG(hw, E1000_RCTL);
- /* Do not disable if ever enabled on this hardware */
- if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
+ /* Do not disable if ever enabled on this hardware. */
+ if (hw->mac.type != e1000_82574 &&
+ hw->mac.type != e1000_82583)
E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
- /* Setup the Receive Control Register */
+ /* Setup the Receive Control Register. */
rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
-
- /* Do not store bad packets */
rctl &= ~E1000_RCTL_SBP;
- /* Enable Long Packet receive */
- if (if_getmtu(ifp) > ETHERMTU)
+ if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU)
rctl |= E1000_RCTL_LPE;
else
rctl &= ~E1000_RCTL_LPE;
-
- /* Strip the CRC */
if (!em_disable_crc_stripping)
rctl |= E1000_RCTL_SECRC;
@@ -5390,7 +5998,7 @@ em_initialize_receive_unit(if_ctx_t ctx)
sc->rx_int_delay.value);
}
- if (hw->mac.type >= em_mac_min && !sc->vf_ifp) {
+ if (hw->mac.type >= em_mac_min) {
uint32_t rfctl;
/* Use extended rx descriptor formats */
rfctl = E1000_READ_REG(hw, E1000_RFCTL);
@@ -5410,55 +6018,52 @@ em_initialize_receive_unit(if_ctx_t ctx)
E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
}
- /*
- * Set up L3 and L4 csum Rx descriptor offloads only on Physical
- * Functions. Virtual Functions have no access to this register.
- */
- if (!sc->vf_ifp) {
- rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
- if (if_getcapenable(ifp) & IFCAP_RXCSUM) {
- rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL;
- if (hw->mac.type > e1000_82575)
- rxcsum |= E1000_RXCSUM_CRCOFL;
- else if (hw->mac.type < em_mac_min &&
- if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6)
- rxcsum |= E1000_RXCSUM_IPV6OFL;
- } else {
- rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
- if (hw->mac.type > e1000_82575)
- rxcsum &= ~E1000_RXCSUM_CRCOFL;
- else if (hw->mac.type < em_mac_min)
- rxcsum &= ~E1000_RXCSUM_IPV6OFL;
- }
+ rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
+ if (if_getcapenable(ifp) & IFCAP_RXCSUM) {
+ rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL;
+ if (hw->mac.type > e1000_82575)
+ rxcsum |= E1000_RXCSUM_CRCOFL;
+ else if (hw->mac.type < em_mac_min &&
+ if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6)
+ rxcsum |= E1000_RXCSUM_IPV6OFL;
+ } else {
+ rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
+ if (hw->mac.type > e1000_82575)
+ rxcsum &= ~E1000_RXCSUM_CRCOFL;
+ else if (hw->mac.type < em_mac_min)
+ rxcsum &= ~E1000_RXCSUM_IPV6OFL;
+ }
- if (sc->rx_num_queues > 1) {
- /* RSS hash needed in the Rx descriptor */
- rxcsum |= E1000_RXCSUM_PCSD;
+ if (sc->rx_num_queues > 1) {
+ /* RSS hash needed in the Rx descriptor */
+ rxcsum |= E1000_RXCSUM_PCSD;
- if (hw->mac.type >= igb_mac_min)
- igb_initialize_rss_mapping(sc);
- else
- em_initialize_rss_mapping(sc);
- }
- E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
+ if (hw->mac.type >= igb_mac_min)
+ igb_initialize_rss_mapping(sc);
+ else
+ em_initialize_rss_mapping(sc);
}
+ E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
- for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; i++, que++) {
+ for (i = 0, que = sc->rx_queues;
+ hw->mac.type < igb_mac_min && i < sc->rx_num_queues;
+ i++, que++) {
struct rx_ring *rxr = &que->rxr;
/* Setup the Base and Length of the Rx Descriptor Ring */
u64 bus_addr = rxr->rx_paddr;
+ uint32_t qid = rxr->me;
#if 0
u32 rdt = sc->rx_num_queues -1; /* default */
#endif
- E1000_WRITE_REG(hw, E1000_RDLEN(i),
+ E1000_WRITE_REG(hw, E1000_RDLEN(qid),
scctx->isc_nrxd[0] *
sizeof(union e1000_rx_desc_extended));
- E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
- E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
+ E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32));
+ E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr);
/* Setup the Head and Tail Descriptor Pointers */
- E1000_WRITE_REG(hw, E1000_RDH(i), 0);
- E1000_WRITE_REG(hw, E1000_RDT(i), 0);
+ E1000_WRITE_REG(hw, E1000_RDH(qid), 0);
+ E1000_WRITE_REG(hw, E1000_RDT(qid), 0);
}
/* Increase receive-descriptor prefetching for integrated jumbo MACs. */
@@ -5483,24 +6088,21 @@ em_initialize_receive_unit(if_ctx_t ctx)
E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
}
} else if (hw->mac.type >= igb_mac_min) {
- u32 psize, srrctl = 0;
+ bool drop;
+ u32 psize;
- if (if_getmtu(ifp) > ETHERMTU) {
+ if (igb_iov_enabled(sc)) {
+ E1000_WRITE_REG(hw, E1000_RLPML,
+ IGB_IOV_MAX_FRAME_SIZE);
+ } else if (if_getmtu(ifp) > ETHERMTU) {
psize = scctx->isc_max_frame_size;
/* are we on a vlan? */
if (if_vlantrunkinuse(ifp))
psize += VLAN_TAG_SIZE;
- if (sc->vf_ifp)
- e1000_rlpml_set_vf(hw, psize);
- else
- E1000_WRITE_REG(hw, E1000_RLPML, psize);
+ E1000_WRITE_REG(hw, E1000_RLPML, psize);
}
- /* Set maximum packet buffer len */
- srrctl |= (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
- E1000_SRRCTL_BSIZEPKT_SHIFT;
-
/*
* If TX flow control is disabled and there's >1 queue
* defined, enable DROP.
@@ -5508,43 +6110,11 @@ em_initialize_receive_unit(if_ctx_t ctx)
* This drops frames rather than hanging the RX MAC for all
* queues.
*/
- if ((sc->rx_num_queues > 1) &&
+ drop = igb_iov_enabled(sc) ||
+ ((sc->rx_num_queues > 1) &&
(sc->fc == e1000_fc_none ||
- sc->fc == e1000_fc_rx_pause)) {
- srrctl |= E1000_SRRCTL_DROP_EN;
- }
- /* Setup the Base and Length of the Rx Descriptor Rings */
- for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues;
- i++, que++) {
- struct rx_ring *rxr = &que->rxr;
- u64 bus_addr = rxr->rx_paddr;
- u32 rxdctl;
-
-#ifdef notyet
- /* Configure for header split? -- ignore for now */
- rxr->hdr_split = igb_header_split;
-#else
- srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
-#endif
-
- rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
- E1000_WRITE_REG(hw, E1000_RXDCTL(i),
- rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
- E1000_WRITE_FLUSH(hw);
-
- E1000_WRITE_REG(hw, E1000_RDLEN(i),
- scctx->isc_nrxd[0] *
- sizeof(struct e1000_rx_desc));
- E1000_WRITE_REG(hw, E1000_RDBAH(i),
- (uint32_t)(bus_addr >> 32));
- E1000_WRITE_REG(hw, E1000_RDBAL(i),
- (uint32_t)bus_addr);
- E1000_WRITE_REG(hw, E1000_RDH(i), 0);
- E1000_WRITE_REG(hw, E1000_RDT(i), 0);
- E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
- rxdctl = igb_rxdctl(sc, rxdctl);
- E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
- }
+ sc->fc == e1000_fc_rx_pause));
+ igb_initialize_receive_rings(ctx, drop);
} else if (hw->mac.type >= e1000_pch2lan) {
if (if_getmtu(ifp) > ETHERMTU)
e1000_lv_jumbo_workaround_ich8lan(hw, true);
@@ -5589,37 +6159,75 @@ static void
em_if_vlan_register(if_ctx_t ctx, u16 vtag)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
- u32 index, bit;
+ bool present;
+ u32 index, mask;
+
+ /* Hyper-V supports host-assigned access VLANs, not guest VLANs. */
+ if (igbv_is_hyperv(sc))
+ return;
index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- sc->shadow_vfta[index] |= (1 << bit);
- ++sc->num_vlans;
- if (!sc->vf_ifp)
- em_if_vlan_filter_write(sc, index);
- else
- /*
- * Physical funtion may reject registering VLAN
- * but we have no way to inform the stack
- * about that.
- */
- e1000_vfta_set_vf(&sc->hw, vtag, true);
+ mask = 1U << (vtag & 0x1F);
+ present = (sc->shadow_vfta[index] & mask) != 0;
+ /*
+ * On a VF, record registration intent for replay even if the PF is not
+ * ready to accept it yet.
+ */
+ sc->shadow_vfta[index] |= mask;
+ sc->vf_vfta_stale[index] &= ~mask;
+ if (!present)
+ ++sc->num_vlans;
+ if (sc->vf_ifp &&
+ e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) {
+ igbv_vlan_retry_add(sc, vtag);
+ device_printf(sc->dev,
+ "VF VLAN %u add request failed\n", vtag);
+ } else if (sc->vf_ifp)
+ igbv_vlan_retry_clear(sc, vtag);
+ if (!sc->vf_ifp) {
+ if (igb_iov_enabled(sc))
+ igb_iov_rebuild_vlan(sc);
+ else
+ em_if_vlan_filter_write(sc, index);
+ }
}
static void
em_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
- u32 index, bit;
+ bool present;
+ u32 index, mask;
+
+ if (igbv_is_hyperv(sc))
+ return;
index = (vtag >> 5) & 0x7F;
- bit = vtag & 0x1F;
- sc->shadow_vfta[index] &= ~(1 << bit);
- --sc->num_vlans;
- if (!sc->vf_ifp)
- em_if_vlan_filter_write(sc, index);
- else
- e1000_vfta_set_vf(&sc->hw, vtag, false);
+ mask = 1U << (vtag & 0x1F);
+ present = (sc->shadow_vfta[index] & mask) != 0;
+ if (sc->vf_ifp)
+ igbv_vlan_retry_clear(sc, vtag);
+ if (sc->vf_ifp &&
+ e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) {
+ device_printf(sc->dev,
+ "VF VLAN %u remove request failed\n", vtag);
+ /*
+ * Hardware might still admit this VID. Preserve its receive
+ * tag until a successful VF reset proves the stale filter gone.
+ */
+ sc->vf_vfta_stale[index] |= mask;
+ } else {
+ sc->vf_vfta_stale[index] &= ~mask;
+ }
+ sc->shadow_vfta[index] &= ~mask;
+ if (present)
+ --sc->num_vlans;
+ if (!sc->vf_ifp) {
+ if (igb_iov_enabled(sc))
+ igb_iov_rebuild_vlan(sc);
+ else
+ em_if_vlan_filter_write(sc, index);
+ }
}
static bool
@@ -5703,15 +6311,46 @@ em_setup_vlan_hw_support(if_ctx_t ctx)
struct e1000_softc *sc = iflib_get_softc(ctx);
struct e1000_hw *hw = &sc->hw;
if_t ifp = iflib_get_ifp(ctx);
- u32 reg;
+ s32 error;
+ u32 max_frame_size, reg;
+ u16 vid;
+ int restore_failures;
+
+ /* Hyper-V programs the VF's receive limit and VLAN membership. */
+ if (igbv_is_hyperv(sc))
+ return;
/*
* Only PFs have control over VLAN HW filtering
* configuration. VFs have to act as if it's always
* enabled.
*/
- if (sc->vf_ifp)
+ if (sc->vf_ifp) {
+ max_frame_size = min(sc->shared->isc_max_frame_size +
+ VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE);
+ e1000_rlpml_set_vf(hw, max_frame_size);
+ restore_failures = 0;
+ for (vid = 0; vid < 4096; vid++) {
+ if ((sc->shadow_vfta[vid >> 5] &
+ (1U << (vid & 0x1f))) == 0)
+ continue;
+ /*
+ * Desired state remains in shadow_vfta for the next
+ * replay if the PF mailbox is absent during reset.
+ */
+ error = e1000_vfta_set_vf(hw, vid, true);
+ if (error != E1000_SUCCESS) {
+ igbv_vlan_retry_add(sc, vid);
+ restore_failures++;
+ } else
+ igbv_vlan_retry_clear(sc, vid);
+ }
+ if (restore_failures != 0)
+ device_printf(sc->dev,
+ "VF VLAN restore failed for %d VIDs; retrying\n",
+ restore_failures);
return;
+ }
if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING &&
!em_disable_crc_stripping) {
@@ -5724,11 +6363,25 @@ em_setup_vlan_hw_support(if_ctx_t ctx)
E1000_WRITE_REG(hw, E1000_CTRL, reg);
}
- /* If we aren't doing HW filtering, we're done */
+ /*
+ * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN
+ * filtering is disabled, the IOV VLAN rebuild instead makes the PF
+ * VLAN-promiscuous without disabling the global filter.
+ */
if (!em_if_vlan_filter_capable(ctx)) {
- em_if_vlan_filter_disable(sc);
+ if (igb_iov_enabled(sc)) {
+#ifdef PCI_IOV
+ sc->iov_pf_vlan_promisc = true;
+#endif
+ em_if_vlan_filter_enable(sc);
+ } else
+ em_if_vlan_filter_disable(sc);
return;
}
+#ifdef PCI_IOV
+ if (igb_iov_enabled(sc))
+ sc->iov_pf_vlan_promisc = false;
+#endif
/*
* A soft reset zero's out the VFTA, so
@@ -5775,24 +6428,31 @@ igb_if_intr_enable(if_ctx_t ctx)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
struct e1000_hw *hw = &sc->hw;
- u32 mask;
+ u32 mask, reg;
if (igb_device_reset_pending(sc))
return;
if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) {
mask = (sc->que_mask | sc->link_mask);
- E1000_WRITE_REG(hw, E1000_EIAC, mask);
- E1000_WRITE_REG(hw, E1000_EIAM, mask);
+ /*
+ * VF interrupt controls are also mapped into these registers.
+ * Preserve them and change only the PF vectors we own.
+ */
+ reg = E1000_READ_REG(hw, E1000_EIAC);
+ E1000_WRITE_REG(hw, E1000_EIAC, reg | mask);
+ reg = E1000_READ_REG(hw, E1000_EIAM);
+ E1000_WRITE_REG(hw, E1000_EIAM, reg | mask);
+ igb_iov_intr_drain_stale(sc);
E1000_WRITE_REG(hw, E1000_EIMS, mask);
E1000_WRITE_REG(hw, E1000_IMS,
E1000_IMS_LSC | igb_device_reset_intr_mask(sc) |
+ igb_iov_intr_mask(sc) |
em_fatal_error_intr_mask(sc));
} else {
mask = IMS_ENABLE_MASK | igb_device_reset_intr_mask(sc) |
em_fatal_error_intr_mask(sc);
/* Reading ICR masks every shared interrupt before the filter runs. */
- if (!sc->vf_ifp)
- E1000_WRITE_REG(hw, E1000_IAM, mask);
+ E1000_WRITE_REG(hw, E1000_IAM, mask);
E1000_WRITE_REG(hw, E1000_IMS, mask);
}
E1000_WRITE_FLUSH(hw);
@@ -5803,15 +6463,26 @@ igb_if_intr_disable(if_ctx_t ctx)
{
struct e1000_softc *sc = iflib_get_softc(ctx);
struct e1000_hw *hw = &sc->hw;
+ u32 mask, reg;
/* This is the first CTX-owned register access after ICR.DRSTA. */
- if (!sc->vf_ifp)
- igb_prepare_device_reset(sc);
+ igb_prepare_device_reset(sc);
if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) {
- E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff);
- E1000_WRITE_REG(hw, E1000_EIAC, 0);
- } else if (!sc->vf_ifp)
+ /*
+ * Do not use a blanket EIMC write here. VF interrupt controls
+ * are mapped into the same PF register space, so clearing bits
+ * we do not own can leave running VFs with interrupts masked.
+ * Before initial queue configuration the owned mask is zero
+ * because this driver has not enabled a vector yet.
+ */
+ mask = (sc->que_mask | sc->link_mask);
+ reg = E1000_READ_REG(hw, E1000_EIAM);
+ E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask);
+ E1000_WRITE_REG(hw, E1000_EIMC, mask);
+ reg = E1000_READ_REG(hw, E1000_EIAC);
+ E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask);
+ } else
E1000_WRITE_REG(hw, E1000_IAM, 0);
E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
E1000_WRITE_FLUSH(hw);
@@ -5916,16 +6587,13 @@ em_release_hw_control(struct e1000_softc *sc)
return;
}
-static int
-em_is_valid_ether_addr(u8 *addr)
+bool
+em_is_valid_ether_addr(const u8 *addr)
{
- char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
+ static const u8 zero_addr[ETHER_ADDR_LEN];
- if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
- return (false);
- }
-
- return (true);
+ return (!ETHER_IS_MULTICAST(addr) &&
+ memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0);
}
static bool
@@ -5991,8 +6659,6 @@ em_get_wakeup(if_ctx_t ctx)
switch (sc->hw.mac.type) {
case e1000_82542:
case e1000_82543:
- case e1000_vfadapt:
- case e1000_vfadapt_i350:
break;
case e1000_82544:
error = e1000_read_nvm(&sc->hw,
@@ -6642,7 +7308,7 @@ em_disable_aspm(struct e1000_softc *sc)
* Update the board statistics counters.
*
**********************************************************************/
-static void
+void
em_update_stats_counters(struct e1000_softc *sc)
{
struct e1000_hw_stats *stats;
@@ -6790,15 +7456,81 @@ em_mac_has_eee(enum e1000_mac_type type)
}
static void
-em_update_vf_stats_counters(struct e1000_softc *sc)
+em_initialize_vf_stats(struct e1000_softc *sc)
{
struct e1000_vf_stats *stats;
- if (sc->link_speed == 0)
+ stats = &sc->ustats.vf_stats;
+ *stats = (struct e1000_vf_stats){};
+ em_rebase_vf_stats(sc);
+}
+
+static void
+em_rebase_vf_stats(struct e1000_softc *sc)
+{
+ struct e1000_vf_stats *stats;
+ bool hyperv = igbv_is_hyperv(sc);
+
+ sc->vf_stats_valid = false;
+ if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff)
return;
+ /*
+ * A PF reset starts a new VF counter epoch. Preserve the accumulated
+ * totals while establishing a new raw baseline so the reset is not
+ * mistaken for a 32-bit wrap.
+ */
stats = &sc->ustats.vf_stats;
+#define INIT_VF_REG(reg, name) do { \
+ stats->last_##name = E1000_READ_REG(&sc->hw, reg); \
+} while (0)
+ INIT_VF_REG(E1000_VFGPRC, gprc);
+ INIT_VF_REG(E1000_VFGORC, gorc);
+ INIT_VF_REG(E1000_VFGPTC, gptc);
+ INIT_VF_REG(E1000_VFGOTC, gotc);
+ /*
+ * I350 specification update erratum 31 says VFMPRC is not
+ * accessible from VF memory. The 0xf3c register remains valid on
+ * 82576 VFs, but must not be read on vfadapt_i350.
+ */
+ if (sc->hw.mac.type == e1000_vfadapt)
+ INIT_VF_REG(E1000_VFMPRC, mprc);
+ else
+ stats->last_mprc = 0;
+ INIT_VF_REG(E1000_VFGOTLBC, gotlbc);
+ INIT_VF_REG(E1000_VFGPTLBC, gptlbc);
+ INIT_VF_REG(E1000_VFGORLBC, gorlbc);
+ INIT_VF_REG(E1000_VFGPRLBC, gprlbc);
+#undef INIT_VF_REG
+ sc->vf_stats_valid = !hyperv ||
+ E1000_READ_REG(&sc->hw, E1000_STATUS) != 0xffffffff;
+}
+
+static void
+em_update_vf_stats_counters(struct e1000_softc *sc)
+{
+ struct e1000_vf_stats sample;
+ struct e1000_vf_stats *stats;
+ bool hyperv, reset;
+ hyperv = igbv_is_hyperv(sc);
+ if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) {
+ sc->vf_stats_valid = false;
+ return;
+ }
+ reset = hyperv && e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS;
+ if (hyperv && !sc->vf_stats_valid)
+ reset = true;
+ if (hyperv && (E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) &
+ E1000_TXDCTL_QUEUE_ENABLE) == 0)
+ reset = true;
+ sample = sc->ustats.vf_stats;
+ stats = &sample;
+
+ /*
+ * Internal VF loopback traffic can continue without physical link,
+ * so sample the counters regardless of link state.
+ */
UPDATE_VF_REG(E1000_VFGPRC,
stats->last_gprc, stats->gprc);
UPDATE_VF_REG(E1000_VFGORC,
@@ -6807,8 +7539,40 @@ em_update_vf_stats_counters(struct e1000_softc *sc)
stats->last_gptc, stats->gptc);
UPDATE_VF_REG(E1000_VFGOTC,
stats->last_gotc, stats->gotc);
- UPDATE_VF_REG(E1000_VFMPRC,
- stats->last_mprc, stats->mprc);
+ if (sc->hw.mac.type == e1000_vfadapt)
+ UPDATE_VF_REG(E1000_VFMPRC,
+ stats->last_mprc, stats->mprc);
+ UPDATE_VF_REG(E1000_VFGOTLBC,
+ stats->last_gotlbc, stats->gotlbc);
+ UPDATE_VF_REG(E1000_VFGPTLBC,
+ stats->last_gptlbc, stats->gptlbc);
+ UPDATE_VF_REG(E1000_VFGORLBC,
+ stats->last_gorlbc, stats->gorlbc);
+ UPDATE_VF_REG(E1000_VFGPRLBC,
+ stats->last_gprlbc, stats->gprlbc);
+ /*
+ * Hyper-V can reset the VF without a native mailbox handshake. Do not
+ * count that counter clear as a 32-bit wrap, including a reset during
+ * this sweep. The PF can also clear counters while blocking a queue
+ * for MDD without leaving a reset indication. Hyper-V has one queue;
+ * a disabled queue cannot supply a valid running counter epoch.
+ */
+ if (hyperv) {
+ if (e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS)
+ reset = true;
+ if ((E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) &
+ E1000_TXDCTL_QUEUE_ENABLE) == 0)
+ reset = true;
+ if (E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) {
+ /* Rebase on a good sample before accounting further deltas. */
+ sc->vf_stats_valid = false;
+ return;
+ }
+ }
+ if (reset)
+ em_rebase_vf_stats(sc);
+ else
+ sc->ustats.vf_stats = sample;
}
static uint64_t
@@ -6883,6 +7647,38 @@ em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
return (sysctl_handle_int(oidp, &val, 0, req));
}
+enum em_ring_register {
+ EM_RING_HEAD,
+ EM_RING_TAIL,
+};
+
+/* Queue register addresses can change when the PF enters IOV mode. */
+static int
+em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct tx_ring *txr;
+ u_int reg, val;
+
+ txr = oidp->oid_arg1;
+ reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) :
+ E1000_TDT(txr->me);
+ val = E1000_READ_REG(&txr->sc->hw, reg);
+ return (sysctl_handle_int(oidp, &val, 0, req));
+}
+
+static int
+em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct rx_ring *rxr;
+ u_int reg, val;
+
+ rxr = oidp->oid_arg1;
+ reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) :
+ E1000_RDT(rxr->me);
+ val = E1000_READ_REG(&rxr->sc->hw, reg);
+ return (sysctl_handle_int(oidp, &val, 0, req));
+}
+
/* Per queue holdoff interrupt rate handler */
static int
em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
@@ -6968,20 +7764,22 @@ em_add_hw_stats(struct e1000_softc *sc)
SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
CTLFLAG_RD, &sc->rx_overruns,
"RX overruns");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
- CTLTYPE_UINT | CTLFLAG_RD,
- sc, E1000_CTRL, em_sysctl_reg_handler, "IU",
- "Device Control Register");
- SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
- CTLTYPE_UINT | CTLFLAG_RD,
- sc, E1000_RCTL, em_sysctl_reg_handler, "IU",
- "Receiver Control Register");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
- CTLFLAG_RD, &sc->hw.fc.high_water, 0,
- "Flow Control High Watermark");
- SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
- CTLFLAG_RD, &sc->hw.fc.low_water, 0,
- "Flow Control Low Watermark");
+ if (!sc->vf_ifp) {
+ SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
+ CTLTYPE_UINT | CTLFLAG_RD,
+ sc, E1000_CTRL, em_sysctl_reg_handler, "IU",
+ "Device Control Register");
+ SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
+ CTLTYPE_UINT | CTLFLAG_RD,
+ sc, E1000_RCTL, em_sysctl_reg_handler, "IU",
+ "Receiver Control Register");
+ SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
+ CTLFLAG_RD, &sc->hw.fc.high_water, 0,
+ "Flow Control High Watermark");
+ SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
+ CTLFLAG_RD, &sc->hw.fc.low_water, 0,
+ "Flow Control Low Watermark");
+ }
for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) {
struct tx_ring *txr = &tx_que->txr;
@@ -6996,12 +7794,12 @@ em_add_hw_stats(struct e1000_softc *sc)
"IU", "Interrupt Rate");
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
- CTLTYPE_UINT | CTLFLAG_RD, sc,
- E1000_TDH(txr->me), em_sysctl_reg_handler, "IU",
+ CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD,
+ em_sysctl_tx_ring_handler, "IU",
"Transmit Descriptor Head");
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, sc,
- E1000_TDT(txr->me), em_sysctl_reg_handler, "IU",
+ CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL,
+ em_sysctl_tx_ring_handler, "IU",
"Transmit Descriptor Tail");
SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
CTLFLAG_RD, &txr->tx_irq,
@@ -7021,12 +7819,12 @@ em_add_hw_stats(struct e1000_softc *sc)
"IU", "Interrupt Rate");
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
- CTLTYPE_UINT | CTLFLAG_RD, sc,
- E1000_RDH(rxr->me), em_sysctl_reg_handler, "IU",
+ CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD,
+ em_sysctl_rx_ring_handler, "IU",
"Receive Descriptor Head");
SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
- CTLTYPE_UINT | CTLFLAG_RD, sc,
- E1000_RDT(rxr->me), em_sysctl_reg_handler, "IU",
+ CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL,
+ em_sysctl_rx_ring_handler, "IU",
"Receive Descriptor Tail");
SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
CTLFLAG_RD, &rxr->rx_irq,
@@ -7057,9 +7855,35 @@ em_add_hw_stats(struct e1000_softc *sc)
SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
CTLFLAG_RD, &vfstats->gotc,
"Good Octets Transmitted");
- SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
- CTLFLAG_RD, &vfstats->mprc,
- "Multicast Packets Received");
+ if (sc->hw.mac.type == e1000_vfadapt) {
+ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO,
+ "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc,
+ "Multicast Packets Received");
+ }
+ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO,
+ "loopback_good_pkts_recvd",
+ CTLFLAG_RD, &vfstats->gprlbc,
+ "Good Loopback Packets Received");
+ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO,
+ "loopback_good_pkts_txd",
+ CTLFLAG_RD, &vfstats->gptlbc,
+ "Good Loopback Packets Transmitted");
+ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO,
+ "loopback_good_octets_recvd",
+ CTLFLAG_RD, &vfstats->gorlbc,
+ "Good Loopback Octets Received");
+ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO,
+ "loopback_good_octets_txd",
+ CTLFLAG_RD, &vfstats->gotlbc,
+ "Good Loopback Octets Transmitted");
+ SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO,
+ "rx_csum_offload_good",
+ CTLFLAG_RD, &sc->rx_csum_good,
+ "Receive Checksum Offload Successes");
+ SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO,
+ "rx_csum_offload_errors",
+ CTLFLAG_RD, &sc->rx_csum_errors,
+ "Receive Checksum Offload Errors");
return;
}
@@ -7598,7 +8422,12 @@ em_print_nvm_info(struct e1000_softc *sc)
j = 0; ++row;
printf("\n0x00%x0 ",row);
}
- e1000_read_nvm(hw, i, 1, &eeprom_data);
+ eeprom_data = 0;
+ if (e1000_read_nvm(hw, i, 1, &eeprom_data) !=
+ E1000_SUCCESS) {
+ printf("\nNVM read failed at offset %#x\n", i);
+ break;
+ }
printf("%04x ", eeprom_data);
}
sx_xunlock(iflib_ctx_lock);
@@ -7875,6 +8704,8 @@ static void
em_print_debug_info(struct e1000_softc *sc)
{
device_t dev = iflib_get_dev(sc->ctx);
+ struct tx_ring *txr;
+ struct rx_ring *rxr;
if (sc->tx_queues == NULL || sc->rx_queues == NULL) {
device_printf(dev, "queue state is unavailable\n");
@@ -7884,17 +8715,19 @@ em_print_debug_info(struct e1000_softc *sc)
iflib_is_running(sc->ctx) ? "open" : "closed");
for (int i = 0; i < sc->tx_num_queues; i++) {
+ txr = &sc->tx_queues[i].txr;
device_printf(dev, "TX Queue %d ------\n", i);
device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
- E1000_READ_REG(&sc->hw, E1000_TDH(i)),
- E1000_READ_REG(&sc->hw, E1000_TDT(i)));
+ E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)),
+ E1000_READ_REG(&sc->hw, E1000_TDT(txr->me)));
}
for (int j = 0; j < sc->rx_num_queues; j++) {
+ rxr = &sc->rx_queues[j].rxr;
device_printf(dev, "RX Queue %d ------\n", j);
device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
- E1000_READ_REG(&sc->hw, E1000_RDH(j)),
- E1000_READ_REG(&sc->hw, E1000_RDT(j)));
+ E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)),
+ E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me)));
}
}
diff --git a/sys/dev/e1000/if_em.h b/sys/dev/e1000/if_em.h
index 7c6354c04c74..c0934b65cb1f 100644
--- a/sys/dev/e1000/if_em.h
+++ b/sys/dev/e1000/if_em.h
@@ -48,6 +48,7 @@
#endif
#include <sys/buf_ring.h>
#include <sys/bus.h>
+#include <sys/callout.h>
#include <sys/endian.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
@@ -95,6 +96,9 @@
#include "e1000_82571.h"
#include "ifdi_if.h"
+struct igb_vf;
+struct igb_vf_mac_filter;
+
/* Tunables */
/*
@@ -133,6 +137,7 @@
#define EM_DEFAULT_RXD 1024
#define EM_DEFAULT_MULTI_RXD 4096
#define IGB_MAX_RXD 4096
+#define IGB_MAX_FRAME_SIZE 9234
/*
* EM_TIDV - Transmit Interrupt Delay Value
@@ -412,11 +417,8 @@
#define UPDATE_VF_REG(reg, last, cur) \
do { \
u32 new = E1000_READ_REG(&sc->hw, reg); \
- if (new < last) \
- cur += 0x100000000LL; \
+ cur += (u32)(new - last); \
last = new; \
- cur &= 0xFFFFFFFF00000000LL; \
- cur |= new; \
} while (0)
struct e1000_softc;
@@ -612,6 +614,10 @@ struct e1000_softc {
** to repopulate it.
*/
u32 shadow_vfta[EM_VFTA_SIZE];
+ u32 vf_vfta_stale[EM_VFTA_SIZE];
+ u32 vf_vfta_retry[EM_VFTA_SIZE];
+ sbintime_t vf_vlan_retry_deadline;
+ u16 vf_vlan_retry_cursor;
/* Info about the interface */
enum em_link_state link_state;
@@ -624,6 +630,7 @@ struct e1000_softc {
int link_mask;
int tso_automasked;
u32 phy_hang_count;
+ u32 promisc_pending;
u32 stats_pending;
u32 device_reset_state;
u32 fatal_error_state;
@@ -652,6 +659,29 @@ struct e1000_softc {
u64 uncorrected_error_dma_count;
u64 uncorrected_error_pcie_count;
+#ifdef PCI_IOV
+ struct igb_vf *vfs;
+ struct igb_vf_mac_filter *vf_mac_filters;
+ struct callout iov_mbx_retry;
+ u32 iov_vfta[EM_VFTA_SIZE];
+ u32 iov_mdd_cause;
+ u32 iov_pending;
+ u32 iov_spoof_pending;
+ u32 iov_blocked_pending;
+ u32 iov_intr_drain_pending;
+ u32 iov_teardown;
+ struct timeval iov_last_mdd_log;
+ u16 num_vfs;
+ u16 num_vf_mac_filters;
+ u16 pool;
+ bool iov_hw_active;
+ bool iov_mta_valid;
+ bool iov_mbx_retry_initialized;
+ bool iov_pf_mdd_blocked;
+ bool iov_pf_vlan_promisc;
+ bool iov_vfta_valid;
+#endif
+
u64 que_mask;
/* We need to store this at attach due to e1000 hw/sw locking model */
@@ -667,15 +697,83 @@ struct e1000_softc {
unsigned long dropped_pkts;
unsigned long link_irq;
unsigned long rx_overruns;
+ u64 rx_csum_good;
+ u64 rx_csum_errors;
union {
struct e1000_hw_stats stats; /* !sc->vf_ifp */
struct e1000_vf_stats vf_stats; /* sc->vf_ifp */
} ustats;
+ struct callout vf_queue_retry;
+ struct callout vf_mbx_retry;
+ struct timeval vf_last_queue_log;
+ struct timeval vf_last_mbx_log;
+ u32 vf_queue_retry_new_epoch;
+ u32 vf_queue_retry_pending;
+ u32 vf_mbx_ready;
+ u32 vf_mbx_retry_pending;
u16 vf_ifp;
+ u8 vf_queue_failures;
+ u8 vf_mbx_retry_stage;
+ bool vf_queue_gave_up;
+ bool vf_queue_retry_initialized;
+ bool vf_mbx_retry_initialized;
+ bool vf_queues_sanitized;
+ bool vf_reset_pending;
+ bool vf_stats_valid;
+ /* A PF can retain auxiliary filters across a VF reset. */
+ bool vf_uc_filters_set;
};
+static inline bool
+igbv_is_hyperv(const struct e1000_softc *sc)
+{
+
+ return (sc->vf_ifp &&
+ (sc->hw.device_id == E1000_DEV_ID_82576_VF_HV ||
+ sc->hw.device_id == E1000_DEV_ID_I350_VF_HV));
+}
+
+/*
+ * Shared PF/VF mechanisms and VF policy entry points. The latter live in
+ * if_igbv.c so the VF method table cannot accidentally select PF policy.
+ */
+int em_if_attach_pre(if_ctx_t);
+int em_if_attach_post(if_ctx_t);
+void em_add_device_sysctls(struct e1000_softc *);
+int em_if_set_promisc_impl(if_ctx_t, int);
+bool em_is_valid_ether_addr(const u8 *);
+void em_initialize_transmit_rings(if_ctx_t);
+void em_update_stats_counters(struct e1000_softc *);
+void igb_initialize_receive_rings(if_ctx_t, bool);
+
+int igbv_get_regs(SYSCTL_HANDLER_ARGS);
+int igbv_if_attach_pre(if_ctx_t);
+int igbv_if_attach_post(if_ctx_t);
+int igbv_if_media_change(if_ctx_t);
+void igbv_init_hv_ops(struct e1000_hw *);
+void igbv_if_intr_enable(if_ctx_t);
+void igbv_if_intr_disable(if_ctx_t);
+void igbv_if_update_admin_status(if_ctx_t);
+void igbv_initialize_receive_unit(if_ctx_t);
+void igbv_initialize_transmit_unit(if_ctx_t);
+void igbv_mbx_retry_detach(struct e1000_softc *);
+void igbv_mbx_retry_failed(if_ctx_t);
+void igbv_mbx_retry_prepare(struct e1000_softc *);
+void igbv_mbx_retry_stop(struct e1000_softc *);
+void igbv_queue_retry_detach(struct e1000_softc *);
+void igbv_queue_retry_failed(if_ctx_t);
+void igbv_queue_retry_prepare(struct e1000_softc *);
+void igbv_queue_retry_stop(struct e1000_softc *);
+void igbv_reconcile_mac(struct e1000_softc *, if_t);
+bool igbv_reset(if_ctx_t);
+bool igbv_sanitize_queues(struct e1000_softc *);
+void igbv_log_reset_failure(struct e1000_softc *, s32, bool);
+void igbv_update_uc_addr_list(struct e1000_softc *, if_t);
+void igbv_vlan_retry_add(struct e1000_softc *, u16);
+void igbv_vlan_retry_clear(struct e1000_softc *, u16);
+
/********************************************************************************
* vendor_info_array
*
diff --git a/sys/dev/e1000/if_igb_iov.c b/sys/dev/e1000/if_igb_iov.c
new file mode 100644
index 000000000000..48edf18f02b4
--- /dev/null
+++ b/sys/dev/e1000/if_igb_iov.c
@@ -0,0 +1,2268 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2010-2016, Intel Corporation
+ * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
+ */
+
+#include "if_em.h"
+#include "if_igb_iov.h"
+
+#ifdef PCI_IOV
+
+#include <sys/iov.h>
+#include <sys/sdt.h>
+#include <sys/time.h>
+
+#include <net/if_vf_status.h>
+
+#define IGB_IOV_RAH_POOLSEL_SHIFT 18
+#define IGB_IOV_RAH_POOLSEL_MASK (0xffU << IGB_IOV_RAH_POOLSEL_SHIFT)
+#define IGB_IOV_MAX_MAC_FILTERS 3
+#define IGB_IOV_MAX_MC_HASHES 30
+#define IGB_IOV_MBX_RETRY_COUNT 6
+/* Allow two complete 31-VID replays, then sustain eight additions/second. */
+#define IGB_IOV_VLAN_CHANGE_BURST 64
+#define IGB_IOV_VLAN_CHANGE_INTERVAL (SBT_1S / 8)
+/* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */
+#define IGB_82576_VMOLR_RSV (1U << 31)
+#define IGB_82576_LVMMC_BLOCK_MASK 0x1c
+#define IGB_82576_NUM_QUEUES 16
+#define IGB_82576_QUEUE_MASK 0xffff
+#define IGB_82576_STAGGERED_QUEUE_SHIFT 8
+#define IGB_82576_VF_QUEUE_STRIDE 8
+#define IGB_82576_VF_QUEUES 2
+#define IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE (1U << 2)
+#define IGB_I350_LVMMC_MAC_VLAN_SPOOF (1U << 25)
+#define IGB_I350_LVMMC_LAST_Q_SHIFT 29
+#define IGB_I350_LVMMC_LAST_Q_MASK 0x7
+#define IGB_I350_NUM_QUEUES 8
+#define IGB_I350_QUEUE_MASK 0xff
+#define IGB_I350_RESET_ACK_TIMEOUT (100 * SBT_1MS)
+#define IGB_I350_VF_QUEUES 1
+#define IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES 10
+#define IGB_IOV_QUEUE_DISABLE_DELAY_US 10
+#define IGB_IOV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
+#define IGB_IOV_QUEUE_DISABLE_RETRIES 20
+#define IGB_IOV_VF_QUEUES_MAX 2
+
+#define IGB_VF_CTS (1U << 0)
+#define IGB_VF_CAP_MAC (1U << 1)
+#define IGB_VF_ACTIVE (1U << 2)
+#define IGB_VF_MAC_ANTI_SPOOF (1U << 3)
+#define IGB_VF_ALLOW_PROMISC (1U << 4)
+#define IGB_VF_UCAST_PROMISC (1U << 5)
+#define IGB_VF_MCAST_PROMISC (1U << 6)
+#define IGB_VF_MCAST_OVERFLOW (1U << 7)
+#define IGB_VF_MCAST_OVERFLOW_WARNED (1U << 8)
+#define IGB_VF_MDD_BLOCKED (1U << 9)
+#define IGB_VF_MBX_PENDING (1U << 10)
+/*
+ * After bounded PFU retries, suppress future or overlapping VF requests until
+ * RST/VFLR starts a new mailbox epoch. Intel VF drivers assert CTRL.RST
+ * before sending their mailbox reset request.
+ */
+#define IGB_VF_MBX_GAVE_UP (1U << 11)
+#define IGB_VF_MDD_NOTIFY_PENDING (1U << 12)
+
+struct igb_vf {
+ u32 flags;
+ struct timeval last_nack;
+ struct timeval last_mbx_log;
+ struct timeval last_spoof_log;
+ struct timeval last_mdd_log;
+ struct timeval last_queue_log;
+ sbintime_t mbx_retry_at;
+ sbintime_t mdd_notify_at;
+ sbintime_t vlan_token_time;
+ u16 pool;
+ u16 rar_index;
+ u16 max_frame_size;
+ u16 mc_count;
+ u16 vlan_count;
+ u16 vlan_tokens;
+ u16 default_vlan;
+ u8 mbx_retry_count;
+ u8 mac[ETHER_ADDR_LEN];
+ u16 mc_hashes[IGB_IOV_MAX_MC_HASHES];
+ u32 vlans[EM_VFTA_SIZE];
+};
+
+struct igb_vf_mac_filter {
+ bool active;
+ u16 pool;
+ u16 rar_index;
+ u8 mac[ETHER_ADDR_LEN];
+};
+
+MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations");
+
+/*
+ * These logical-write probes let hardware tests verify the elision policy.
+ * e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so
+ * the probes intentionally count calls made by the rebuild rather than MMIO
+ * transactions. The state probe exposes the final software images while the
+ * stack arrays are still live.
+ */
+SDT_PROVIDER_DEFINE(igb_iov);
+SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear,
+ "struct e1000_softc *", "u_int", "uint32_t");
+SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write,
+ "struct e1000_softc *", "u_int", "uint32_t");
+SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set,
+ "struct e1000_softc *", "u_int", "uint32_t");
+SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state,
+ "struct e1000_softc *", "uint32_t *", "uint32_t *");
+SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr,
+ "struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t");
+
+static const struct timeval igb_iov_nack_interval = { 2, 0 };
+static const struct timeval igb_iov_mbx_log_interval = { 2, 0 };
+static const struct timeval igb_iov_spoof_log_interval = { 2, 0 };
+static const struct timeval igb_iov_mdd_log_interval = { 2, 0 };
+static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2;
+static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = {
+ SBT_1MS,
+ 2 * SBT_1MS,
+ 4 * SBT_1MS,
+ 8 * SBT_1MS,
+ 16 * SBT_1MS,
+ 32 * SBT_1MS,
+};
+
+static void igb_iov_clear_mac_filters(struct e1000_softc *,
+ const struct igb_vf *);
+static bool igb_iov_mac_in_use(struct e1000_softc *, const u8 *,
+ const struct igb_vf *);
+static bool igb_iov_vlan_present(struct e1000_softc *, u16, bool);
+static int igb_iov_vlan_unique_count(struct e1000_softc *, bool);
+
+static void
+igb_iov_mbx_retry_callout(void *arg)
+{
+ struct e1000_softc *sc;
+
+ sc = arg;
+ /*
+ * Mailbox service is serialized by iflib's context lock. The
+ * callout only re-enters through the ordinary admin task.
+ */
+ iflib_admin_intr_deferred(sc->ctx);
+}
+
+static u_int
+igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
+{
+ u8 *mta;
+
+ if (idx == MAX_NUM_MULTICAST_ADDRESSES)
+ return (0);
+ mta = arg;
+ memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN);
+ return (1);
+}
+
+static bool
+igb_iov_pf_vlan_promisc(struct e1000_softc *sc)
+{
+ if_t ifp;
+
+ ifp = iflib_get_ifp(sc->ctx);
+ return (sc->iov_pf_vlan_promisc ||
+ (if_getflags(ifp) & IFF_PROMISC) != 0);
+}
+
+static bool
+igb_iov_mac_valid(const u8 *mac)
+{
+ static const u8 zero[ETHER_ADDR_LEN];
+
+ return (!ETHER_IS_MULTICAST(mac) &&
+ memcmp(mac, zero, ETHER_ADDR_LEN) != 0);
+}
+
+static bool
+igb_iov_nack_allowed(struct igb_vf *vf)
+{
+ return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0);
+}
+
+static void
+igb_iov_reset_vlan_rate(struct igb_vf *vf)
+{
+
+ vf->vlan_token_time = getsbinuptime();
+ vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST;
+}
+
+static bool
+igb_iov_vlan_add_allowed(struct igb_vf *vf)
+{
+ sbintime_t elapsed, now;
+ uint64_t refill;
+
+ now = getsbinuptime();
+ elapsed = now - vf->vlan_token_time;
+ if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) {
+ refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL;
+ vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST,
+ vf->vlan_tokens + refill);
+ vf->vlan_token_time = now;
+ }
+ if (vf->vlan_tokens == 0)
+ return (false);
+ vf->vlan_tokens--;
+ return (true);
+}
+
+static u32
+igb_iov_reply_header(u32 request, bool cts, bool ack)
+{
+ u32 reply, type;
+
+ type = request & 0xffff;
+ if (type == E1000_VF_SET_MAC_ADDR &&
+ (request & E1000_VT_MSGINFO_MASK) != 0)
+ reply = request;
+ else
+ reply = type;
+ reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK |
+ E1000_VT_MSGTYPE_CTS);
+ if (cts)
+ reply |= E1000_VT_MSGTYPE_CTS;
+ reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK;
+ return (reply);
+}
+
+bool
+igb_iov_supported(const struct e1000_softc *sc)
+{
+ switch (sc->hw.mac.type) {
+ case e1000_82576:
+ case e1000_i350:
+ return (true);
+ default:
+ return (false);
+ }
+}
+
+bool
+igb_iov_enabled(const struct e1000_softc *sc)
+{
+ return (sc->num_vfs != 0);
+}
+
+int
+igb_iov_attach(struct e1000_softc *sc)
+{
+ nvlist_t *pf_schema, *vf_schema;
+ int error, iov_pos;
+
+ if (!igb_iov_supported(sc))
+ return (0);
+ if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0)
+ return (0);
+
+ pf_schema = pci_iov_schema_alloc_node();
+ vf_schema = pci_iov_schema_alloc_node();
+ pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
+ pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
+ IOV_SCHEMA_HASDEFAULT, true);
+ pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
+ IOV_SCHEMA_HASDEFAULT, false);
+ pci_iov_schema_add_bool(vf_schema, "allow-promisc",
+ IOV_SCHEMA_HASDEFAULT, false);
+ pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT,
+ VF_VLAN_TRUNK);
+
+ error = pci_iov_attach(sc->dev, pf_schema, vf_schema);
+ if (error != 0)
+ device_printf(sc->dev,
+ "failed to attach SR-IOV configuration interface: %d\n",
+ error);
+ else {
+ callout_init(&sc->iov_mbx_retry, 1);
+ sc->iov_mbx_retry_initialized = true;
+ }
+ return (error);
+}
+
+void
+igb_iov_detach(struct e1000_softc *sc)
+{
+
+ if (!sc->iov_mbx_retry_initialized)
+ return;
+ callout_drain(&sc->iov_mbx_retry);
+ sc->iov_mbx_retry_initialized = false;
+}
+
+static u32
+igb_iov_active_mask(struct e1000_softc *sc)
+{
+ u32 mask;
+ int i;
+
+ mask = 0;
+ for (i = 0; i < sc->num_vfs; i++)
+ if (sc->vfs[i].flags & IGB_VF_ACTIVE)
+ mask |= 1U << i;
+ return (mask);
+}
+
+static void
+igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool)
+{
+ struct e1000_hw *hw;
+ u32 rah;
+
+ hw = &sc->hw;
+ e1000_rar_set(hw, __DECONST(u8 *, mac), rar);
+ rah = E1000_READ_REG(hw, E1000_RAH(rar));
+ rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
+ rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool);
+ E1000_WRITE_REG(hw, E1000_RAH(rar), rah);
+}
+
+static void
+igb_iov_clear_rar(struct e1000_softc *sc, u16 rar)
+{
+ u8 zero[ETHER_ADDR_LEN] = {};
+
+ e1000_rar_set(&sc->hw, zero, rar);
+}
+
+static void
+igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf)
+{
+ struct igb_vf_mac_filter *filter;
+ int i;
+
+ for (i = 0; i < sc->num_vf_mac_filters; i++) {
+ filter = &sc->vf_mac_filters[i];
+ if (!filter->active || filter->pool != vf->pool)
+ continue;
+ igb_iov_clear_rar(sc, filter->rar_index);
+ filter->active = false;
+ memset(filter->mac, 0, sizeof(filter->mac));
+ }
+}
+
+static u32
+igb_iov_switch_reg(struct e1000_softc *sc)
+{
+ return (sc->hw.mac.type == e1000_82576 ?
+ E1000_DTXSWC : E1000_TXSWC);
+}
+
+static void
+igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ struct e1000_hw *hw;
+ u32 reg, value;
+
+ hw = &sc->hw;
+ reg = igb_iov_switch_reg(sc);
+ value = E1000_READ_REG(hw, reg);
+ value &= ~((1U << vf->pool) |
+ (1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
+ if (vf->flags & IGB_VF_MAC_ANTI_SPOOF)
+ value |= 1U << vf->pool;
+ if (vf->flags & IGB_VF_ACTIVE)
+ value |= 1U <<
+ (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT);
+ E1000_WRITE_REG(hw, reg, value);
+}
+
+static void
+igb_iov_set_uta(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ bool enable;
+ int i;
+
+ if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576)
+ return;
+
+ hw = &sc->hw;
+ enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) &
+ E1000_VMOLR_ROPE) != 0;
+ for (i = 0; i < sc->num_vfs; i++)
+ if ((sc->vfs[i].flags &
+ (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) ==
+ (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) {
+ enable = true;
+ break;
+ }
+
+ for (i = 0; i < MAX_MTA_REG; i++)
+ E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i,
+ enable ? 0xffffffffU : 0);
+}
+
+static void
+igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool,
+ bool strip_vlan, bool hide_vlan, bool vf_pool)
+{
+ struct e1000_hw *hw;
+ u32 dvmolr;
+
+ hw = &sc->hw;
+ if (hw->mac.type != e1000_i350)
+ return;
+
+ dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool));
+ dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN |
+ E1000_DVMOLR_STRCRC);
+ if (hide_vlan)
+ dvmolr |= E1000_DVMOLR_HIDVLAN;
+ if (strip_vlan)
+ dvmolr |= E1000_DVMOLR_STRVLAN;
+ if (vf_pool || strip_vlan ||
+ (E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0)
+ dvmolr |= E1000_DVMOLR_STRCRC;
+ E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr);
+}
+
+static void
+igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ struct e1000_hw *hw;
+ u32 max_frame_size, vmolr, vmvir;
+
+ hw = &sc->hw;
+ max_frame_size = vf->max_frame_size;
+ if (vf->vlan_count != 0)
+ max_frame_size = min(max_frame_size + VLAN_TAG_SIZE,
+ IGB_IOV_MAX_FRAME_SIZE);
+ vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool));
+ vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE |
+ E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE |
+ E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN);
+ vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE |
+ (max_frame_size & E1000_VMOLR_RLPML_MASK);
+ if (vf->default_vlan == 0)
+ vmolr |= E1000_VMOLR_AUPE;
+ if (vf->mc_count != 0 &&
+ (vf->flags & (IGB_VF_MCAST_PROMISC |
+ IGB_VF_MCAST_OVERFLOW)) == 0)
+ vmolr |= E1000_VMOLR_ROMPE;
+ if (hw->mac.type == e1000_82576)
+ vmolr |= IGB_82576_VMOLR_RSV;
+
+ if (vf->flags & IGB_VF_UCAST_PROMISC) {
+ if (hw->mac.type == e1000_82576)
+ vmolr |= E1000_VMOLR_ROPE;
+ else
+ vmolr |= E1000_VMOLR_UPE;
+ }
+ /*
+ * The mailbox can describe only 30 hashes. Fall back to receiving all
+ * multicast within the VF's VLAN membership when that list overflows.
+ */
+ if ((vf->flags & (IGB_VF_MCAST_PROMISC |
+ IGB_VF_MCAST_OVERFLOW)) != 0)
+ vmolr |= E1000_VMOLR_MPME;
+ if (hw->mac.type == e1000_82576 && vf->vlan_count != 0)
+ vmolr |= E1000_VMOLR_STRVLAN;
+ /* A nonzero default VLAN makes this VF an untagged access port. */
+ if (vf->default_vlan == 0)
+ vmvir = 0;
+ else
+ vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT;
+
+ E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr);
+ E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir);
+ igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0,
+ vf->default_vlan != 0, true);
+}
+
+static void
+igb_iov_configure_pf_vmolr(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ if_t ifp;
+ bool strip_vlan;
+ u32 max_frame_size;
+ u32 old_vmolr, vmolr;
+
+ hw = &sc->hw;
+ ifp = iflib_get_ifp(sc->ctx);
+ max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE,
+ IGB_IOV_MAX_FRAME_SIZE);
+ strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0;
+ old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
+ vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE |
+ E1000_VMOLR_LPE |
+ (max_frame_size & E1000_VMOLR_RLPML_MASK);
+ if (hw->mac.type == e1000_82576) {
+ vmolr |= IGB_82576_VMOLR_RSV;
+ if (strip_vlan)
+ vmolr |= E1000_VMOLR_STRVLAN;
+ } else
+ vmolr |= old_vmolr & E1000_VMOLR_VPE;
+
+ if (if_getflags(ifp) & IFF_PROMISC) {
+ if (hw->mac.type == e1000_82576)
+ vmolr |= E1000_VMOLR_ROPE;
+ else
+ vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE;
+ vmolr |= E1000_VMOLR_MPME;
+ } else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 ||
+ if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES)
+ vmolr |= E1000_VMOLR_MPME;
+ else if (if_llmaddr_count(ifp) != 0)
+ vmolr |= E1000_VMOLR_ROMPE;
+
+ E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
+ igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false);
+}
+
+void
+igb_iov_update_pf_vmolr(struct e1000_softc *sc)
+{
+ if (!igb_iov_enabled(sc))
+ return;
+
+ igb_iov_configure_pf_vmolr(sc);
+ igb_iov_set_uta(sc);
+}
+
+u32
+igb_iov_intr_mask(const struct e1000_softc *sc)
+{
+ if (!sc->iov_hw_active)
+ return (0);
+ return (E1000_IMS_VMMB | E1000_IMS_MDDET);
+}
+
+void
+igb_iov_intr_drain_stale(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ u32 icr;
+
+ if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0)
+ return;
+ hw = &sc->hw;
+ /*
+ * Consume setup-time diagnostic state at the actual transition from
+ * masked to armed. Read ICR last so an event arriving after the drain
+ * remains pending and is delivered when the caller enables MDDET.
+ */
+ (void)E1000_READ_REG(hw, E1000_LVMMC);
+ if (hw->mac.type == e1000_82576)
+ (void)E1000_READ_REG(hw, E1000_WVBR);
+ icr = E1000_READ_REG(hw, E1000_ICR);
+ /*
+ * em_if_init() injects LSC after IOV setup to close the post-reset
+ * link race. Preserve that cause across this MDDET-specific drain.
+ */
+ if (__predict_true(icr != 0xffffffff) &&
+ (icr & E1000_ICR_LSC) != 0)
+ E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC);
+}
+
+static void
+igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc)
+{
+
+ /*
+ * I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or
+ * anti-spoofing is active. The shadow is therefore authoritative
+ * until a reset or another independent hardware writer invalidates
+ * it. Readback cannot reliably audit a stale-but-valid shadow on
+ * this part, so keep all shadow mutation in these two helpers.
+ */
+ memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta));
+ sc->iov_vfta_valid = false;
+}
+
+static void
+igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta)
+{
+
+ memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta));
+ sc->iov_vfta_valid = true;
+}
+
+static void
+igb_iov_notify_vfs_reset(struct e1000_softc *sc)
+{
+ struct igb_vf *vf;
+ struct e1000_hw *hw;
+ sbintime_t deadline;
+ u32 msg, pending, undelivered;
+ int i;
+
+ hw = &sc->hw;
+ /*
+ * Process VFLRs first and wait only for VFs that completed their
+ * mailbox handshake. An unattached VF has nobody who can acknowledge.
+ */
+ igb_iov_handle_mbx(sc);
+ pending = 0;
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) ==
+ (IGB_VF_ACTIVE | IGB_VF_CTS))
+ pending |= 1U << i;
+ }
+ if (pending == 0)
+ return;
+
+ /*
+ * I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a
+ * mailbox warning before the PF asserts CTRL.RST.
+ *
+ * The mailbox pass above drained requests and stale acknowledgements.
+ * A VF read of the new notification sets its ACK bit.
+ */
+ undelivered = 0;
+ for (i = 0; i < sc->num_vfs; i++) {
+ if ((pending & (1U << i)) == 0)
+ continue;
+ msg = E1000_PF_CONTROL_MSG;
+ if (e1000_write_mbx(hw, &msg, 1, i) != 0) {
+ undelivered |= 1U << i;
+ pending &= ~(1U << i);
+ }
+ }
+ if (undelivered != 0)
+ device_printf(sc->dev,
+ "could not deliver reset warning to VF mask %#x\n",
+ undelivered);
+
+ deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT;
+ while (pending != 0 && getsbinuptime() < deadline) {
+ for (i = 0; i < sc->num_vfs; i++) {
+ if ((pending & (1U << i)) != 0 &&
+ e1000_check_for_ack(hw, i) == 0)
+ pending &= ~(1U << i);
+ }
+ if (pending != 0)
+ pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK);
+ }
+ if (pending != 0)
+ device_printf(sc->dev,
+ "VF reset acknowledgement timed out for mask %#x\n",
+ pending);
+}
+
+void
+igb_iov_reset_prepare(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ u32 mask;
+
+ if (sc->iov_hw_active) {
+ hw = &sc->hw;
+ if (atomic_load_acq_32(&sc->iov_teardown) == 0) {
+ if (hw->mac.type == e1000_i350)
+ igb_iov_notify_vfs_reset(sc);
+ else
+ igb_iov_ping_all_vfs(sc);
+ }
+
+ /* Stop VF DMA before the PF asserts CTRL.RST. */
+ mask = 1U << sc->pool;
+ E1000_WRITE_REG(hw, E1000_VFRE, mask);
+ E1000_WRITE_REG(hw, E1000_VFTE, mask);
+ E1000_WRITE_FLUSH(hw);
+ }
+ sc->iov_hw_active = false;
+ if (sc->iov_mbx_retry_initialized)
+ callout_stop(&sc->iov_mbx_retry);
+ sc->iov_mta_valid = false;
+ igb_iov_vfta_shadow_invalidate(sc);
+ atomic_readandclear_32(&sc->iov_mdd_cause);
+ atomic_readandclear_32(&sc->iov_pending);
+ atomic_readandclear_32(&sc->iov_spoof_pending);
+ atomic_readandclear_32(&sc->iov_blocked_pending);
+ /*
+ * Normal iflib initialization prepares the reset before
+ * igb_iov_initialize() requests this drain. Preserve a still-pending
+ * I350 request across a later stop or repeated preparation so the next
+ * interrupt arm consumes it. Other families retain the ordinary
+ * stop-time cleanup.
+ */
+ if (sc->hw.mac.type != e1000_i350)
+ atomic_readandclear_32(&sc->iov_intr_drain_pending);
+}
+
+void
+igb_iov_rebuild_mta(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ struct igb_vf *vf;
+ u32 hash_bit, hash_reg, hash_value;
+ u32 mta[MAX_MTA_REG] = {};
+ u16 hash;
+ bool changed;
+ int i, j, mcnt;
+
+ if (!igb_iov_enabled(sc))
+ return;
+
+ hw = &sc->hw;
+ memset(sc->mta, 0,
+ ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
+ mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx),
+ igb_iov_copy_maddr, sc->mta);
+ mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES);
+ for (i = 0; i < mcnt; i++) {
+ hash_value = e1000_hash_mc_addr(hw,
+ &sc->mta[i * ETHER_ADDR_LEN]);
+ hash_reg = (hash_value >> 5) &
+ (hw->mac.mta_reg_count - 1);
+ hash_bit = hash_value & 0x1f;
+ mta[hash_reg] |= 1U << hash_bit;
+ }
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ for (j = 0; j < vf->mc_count; j++) {
+ hash = vf->mc_hashes[j] & 0xfff;
+ mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |=
+ 1U << (hash & 0x1f);
+ }
+ }
+
+ changed = false;
+ for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) {
+ if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i])
+ continue;
+ hw->mac.mta_shadow[i] = mta[i];
+ E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]);
+ changed = true;
+ }
+ if (changed)
+ E1000_WRITE_FLUSH(hw);
+ sc->iov_mta_valid = true;
+}
+
+static int
+igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool,
+ bool preserve_only)
+{
+ int free_slot, i;
+
+ free_slot = -1;
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
+ if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
+ (vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) {
+ vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool);
+ return (0);
+ }
+ if (free_slot == -1 &&
+ (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0)
+ free_slot = i;
+ }
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
+ if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
+ (old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid &&
+ (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) {
+ free_slot = i;
+ break;
+ }
+ if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE))
+ return (ENOENT);
+ if (free_slot == -1)
+ return (ENOSPC);
+
+ vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid |
+ (1U << (E1000_VLVF_POOLSEL_SHIFT + pool));
+ return (0);
+}
+
+void
+igb_iov_rebuild_vlan(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ struct igb_vf *vf;
+ u32 old_vlvf[E1000_VLVF_ARRAY_SIZE];
+ u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE];
+ u32 vlvf[E1000_VLVF_ARRAY_SIZE];
+ u32 old_vfta, rctl, vmolr;
+ bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf;
+ bool vfta_changed, vlvf_changed;
+ int i, vid;
+
+ if (!igb_iov_enabled(sc))
+ return;
+
+ hw = &sc->hw;
+ rctl = E1000_READ_REG(hw, E1000_RCTL);
+ rctl &= ~E1000_RCTL_CFIEN;
+ rctl |= E1000_RCTL_VFE;
+ E1000_WRITE_REG(hw, E1000_RCTL, rctl);
+ memcpy(vfta, sc->shadow_vfta, sizeof(vfta));
+ memset(vlvf, 0, sizeof(vlvf));
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
+ old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i));
+
+ pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc);
+ pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 &&
+ igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE;
+ preserve_pf = !pf_vlan_promisc && !pf_overflow;
+
+ /* First keep every surviving VF mapping in its current slot. */
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ for (vid = 0; vid < 4096; vid++) {
+ if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
+ 0)
+ continue;
+ (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
+ vf->pool, true);
+ }
+ }
+
+ /*
+ * Preserve PF mappings unless I350 needs their slots for VFs.
+ * PF-only VLANs on 82576 intentionally have no VLVF mapping and
+ * reach the default PF pool after passing the global VFTA.
+ */
+ if (preserve_pf)
+ for (vid = 0; vid < 4096; vid++) {
+ if ((sc->shadow_vfta[vid >> 5] &
+ (1U << (vid & 0x1f))) == 0)
+ continue;
+ if (hw->mac.type == e1000_82576 &&
+ !igb_iov_vlan_present(sc, vid, false))
+ continue;
+ (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
+ sc->pool, true);
+ }
+
+ /* Allocate new VF mappings before PF mappings. */
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ for (vid = 0; vid < 4096; vid++) {
+ if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
+ 0)
+ continue;
+ if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
+ vf->pool, false) == 0)
+ vfta[vid >> 5] |= 1U << (vid & 0x1f);
+ }
+ igb_iov_configure_vmolr(sc, vf);
+ }
+ if (!pf_vlan_promisc)
+ for (vid = 0; vid < 4096; vid++) {
+ if ((sc->shadow_vfta[vid >> 5] &
+ (1U << (vid & 0x1f))) == 0)
+ continue;
+ /*
+ * With no VLVF match, 82576 sends a globally admitted
+ * VLAN to the default PF pool. A VLVF entry is needed
+ * only when this VLAN is also assigned to a VF.
+ */
+ if (hw->mac.type == e1000_82576 &&
+ !igb_iov_vlan_present(sc, vid, false))
+ continue;
+ if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
+ sc->pool, false) != 0)
+ pf_overflow = true;
+ }
+
+ if (pf_vlan_promisc) {
+ memset(vfta, 0xff, sizeof(vfta));
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
+ if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0)
+ vlvf[i] |= 1U <<
+ (E1000_VLVF_POOLSEL_SHIFT + sc->pool);
+ }
+
+ /*
+ * Establish the PF fallback before an overflowing I350 rebuild can
+ * displace one of its old VLVF mappings.
+ */
+ vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
+ vmolr &= ~E1000_VMOLR_VPE;
+ if (hw->mac.type == e1000_i350 &&
+ (pf_overflow || pf_vlan_promisc))
+ vmolr |= E1000_VMOLR_VPE;
+ E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
+
+ /*
+ * Remove global VFTA membership before removing a VLAN entirely, and
+ * add a VLVF mapping before globally admitting a new VF VLAN. A
+ * transition to a PF-only VLAN deliberately retains VFTA membership
+ * and falls through to the default PF pool.
+ */
+ force_vfta = hw->mac.type == e1000_i350 &&
+ !sc->iov_vfta_valid;
+ vfta_changed = false;
+ for (i = 0; i < EM_VFTA_SIZE; i++) {
+ /*
+ * I350 erratum 20 makes VFTA reads unreliable while VMDq
+ * loopback or anti-spoofing is active. Its ten-write
+ * workaround is already in e1000_write_vfta_i350(). Force a
+ * complete clear when the authoritative shadow is invalid;
+ * 82576 can safely diff against its live register contents.
+ */
+ if (hw->mac.type == e1000_i350)
+ old_vfta = force_vfta ? 0 : sc->iov_vfta[i];
+ else
+ old_vfta =
+ E1000_READ_REG_ARRAY(hw, E1000_VFTA, i);
+ effective_vfta[i] = old_vfta & vfta[i];
+ if (force_vfta || effective_vfta[i] != old_vfta) {
+ SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear,
+ sc, i, effective_vfta[i]);
+ e1000_write_vfta(hw, i, effective_vfta[i]);
+ vfta_changed = true;
+ }
+ }
+ if (vfta_changed)
+ E1000_WRITE_FLUSH(hw);
+ vlvf_changed = false;
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
+ if (vlvf[i] != old_vlvf[i]) {
+ SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write,
+ sc, i, vlvf[i]);
+ E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]);
+ vlvf_changed = true;
+ }
+ if (vlvf_changed)
+ E1000_WRITE_FLUSH(hw);
+ vfta_changed = false;
+ for (i = 0; i < EM_VFTA_SIZE; i++)
+ if (vfta[i] != effective_vfta[i]) {
+ SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set,
+ sc, i, vfta[i]);
+ e1000_write_vfta(hw, i, vfta[i]);
+ vfta_changed = true;
+ }
+ if (vfta_changed)
+ E1000_WRITE_FLUSH(hw);
+ SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf);
+ igb_iov_vfta_shadow_store(sc, vfta);
+}
+
+static bool
+igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf)
+{
+ int i;
+
+ if (include_pf &&
+ (sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0)
+ return (true);
+ for (i = 0; i < sc->num_vfs; i++)
+ if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
+ (sc->vfs[i].vlans[vid >> 5] &
+ (1U << (vid & 0x1f))) != 0)
+ return (true);
+ return (false);
+}
+
+static int
+igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf)
+{
+ u32 vlans;
+ int count, i, word;
+
+ count = 0;
+ for (word = 0; word < EM_VFTA_SIZE; word++) {
+ vlans = include_pf ? sc->shadow_vfta[word] : 0;
+ for (i = 0; i < sc->num_vfs; i++)
+ if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0)
+ vlans |= sc->vfs[i].vlans[word];
+ count += bitcount32(vlans);
+ }
+ return (count);
+}
+
+static int
+igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid,
+ bool add)
+{
+ u32 bit;
+ bool present;
+
+ bit = 1U << (vid & 0x1f);
+ present = (vf->vlans[vid >> 5] & bit) != 0;
+ if (vid == 0) {
+ if (!present) {
+ vf->vlans[0] |= 1U;
+ igb_iov_rebuild_vlan(sc);
+ }
+ return (0);
+ }
+ if (add == present)
+ return (0);
+
+ /*
+ * Removals always reduce privilege and remain available. Charge only
+ * additions, which a hostile VF must alternate with removals to force
+ * repeated global VLAN rebuilds.
+ */
+ if (add && !igb_iov_vlan_present(sc, vid, false) &&
+ igb_iov_vlan_unique_count(sc, false) >=
+ E1000_VLVF_ARRAY_SIZE)
+ return (ENOSPC);
+ if (add && !igb_iov_vlan_add_allowed(vf))
+ return (EBUSY);
+
+ if (add) {
+ vf->vlans[vid >> 5] |= bit;
+ vf->vlan_count++;
+ } else {
+ vf->vlans[vid >> 5] &= ~bit;
+ vf->vlan_count--;
+ }
+ igb_iov_rebuild_vlan(sc);
+ return (0);
+}
+
+static void
+igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ bool update_uta;
+
+ update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0;
+ vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC |
+ IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW |
+ IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP |
+ IGB_VF_MDD_NOTIFY_PENDING);
+ vf->mbx_retry_at = 0;
+ vf->mdd_notify_at = 0;
+ vf->mbx_retry_count = 0;
+ /*
+ * A reset starts a new mailbox epoch. Permit one immediate NACK so a
+ * premature non-reset request does not wait for its posted-read
+ * timeout.
+ */
+ memset(&vf->last_nack, 0, sizeof(vf->last_nack));
+ vf->max_frame_size = ETHER_MAX_LEN;
+ vf->mc_count = 0;
+ vf->vlan_count = 0;
+ memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes));
+ memset(vf->vlans, 0, sizeof(vf->vlans));
+ /* Preserve the administrative access VLAN across VF and PF resets. */
+ if (vf->default_vlan == 0)
+ vf->vlans[0] = 1U;
+ else {
+ vf->vlans[vf->default_vlan >> 5] =
+ 1U << (vf->default_vlan & 0x1f);
+ vf->vlan_count = 1;
+ }
+ igb_iov_configure_vmolr(sc, vf);
+ if (update_uta)
+ igb_iov_set_uta(sc);
+}
+
+static bool
+igb_iov_vf_vlan_is_default(const struct igb_vf *vf)
+{
+ u32 expected;
+ int i;
+
+ for (i = 0; i < EM_VFTA_SIZE; i++) {
+ expected = 0;
+ if (i == vf->default_vlan >> 5)
+ expected = 1U << (vf->default_vlan & 0x1f);
+ if (vf->vlans[i] != expected)
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+igb_iov_sanitize_vf_queues(struct e1000_softc *sc,
+ struct igb_vf *vf)
+{
+ struct e1000_hw *hw;
+ u16 qid[IGB_IOV_VF_QUEUES_MAX];
+ u32 rxdctl, txdctl;
+ int i, nqueues, retry;
+
+ hw = &sc->hw;
+ switch (hw->mac.type) {
+ case e1000_82576:
+ nqueues = IGB_82576_VF_QUEUES;
+ qid[0] = vf->pool;
+ qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE;
+ break;
+ case e1000_i350:
+ nqueues = IGB_I350_VF_QUEUES;
+ qid[0] = vf->pool;
+ break;
+ default:
+ return (true);
+ }
+
+ /*
+ * I350 maps pool n to queue n. 82576 gives VF n physical queues n
+ * and n + 8, so both retained queue configurations must be cleared.
+ */
+ for (i = 0; i < nqueues; i++)
+ KASSERT(qid[i] < (hw->mac.type == e1000_82576 ?
+ IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES),
+ ("%s: invalid VF queue %u", __func__, qid[i]));
+
+ /*
+ * The 82576 and I350 specification updates, Software Clarification 3,
+ * note that VFLR does not reset the VF queue configuration. Clear the
+ * PF-programmable state before acknowledging the reset so a new VF
+ * owner cannot inherit it, particularly a descriptor-head write-back
+ * DMA address. The new VF driver initializes its active ring pointers
+ * during queue setup.
+ *
+ * Disable every queue first, then wait for outstanding DMA activity to
+ * stop before clearing TDWBAL/H and the remaining retained state.
+ * Spin only for the normal fast transition, then sleep so a VF that
+ * keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms.
+ */
+ for (i = 0; i < nqueues; i++) {
+ E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0);
+ E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0);
+ }
+ E1000_WRITE_FLUSH(hw);
+ for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) {
+ for (i = 0; i < nqueues; i++) {
+ rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i]));
+ txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i]));
+ if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
+ (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
+ break;
+ }
+ if (i == nqueues)
+ break;
+ if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) {
+ if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES)
+ DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US);
+ else
+ pause_sbt("igbqds",
+ IGB_IOV_QUEUE_DISABLE_PAUSE, 0,
+ C_PREL(1));
+ }
+ }
+ if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) {
+ if (ratecheck(&vf->last_queue_log,
+ &igb_iov_mbx_log_interval))
+ device_printf(sc->dev,
+ "could not disable queues for VF %u; "
+ "reset deferred\n", vf->pool);
+ return (false);
+ }
+
+ for (i = 0; i < nqueues; i++) {
+ E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0);
+ E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0);
+ E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0);
+ E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0);
+ E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0);
+ }
+ E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0);
+ E1000_WRITE_FLUSH(hw);
+ return (true);
+}
+
+static bool
+igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf,
+ bool reset_intrs)
+{
+ struct e1000_hw *hw;
+ bool rebuild_mta, rebuild_vlan, sanitized;
+ u32 reg;
+
+ hw = &sc->hw;
+ rebuild_mta = vf->mc_count != 0;
+ rebuild_vlan = !igb_iov_vf_vlan_is_default(vf);
+ reg = E1000_READ_REG(hw, E1000_VFTE);
+ E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool));
+ reg = E1000_READ_REG(hw, E1000_VFRE);
+ E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool));
+ if (reset_intrs)
+ E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool),
+ E1000_VTCTRL_RST);
+ sanitized = igb_iov_sanitize_vf_queues(sc, vf);
+ E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0);
+ igb_iov_clear_mac_filters(sc, vf);
+ igb_iov_clear_rar(sc, vf->rar_index);
+ igb_iov_reset_vf_state(sc, vf);
+ if (rebuild_mta)
+ igb_iov_rebuild_mta(sc);
+ if (rebuild_vlan)
+ igb_iov_rebuild_vlan(sc);
+ return (sanitized);
+}
+
+static bool
+igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ return (igb_iov_reset_event_common(sc, vf, true));
+}
+
+static void
+igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ /*
+ * VTCTRL.RST clears the VF's queue-enable and interrupt registers
+ * (I350 section 8.28.1). It therefore also removes the admin-vector
+ * route needed to deliver the reset notification below. MDD recovery
+ * explicitly permits toggling VFTE instead (section 7.8.3.8.3).
+ *
+ * Leave the interrupt registers intact, keep VFTE/VFRE disabled until
+ * the VF completes a new reset handshake, and use the no-CTS control
+ * message to make the guest reinitialize. FreeBSD and DPDK consume
+ * that message directly; Linux ACKs it and the PF's non-CTS ACK path
+ * replies with the NACK that schedules igbvf's reset task.
+ *
+ * Sanitization failure leaves the pool disabled. The VF reset
+ * handshake retries it and is NACKed while a queue remains active.
+ */
+ (void)igb_iov_reset_event_common(sc, vf, false);
+}
+
+static void
+igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ struct e1000_hw *hw;
+ u32 msg[3], reg;
+
+ hw = &sc->hw;
+ if (!igb_iov_reset_event(sc, vf)) {
+ msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
+ e1000_write_mbx(hw, msg, 1, vf->pool);
+ return;
+ }
+ igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
+ igb_iov_set_anti_spoof(sc, vf);
+
+ reg = E1000_READ_REG(hw, E1000_VFTE);
+ E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool));
+ reg = E1000_READ_REG(hw, E1000_VFRE);
+ E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool));
+ /*
+ * 82576's WVBR blocked bitmap is read-clear, so the reset handshake
+ * completes that event's lifetime. I350 MDFB might be read-only;
+ * re-arm its edge latch only after a valid MDFB sample reads clear.
+ */
+ if (hw->mac.type == e1000_82576)
+ vf->flags &= ~IGB_VF_MDD_BLOCKED;
+ vf->flags |= IGB_VF_CTS;
+
+ memset(msg, 0, sizeof(msg));
+ msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
+ memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN);
+ e1000_write_mbx(hw, msg, 3, vf->pool);
+}
+
+static int
+igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
+{
+ struct igb_vf_mac_filter *filter, *free_filter;
+ const u8 *mac;
+ u32 info;
+ int count, i;
+
+ info = msg[0] & E1000_VT_MSGINFO_MASK;
+ if (info == E1000_VF_MAC_FILTER_CLR) {
+ igb_iov_clear_mac_filters(sc, vf);
+ return (0);
+ }
+ if (info != E1000_VF_MAC_FILTER_ADD)
+ return (EINVAL);
+ if ((vf->flags & IGB_VF_CAP_MAC) == 0)
+ return (EPERM);
+
+ mac = (const u8 *)&msg[1];
+ if (!igb_iov_mac_valid(mac))
+ return (EINVAL);
+ if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0)
+ return (0);
+
+ count = 0;
+ free_filter = NULL;
+ for (i = 0; i < sc->num_vf_mac_filters; i++) {
+ filter = &sc->vf_mac_filters[i];
+ if (!filter->active) {
+ if (free_filter == NULL)
+ free_filter = filter;
+ continue;
+ }
+ if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0)
+ continue;
+ return (filter->pool == vf->pool ? 0 : EADDRINUSE);
+ }
+ for (i = 0; i < sc->num_vf_mac_filters; i++)
+ if (sc->vf_mac_filters[i].active &&
+ sc->vf_mac_filters[i].pool == vf->pool)
+ count++;
+ if (igb_iov_mac_in_use(sc, mac, vf))
+ return (EADDRINUSE);
+ if (count >= IGB_IOV_MAX_MAC_FILTERS)
+ return (ENOSPC);
+ if (free_filter == NULL)
+ return (ENOSPC);
+
+ free_filter->active = true;
+ free_filter->pool = vf->pool;
+ memcpy(free_filter->mac, mac, ETHER_ADDR_LEN);
+ igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool);
+ return (0);
+}
+
+static int
+igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
+{
+ u8 *mac;
+
+ if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0)
+ return (igb_iov_set_mac_filter(sc, vf, msg));
+
+ mac = (u8 *)&msg[1];
+ if (!igb_iov_mac_valid(mac))
+ return (EINVAL);
+ if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
+ !(vf->flags & IGB_VF_CAP_MAC))
+ return (EPERM);
+ if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
+ igb_iov_mac_in_use(sc, mac, vf))
+ return (EADDRINUSE);
+
+ memcpy(vf->mac, mac, ETHER_ADDR_LEN);
+ igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
+ return (0);
+}
+
+static int
+igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
+{
+ u16 hashes[IGB_IOV_MAX_MC_HASHES] = {};
+ bool overflow;
+ int count, i;
+
+ count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >>
+ E1000_VT_MSGINFO_SHIFT;
+ overflow = count > IGB_IOV_MAX_MC_HASHES ||
+ (msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0;
+ count = min(count, IGB_IOV_MAX_MC_HASHES);
+ for (i = 0; i < count; i++)
+ hashes[i] =
+ (msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff;
+ if (vf->mc_count == count &&
+ ((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow &&
+ memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0)
+ return (0);
+ memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes));
+ vf->mc_count = count;
+ if (overflow)
+ vf->flags |= IGB_VF_MCAST_OVERFLOW;
+ else
+ vf->flags &= ~IGB_VF_MCAST_OVERFLOW;
+ if (overflow &&
+ (vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) {
+ vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED;
+ device_printf(sc->dev,
+ "VF %u multicast list exceeds 30 entries; "
+ "enabling all-multicast reception\n", vf->pool);
+ }
+ igb_iov_configure_vmolr(sc, vf);
+ igb_iov_rebuild_mta(sc);
+ return (0);
+}
+
+static int
+igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
+{
+ u32 size;
+
+ size = msg[1];
+ if (size < ETHER_MIN_LEN)
+ return (EINVAL);
+ vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE);
+ igb_iov_configure_vmolr(sc, vf);
+ return (0);
+}
+
+static int
+igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg)
+{
+ u32 mode;
+
+ mode = msg & E1000_VT_MSGINFO_MASK;
+ if (mode & ~(E1000_VF_SET_PROMISC_UNICAST |
+ E1000_VF_SET_PROMISC_MULTICAST))
+ return (EINVAL);
+ if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC))
+ return (EPERM);
+
+ vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC);
+ if (mode & E1000_VF_SET_PROMISC_UNICAST)
+ vf->flags |= IGB_VF_UCAST_PROMISC;
+ if (mode & E1000_VF_SET_PROMISC_MULTICAST)
+ vf->flags |= IGB_VF_MCAST_PROMISC;
+ igb_iov_configure_vmolr(sc, vf);
+ igb_iov_set_uta(sc);
+ return (0);
+}
+
+static bool
+igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ struct e1000_hw *hw;
+ u32 msg[E1000_VFMAILBOX_SIZE], type;
+ int error;
+
+ hw = &sc->hw;
+ memset(msg, 0, sizeof(msg));
+ if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0)
+ return (false);
+ vf->flags &= ~IGB_VF_MBX_PENDING;
+ vf->mbx_retry_at = 0;
+ vf->mbx_retry_count = 0;
+
+ if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) {
+ e1000_unlock_mbx(hw, vf->pool);
+ return (true);
+ }
+ if (msg[0] == E1000_VF_RESET) {
+ igb_iov_reset_msg(sc, vf);
+ return (true);
+ }
+ if (!(vf->flags & IGB_VF_CTS)) {
+ if (igb_iov_nack_allowed(vf)) {
+ msg[0] = igb_iov_reply_header(msg[0], false, false);
+ e1000_write_mbx(hw, msg, 1, vf->pool);
+ } else
+ e1000_unlock_mbx(hw, vf->pool);
+ return (true);
+ }
+
+ type = msg[0] & 0xffff;
+ switch (type) {
+ case E1000_VF_SET_MAC_ADDR:
+ error = igb_iov_set_mac(sc, vf, msg);
+ break;
+ case E1000_VF_SET_MULTICAST:
+ error = igb_iov_set_multicast(sc, vf, msg);
+ break;
+ case E1000_VF_SET_VLAN:
+ if (vf->default_vlan != 0)
+ error = EPERM;
+ else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0)
+ error = EINVAL;
+ else
+ error = igb_iov_set_vlan(sc, vf,
+ msg[1] & E1000_VLVF_VLANID_MASK,
+ (msg[0] & E1000_VF_SET_VLAN_ADD) != 0);
+ break;
+ case E1000_VF_SET_LPE:
+ error = igb_iov_set_lpe(sc, vf, msg);
+ break;
+ case E1000_VF_SET_PROMISC:
+ error = igb_iov_set_promisc(sc, vf, msg[0]);
+ break;
+ default:
+ error = EOPNOTSUPP;
+ break;
+ }
+
+ msg[0] = igb_iov_reply_header(msg[0], true, error == 0);
+ e1000_write_mbx(hw, msg, 1, vf->pool);
+ return (true);
+}
+
+static sbintime_t
+igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf,
+ sbintime_t now)
+{
+ sbintime_t delay;
+
+ if ((vf->flags & IGB_VF_MBX_PENDING) == 0)
+ return (0);
+ if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at)
+ return (vf->mbx_retry_at);
+ if (igb_iov_process_msg(sc, vf))
+ return (0);
+
+ now = getsbinuptime();
+ if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) {
+ delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++];
+ vf->mbx_retry_at = now + delay;
+ return (vf->mbx_retry_at);
+ }
+
+ vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING);
+ vf->flags |= IGB_VF_MBX_GAVE_UP;
+ vf->mbx_retry_at = 0;
+ if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval))
+ device_printf(sc->dev,
+ "mailbox remained busy for VF %u; CTS revoked\n",
+ vf->pool);
+ return (0);
+}
+
+void
+igb_iov_handle_mbx(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ struct igb_vf *vf;
+ sbintime_t delay, next_retry_at, now, retry_at;
+ u32 msg;
+ int i;
+
+ if (!sc->iov_hw_active)
+ return;
+
+ hw = &sc->hw;
+ next_retry_at = 0;
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ now = getsbinuptime();
+ if (e1000_check_for_rst(hw, vf->pool) == 0) {
+ /*
+ * The old VF is gone. A new owner's reset handshake
+ * reruns sanitization before enabling its pool.
+ */
+ (void)igb_iov_reset_event(sc, vf);
+ }
+ if ((vf->flags &
+ (IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 &&
+ e1000_check_for_msg(hw, vf->pool) == 0) {
+ vf->flags |= IGB_VF_MBX_PENDING;
+ vf->mbx_retry_at = 0;
+ vf->mbx_retry_count = 0;
+ }
+ retry_at = igb_iov_service_pending_mbx(sc, vf, now);
+ if (retry_at != 0 &&
+ (next_retry_at == 0 || retry_at < next_retry_at))
+ next_retry_at = retry_at;
+ if (e1000_check_for_ack(hw, vf->pool) == 0 &&
+ !(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) {
+ msg = E1000_VT_MSGTYPE_NACK;
+ e1000_write_mbx(hw, &msg, 1, vf->pool);
+ }
+ }
+ if (next_retry_at != 0) {
+ delay = next_retry_at - getsbinuptime();
+ if (delay <= 0)
+ delay = SBT_1MS;
+ callout_reset_sbt(&sc->iov_mbx_retry, delay, 0,
+ igb_iov_mbx_retry_callout, sc, C_PREL(1));
+ }
+}
+
+static bool
+igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf)
+{
+ u32 msg;
+
+ /*
+ * MDD recovery preserves the VF's admin-vector configuration. Send
+ * the same no-CTS control message used for PF reset notification so
+ * the VF discards its state and completes a new reset handshake.
+ * A failed write is retried from the timer-driven admin pass; the VF's
+ * transmit watchdog remains the final fallback when traffic is still
+ * queued and notification never succeeds.
+ */
+ msg = E1000_PF_CONTROL_MSG;
+ if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) {
+ vf->mdd_notify_at =
+ getsbinuptime() + igb_iov_mdd_notify_retry;
+ if (ratecheck(&vf->last_mbx_log,
+ &igb_iov_mbx_log_interval))
+ device_printf(sc->dev,
+ "could not notify VF %u of malicious-driver "
+ "reset; will retry\n", vf->pool);
+ return (false);
+ }
+ vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING;
+ vf->mdd_notify_at = 0;
+ return (true);
+}
+
+void
+igb_iov_handle_mdd(struct e1000_softc *sc)
+{
+ struct igb_vf *vf;
+ u32 blocked, cleared, handled, lvmmc;
+ u32 readback, spoofed;
+ bool mdfb_valid, pending;
+ int i;
+
+ pending = atomic_readandclear_32(&sc->iov_pending) != 0;
+ lvmmc = pending ?
+ atomic_readandclear_32(&sc->iov_mdd_cause) : 0;
+ if (!sc->iov_hw_active) {
+ atomic_readandclear_32(&sc->iov_spoof_pending);
+ atomic_readandclear_32(&sc->iov_blocked_pending);
+ return;
+ }
+
+ blocked = 0;
+ handled = 0;
+ mdfb_valid = false;
+ if (sc->hw.mac.type == e1000_i350) {
+ u32 mdfb;
+
+ spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
+ /*
+ * I350 reports ordinary MAC/VLAN spoofing through the
+ * interrupt-time LVMMC snapshot rather than WVBR. The
+ * filter accumulates Last_Q into iov_spoof_pending so events
+ * from different VFs coalesce safely until this timer-driven
+ * admin pass.
+ */
+ spoofed &= IGB_I350_QUEUE_MASK;
+ /*
+ * Sample MDFB on every admin pass so a blocked queue is not
+ * mislabeled as an ordinary spoof when no MDDET observation
+ * is pending.
+ */
+ mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB);
+ if (__predict_false(mdfb == 0xffffffff))
+ mdfb = 0;
+ else {
+ mdfb &= IGB_I350_QUEUE_MASK;
+ mdfb_valid = true;
+ }
+ /*
+ * I350 SDM sections 8.14.10 and 8.14.11: WVBR reports
+ * spoof and malicious-driver events, while MDFB identifies
+ * the queues actually blocked for malicious behavior.
+ */
+ spoofed &= ~mdfb;
+ blocked = mdfb;
+ if (blocked != 0 && lvmmc == 0)
+ lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC);
+ /*
+ * A failed diagnostic read does not invalidate the
+ * blocked-queue bitmap that was read successfully above.
+ */
+ if (__predict_false(lvmmc == 0xffffffff))
+ lvmmc = 0;
+ /*
+ * MDFB is authoritative for queues stopped by malicious-driver
+ * detection. LVMMC reports causes such as VLAN IERR and
+ * Mal_PF, but its Last_Q field does not establish that a queue
+ * was blocked. Do not manufacture a blocked bit when MDFB is
+ * clear.
+ */
+ } else {
+ if (!pending)
+ return;
+ /*
+ * WVBR is read-clear and does not preserve every queue across
+ * multiple MDDET interrupts. The interrupt filter snapshots and
+ * accumulates its pool bitmaps before this deferred admin pass.
+ */
+ spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
+ blocked = atomic_readandclear_32(&sc->iov_blocked_pending);
+ /* A blocked-queue classification dominates its WVBR low bit. */
+ spoofed &= ~blocked;
+ }
+
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 &&
+ getsbinuptime() >= vf->mdd_notify_at)
+ (void)igb_iov_notify_vf_mdd_reset(sc, vf);
+ /*
+ * An invalid MDFB sample must neither report a new edge nor
+ * masquerade as evidence that an old edge has cleared.
+ */
+ if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
+ (blocked & (1U << i)) == 0)
+ vf->flags &= ~IGB_VF_MDD_BLOCKED;
+ if ((spoofed & (1U << i)) != 0 &&
+ ratecheck(&vf->last_spoof_log,
+ &igb_iov_spoof_log_interval))
+ device_printf(sc->dev,
+ "spoof event detected from VF %u; packet dropped\n",
+ vf->pool);
+ if ((blocked & (1U << i)) == 0)
+ continue;
+ if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0)
+ continue;
+ vf->flags |= IGB_VF_MDD_BLOCKED;
+ if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval))
+ device_printf(sc->dev,
+ "malicious-driver event 0x%08x from VF %u; "
+ "resetting VF\n", lvmmc, vf->pool);
+ igb_iov_mdd_reset_event(sc, vf);
+ vf->flags |= IGB_VF_MDD_NOTIFY_PENDING;
+ (void)igb_iov_notify_vf_mdd_reset(sc, vf);
+ handled |= 1U << i;
+ }
+ if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
+ (blocked & (1U << sc->pool)) == 0)
+ sc->iov_pf_mdd_blocked = false;
+ if ((blocked & (1U << sc->pool)) != 0 &&
+ (sc->hw.mac.type != e1000_i350 ||
+ !sc->iov_pf_mdd_blocked)) {
+ if (sc->hw.mac.type == e1000_i350)
+ sc->iov_pf_mdd_blocked = true;
+ if (ratecheck(&sc->iov_last_mdd_log,
+ &igb_iov_mdd_log_interval))
+ device_printf(sc->dev,
+ "malicious-driver event 0x%08x from PF queue; "
+ "resetting PF\n", lvmmc);
+ iflib_request_reset(sc->ctx);
+ iflib_admin_intr_deferred(sc->ctx);
+ handled |= 1U << sc->pool;
+ }
+ if (sc->hw.mac.type == e1000_i350 && handled != 0) {
+ /*
+ * I350 documentation conflicts: the register summary calls
+ * MDFB RWS while the detailed field table calls it RO. I350
+ * silicon clears a blocked bit when software writes it back.
+ * Write only bits whose recovery was initiated. If a revision
+ * instead implements MDFB as RO, the edge latch above prevents
+ * a reset loop and this one transition-time write is harmless.
+ */
+ E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled);
+ E1000_WRITE_FLUSH(&sc->hw);
+ /*
+ * Rearm from observed hardware state instead of waiting for
+ * the next admin pass. The PF context lock prevents a reset
+ * handshake from re-enabling the VF before this readback. A
+ * write-to-clear part reports zero; a read-only part retains
+ * the bit and therefore retains the one-shot edge latch.
+ */
+ readback = E1000_READ_REG(&sc->hw, E1000_MDFB);
+ if (__predict_false(readback == 0xffffffff))
+ cleared = 0;
+ else
+ cleared = handled &
+ ~(readback & IGB_I350_QUEUE_MASK);
+ for (i = 0; i < sc->num_vfs; i++)
+ if ((cleared & (1U << i)) != 0)
+ sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED;
+ if ((cleared & (1U << sc->pool)) != 0)
+ sc->iov_pf_mdd_blocked = false;
+ }
+ if (sc->hw.mac.type == e1000_i350) {
+ /*
+ * I350 can retain EICR.OTHER without delivering the admin MSI-X
+ * even though its EIMS and legacy IMS bits remain enabled. Kick
+ * the already-enabled vector on each admin pass so its filter
+ * consumes any retained ICR/LVMMC cause. A synthetic interrupt
+ * with no legacy cause is handled entirely by the filter.
+ */
+ E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask);
+ E1000_WRITE_FLUSH(&sc->hw);
+ }
+}
+
+void
+igb_iov_mdd_event(struct e1000_softc *sc)
+{
+ u32 blocked, cause, queues, queue, spoofed, wvbr;
+
+ /*
+ * LVMMC is clear-on-read. Preserve it in the interrupt filter, as
+ * Intel's igb driver does, rather than deferring the only copy.
+ */
+ cause = E1000_READ_REG(&sc->hw, E1000_LVMMC);
+ if (__predict_false(cause == 0xffffffff))
+ return;
+ if (sc->hw.mac.type == e1000_82576) {
+ /*
+ * Snapshot WVBR in the interrupt filter. Waiting for the admin
+ * task loses all but the last of back-to-back VF MDD events on
+ * 82576. Convert the staggered queue map into pool bits and OR
+ * each observation into software latches for deferred recovery.
+ */
+ wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR);
+ if (__predict_false(wvbr == 0xffffffff)) {
+ spoofed = 0;
+ blocked = 0;
+ } else {
+ queues = wvbr & IGB_82576_QUEUE_MASK;
+ spoofed = (queues & 0xff) |
+ (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
+ queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK;
+ blocked = (queues & 0xff) |
+ (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
+ }
+ SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed,
+ blocked);
+ /*
+ * 82576 can report a coalesced block-class event with all affected
+ * queues in WVBR's low half and no high-half blocked bits. If an
+ * ordinary spoof shares that snapshot, the register has no per-queue
+ * cause information. Deliberately fail closed by recovering every
+ * low-half queue; this can reset a spoof-only sibling, but avoids
+ * stranding a blocked VF. LVMMC.Last_Q identifies only the final
+ * event and lost simultaneous blocked VFs on tested silicon.
+ */
+ if (blocked == 0 &&
+ (cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) {
+ blocked = spoofed;
+ if (blocked == 0) {
+ queue = (cause >> 16) & 0xf;
+ blocked = 1U << (queue & 0x7);
+ }
+ }
+ if (spoofed != 0)
+ atomic_set_32(&sc->iov_spoof_pending, spoofed);
+ if (blocked != 0)
+ atomic_set_32(&sc->iov_blocked_pending, blocked);
+ }
+ if (sc->hw.mac.type == e1000_i350 &&
+ (cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) {
+ queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) &
+ IGB_I350_LVMMC_LAST_Q_MASK;
+ /*
+ * FreeBSD assigns one queue to each VF pool, so Last_Q is
+ * also the VF number. Preserve all VFs observed before the
+ * timer pass, and do not overwrite an unrelated blocked
+ * queue's diagnostic with this non-blocking spoof event.
+ */
+ atomic_set_32(&sc->iov_spoof_pending, 1U << queue);
+ return;
+ }
+ atomic_store_rel_32(&sc->iov_mdd_cause, cause);
+ atomic_set_32(&sc->iov_pending, 1);
+}
+
+void
+igb_iov_ping_all_vfs(struct e1000_softc *sc)
+{
+ struct igb_vf *vf;
+ u32 msg;
+ int i;
+
+ if (!sc->iov_hw_active)
+ return;
+
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ msg = E1000_PF_CONTROL_MSG;
+ if (vf->flags & IGB_VF_CTS)
+ msg |= E1000_VT_MSGTYPE_CTS;
+ e1000_write_mbx(&sc->hw, &msg, 1, vf->pool);
+ }
+}
+
+void
+igb_iov_initialize(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ struct igb_vf *vf;
+ u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl;
+ int i;
+
+ if (sc->num_vfs == 0)
+ return;
+
+ hw = &sc->hw;
+ atomic_readandclear_32(&sc->iov_mdd_cause);
+ atomic_readandclear_32(&sc->iov_pending);
+ atomic_readandclear_32(&sc->iov_spoof_pending);
+ atomic_readandclear_32(&sc->iov_blocked_pending);
+ /* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */
+ E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ);
+
+ vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL);
+ vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
+ E1000_VT_CTL_DISABLE_DEF_POOL);
+ vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT;
+ vt_ctl |= E1000_VT_CTL_VM_REPL_EN;
+ E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl);
+
+ mask = 1U << sc->pool;
+ E1000_WRITE_REG(hw, E1000_VFRE, mask);
+ E1000_WRITE_REG(hw, E1000_VFTE, mask);
+ /* A VF without RX descriptors must not block any other pool. */
+ E1000_WRITE_REG(hw, E1000_QDE,
+ hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES);
+ e1000_vmdq_set_loopback_pf(hw, true);
+ dtxctl = E1000_READ_REG(hw, E1000_DTXCTL);
+ dtxctl |= E1000_DTXCTL_MDP_EN;
+ if (hw->mac.type == e1000_82576) {
+ dtxctl |= E1000_DTXCTL_VLAN_ADDED |
+ E1000_DTXCTL_SPOOF_INT;
+ rplolr = E1000_READ_REG(hw, E1000_RPLOLR);
+ rplolr |= E1000_RPLOLR_STRVLAN;
+ E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr);
+ } else {
+ /*
+ * I350 SDM section 8.12.5 defines this field with inverted
+ * polarity: setting it keeps an ordinary spoof from disabling
+ * the VF queue. Enable its notification as well. I350
+ * hardware reports the VF in LVMMC.Last_Q (WVBR remains zero);
+ * the moderated admin vector captures that value, while
+ * timer-driven administration and per-VF ratecheck bound the
+ * work and console output.
+ */
+ dtxctl |= E1000_DTXCTL_SPOOF_INT |
+ IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE;
+ }
+ E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl);
+
+ igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool);
+ igb_iov_configure_pf_vmolr(sc);
+ igb_iov_set_uta(sc);
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ if (!(vf->flags & IGB_VF_ACTIVE))
+ continue;
+ /*
+ * A PF-wide reset is trusted and can require a complete guest
+ * replay. Guest-controlled RESET and VFLR do not refill this
+ * allowance.
+ */
+ igb_iov_reset_vlan_rate(vf);
+ igb_iov_clear_mac_filters(sc, vf);
+ igb_iov_reset_vf_state(sc, vf);
+ igb_iov_clear_rar(sc, vf->rar_index);
+ igb_iov_set_anti_spoof(sc, vf);
+ }
+ igb_iov_rebuild_mta(sc);
+ igb_iov_rebuild_vlan(sc);
+
+ rctl = E1000_READ_REG(hw, E1000_RCTL);
+ E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE);
+ E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
+
+ ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
+ E1000_WRITE_REG(hw, E1000_CTRL_EXT,
+ ctrl_ext | E1000_CTRL_EXT_PFRSTD);
+ E1000_WRITE_FLUSH(hw);
+ /*
+ * MDDET remains masked until iov_hw_active is published and iflib
+ * rearms the admin vector. Programming the per-pool policy above can
+ * leave a setup-time MDDET observation in the read-clear registers.
+ * If that stale cause is carried across the unmask, a later ordinary
+ * spoof can update LVMMC without generating a new interrupt edge.
+ *
+ * Drain only after all IOV policy is installed and before exposing the
+ * active state. Mailbox requests are also serviced by the periodic
+ * admin pass, and ping_all_vfs() below supplies a fresh notification.
+ */
+ /*
+ * Clear the setup-time interrupt latch before its diagnostic state.
+ * I350 does not reliably generate the next MDDET edge when LVMMC is
+ * consumed while ICR.MDDET remains latched. This differs deliberately
+ * from the final arm-time drain, where ICR is read last so a later event
+ * remains pending for the unmask.
+ */
+ if (hw->mac.type == e1000_i350)
+ (void)E1000_READ_REG(hw, E1000_ICR);
+ (void)E1000_READ_REG(hw, E1000_LVMMC);
+ if (hw->mac.type == e1000_82576)
+ (void)E1000_READ_REG(hw, E1000_WVBR);
+ if (hw->mac.type != e1000_i350)
+ (void)E1000_READ_REG(hw, E1000_ICR);
+ atomic_readandclear_32(&sc->iov_mdd_cause);
+ atomic_readandclear_32(&sc->iov_pending);
+ atomic_readandclear_32(&sc->iov_spoof_pending);
+ atomic_readandclear_32(&sc->iov_blocked_pending);
+ atomic_store_rel_32(&sc->iov_intr_drain_pending, 1);
+ sc->iov_hw_active = true;
+ igb_iov_ping_all_vfs(sc);
+}
+
+int
+igb_iov_validate(struct e1000_softc *sc, u16 num_vfs)
+{
+ if (!igb_iov_supported(sc))
+ return (ENXIO);
+ /* One of the eight hardware pools is reserved for the PF. */
+ if (num_vfs == 0 || num_vfs > MAX_NUM_VFS)
+ return (EINVAL);
+ if (sc->vfs != NULL)
+ return (EBUSY);
+ if (sc->intr_type != IFLIB_INTR_MSIX) {
+ device_printf(sc->dev, "SR-IOV requires MSI-X\n");
+ return (ENOTSUP);
+ }
+ if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) {
+ device_printf(sc->dev,
+ "SR-IOV requires one PF TX and RX queue; set "
+ "dev.igb.%d.iflib.override_ntxqs=1 and "
+ "dev.igb.%d.iflib.override_nrxqs=1 before attach\n",
+ device_get_unit(sc->dev), device_get_unit(sc->dev));
+ return (EINVAL);
+ }
+ return (0);
+}
+
+int
+igb_if_vf_status(if_ctx_t ctx, struct if_vf_status **statusp)
+{
+ struct e1000_softc *sc;
+ struct igb_vf *vf;
+ struct if_vf_info *info;
+ struct if_vf_status *status;
+ u_int num_queues;
+ int i;
+
+ sc = iflib_get_softc(ctx);
+ if (!sc->iov_mbx_retry_initialized)
+ return (EOPNOTSUPP);
+ num_queues = sc->hw.mac.type == e1000_82576 ?
+ IGB_82576_VF_QUEUES : IGB_I350_VF_QUEUES;
+ status = if_vf_status_alloc(sc->num_vfs);
+ if (status == NULL)
+ return (ENOMEM);
+ for (i = 0; i < sc->num_vfs; i++) {
+ vf = &sc->vfs[i];
+ info = &status->vfs[i];
+ info->fields = IFVF_F_CONFIGURED | IFVF_F_INITIALIZED |
+ IFVF_F_VLAN_MODE | IFVF_F_VLAN_COUNT |
+ IFVF_F_NUM_TX_QUEUES | IFVF_F_NUM_RX_QUEUES |
+ IFVF_F_ALLOW_SET_MAC |
+ IFVF_F_ALLOW_SET_VLAN | IFVF_F_MAC_ANTI_SPOOF |
+ IFVF_F_ALLOW_PROMISC | IFVF_F_TRAFFIC_ALLOWED |
+ IFVF_F_FAULT_BLOCKED;
+ info->index = i;
+ info->configured = (vf->flags & IGB_VF_ACTIVE) != 0;
+ info->initialized = sc->iov_hw_active &&
+ (vf->flags & IGB_VF_CTS) != 0;
+ if (!ETHER_IS_ZERO(vf->mac)) {
+ memcpy(info->mac, vf->mac, sizeof(info->mac));
+ info->fields |= IFVF_F_MAC;
+ }
+ if (vf->default_vlan == 0)
+ info->vlan_mode = IFVF_VLAN_TRUNK;
+ else {
+ info->vlan_mode = IFVF_VLAN_ACCESS;
+ info->vlan = vf->default_vlan;
+ info->vlan_pcp = 0;
+ info->vlan_proto = ETHERTYPE_VLAN;
+ info->fields |= IFVF_F_VLAN | IFVF_F_VLAN_PCP |
+ IFVF_F_VLAN_PROTO;
+ }
+ info->vlan_count = vf->vlan_count;
+ info->tx_queue_count = num_queues;
+ info->rx_queue_count = num_queues;
+ info->allow_set_mac = (vf->flags & IGB_VF_CAP_MAC) != 0;
+ /* Access VFs cannot manage VLAN membership through the mailbox. */
+ info->allow_set_vlan = vf->default_vlan == 0;
+ info->mac_anti_spoof =
+ (vf->flags & IGB_VF_MAC_ANTI_SPOOF) != 0;
+ info->allow_promisc =
+ (vf->flags & IGB_VF_ALLOW_PROMISC) != 0;
+ info->fault_blocked =
+ (vf->flags & IGB_VF_MDD_BLOCKED) != 0;
+ info->traffic_allowed = info->configured &&
+ !info->fault_blocked;
+ }
+ *statusp = status;
+ return (0);
+}
+
+int
+igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config)
+{
+ struct e1000_softc *sc;
+ int error, i;
+
+ sc = iflib_get_softc(ctx);
+ (void)config;
+ /*
+ * This callback may run while the PF is down. Record the software
+ * layout here; igb_iov_initialize() programs it during interface init.
+ */
+ atomic_store_rel_32(&sc->iov_teardown, 0);
+ error = igb_iov_validate(sc, num_vfs);
+ if (error != 0)
+ return (error);
+
+ sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV,
+ M_WAITOK | M_ZERO);
+ sc->num_vf_mac_filters =
+ sc->hw.mac.rar_entry_count - num_vfs - 1;
+ sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters,
+ sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO);
+ for (i = 0; i < sc->num_vf_mac_filters; i++)
+ sc->vf_mac_filters[i].rar_index = i + 1;
+ sc->pool = num_vfs;
+ sc->iov_mta_valid = false;
+ sc->iov_pf_mdd_blocked = false;
+ sc->tx_queues[0].txr.me = sc->pool;
+ sc->rx_queues[0].rxr.me = sc->pool;
+ e1000_init_mbx_params_pf(&sc->hw);
+ sc->num_vfs = num_vfs;
+ return (0);
+}
+
+void
+igb_if_iov_uninit(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+ u32 mask, rah;
+ int error, i, iov_pos;
+ u16 iov_ctl;
+
+ sc = iflib_get_softc(ctx);
+ if (sc->vfs == NULL)
+ return;
+ hw = &sc->hw;
+ sc->iov_hw_active = false;
+ if (sc->iov_mbx_retry_initialized)
+ callout_drain(&sc->iov_mbx_retry);
+
+ E1000_WRITE_REG(hw, E1000_MBVFIMR, 0);
+ mask = 1U << sc->pool;
+ E1000_WRITE_REG(hw, E1000_VFRE, mask);
+ E1000_WRITE_REG(hw, E1000_VFTE, mask);
+
+ /*
+ * pci_iov(4) invokes the driver before it clears VF Enable. Quiesce
+ * the VFs and clear it here so that 82576's queue-reuse interval is
+ * measured from the actual IOV-disable event.
+ */
+ error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos);
+ if (error == 0) {
+ iov_ctl = pci_read_config(sc->dev,
+ iov_pos + PCIR_SRIOV_CTL, 2);
+ iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
+ pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL,
+ iov_ctl, 2);
+ if (hw->mac.type == e1000_82576) {
+ pause("igbiov", MAX(1, howmany(hz, 10)));
+ E1000_WRITE_REG(hw, E1000_IOVCTL,
+ E1000_IOVCTL_REUSE_VFQ);
+ E1000_WRITE_FLUSH(hw);
+ pause("igbiov", MAX(1, howmany(hz, 10)));
+ }
+ } else
+ device_printf(sc->dev,
+ "could not disable PCI SR-IOV before queue reuse: %d\n",
+ error);
+
+ E1000_WRITE_REG(hw, E1000_VT_CTL, 0);
+ e1000_vmdq_set_loopback_pf(hw, false);
+ e1000_vmdq_set_anti_spoofing_pf(hw, false, 0);
+ for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
+ E1000_WRITE_REG(hw, E1000_VLVF(i), 0);
+ for (i = 0; i < sc->num_vfs; i++)
+ if (sc->vfs[i].flags & IGB_VF_ACTIVE)
+ igb_iov_clear_rar(sc, sc->vfs[i].rar_index);
+ for (i = 0; i < sc->num_vf_mac_filters; i++)
+ if (sc->vf_mac_filters[i].active)
+ igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index);
+ rah = E1000_READ_REG(hw, E1000_RAH(0));
+ rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
+ E1000_WRITE_REG(hw, E1000_RAH(0), rah);
+
+ free(sc->vfs, M_IGB_IOV);
+ free(sc->vf_mac_filters, M_IGB_IOV);
+ sc->vfs = NULL;
+ sc->vf_mac_filters = NULL;
+ sc->num_vfs = 0;
+ sc->num_vf_mac_filters = 0;
+ sc->pool = 0;
+ sc->iov_mta_valid = false;
+ sc->iov_pf_mdd_blocked = false;
+ sc->iov_pf_vlan_promisc = false;
+ igb_iov_vfta_shadow_invalidate(sc);
+ sc->tx_queues[0].txr.me = 0;
+ sc->rx_queues[0].rxr.me = 0;
+ atomic_readandclear_32(&sc->iov_mdd_cause);
+ atomic_readandclear_32(&sc->iov_pending);
+ atomic_readandclear_32(&sc->iov_spoof_pending);
+ atomic_readandclear_32(&sc->iov_blocked_pending);
+ atomic_readandclear_32(&sc->iov_intr_drain_pending);
+ atomic_store_rel_32(&sc->iov_teardown, 0);
+}
+
+static bool
+igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac,
+ const struct igb_vf *skip)
+{
+ int i;
+
+ if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0)
+ return (true);
+ for (i = 0; i < sc->num_vfs; i++)
+ if (&sc->vfs[i] != skip &&
+ (sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
+ memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0)
+ return (true);
+ for (i = 0; i < sc->num_vf_mac_filters; i++)
+ if (sc->vf_mac_filters[i].active &&
+ memcmp(sc->vf_mac_filters[i].mac, mac,
+ ETHER_ADDR_LEN) == 0)
+ return (true);
+ return (false);
+}
+
+int
+igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config)
+{
+ struct e1000_softc *sc;
+ struct igb_vf *vf;
+ struct ether_addr generated;
+ const void *mac;
+ char nameunit[IFNAMSIZ + sizeof("-vf65535")];
+ size_t mac_size;
+ uint64_t configured_vlan;
+ u16 vlan;
+
+ sc = iflib_get_softc(ctx);
+ if (vfnum >= sc->num_vfs)
+ return (EINVAL);
+ vf = &sc->vfs[vfnum];
+ if (vf->flags & IGB_VF_ACTIVE)
+ return (EBUSY);
+
+ configured_vlan = nvlist_get_number(config, "vlan");
+ if (configured_vlan > VF_VLAN_TRUNK)
+ return (EINVAL);
+ vlan = configured_vlan;
+ if (vlan == 0)
+ return (ENOTSUP);
+ if (vlan == VF_VLAN_TRUNK)
+ vlan = 0;
+ if (!igb_iov_vlan_present(sc, vlan, false) &&
+ igb_iov_vlan_unique_count(sc, false) >=
+ E1000_VLVF_ARRAY_SIZE)
+ return (ENOSPC);
+
+ vf->pool = vfnum;
+ vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1);
+ vf->max_frame_size = ETHER_MAX_LEN;
+ vf->default_vlan = vlan;
+ igb_iov_reset_vlan_rate(vf);
+ if (nvlist_exists_binary(config, "mac-addr")) {
+ mac = nvlist_get_binary(config, "mac-addr", &mac_size);
+ if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac))
+ return (EINVAL);
+ if (igb_iov_mac_in_use(sc, mac, vf))
+ return (EADDRINUSE);
+ memcpy(vf->mac, mac, ETHER_ADDR_LEN);
+ } else {
+ snprintf(nameunit, sizeof(nameunit), "%s-vf%u",
+ device_get_nameunit(sc->dev), vfnum);
+ ether_gen_addr_byname(nameunit, &generated);
+ memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN);
+ if (igb_iov_mac_in_use(sc, vf->mac, vf))
+ return (EADDRINUSE);
+ }
+ if (nvlist_get_bool(config, "allow-set-mac"))
+ vf->flags |= IGB_VF_CAP_MAC;
+ if (nvlist_get_bool(config, "mac-anti-spoof"))
+ vf->flags |= IGB_VF_MAC_ANTI_SPOOF;
+ if (nvlist_get_bool(config, "allow-promisc"))
+ vf->flags |= IGB_VF_ALLOW_PROMISC;
+ vf->flags |= IGB_VF_ACTIVE;
+
+ igb_iov_reset_vf_state(sc, vf);
+ igb_iov_set_anti_spoof(sc, vf);
+ igb_iov_rebuild_vlan(sc);
+ E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
+ return (0);
+}
+
+#endif /* PCI_IOV */
diff --git a/sys/dev/e1000/if_igb_iov.h b/sys/dev/e1000/if_igb_iov.h
new file mode 100644
index 000000000000..fd8053d9417f
--- /dev/null
+++ b/sys/dev/e1000/if_igb_iov.h
@@ -0,0 +1,61 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2010-2016, Intel Corporation
+ * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
+ */
+
+#ifndef _IF_IGB_IOV_H_
+#define _IF_IGB_IOV_H_
+
+#define IGB_IOV_MAX_FRAME_SIZE 0x2600
+
+#ifdef PCI_IOV
+
+#include <sys/nv.h>
+#include <sys/iov_schema.h>
+#include <dev/pci/pci_iov.h>
+
+int igb_iov_attach(struct e1000_softc *);
+void igb_iov_detach(struct e1000_softc *);
+bool igb_iov_supported(const struct e1000_softc *);
+bool igb_iov_enabled(const struct e1000_softc *);
+int igb_iov_validate(struct e1000_softc *, u16);
+int igb_if_iov_init(if_ctx_t, u16, const nvlist_t *);
+void igb_if_iov_uninit(if_ctx_t);
+int igb_if_iov_vf_add(if_ctx_t, u16, const nvlist_t *);
+struct if_vf_status;
+int igb_if_vf_status(if_ctx_t, struct if_vf_status **);
+void igb_iov_initialize(struct e1000_softc *);
+void igb_iov_handle_mbx(struct e1000_softc *);
+void igb_iov_handle_mdd(struct e1000_softc *);
+void igb_iov_mdd_event(struct e1000_softc *);
+void igb_iov_ping_all_vfs(struct e1000_softc *);
+void igb_iov_reset_prepare(struct e1000_softc *);
+u32 igb_iov_intr_mask(const struct e1000_softc *);
+void igb_iov_intr_drain_stale(struct e1000_softc *);
+void igb_iov_rebuild_mta(struct e1000_softc *);
+void igb_iov_rebuild_vlan(struct e1000_softc *);
+void igb_iov_update_pf_vmolr(struct e1000_softc *);
+
+#else
+
+#define igb_iov_attach(_sc) ((void)(_sc), 0)
+#define igb_iov_detach(_sc) ((void)(_sc))
+#define igb_iov_supported(_sc) (false)
+#define igb_iov_enabled(_sc) (false)
+#define igb_iov_initialize(_sc)
+#define igb_iov_handle_mbx(_sc)
+#define igb_iov_handle_mdd(_sc)
+#define igb_iov_mdd_event(_sc)
+#define igb_iov_ping_all_vfs(_sc)
+#define igb_iov_reset_prepare(_sc)
+#define igb_iov_intr_mask(_sc) (0)
+#define igb_iov_intr_drain_stale(_sc) ((void)(_sc))
+#define igb_iov_rebuild_mta(_sc)
+#define igb_iov_rebuild_vlan(_sc)
+#define igb_iov_update_pf_vmolr(_sc)
+
+#endif
+
+#endif /* _IF_IGB_IOV_H_ */
diff --git a/sys/dev/e1000/if_igbv.c b/sys/dev/e1000/if_igbv.c
new file mode 100644
index 000000000000..eafae1250053
--- /dev/null
+++ b/sys/dev/e1000/if_igbv.c
@@ -0,0 +1,973 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2001-2024, Intel Corporation
+ * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "if_em.h"
+
+#include <sys/sbuf.h>
+
+#define IGBV_82576_QUEUES 2
+#define IGBV_I350_QUEUES 1
+#define IGBV_MAX_MAC_FILTERS 3
+#define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10
+#define IGBV_QUEUE_DISABLE_DELAY_US 10
+#define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
+#define IGBV_QUEUE_DISABLE_RETRIES 20
+#define IGBV_QUEUE_SANITIZE_ATTEMPTS 3
+#define IGBV_VLAN_RETRY_BATCH 4
+#define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S)
+
+/* Reading these six configuration bytes performs the Hyper-V VF handshake. */
+#define IGBV_HV_RESET_OFFSET 0x201
+
+static const struct timeval igbv_queue_log_interval = { 2, 0 };
+static const struct timeval igbv_mbx_log_interval = { 60, 0 };
+static const sbintime_t igbv_queue_retry_delay[] = {
+ 100 * SBT_1MS,
+ 500 * SBT_1MS,
+};
+static const sbintime_t igbv_mbx_retry_delay[] = {
+ 250 * SBT_1MS,
+ 1 * SBT_1S,
+ 4 * SBT_1S,
+ 8 * SBT_1S,
+};
+_Static_assert(nitems(igbv_queue_retry_delay) + 1 ==
+ IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay");
+
+struct igb_vf_uc_addr_list {
+ struct e1000_softc *sc;
+ u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN];
+};
+
+static bool igbv_tx_pending(struct e1000_softc *);
+static bool igbv_vlan_retry_pending(const struct e1000_softc *);
+static void igbv_vlan_retry_tick(struct e1000_softc *);
+
+static s32
+igbv_hv_read_mac_addr(struct e1000_hw *hw)
+{
+
+ if (!em_is_valid_ether_addr(hw->mac.perm_addr))
+ return (-E1000_ERR_MAC_INIT);
+ memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
+ return (E1000_SUCCESS);
+}
+
+static s32
+igbv_hv_reset_hw(struct e1000_hw *hw)
+{
+ struct e1000_osdep *osdep;
+ u8 addr[ETHER_ADDR_LEN];
+ u32 ctrl;
+ int i;
+
+ osdep = hw->back;
+ hw->mac.get_link_status = true;
+ memset(hw->mac.perm_addr, 0, ETHER_ADDR_LEN);
+ /* Hyper-V does not service the native VF posted-message protocol. */
+ hw->mbx.timeout = 0;
+ ctrl = E1000_READ_REG(hw, E1000_CTRL);
+ if (ctrl == 0xffffffff)
+ return (-E1000_ERR_RESET);
+ /* Reset VF-local state before the Hyper-V host reset/MAC exchange. */
+ E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST);
+ E1000_WRITE_FLUSH(hw);
+ for (i = 0; i < E1000_VF_INIT_TIMEOUT; i++) {
+ if (hw->mbx.ops.check_for_rst(hw, 0) != E1000_SUCCESS)
+ break;
+ DELAY(5);
+ }
+
+ /*
+ * The PF handles 0x201..0x206 as a reset/MAC exchange, including
+ * partial reads. Do not use these bytes for routine status polling.
+ * The host-assigned address also identifies the matching hn interface.
+ */
+ for (i = 0; i < ETHER_ADDR_LEN; i++)
+ addr[i] = pci_read_config(osdep->dev, IGBV_HV_RESET_OFFSET + i, 1);
+ if (!em_is_valid_ether_addr(addr))
+ return (-E1000_ERR_MAC_INIT);
+ memcpy(hw->mac.perm_addr, addr, ETHER_ADDR_LEN);
+ return (E1000_SUCCESS);
+}
+
+static s32
+igbv_hv_check_for_link(struct e1000_hw *hw)
+{
+ u32 status;
+
+ /* Leave reset indications for the statistics counter-epoch check. */
+ status = E1000_READ_REG(hw, E1000_STATUS);
+ hw->mac.get_link_status = true;
+ if (status == 0xffffffff)
+ return (-E1000_ERR_MAC_INIT);
+ /* Poll even after link-up so a host port link-down cannot stay cached. */
+ hw->mac.get_link_status = (status & E1000_STATUS_LU) == 0;
+ return (E1000_SUCCESS);
+}
+
+static int
+igbv_hv_rar_set(struct e1000_hw *hw, u8 *addr, u32 index)
+{
+ int error;
+
+ error = index == 0 &&
+ memcmp(addr, hw->mac.perm_addr, ETHER_ADDR_LEN) == 0 ?
+ E1000_SUCCESS : -E1000_ERR_MAC_INIT;
+ if (igbv_hv_read_mac_addr(hw) != E1000_SUCCESS)
+ return (-E1000_ERR_MAC_INIT);
+ return (error);
+}
+
+static void
+igbv_hv_update_mc_addr_list(struct e1000_hw *hw __unused,
+ u8 *addrs __unused, u32 count __unused)
+{
+
+ /* The host owns receive-filter policy for the synthetic/VF pair. */
+}
+
+void
+igbv_init_hv_ops(struct e1000_hw *hw)
+{
+
+ /* Install after the ordinary VF MAC and mailbox parameter setup. */
+ hw->mac.ops.reset_hw = igbv_hv_reset_hw;
+ /* Native init_hw calls rar_set_vf directly, bypassing the RAR op. */
+ hw->mac.ops.init_hw = igbv_hv_read_mac_addr;
+ hw->mac.ops.read_mac_addr = igbv_hv_read_mac_addr;
+ hw->mac.ops.rar_set = igbv_hv_rar_set;
+ hw->mac.ops.check_for_link = igbv_hv_check_for_link;
+ hw->mac.ops.update_mc_addr_list = igbv_hv_update_mc_addr_list;
+}
+
+static void
+igbv_queue_retry_callout(void *arg)
+{
+ struct e1000_softc *sc;
+ if_t ifp;
+
+ sc = arg;
+ if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0)
+ return;
+ ifp = iflib_get_ifp(sc->ctx);
+ if ((if_getflags(ifp) & IFF_UP) == 0) {
+ atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
+ return;
+ }
+ iflib_request_reset_if_up(sc->ctx);
+ iflib_admin_intr_deferred(sc->ctx);
+}
+
+/*
+ * A missing PF can make the posted reset handshake wait for a full mailbox
+ * timeout. Keep that work out of stopped status paths. An administratively
+ * up VF retries complete initialization with an exponential delay capped at
+ * eight seconds, so it recovers without creating a tight mailbox poller.
+ */
+static void
+igbv_mbx_retry_callout(void *arg)
+{
+ struct e1000_softc *sc;
+ if_t ifp;
+
+ sc = arg;
+ if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 ||
+ atomic_load_acq_32(&sc->vf_mbx_ready) != 0 ||
+ iflib_in_detach(sc->ctx))
+ return;
+ ifp = iflib_get_ifp(sc->ctx);
+ if ((if_getflags(ifp) & IFF_UP) == 0)
+ return;
+
+ iflib_request_reset_if_up(sc->ctx);
+ iflib_admin_intr_deferred(sc->ctx);
+}
+
+static const char *
+igbv_reset_error_desc(s32 error)
+{
+
+ switch (error) {
+ case -E1000_ERR_RESET:
+ return ("PF reset acknowledgement timed out");
+ case -E1000_ERR_MAC_INIT:
+ return ("PF returned an invalid VF reset response");
+ case -E1000_ERR_MBX:
+ return ("PF mailbox reset exchange failed");
+ default:
+ return ("VF reset handshake failed");
+ }
+}
+
+void
+igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching)
+{
+
+ /* Report each backoff stage, then limit the steady eight-second retry. */
+ if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 &&
+ !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval))
+ return;
+ device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error),
+ error, attaching ? "; continuing attach" : "");
+}
+
+void
+igbv_mbx_retry_detach(struct e1000_softc *sc)
+{
+
+ if (!sc->vf_mbx_retry_initialized)
+ return;
+ atomic_readandclear_32(&sc->vf_mbx_retry_pending);
+ callout_drain(&sc->vf_mbx_retry);
+ sc->vf_mbx_retry_initialized = false;
+}
+
+void
+igbv_mbx_retry_prepare(struct e1000_softc *sc)
+{
+
+ if (!sc->vf_mbx_retry_initialized)
+ return;
+ atomic_readandclear_32(&sc->vf_mbx_retry_pending);
+ callout_drain(&sc->vf_mbx_retry);
+}
+
+void
+igbv_mbx_retry_stop(struct e1000_softc *sc)
+{
+ if_t ifp;
+
+ if (!sc->vf_mbx_retry_initialized)
+ return;
+ atomic_readandclear_32(&sc->vf_mbx_retry_pending);
+ callout_drain(&sc->vf_mbx_retry);
+ ifp = iflib_get_ifp(sc->ctx);
+ if ((if_getflags(ifp) & IFF_UP) == 0)
+ sc->vf_mbx_retry_stage = 0;
+}
+
+void
+igbv_mbx_retry_failed(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ if_t ifp;
+ sbintime_t delay;
+ u_int stage;
+
+ sc = iflib_get_softc(ctx);
+ atomic_store_rel_32(&sc->vf_mbx_ready, 0);
+ sc->link_speed = 0;
+ sc->link_duplex = 0;
+ if (sc->link_state != EM_LINK_STATE_DOWN) {
+ sc->link_state = EM_LINK_STATE_DOWN;
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ }
+ iflib_init_failed(ctx);
+
+ ifp = iflib_get_ifp(ctx);
+ if (!sc->vf_mbx_retry_initialized ||
+ (if_getflags(ifp) & IFF_UP) == 0)
+ return;
+ stage = sc->vf_mbx_retry_stage;
+ if (stage >= nitems(igbv_mbx_retry_delay))
+ stage = nitems(igbv_mbx_retry_delay) - 1;
+ delay = igbv_mbx_retry_delay[stage];
+ if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay))
+ sc->vf_mbx_retry_stage++;
+ atomic_set_32(&sc->vf_mbx_retry_pending, 1);
+ callout_reset_sbt(&sc->vf_mbx_retry, delay, 0,
+ igbv_mbx_retry_callout, sc, C_PREL(1));
+}
+
+static void
+igbv_mbx_retry_succeeded(struct e1000_softc *sc)
+{
+
+ atomic_store_rel_32(&sc->vf_mbx_ready, 1);
+ atomic_readandclear_32(&sc->vf_mbx_retry_pending);
+ if (sc->vf_mbx_retry_initialized)
+ callout_stop(&sc->vf_mbx_retry);
+ sc->vf_mbx_retry_stage = 0;
+ sc->vf_last_mbx_log.tv_sec = 0;
+ sc->vf_last_mbx_log.tv_usec = 0;
+}
+
+void
+igbv_queue_retry_detach(struct e1000_softc *sc)
+{
+
+ if (!sc->vf_queue_retry_initialized)
+ return;
+ atomic_readandclear_32(&sc->vf_queue_retry_pending);
+ callout_drain(&sc->vf_queue_retry);
+ sc->vf_queue_retry_initialized = false;
+}
+
+void
+igbv_queue_retry_stop(struct e1000_softc *sc)
+{
+
+ if (!sc->vf_queue_retry_initialized)
+ return;
+ if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0)
+ atomic_set_32(&sc->vf_queue_retry_new_epoch, 1);
+ callout_stop(&sc->vf_queue_retry);
+}
+
+void
+igbv_queue_retry_prepare(struct e1000_softc *sc)
+{
+ bool new_epoch;
+
+ new_epoch =
+ atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0;
+ if (!sc->vf_queue_gave_up && !new_epoch)
+ return;
+ sc->vf_queue_failures = 0;
+ sc->vf_queue_gave_up = false;
+}
+
+static void
+igbv_queue_retry_succeeded(struct e1000_softc *sc)
+{
+
+ atomic_readandclear_32(&sc->vf_queue_retry_pending);
+ atomic_readandclear_32(&sc->vf_queue_retry_new_epoch);
+ if (sc->vf_queue_retry_initialized)
+ callout_stop(&sc->vf_queue_retry);
+ sc->vf_queue_failures = 0;
+ sc->vf_queue_gave_up = false;
+}
+
+void
+igbv_queue_retry_failed(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ sbintime_t delay;
+
+ sc = iflib_get_softc(ctx);
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+
+ if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS)
+ sc->vf_queue_failures++;
+ if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) {
+ delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1];
+ atomic_set_32(&sc->vf_queue_retry_pending, 1);
+ callout_reset_sbt(&sc->vf_queue_retry, delay, 0,
+ igbv_queue_retry_callout, sc, C_PREL(1));
+ } else if (!sc->vf_queue_gave_up) {
+ atomic_readandclear_32(&sc->vf_queue_retry_pending);
+ callout_stop(&sc->vf_queue_retry);
+ sc->vf_queue_gave_up = true;
+ device_printf(sc->dev,
+ "retained VF queues remained active after %u attempts; "
+ "interface left down; toggle it down/up to retry\n",
+ sc->vf_queue_failures);
+ }
+
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ iflib_init_failed(ctx);
+}
+
+void
+igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid)
+{
+ bool pending;
+
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+ pending = igbv_vlan_retry_pending(sc);
+ sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f);
+ /* Bound the whole batch from its first failure, not each new VID. */
+ if (!pending)
+ sc->vf_vlan_retry_deadline =
+ getsbinuptime() + IGBV_VLAN_RETRY_WINDOW;
+}
+
+void
+igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid)
+{
+
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+ sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f));
+}
+
+static bool
+igbv_vlan_retry_pending(const struct e1000_softc *sc)
+{
+ int i;
+
+ for (i = 0; i < EM_VFTA_SIZE; i++)
+ if (sc->vf_vfta_retry[i] != 0)
+ return (true);
+ return (false);
+}
+
+static void
+igbv_vlan_retry_tick(struct e1000_softc *sc)
+{
+ u32 bit;
+ u16 vid;
+ int attempts, i, remaining;
+
+ if (!igbv_vlan_retry_pending(sc)) {
+ sc->vf_vlan_retry_deadline = 0;
+ return;
+ }
+ if (getsbinuptime() >= sc->vf_vlan_retry_deadline) {
+ remaining = 0;
+ for (i = 0; i < EM_VFTA_SIZE; i++)
+ remaining += bitcount32(sc->vf_vfta_retry[i]);
+ memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
+ sc->vf_vlan_retry_deadline = 0;
+ device_printf(sc->dev,
+ "VF VLAN restore retries exhausted for %d VIDs\n",
+ remaining);
+ return;
+ }
+
+ /*
+ * The mailbox NACK does not distinguish a transient PF rate limit
+ * from permanent VLVF exhaustion. Retry at the PF's sustained
+ * allowance, but bound the entire recovery window so ENOSPC cannot
+ * create a permanent mailbox poller.
+ */
+ for (attempts = 0, i = 0;
+ attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) {
+ vid = sc->vf_vlan_retry_cursor;
+ sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff;
+ bit = 1U << (vid & 0x1f);
+ if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0)
+ continue;
+ attempts++;
+ if ((sc->shadow_vfta[vid >> 5] & bit) == 0 ||
+ e1000_vfta_set_vf(&sc->hw, vid, true) ==
+ E1000_SUCCESS)
+ sc->vf_vfta_retry[vid >> 5] &= ~bit;
+ }
+ if (!igbv_vlan_retry_pending(sc))
+ sc->vf_vlan_retry_deadline = 0;
+}
+
+int
+igbv_if_attach_pre(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ device_t dev;
+ int error;
+
+ dev = iflib_get_dev(ctx);
+ if (pci_msix_count(dev) < 2) {
+ device_printf(dev, "VF operation requires two MSI-X vectors\n");
+ return (ENXIO);
+ }
+ error = em_if_attach_pre(ctx);
+ if (error != 0)
+ return (error);
+
+ sc = iflib_get_softc(ctx);
+ callout_init(&sc->vf_queue_retry, 1);
+ sc->vf_queue_retry_initialized = true;
+ callout_init(&sc->vf_mbx_retry, 1);
+ sc->vf_mbx_retry_initialized = true;
+
+ KASSERT(sc->vf_ifp &&
+ (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0,
+ ("%s: igbv attached without VF policy", __func__));
+ return (0);
+}
+
+int
+igbv_if_attach_post(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ int error;
+
+ sc = iflib_get_softc(ctx);
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+ if (sc->intr_type != IFLIB_INTR_MSIX) {
+ device_printf(sc->dev, "VF operation requires MSI-X\n");
+ return (ENXIO);
+ }
+ error = em_if_attach_post(ctx);
+ if (error != 0)
+ return (error);
+
+ /*
+ * Attach failures can leave the device sysctl tree registered when
+ * hw.bus.disable_failed_devices is set. Do not publish handlers with
+ * softc arguments until iflib has successfully allocated MSI-X.
+ */
+ em_add_device_sysctls(sc);
+ return (0);
+}
+
+int
+igbv_if_media_change(if_ctx_t ctx __unused)
+{
+
+ return (EOPNOTSUPP);
+}
+
+void
+igbv_if_update_admin_status(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+ device_t dev;
+ bool link_check, timer_tick;
+
+ sc = iflib_get_softc(ctx);
+ hw = &sc->hw;
+ dev = iflib_get_dev(ctx);
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+
+ if (!iflib_is_running(ctx) ||
+ !sc->vf_queues_sanitized ||
+ atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
+ if (sc->link_state != EM_LINK_STATE_DOWN) {
+ sc->link_speed = 0;
+ sc->link_duplex = 0;
+ sc->link_state = EM_LINK_STATE_DOWN;
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ }
+ return;
+ }
+
+ if (igbv_is_hyperv(sc)) {
+ u32 status, txdctl;
+
+ if (sc->vf_reset_pending)
+ return;
+ txdctl = E1000_READ_REG(hw, E1000_TXDCTL(0));
+ if (txdctl != 0xffffffff &&
+ (txdctl & E1000_TXDCTL_QUEUE_ENABLE) == 0) {
+ /* Carrier can remain up after the PF disables queue DMA. */
+ sc->vf_stats_valid = false;
+ sc->link_speed = sc->link_duplex = 0;
+ if (sc->link_state != EM_LINK_STATE_DOWN) {
+ sc->link_state = EM_LINK_STATE_DOWN;
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ }
+ /*
+ * Do not reset while carrier is down. Limit requests
+ * if the host keeps the queue disabled with carrier up.
+ */
+ status = E1000_READ_REG(hw, E1000_STATUS);
+ if (status != 0xffffffff &&
+ (status & E1000_STATUS_LU) != 0 &&
+ ratecheck(&sc->vf_last_queue_log,
+ &igbv_queue_log_interval)) {
+ device_printf(dev,
+ "Hyper-V VF queue disabled; requesting recovery\n");
+ sc->vf_reset_pending = true;
+ iflib_request_reset_if_up(ctx);
+ iflib_admin_intr_deferred(ctx);
+ }
+ return;
+ }
+ }
+
+ if (!sc->vf_reset_pending &&
+ atomic_readandclear_32(&sc->promisc_pending) != 0)
+ (void)em_if_set_promisc_impl(ctx,
+ if_getflags(iflib_get_ifp(ctx)));
+
+ if (e1000_check_for_link(hw) != E1000_SUCCESS &&
+ !sc->vf_reset_pending) {
+ sc->vf_reset_pending = true;
+ iflib_request_reset(ctx);
+ iflib_admin_intr_deferred(ctx);
+ }
+ link_check = !hw->mac.get_link_status;
+
+ if (link_check &&
+ (sc->link_state == EM_LINK_STATE_DOWN ||
+ sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) {
+ e1000_get_speed_and_duplex(hw, &sc->link_speed,
+ &sc->link_duplex);
+ if (bootverbose)
+ device_printf(dev, "Link is up %d Mbps %s\n",
+ sc->link_speed,
+ sc->link_duplex == FULL_DUPLEX ?
+ "Full Duplex" : "Half Duplex");
+ sc->link_state = EM_LINK_STATE_UP;
+ iflib_link_state_change(ctx, LINK_STATE_UP,
+ IF_Mbps(sc->link_speed));
+ } else if (!link_check &&
+ (sc->link_state == EM_LINK_STATE_UP ||
+ sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) {
+ sc->link_speed = 0;
+ sc->link_duplex = 0;
+ sc->link_state = EM_LINK_STATE_DOWN;
+ iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
+ }
+
+ /*
+ * A VF stops transmit DMA when its PF reports link down. Reset if
+ * descriptors remain queued so they cannot be sent stale when carrier
+ * returns, matching the periodic check in Linux igbvf.
+ */
+ if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) {
+ sc->vf_reset_pending = true;
+ iflib_request_reset(ctx);
+ iflib_admin_intr_deferred(ctx);
+ }
+ /* em_if_init() establishes a new counter baseline after the reset. */
+ timer_tick = !sc->vf_reset_pending &&
+ atomic_readandclear_32(&sc->stats_pending) != 0;
+ if (timer_tick) {
+ em_update_stats_counters(sc);
+ /* iflib closes admission before stop; do not replay after reset. */
+ if (iflib_is_running(ctx))
+ igbv_vlan_retry_tick(sc);
+ }
+}
+
+static bool
+igbv_tx_pending(struct e1000_softc *sc)
+{
+ struct tx_ring *txr;
+ u32 head, tail;
+
+ for (int i = 0; i < sc->tx_num_queues; i++) {
+ txr = &sc->tx_queues[i].txr;
+ head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me));
+ tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me));
+ if (head != tail)
+ return (true);
+ }
+ return (false);
+}
+
+bool
+igbv_sanitize_queues(struct e1000_softc *sc)
+{
+ struct e1000_hw *hw;
+ u32 rxdctl, txdctl;
+ int i, nqueues, retry;
+
+ hw = &sc->hw;
+ switch (hw->mac.type) {
+ case e1000_vfadapt:
+ nqueues = IGBV_82576_QUEUES;
+ break;
+ case e1000_vfadapt_i350:
+ nqueues = IGBV_I350_QUEUES;
+ break;
+ default:
+ return (true);
+ }
+
+ /*
+ * The 82576 and I350 specification updates, Software Clarification 3,
+ * note that VFLR leaves this queue configuration intact. Clear it
+ * before programming the new rings so igbv does not depend on its PF
+ * to sanitize state left by a previous VF owner. igbv uses only queue
+ * zero, but must also clear the unused second 82576 queue.
+ *
+ * Disable every queue first and wait for outstanding DMA activity to
+ * stop before programming TDWBAL/H. Spin only for the normal fast
+ * transition, then sleep until the bounded deadline.
+ */
+ for (i = 0; i < nqueues; i++) {
+ E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0);
+ E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0);
+ }
+ E1000_WRITE_FLUSH(hw);
+ for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) {
+ for (i = 0; i < nqueues; i++) {
+ rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
+ txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
+ if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
+ (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
+ break;
+ }
+ if (i == nqueues)
+ break;
+ if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) {
+ if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES)
+ DELAY(IGBV_QUEUE_DISABLE_DELAY_US);
+ else
+ pause_sbt("igbvqds",
+ IGBV_QUEUE_DISABLE_PAUSE, 0,
+ C_PREL(1));
+ }
+ }
+ if (retry == IGBV_QUEUE_DISABLE_RETRIES) {
+ if (ratecheck(&sc->vf_last_queue_log,
+ &igbv_queue_log_interval))
+ device_printf(sc->dev,
+ "could not disable retained VF queues; "
+ "reset deferred\n");
+ return (false);
+ }
+
+ for (i = 0; i < nqueues; i++) {
+ E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0);
+ E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0);
+ E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0);
+ E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0);
+ E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0);
+ }
+ E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0);
+ E1000_WRITE_FLUSH(hw);
+ return (true);
+}
+
+bool
+igbv_reset(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+ s32 error;
+
+ sc = iflib_get_softc(ctx);
+ hw = &sc->hw;
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+
+ /*
+ * Receive-buffer allocation and flow control are port resources owned
+ * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size.
+ */
+ sc->pba = 0;
+ hw->fc = (struct e1000_fc_info){
+ .current_mode = e1000_fc_none,
+ .requested_mode = e1000_fc_none,
+ };
+
+ error = e1000_reset_hw(hw);
+ atomic_store_rel_32(&sc->vf_mbx_ready, 0);
+ sc->vf_queues_sanitized = igbv_sanitize_queues(sc);
+ if (!sc->vf_queues_sanitized) {
+ return (false);
+ }
+ igbv_queue_retry_succeeded(sc);
+ if (error != E1000_SUCCESS) {
+ igbv_log_reset_failure(sc, error, false);
+ return (false);
+ }
+ memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale));
+ memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
+ sc->vf_vlan_retry_deadline = 0;
+ sc->vf_vlan_retry_cursor = 0;
+ if (e1000_init_hw(hw) < 0) {
+ device_printf(sc->dev, "Hardware Initialization Failed\n");
+ return (false);
+ }
+ e1000_check_for_link(hw);
+ igbv_mbx_retry_succeeded(sc);
+ return (true);
+}
+
+void
+igbv_initialize_transmit_unit(if_ctx_t ctx)
+{
+
+ KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
+ ("%s called for a PF", __func__));
+ em_initialize_transmit_rings(ctx);
+}
+
+void
+igbv_initialize_receive_unit(if_ctx_t ctx)
+{
+
+ KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp,
+ ("%s called for a PF", __func__));
+ igb_initialize_receive_rings(ctx, true);
+}
+
+void
+igbv_if_intr_enable(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+ u32 mask;
+
+ sc = iflib_get_softc(ctx);
+ hw = &sc->hw;
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+ if (!sc->vf_queues_sanitized ||
+ atomic_load_acq_32(&sc->vf_mbx_ready) == 0)
+ return;
+ mask = sc->que_mask | sc->link_mask;
+
+ E1000_WRITE_REG(hw, E1000_EIAC, mask);
+ E1000_WRITE_REG(hw, E1000_EIAM, mask);
+ E1000_WRITE_REG(hw, E1000_EIMS, mask);
+ E1000_WRITE_FLUSH(hw);
+}
+
+void
+igbv_if_intr_disable(if_ctx_t ctx)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+
+ sc = iflib_get_softc(ctx);
+ hw = &sc->hw;
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+
+ E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff);
+ E1000_WRITE_REG(hw, E1000_EIAC, 0);
+ E1000_WRITE_FLUSH(hw);
+}
+
+int
+igbv_get_regs(SYSCTL_HANDLER_ARGS)
+{
+ struct e1000_softc *sc;
+ struct e1000_hw *hw;
+ struct sbuf *sb;
+ int error;
+
+ sc = (struct e1000_softc *)arg1;
+ hw = &sc->hw;
+ KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ /*
+ * Limited VF register set:
+ * Don't read EICR here because it is clear-on-read. The VF register
+ * file exposes its queue pair at index zero, so this diagnostic does
+ * not depend on the narrower lifetime of iflib's queue arrays.
+ */
+ sbuf_printf(sb, "VF Registers\n");
+ sbuf_printf(sb, "\tVTCTRL\t %08x\n",
+ E1000_READ_REG(hw, E1000_CTRL));
+ sbuf_printf(sb, "\tSTATUS\t %08x\n",
+ E1000_READ_REG(hw, E1000_STATUS));
+ sbuf_printf(sb, "\tRDLEN\t %08x\n",
+ E1000_READ_REG(hw, E1000_RDLEN(0)));
+ sbuf_printf(sb, "\tRDH\t %08x\n",
+ E1000_READ_REG(hw, E1000_RDH(0)));
+ sbuf_printf(sb, "\tRDT\t %08x\n",
+ E1000_READ_REG(hw, E1000_RDT(0)));
+ sbuf_printf(sb, "\tTDLEN\t %08x\n",
+ E1000_READ_REG(hw, E1000_TDLEN(0)));
+ sbuf_printf(sb, "\tTDH\t %08x\n",
+ E1000_READ_REG(hw, E1000_TDH(0)));
+ sbuf_printf(sb, "\tTDT\t %08x\n",
+ E1000_READ_REG(hw, E1000_TDT(0)));
+
+ error = sbuf_finish(sb);
+ sbuf_delete(sb);
+ return (error);
+}
+
+static u_int
+igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx)
+{
+ struct igb_vf_uc_addr_list *list;
+ const u8 *addr;
+
+ list = arg;
+ addr = (const u8 *)LLADDR(sdl);
+ if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
+ return (0);
+ if (idx < IGBV_MAX_MAC_FILTERS)
+ memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN);
+ return (1);
+}
+
+void
+igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp)
+{
+ struct igb_vf_uc_addr_list list = {
+ .sc = sc,
+ };
+ u_int count;
+
+ if (igbv_is_hyperv(sc))
+ return;
+
+ count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list);
+ if (count > IGBV_MAX_MAC_FILTERS) {
+ device_printf(sc->dev,
+ "too many secondary unicast addresses; maximum is %u\n",
+ IGBV_MAX_MAC_FILTERS);
+ }
+ if (count == 0 && !sc->vf_uc_filters_set)
+ return;
+ /*
+ * Linux igb PFs validate the address field before dispatching the CLR
+ * subcommand. Supply the primary address rather than the zero payload
+ * used by igbvf so those PFs actually remove the old filters. FreeBSD
+ * PFs dispatch CLR before inspecting the otherwise-ignored address.
+ */
+ if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR,
+ sc->hw.mac.addr) != E1000_SUCCESS) {
+ device_printf(sc->dev,
+ "VF secondary unicast filter clear request failed\n");
+ return;
+ }
+ sc->vf_uc_filters_set = false;
+ if (count > IGBV_MAX_MAC_FILTERS)
+ return;
+
+ for (u_int i = 0; i < count; i++) {
+ if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD,
+ list.addrs[i]) != E1000_SUCCESS) {
+ device_printf(sc->dev,
+ "VF secondary unicast filter add request failed "
+ "for %6D\n", list.addrs[i], ":");
+ } else
+ sc->vf_uc_filters_set = true;
+ usec_delay(200);
+ }
+}
+
+void
+igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp)
+{
+ u8 *lladdr;
+
+ if (!em_is_valid_ether_addr(sc->hw.mac.addr))
+ return;
+ lladdr = (u8 *)if_getlladdr(ifp);
+ if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
+ return;
+
+ device_printf(sc->dev,
+ "PF rejected or replaced the requested MAC; using %6D\n",
+ sc->hw.mac.addr, ":");
+ /*
+ * if_setlladdr() would re-enter the driver's address-change path.
+ * Initialization already holds the context lock, so update the
+ * storage directly and issue the notification it would have sent.
+ */
+ memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN);
+
+ CURVNET_SET_QUIET(if_getvnet(ifp));
+ EVENTHANDLER_INVOKE(iflladdr_event, ifp);
+ CURVNET_RESTORE();
+}
diff --git a/sys/dev/e1000/igb_txrx.c b/sys/dev/e1000/igb_txrx.c
index 07a01274064c..dd4d1e5b5974 100644
--- a/sys/dev/e1000/igb_txrx.c
+++ b/sys/dev/e1000/igb_txrx.c
@@ -56,7 +56,14 @@ static int igb_tx_ctx_setup(struct tx_ring *, if_pkt_info_t, uint32_t *,
static int igb_tso_setup(struct tx_ring *, if_pkt_info_t, uint32_t *,
uint32_t *);
-static void igb_rx_checksum(uint32_t, if_rxd_info_t, uint32_t);
+enum igb_rx_csum_status {
+ IGB_RX_CSUM_NONE,
+ IGB_RX_CSUM_GOOD,
+ IGB_RX_CSUM_ERROR,
+};
+
+static enum igb_rx_csum_status igb_rx_checksum(uint32_t, if_rxd_info_t,
+ uint32_t);
static int igb_determine_rsstype(uint16_t);
extern void igb_if_enable_intr(if_ctx_t);
@@ -73,6 +80,29 @@ struct if_txrx igb_txrx = {
.ift_legacy_intr = em_intr
};
+static bool
+igb_vf_vlan_registered(const struct e1000_softc *sc, u16 vtag)
+{
+ u32 vlans;
+ u16 vid;
+
+ /*
+ * 82576 strips an administrative access VLAN but still reports it in
+ * the descriptor. Like Linux igbvf, expose tags only when the VF
+ * requested that VID from the PF.
+ */
+ if (sc->hw.mac.type != e1000_vfadapt &&
+ sc->hw.mac.type != e1000_vfadapt_i350)
+ return (true);
+ vid = EVL_VLANOFTAG(vtag);
+ /*
+ * A trunk VF is an implicit member of VID 0, so retain priority-tag
+ * metadata without requiring vlan(4) to register a VID-0 interface.
+ */
+ vlans = sc->shadow_vfta[vid >> 5] | sc->vf_vfta_stale[vid >> 5];
+ return (vid == 0 || (vlans & (1U << (vid & 0x1f))) != 0);
+}
+
/**********************************************************************
*
* Setup work for hardware segmentation offload (TSO) on
@@ -459,6 +489,7 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
uint16_t pkt_info, len;
uint32_t ptype, staterr;
+ enum igb_rx_csum_status csum_status;
int i, cidx;
bool eop;
@@ -508,8 +539,15 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
rxr->rx_bytes += ri->iri_len;
rxr->rx_packets++;
- if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
- igb_rx_checksum(staterr, ri, ptype);
+ if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) {
+ csum_status = igb_rx_checksum(staterr, ri, ptype);
+ if (sc->vf_ifp) {
+ if (csum_status == IGB_RX_CSUM_GOOD)
+ sc->rx_csum_good++;
+ else if (csum_status == IGB_RX_CSUM_ERROR)
+ sc->rx_csum_errors++;
+ }
+ }
if (staterr & E1000_RXD_STAT_VP) {
if (((sc->hw.mac.type == e1000_i350) ||
@@ -519,7 +557,8 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
ri->iri_vtag = be16toh(rxd->wb.upper.vlan);
else
ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
- ri->iri_flags |= M_VLANTAG;
+ if (igb_vf_vlan_registered(sc, ri->iri_vtag))
+ ri->iri_flags |= M_VLANTAG;
}
ri->iri_flowid =
@@ -537,19 +576,19 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
* doesn't spend time verifying the checksum.
*
*********************************************************************/
-static void
+static enum igb_rx_csum_status
igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
{
uint16_t status = (uint16_t)staterr;
uint8_t errors = (uint8_t)(staterr >> 24);
if (__predict_false(status & E1000_RXD_STAT_IXSM))
- return;
+ return (IGB_RX_CSUM_NONE);
/* If there is a layer 3 or 4 error we are done */
if (__predict_false(errors &
(E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE)))
- return;
+ return (IGB_RX_CSUM_ERROR);
/* IP Checksum Good */
if (status & E1000_RXD_STAT_IPCS)
@@ -569,6 +608,8 @@ igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
ri->iri_csum_data = htons(0xffff);
}
}
+
+ return (IGB_RX_CSUM_GOOD);
}
/********************************************************************
diff --git a/sys/modules/em/Makefile b/sys/modules/em/Makefile
index aad7cb07a31f..83d3242a2edb 100644
--- a/sys/modules/em/Makefile
+++ b/sys/modules/em/Makefile
@@ -1,10 +1,10 @@
.PATH: ${SRCTOP}/sys/dev/e1000
KMOD = if_em
-SRCS = device_if.h bus_if.h pci_if.h opt_ddb.h opt_inet.h \
+SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h opt_ddb.h opt_inet.h \
opt_inet6.h opt_rss.h ifdi_if.h
SRCS += $(CORE_SRC) $(LEGACY_SRC)
SRCS += $(COMMON_SHARED) $(LEGACY_SHARED) $(PCIE_SHARED)
-CORE_SRC = if_em.c em_txrx.c e1000_osdep.c
+CORE_SRC = if_em.c if_igbv.c if_igb_iov.c em_txrx.c e1000_osdep.c
CORE_SRC += igb_txrx.c
COMMON_SHARED = e1000_api.c e1000_base.c e1000_phy.c e1000_nvm.c e1000_mac.c \
e1000_manage.c e1000_vf.c e1000_mbx.c e1000_i210.c
@@ -19,6 +19,7 @@ CFLAGS += -I${SRCTOP}/sys/dev/e1000
# DEVICE_POLLING for a non-interrupt-driven method
#CFLAGS += -DDEVICE_POLLING
-SYMLINKS= ${KMOD}.ko ${KMODDIR}/if_igb.ko
+SYMLINKS= ${KMOD}.ko ${KMODDIR}/if_igb.ko \
+ ${KMOD}.ko ${KMODDIR}/if_igbv.ko
.include <bsd.kmod.mk>
generated by cgit v1.3 (git 2.53.0) at 2026-11-09 03:37:55 +0000