diff options
Diffstat (limited to 'sys/dev/e1000')
| -rw-r--r-- | sys/dev/e1000/e1000_82542.c | 39 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_82575.h | 3 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_hw.h | 3 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_mbx.c | 107 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_mbx.h | 5 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_osdep.h | 114 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_regs.h | 1 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_vf.c | 96 | ||||
| -rw-r--r-- | sys/dev/e1000/e1000_vf.h | 6 | ||||
| -rw-r--r-- | sys/dev/e1000/em_txrx.c | 42 | ||||
| -rw-r--r-- | sys/dev/e1000/if_em.c | 2142 | ||||
| -rw-r--r-- | sys/dev/e1000/if_em.h | 209 | ||||
| -rw-r--r-- | sys/dev/e1000/if_igb_iov.c | 2261 | ||||
| -rw-r--r-- | sys/dev/e1000/if_igb_iov.h | 60 | ||||
| -rw-r--r-- | sys/dev/e1000/if_igbv.c | 836 | ||||
| -rw-r--r-- | sys/dev/e1000/igb_txrx.c | 91 |
16 files changed, 5343 insertions, 672 deletions
diff --git a/sys/dev/e1000/e1000_82542.c b/sys/dev/e1000/e1000_82542.c index e8de9086d05d..0656c6408b30 100644 --- a/sys/dev/e1000/e1000_82542.c +++ b/sys/dev/e1000/e1000_82542.c @@ -317,18 +317,43 @@ static s32 e1000_init_hw_82542(struct e1000_hw *hw) static s32 e1000_setup_link_82542(struct e1000_hw *hw) { struct e1000_mac_info *mac = &hw->mac; - s32 ret_val; + s32 ret_val = E1000_SUCCESS; DEBUGFUNC("e1000_setup_link_82542"); - ret_val = e1000_set_default_fc_generic(hw); - if (ret_val) - goto out; + if (hw->fc.requested_mode == e1000_fc_default) { + ret_val = e1000_set_default_fc_generic(hw); + if (ret_val) + goto out; + } - hw->fc.requested_mode &= ~e1000_fc_tx_pause; + /* 82542 rev 2.0 cannot transmit PAUSE frames. */ + if (hw->revision_id == E1000_REVISION_2) { + switch (hw->fc.requested_mode) { + case e1000_fc_tx_pause: + hw->fc.requested_mode = e1000_fc_none; + break; + case e1000_fc_full: + hw->fc.requested_mode = e1000_fc_rx_pause; + break; + default: + break; + } + } - if (mac->report_tx_early) - hw->fc.requested_mode &= ~e1000_fc_rx_pause; + /* Early transmit reporting is incompatible with receiving PAUSE. */ + if (mac->report_tx_early) { + switch (hw->fc.requested_mode) { + case e1000_fc_rx_pause: + hw->fc.requested_mode = e1000_fc_none; + break; + case e1000_fc_full: + hw->fc.requested_mode = e1000_fc_tx_pause; + break; + default: + break; + } + } /* * Save off the requested flow control mode for use later. Depending diff --git a/sys/dev/e1000/e1000_82575.h b/sys/dev/e1000/e1000_82575.h index 91ac3fe9fcbf..c919a8064476 100644 --- a/sys/dev/e1000/e1000_82575.h +++ b/sys/dev/e1000/e1000_82575.h @@ -402,9 +402,10 @@ enum e1000_promisc_type { e1000_num_promisc_types }; -void e1000_vfta_set_vf(struct e1000_hw *, u16, bool); +s32 e1000_vfta_set_vf(struct e1000_hw *, u16, bool); void e1000_rlpml_set_vf(struct e1000_hw *, u16); s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type type); +s32 e1000_set_uc_addr_vf(struct e1000_hw *, u32, u8 *); void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value); u16 e1000_rxpbs_adjust_82580(u32 data); s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data); diff --git a/sys/dev/e1000/e1000_hw.h b/sys/dev/e1000/e1000_hw.h index b4a9592cd89b..5e918ef83263 100644 --- a/sys/dev/e1000/e1000_hw.h +++ b/sys/dev/e1000/e1000_hw.h @@ -938,13 +938,14 @@ struct e1000_fc_info { struct e1000_mbx_operations { s32 (*init_params)(struct e1000_hw *hw); - s32 (*read)(struct e1000_hw *, u32 *, u16, u16); + s32 (*read)(struct e1000_hw *, u32 *, u16, u16, bool); s32 (*write)(struct e1000_hw *, u32 *, u16, u16); s32 (*read_posted)(struct e1000_hw *, u32 *, u16, u16); s32 (*write_posted)(struct e1000_hw *, u32 *, u16, u16); s32 (*check_for_msg)(struct e1000_hw *, u16); s32 (*check_for_ack)(struct e1000_hw *, u16); s32 (*check_for_rst)(struct e1000_hw *, u16); + s32 (*unlock)(struct e1000_hw *, u16); }; struct e1000_mbx_stats { diff --git a/sys/dev/e1000/e1000_mbx.c b/sys/dev/e1000/e1000_mbx.c index 97097fd777a1..985d315d527e 100644 --- a/sys/dev/e1000/e1000_mbx.c +++ b/sys/dev/e1000/e1000_mbx.c @@ -64,6 +64,17 @@ static s32 e1000_null_mbx_transact(struct e1000_hw E1000_UNUSEDARG *hw, return E1000_SUCCESS; } +static s32 e1000_null_mbx_read(struct e1000_hw E1000_UNUSEDARG *hw, + u32 E1000_UNUSEDARG *msg, + u16 E1000_UNUSEDARG size, + u16 E1000_UNUSEDARG mbx_id, + bool E1000_UNUSEDARG unlock) +{ + DEBUGFUNC("e1000_null_mbx_read"); + + return E1000_SUCCESS; +} + /** * e1000_read_mbx - Reads a message from the mailbox * @hw: pointer to the HW structure @@ -73,7 +84,8 @@ static s32 e1000_null_mbx_transact(struct e1000_hw E1000_UNUSEDARG *hw, * * returns SUCCESS if it successfully read message from buffer **/ -s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id) +s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id, + bool unlock) { struct e1000_mbx_info *mbx = &hw->mbx; s32 ret_val = -E1000_ERR_MBX; @@ -85,7 +97,7 @@ s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id) size = mbx->size; if (mbx->ops.read) - ret_val = mbx->ops.read(hw, msg, size, mbx_id); + ret_val = mbx->ops.read(hw, msg, size, mbx_id, unlock); return ret_val; } @@ -176,6 +188,24 @@ s32 e1000_check_for_rst(struct e1000_hw *hw, u16 mbx_id) } /** + * e1000_unlock_mbx - release mailbox ownership + * @hw: pointer to the HW structure + * @mbx_id: id of mailbox to unlock + **/ +s32 e1000_unlock_mbx(struct e1000_hw *hw, u16 mbx_id) +{ + struct e1000_mbx_info *mbx = &hw->mbx; + s32 ret_val = -E1000_ERR_MBX; + + DEBUGFUNC("e1000_unlock_mbx"); + + if (mbx->ops.unlock) + ret_val = mbx->ops.unlock(hw, mbx_id); + + return (ret_val); +} + +/** * e1000_poll_for_msg - Wait for message notification * @hw: pointer to the HW structure * @mbx_id: id of mailbox to write @@ -261,7 +291,7 @@ s32 e1000_read_posted_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id) /* if ack received read message, otherwise we timed out */ if (!ret_val) - ret_val = mbx->ops.read(hw, msg, size, mbx_id); + ret_val = mbx->ops.read(hw, msg, size, mbx_id, true); out: return ret_val; } @@ -307,11 +337,12 @@ void e1000_init_mbx_ops_generic(struct e1000_hw *hw) { struct e1000_mbx_info *mbx = &hw->mbx; mbx->ops.init_params = e1000_null_ops_generic; - mbx->ops.read = e1000_null_mbx_transact; + mbx->ops.read = e1000_null_mbx_read; mbx->ops.write = e1000_null_mbx_transact; mbx->ops.check_for_msg = e1000_null_mbx_check_for_flag; mbx->ops.check_for_ack = e1000_null_mbx_check_for_flag; mbx->ops.check_for_rst = e1000_null_mbx_check_for_flag; + mbx->ops.unlock = e1000_null_mbx_check_for_flag; mbx->ops.read_posted = e1000_read_posted_mbx; mbx->ops.write_posted = e1000_write_posted_mbx; } @@ -500,7 +531,8 @@ out_no_write: * returns SUCCESS if it successfully read message from buffer **/ static s32 e1000_read_mbx_vf(struct e1000_hw *hw, u32 *msg, u16 size, - u16 E1000_UNUSEDARG mbx_id) + u16 E1000_UNUSEDARG mbx_id, + bool E1000_UNUSEDARG unlock) { s32 ret_val = E1000_SUCCESS; u16 i; @@ -649,28 +681,37 @@ static s32 e1000_obtain_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number) { s32 ret_val = -E1000_ERR_MBX; u32 p2v_mailbox; - int count = 10; DEBUGFUNC("e1000_obtain_mbx_lock_pf"); - do { - /* Take ownership of the buffer */ - E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), - E1000_P2VMAILBOX_PFU); - - /* reserve mailbox for pf use */ - p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); - if (p2v_mailbox & E1000_P2VMAILBOX_PFU) { - ret_val = E1000_SUCCESS; - break; - } - usec_delay(1000); - } while (count-- > 0); + /* + * A VF request releases VFU as it raises REQ. If the VF still owns + * the buffer, leave the request for a later admin pass rather than + * sleeping under the PF's context lock and delaying every other VF. + */ + E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), + E1000_P2VMAILBOX_PFU); + p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); + if (p2v_mailbox & E1000_P2VMAILBOX_PFU) + ret_val = E1000_SUCCESS; return ret_val; } +static s32 +e1000_release_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number) +{ + u32 p2v_mailbox; + + p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); + if (p2v_mailbox & E1000_P2VMAILBOX_PFU) + E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), + p2v_mailbox & ~E1000_P2VMAILBOX_PFU); + + return (E1000_SUCCESS); +} + /** * e1000_write_mbx_pf - Places a message in the mailbox * @hw: pointer to the HW structure @@ -693,6 +734,17 @@ static s32 e1000_write_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size, if (ret_val) goto out_no_write; + /* + * A VF request wins over an asynchronous PF message. Do not clear + * VFREQ here: the PF mailbox handler still needs to consume it. + */ + if (E1000_READ_REG(hw, E1000_MBVFICR) & + (E1000_MBVFICR_VFREQ_VF1 << vf_number)) { + e1000_release_mbx_lock_pf(hw, vf_number); + ret_val = -E1000_ERR_MBX; + goto out_no_write; + } + /* flush msg and acks as we are overwriting the message buffer */ e1000_check_for_msg_pf(hw, vf_number); e1000_check_for_ack_pf(hw, vf_number); @@ -724,7 +776,7 @@ out_no_write: * a message due to a VF request so no polling for message is needed. **/ static s32 e1000_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size, - u16 vf_number) + u16 vf_number, bool unlock) { s32 ret_val; u16 i; @@ -736,12 +788,21 @@ static s32 e1000_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size, if (ret_val) goto out_no_read; + /* + * A second VF request can arrive while the PF retries ownership. PFU + * now keeps VMBMEM stable, so consume any reasserted VFREQ before + * reading the request that it describes. + */ + (void)e1000_check_for_msg_pf(hw, vf_number); + /* copy the message to the mailbox memory buffer */ for (i = 0; i < size; i++) msg[i] = E1000_READ_REG_ARRAY(hw, E1000_VMBMEM(vf_number), i); - /* Acknowledge the message and release buffer */ - E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_ACK); + /* Acknowledge the message and optionally retain PF ownership. */ + E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), + E1000_P2VMAILBOX_ACK | + (unlock ? 0 : E1000_P2VMAILBOX_PFU)); /* update stats */ hw->mbx.stats.msgs_rx++; @@ -776,6 +837,7 @@ s32 e1000_init_mbx_params_pf(struct e1000_hw *hw) mbx->ops.check_for_msg = e1000_check_for_msg_pf; mbx->ops.check_for_ack = e1000_check_for_ack_pf; mbx->ops.check_for_rst = e1000_check_for_rst_pf; + mbx->ops.unlock = e1000_release_mbx_lock_pf; mbx->stats.msgs_tx = 0; mbx->stats.msgs_rx = 0; @@ -787,4 +849,3 @@ s32 e1000_init_mbx_params_pf(struct e1000_hw *hw) return E1000_SUCCESS; } } - diff --git a/sys/dev/e1000/e1000_mbx.h b/sys/dev/e1000/e1000_mbx.h index 61f8d5cbe265..edd74a49f00b 100644 --- a/sys/dev/e1000/e1000_mbx.h +++ b/sys/dev/e1000/e1000_mbx.h @@ -77,6 +77,8 @@ #define E1000_VF_RESET 0x01 /* VF requests reset */ #define E1000_VF_SET_MAC_ADDR 0x02 /* VF requests to set MAC addr */ +#define E1000_VF_MAC_FILTER_CLR (0x01 << E1000_VT_MSGINFO_SHIFT) +#define E1000_VF_MAC_FILTER_ADD (0x02 << E1000_VT_MSGINFO_SHIFT) #define E1000_VF_SET_MULTICAST 0x03 /* VF requests to set MC addr */ #define E1000_VF_SET_MULTICAST_COUNT_MASK (0x1F << E1000_VT_MSGINFO_SHIFT) #define E1000_VF_SET_MULTICAST_OVERFLOW (0x80 << E1000_VT_MSGINFO_SHIFT) @@ -92,13 +94,14 @@ #define E1000_VF_MBX_INIT_TIMEOUT 2000 /* number of retries on mailbox */ #define E1000_VF_MBX_INIT_DELAY 500 /* microseconds between retries */ -s32 e1000_read_mbx(struct e1000_hw *, u32 *, u16, u16); +s32 e1000_read_mbx(struct e1000_hw *, u32 *, u16, u16, bool); s32 e1000_write_mbx(struct e1000_hw *, u32 *, u16, u16); s32 e1000_read_posted_mbx(struct e1000_hw *, u32 *, u16, u16); s32 e1000_write_posted_mbx(struct e1000_hw *, u32 *, u16, u16); s32 e1000_check_for_msg(struct e1000_hw *, u16); s32 e1000_check_for_ack(struct e1000_hw *, u16); s32 e1000_check_for_rst(struct e1000_hw *, u16); +s32 e1000_unlock_mbx(struct e1000_hw *, u16); void e1000_init_mbx_ops_generic(struct e1000_hw *hw); s32 e1000_init_mbx_params_vf(struct e1000_hw *); s32 e1000_init_mbx_params_pf(struct e1000_hw *); diff --git a/sys/dev/e1000/e1000_osdep.h b/sys/dev/e1000/e1000_osdep.h index ba1c8a16fad1..1226c0264333 100644 --- a/sys/dev/e1000/e1000_osdep.h +++ b/sys/dev/e1000/e1000_osdep.h @@ -161,8 +161,111 @@ struct e1000_osdep bus_space_handle_t flash_bus_space_handle; device_t dev; if_ctx_t ctx; + bool vf; + bool vf_82576; }; +#ifdef INVARIANTS +/* + * 82576 and I350 VFs expose a sparse register file. Keep this list local to + * the OS accessors so a PF-only register that leaks into the shared VF path + * fails at its first access instead of returning reserved-register garbage. + * + * The driver intentionally uses only queue pair zero. It also clears the + * retained configuration of the unused second 82576 queue pair after VFLR. + * Expanding VF data-path queue support must extend this predicate from the + * applicable device CSR map. + */ +static __inline bool +e1000_vf_reg_valid(uint32_t reg, bool write, bool vf_82576) +{ + /* VF mailbox memory: 16 dwords beginning at 0x800. */ + if (reg >= 0x00800 && reg <= 0x0083c && (reg & 3) == 0) + return (true); + + /* The three VF MSI-X interrupt-throttling registers. */ + if (reg >= 0x01680 && reg <= 0x01688 && (reg & 3) == 0) + return (true); + + /* Receive queue zero. */ + switch (reg) { + case 0x02800: /* RDBAL */ + case 0x02804: /* RDBAH */ + case 0x02808: /* RDLEN */ + case 0x0280c: /* SRRCTL */ + case 0x02810: /* RDH */ + case 0x02814: /* RXCTL */ + case 0x02818: /* RDT */ + case 0x02828: /* RXDCTL */ + return (true); + } + + /* Transmit queue zero. */ + switch (reg) { + case 0x03800: /* TDBAL */ + case 0x03804: /* TDBAH */ + case 0x03808: /* TDLEN */ + case 0x03810: /* TDH */ + case 0x03814: /* TXCTL */ + case 0x03818: /* TDT */ + case 0x03828: /* TXDCTL */ + case 0x03838: /* TDWBAL */ + case 0x0383c: /* TDWBAH */ + return (true); + } + + /* Retained configuration from 82576 virtual queue one. */ + if (vf_82576) { + switch (reg) { + case 0x0290c: /* SRRCTL(1) */ + case 0x02914: /* RXCTL(1) */ + case 0x02928: /* RXDCTL(1) */ + case 0x03914: /* TXCTL(1) */ + case 0x03928: /* TXDCTL(1) */ + case 0x03938: /* TDWBAL(1) */ + case 0x0393c: /* TDWBAH(1) */ + return (true); + } + } + + /* + * 82576 exposes VFMPRC at 0xf3c. I350 erratum 31 makes + * its corrected 0xf38 address inaccessible to a VF. + */ + if (vf_82576 && reg == 0x00f3c) + return (!write); + + switch (reg) { + case 0x00000: /* CTRL */ + case 0x000c4: /* Legacy ITR, listed but unused by igb VFs */ + case 0x00c40: /* V2PMAILBOX(0) */ + case 0x00f0c: /* VFPSRTYPE */ + case 0x01524: /* EIMS */ + case 0x0152c: /* EIAC */ + case 0x01530: /* EIAM */ + case 0x01700: /* IVAR0 */ + case 0x01740: /* IVAR_MISC */ + return (true); + case 0x01520: /* EICS */ + case 0x01528: /* EIMC */ + return (write); + case 0x00008: /* STATUS */ + case 0x00f10: /* VFGPRC */ + case 0x00f14: /* VFGPTC */ + case 0x00f18: /* VFGORC */ + case 0x00f34: /* VFGOTC */ + case 0x00f40: /* VFGPRLBC */ + case 0x00f44: /* VFGPTLBC */ + case 0x00f48: /* VFGORLBC */ + case 0x00f50: /* VFGOTLBC */ + case 0x01580: /* EICR */ + return (!write); + default: + return (false); + } +} +#endif + #define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \ ? reg : e1000_translate_register_82542(reg)) @@ -185,6 +288,11 @@ e1000_rd32(struct e1000_osdep *osdep, uint32_t reg) KASSERT(reg < osdep->mem_bus_space_size, ("e1000: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size)); +#ifdef INVARIANTS + KASSERT(!osdep->vf || + e1000_vf_reg_valid(reg, false, osdep->vf_82576), + ("e1000: invalid VF register read at %#x", reg)); +#endif return (bus_space_read_4(osdep->mem_bus_space_tag, osdep->mem_bus_space_handle, reg)); @@ -198,6 +306,11 @@ e1000_wr32(struct e1000_osdep *osdep, uint32_t reg, uint32_t value) KASSERT(reg < osdep->mem_bus_space_size, ("e1000: register offset %#jx too large (max is %#jx)", (uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size)); +#ifdef INVARIANTS + KASSERT(!osdep->vf || + e1000_vf_reg_valid(reg, true, osdep->vf_82576), + ("e1000: invalid VF register write at %#x", reg)); +#endif bus_space_write_4(osdep->mem_bus_space_tag, osdep->mem_bus_space_handle, reg, value); @@ -277,4 +390,3 @@ e1000_wr32(struct e1000_osdep *osdep, uint32_t reg, uint32_t value) #endif #endif /* _FREEBSD_OS_H_ */ - diff --git a/sys/dev/e1000/e1000_regs.h b/sys/dev/e1000/e1000_regs.h index 5af1142a1ff0..138155cfc5bc 100644 --- a/sys/dev/e1000/e1000_regs.h +++ b/sys/dev/e1000/e1000_regs.h @@ -286,6 +286,7 @@ #define E1000_TXDMAC 0x03000 /* Tx DMA Control - RW */ #define E1000_KABGTXD 0x03004 /* AFE Band Gap Transmit Ref Data */ #define E1000_PSRTYPE(_i) (0x05480 + ((_i) * 4)) +#define E1000_VFPSRTYPE 0x00F0C #define E1000_RAL(_i) (((_i) <= 15) ? (0x05400 + ((_i) * 8)) : \ (0x054E0 + ((_i - 16) * 8))) #define E1000_RAH(_i) (((_i) <= 15) ? (0x05404 + ((_i) * 8)) : \ diff --git a/sys/dev/e1000/e1000_vf.c b/sys/dev/e1000/e1000_vf.c index 9bcd2798e486..2394c240d18d 100644 --- a/sys/dev/e1000/e1000_vf.c +++ b/sys/dev/e1000/e1000_vf.c @@ -275,24 +275,26 @@ static s32 e1000_reset_hw_vf(struct e1000_hw *hw) usec_delay(5); } - if (timeout) { - /* mailbox timeout can now become active */ - mbx->timeout = E1000_VF_MBX_INIT_TIMEOUT; + if (!timeout) + return -E1000_ERR_RESET; - msgbuf[0] = E1000_VF_RESET; - mbx->ops.write_posted(hw, msgbuf, 1, 0); + /* mailbox timeout can now become active */ + mbx->timeout = E1000_VF_MBX_INIT_TIMEOUT; - msec_delay(10); + msgbuf[0] = E1000_VF_RESET; + ret_val = mbx->ops.write_posted(hw, msgbuf, 1, 0); + if (ret_val) + return ret_val; - /* set our "perm_addr" based on info provided by PF */ - ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); - if (!ret_val) { - if (msgbuf[0] == (E1000_VF_RESET | - E1000_VT_MSGTYPE_ACK)) - memcpy(hw->mac.perm_addr, addr, 6); - else - ret_val = -E1000_ERR_MAC_INIT; - } + msec_delay(10); + + /* set our "perm_addr" based on info provided by PF */ + ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); + if (!ret_val) { + if (msgbuf[0] == (E1000_VF_RESET | E1000_VT_MSGTYPE_ACK)) + memcpy(hw->mac.perm_addr, addr, 6); + else + ret_val = -E1000_ERR_MAC_INIT; } return ret_val; @@ -389,6 +391,41 @@ static void e1000_write_msg_read_ack(struct e1000_hw *hw, } /** + * e1000_set_uc_addr_vf - Add or clear secondary unicast addresses + * @hw: pointer to the HW structure + * @sub_cmd: E1000_VF_MAC_FILTER_ADD or E1000_VF_MAC_FILTER_CLR + * @addr: address to add, or a valid compatibility address when clearing + * + * Uses the secondary-MAC mailbox subprotocol implemented by Linux igbvf. + * Linux igb PFs validate this field before dispatching the clear subcommand, + * even though they do not otherwise use it for a clear request. + **/ +s32 +e1000_set_uc_addr_vf(struct e1000_hw *hw, u32 sub_cmd, u8 *addr) +{ + struct e1000_mbx_info *mbx = &hw->mbx; + u32 msgbuf[3] = {}; + u32 request; + s32 ret_val; + + msgbuf[0] = E1000_VF_SET_MAC_ADDR | sub_cmd; + request = msgbuf[0]; + if (addr != NULL) + memcpy(&msgbuf[1], addr, ETHER_ADDR_LEN); + + ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0); + if (ret_val == E1000_SUCCESS) + ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); + + msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; + if (ret_val == E1000_SUCCESS && + msgbuf[0] == (request | E1000_VT_MSGTYPE_NACK)) + ret_val = -E1000_ERR_NO_SPACE; + + return (ret_val); +} + +/** * e1000_update_mc_addr_list_vf - Update Multicast addresses * @hw: pointer to the HW structure * @mc_addr_list: array of multicast addresses to program @@ -400,7 +437,7 @@ static void e1000_write_msg_read_ack(struct e1000_hw *hw, void e1000_update_mc_addr_list_vf(struct e1000_hw *hw, u8 *mc_addr_list, u32 mc_addr_count) { - u32 msgbuf[E1000_VFMAILBOX_SIZE]; + u32 msgbuf[E1000_VFMAILBOX_SIZE] = {}; u16 *hash_list = (u16 *)&msgbuf[1]; u32 hash_value; u32 i; @@ -442,10 +479,14 @@ void e1000_update_mc_addr_list_vf(struct e1000_hw *hw, * @hw: pointer to the HW structure * @vid: determines the vfta register and bit to set/unset * @set: if true then set bit, else clear bit + * + * Returns success if the PF accepted the request, or an error otherwise. **/ -void e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set) +s32 e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set) { + struct e1000_mbx_info *mbx = &hw->mbx; u32 msgbuf[2]; + s32 ret_val; msgbuf[0] = E1000_VF_SET_VLAN; msgbuf[1] = vid; @@ -453,7 +494,15 @@ void e1000_vfta_set_vf(struct e1000_hw *hw, u16 vid, bool set) if (set) msgbuf[0] |= E1000_VF_SET_VLAN_ADD; - e1000_write_msg_read_ack(hw, msgbuf, 2); + ret_val = mbx->ops.write_posted(hw, msgbuf, 2, 0); + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 1, 0); + if (!ret_val && + ((msgbuf[0] & 0xffff) != E1000_VF_SET_VLAN || + !(msgbuf[0] & E1000_VT_MSGTYPE_ACK))) + ret_val = -E1000_ERR_MAC_INIT; + + return (ret_val); } /** e1000_rlpml_set_vf - Set the maximum receive packet length @@ -559,13 +608,17 @@ static s32 e1000_check_for_link_vf(struct e1000_hw *hw) /* if the read failed it could just be a mailbox collision, best wait * until we are called again and don't report an error */ - if (mbx->ops.read(hw, &in_msg, 1, 0)) + if (mbx->ops.read(hw, &in_msg, 1, 0, true)) goto out; /* if incoming message isn't clear to send we are waiting on response */ if (!(in_msg & E1000_VT_MSGTYPE_CTS)) { - /* message is not CTS and is NACK we have lost CTS status */ - if (in_msg & E1000_VT_MSGTYPE_NACK) + /* + * A NACK or a PF control message without CTS means that the PF + * discarded our state and requires a new VF reset handshake. + */ + if ((in_msg & E1000_VT_MSGTYPE_NACK) != 0 || + (in_msg & 0xffff) == E1000_PF_CONTROL_MSG) ret_val = -E1000_ERR_MAC_INIT; goto out; } @@ -585,4 +638,3 @@ static s32 e1000_check_for_link_vf(struct e1000_hw *hw) out: return ret_val; } - diff --git a/sys/dev/e1000/e1000_vf.h b/sys/dev/e1000/e1000_vf.h index aace8e78ed6a..17116a86b084 100644 --- a/sys/dev/e1000/e1000_vf.h +++ b/sys/dev/e1000/e1000_vf.h @@ -227,13 +227,14 @@ struct e1000_mac_info { struct e1000_mbx_operations { s32 (*init_params)(struct e1000_hw *hw); - s32 (*read)(struct e1000_hw *, u32 *, u16, u16); + s32 (*read)(struct e1000_hw *, u32 *, u16, u16, bool); s32 (*write)(struct e1000_hw *, u32 *, u16, u16); s32 (*read_posted)(struct e1000_hw *, u32 *, u16, u16); s32 (*write_posted)(struct e1000_hw *, u32 *, u16, u16); s32 (*check_for_msg)(struct e1000_hw *, u16); s32 (*check_for_ack)(struct e1000_hw *, u16); s32 (*check_for_rst)(struct e1000_hw *, u16); + s32 (*unlock)(struct e1000_hw *, u16); }; struct e1000_mbx_stats { @@ -290,7 +291,8 @@ enum e1000_promisc_type { /* These functions must be implemented by drivers */ s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value); -void e1000_vfta_set_vf(struct e1000_hw *, u16, bool); +s32 e1000_vfta_set_vf(struct e1000_hw *, u16, bool); void e1000_rlpml_set_vf(struct e1000_hw *, u16); s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type); +s32 e1000_set_uc_addr_vf(struct e1000_hw *, u32, u8 *); #endif /* _E1000_VF_H_ */ diff --git a/sys/dev/e1000/em_txrx.c b/sys/dev/e1000/em_txrx.c index 647255417b3e..dced12770c40 100644 --- a/sys/dev/e1000/em_txrx.c +++ b/sys/dev/e1000/em_txrx.c @@ -96,6 +96,11 @@ em_dump_rs(struct e1000_softc *sc) int16_t rs_cidx; uint8_t status; + if (sc->tx_queues == NULL) { + device_printf(sc->dev, "queue state is unavailable\n"); + return; + } + printf("\n"); ntxd = scctx->isc_ntxd[0]; for (qid = 0; qid < sc->tx_num_queues; qid++) { @@ -461,10 +466,27 @@ em_isc_txd_encap(void *arg, if_pkt_info_t pi) first, pidx_last, i); pi->ipi_new_pidx = i; - /* Sent data accounting for AIM */ + /* + * Sent data accounting for AIM. For TSO, ipi_len is the whole + * unsegmented payload, which is not a size the moderation + * calculation can use. Count the segments the hardware will put on + * the wire and the header each of them carries, so that the average + * it sees is a wire packet. + */ + if (do_tso && pi->ipi_tso_segsz != 0) { + u32 hdrlen, segs; + + hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; + if (pi->ipi_len > hdrlen) { + segs = howmany(pi->ipi_len - hdrlen, pi->ipi_tso_segsz); + txr->tx_bytes += pi->ipi_len + (segs - 1) * hdrlen; + txr->tx_packets += segs; + return (0); + } + } + txr->tx_bytes += pi->ipi_len; ++txr->tx_packets; - return (0); } @@ -476,6 +498,8 @@ em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) struct tx_ring *txr = &que->txr; E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); + if (sc->hw.mac.type >= e1000_82540) + em_aim_publish(txr); } static int @@ -601,6 +625,8 @@ em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, struct rx_ring *rxr = &que->rxr; E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); + if (sc->hw.mac.type >= e1000_82540) + em_aim_publish_rx(rxr); } static int @@ -679,7 +705,6 @@ lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) len = le16toh(rxd->length); ri->iri_len += len; - rxr->rx_bytes += ri->iri_len; eop = (status & E1000_RXD_STAT_EOP) != 0; @@ -701,6 +726,7 @@ lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) i++; } while (!eop); + rxr->rx_bytes += ri->iri_len; rxr->rx_packets++; if (scctx->isc_capenable & IFCAP_RXCSUM) @@ -745,7 +771,6 @@ em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) len = le16toh(rxd->wb.upper.length); ri->iri_len += len; - rxr->rx_bytes += ri->iri_len; eop = (staterr & E1000_RXD_STAT_EOP) != 0; @@ -766,6 +791,7 @@ em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) i++; } while (!eop); + rxr->rx_bytes += ri->iri_len; rxr->rx_packets++; if (scctx->isc_capenable & IFCAP_RXCSUM) @@ -835,7 +861,13 @@ em_determine_rsstype(uint32_t pkt_info) return M_HASHTYPE_RSS_IPV6; case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: return M_HASHTYPE_RSS_TCP_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV4_UDP: + return M_HASHTYPE_RSS_UDP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_UDP: + return M_HASHTYPE_RSS_UDP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_UDP_EX: + return M_HASHTYPE_RSS_UDP_IPV6_EX; default: - return M_HASHTYPE_OPAQUE; + return M_HASHTYPE_NONE; } } diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index 7d7655a7ae6f..a3523315d5c7 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -28,6 +28,7 @@ */ #include "if_em.h" +#include "if_igb_iov.h" #include <sys/sbuf.h> #include <machine/_inttypes.h> @@ -342,8 +343,6 @@ static const pci_vendor_info_t igb_vendor_info_array[] = "Intel(R) PRO/1000 ET 82576 (Quad Copper)"), PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"), - PVID(0x8086, E1000_DEV_ID_82576_VF, - "Intel(R) PRO/1000 82576 Virtual Function"), PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) I340 82580 (Copper)"), PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"), @@ -365,7 +364,6 @@ static const pci_vendor_info_t igb_vendor_info_array[] = PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"), PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"), PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"), - PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) I350 Virtual Function"), PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"), PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) I210 IT (Copper)"), @@ -389,13 +387,29 @@ static const pci_vendor_info_t igb_vendor_info_array[] = PVID_END }; +static const pci_vendor_info_t igbv_vendor_info_array[] = { + PVID(0x8086, E1000_DEV_ID_82576_VF, + "Intel(R) PRO/1000 82576 Virtual Function"), + PVID(0x8086, E1000_DEV_ID_82576_VF_HV, + "Intel(R) PRO/1000 82576 Virtual Function"), + PVID(0x8086, E1000_DEV_ID_I350_VF, + "Intel(R) I350 Virtual Function"), + PVID(0x8086, E1000_DEV_ID_I350_VF_HV, + "Intel(R) I350 Virtual Function"), + PVID_END +}; + /********************************************************************* * Function prototypes *********************************************************************/ static void *em_register(device_t); static void *igb_register(device_t); -static int em_if_attach_pre(if_ctx_t); -static int em_if_attach_post(if_ctx_t); +static void *igbv_register(device_t); +static int igb_device_attach(device_t); +#ifdef PCI_IOV +static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *); +static void igb_device_iov_uninit(device_t); +#endif static int em_if_detach(if_ctx_t); static int em_if_shutdown(if_ctx_t); static int em_if_suspend(if_ctx_t); @@ -417,7 +431,6 @@ static int em_if_mtu_set(if_ctx_t, uint32_t); static void em_if_timer(if_ctx_t, uint16_t); static void em_if_vlan_register(if_ctx_t, u16); static void em_if_vlan_unregister(if_ctx_t, u16); -static void em_if_watchdog_reset(if_ctx_t); static bool em_if_needs_restart(if_ctx_t, enum iflib_restart_event); static void em_identify_hardware(if_ctx_t); @@ -441,15 +454,17 @@ static int igb_if_tx_queue_intr_enable(if_ctx_t, uint16_t); static void em_if_multi_set(if_ctx_t); static void em_if_update_admin_status(if_ctx_t); static void em_if_debug(if_ctx_t); +static void em_initialize_vf_stats(struct e1000_softc *); +static void em_rebase_vf_stats(struct e1000_softc *); static void em_update_vf_stats_counters(struct e1000_softc *); -static void em_update_stats_counters(struct e1000_softc *); static void em_add_hw_stats(struct e1000_softc *); static int em_if_set_promisc(if_ctx_t, int); +static bool em_if_defer_promisc(struct e1000_softc *); static bool em_if_vlan_filter_capable(if_ctx_t); static bool em_if_vlan_filter_used(if_ctx_t); static void em_if_vlan_filter_enable(struct e1000_softc *); static void em_if_vlan_filter_disable(struct e1000_softc *); -static void em_if_vlan_filter_write(struct e1000_softc *); +static void em_if_vlan_filter_write(struct e1000_softc *, int); static void em_setup_vlan_hw_support(if_ctx_t ctx); static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); static void em_print_nvm_info(struct e1000_softc *); @@ -460,9 +475,8 @@ static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); static int em_get_rs(SYSCTL_HANDLER_ARGS); static void em_print_debug_info(struct e1000_softc *); -static int em_is_valid_ether_addr(u8 *); static void em_newitr(struct e1000_softc *, struct em_rx_queue *, - struct tx_ring *, struct rx_ring *); + struct rx_ring *); static bool em_automask_tso(if_ctx_t); static int em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); @@ -493,9 +507,9 @@ static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); static void em_if_led_func(if_ctx_t, int); static int em_get_regs(SYSCTL_HANDLER_ARGS); - static void lem_smartspeed(struct e1000_softc *); static void igb_configure_queues(struct e1000_softc *); +static void igb_initialize_interrupt_rate(struct e1000_softc *); static void em_flush_desc_rings(struct e1000_softc *); @@ -518,6 +532,23 @@ static device_method_t igb_methods[] = { /* Device interface */ DEVMETHOD(device_register, igb_register), DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, igb_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), +#ifdef PCI_IOV + DEVMETHOD(pci_iov_init, igb_device_iov_init), + DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit), + DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), +#endif + DEVMETHOD_END +}; + +static device_method_t igbv_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, igbv_register), + DEVMETHOD(device_probe, iflib_device_probe), DEVMETHOD(device_attach, iflib_device_attach), DEVMETHOD(device_detach, iflib_device_detach), DEVMETHOD(device_shutdown, iflib_device_shutdown), @@ -551,6 +582,18 @@ MODULE_DEPEND(igb, iflib, 1, 1, 1); IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array); +static driver_t igbv_driver = { + "igbv", igbv_methods, sizeof(struct e1000_softc), +}; + +DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0); + +MODULE_DEPEND(igbv, pci, 1, 1, 1); +MODULE_DEPEND(igbv, ether, 1, 1, 1); +MODULE_DEPEND(igbv, iflib, 1, 1, 1); + +IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array); + static device_method_t em_if_methods[] = { DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), DEVMETHOD(ifdi_attach_post, em_if_attach_post), @@ -573,7 +616,6 @@ static device_method_t em_if_methods[] = { DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), DEVMETHOD(ifdi_timer, em_if_timer), - DEVMETHOD(ifdi_watchdog_reset, em_if_watchdog_reset), DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), DEVMETHOD(ifdi_get_counter, em_if_get_counter), @@ -611,7 +653,6 @@ static device_method_t igb_if_methods[] = { DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), DEVMETHOD(ifdi_timer, em_if_timer), - DEVMETHOD(ifdi_watchdog_reset, em_if_watchdog_reset), DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), DEVMETHOD(ifdi_get_counter, em_if_get_counter), @@ -620,6 +661,12 @@ static device_method_t igb_if_methods[] = { DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), DEVMETHOD(ifdi_debug, em_if_debug), DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), +#ifdef PCI_IOV + DEVMETHOD(ifdi_iov_init, igb_if_iov_init), + DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit), + DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add), + DEVMETHOD(ifdi_vf_status, igb_if_vf_status), +#endif DEVMETHOD_END }; @@ -627,6 +674,42 @@ static driver_t igb_if_driver = { "igb_if", igb_if_methods, sizeof(struct e1000_softc) }; +static device_method_t igbv_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre), + DEVMETHOD(ifdi_attach_post, igbv_if_attach_post), + DEVMETHOD(ifdi_detach, em_if_detach), + DEVMETHOD(ifdi_shutdown, em_if_shutdown), + DEVMETHOD(ifdi_suspend, em_if_suspend), + DEVMETHOD(ifdi_resume, em_if_resume), + DEVMETHOD(ifdi_init, em_if_init), + DEVMETHOD(ifdi_stop, em_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable), + DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable), + DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, em_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, em_if_multi_set), + DEVMETHOD(ifdi_media_status, em_if_media_status), + DEVMETHOD(ifdi_media_change, igbv_if_media_change), + DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), + DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), + DEVMETHOD(ifdi_timer, em_if_timer), + DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, em_if_get_counter), + DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), + DEVMETHOD(ifdi_debug, em_if_debug), + DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), + DEVMETHOD_END +}; + +static driver_t igbv_if_driver = { + "igbv_if", igbv_if_methods, sizeof(struct e1000_softc) +}; + /********************************************************************* * Tunable default values. *********************************************************************/ @@ -760,6 +843,40 @@ static struct if_shared_ctx igb_sctx_init = { .isc_ntxd_default = {EM_DEFAULT_TXD}, }; +/* + * igb PFs and igbv VFs share the common datapath implementation. Keep a + * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or + * interrupt modes. + */ +static struct if_shared_ctx igbv_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), + .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = igbv_vendor_info_array, + .isc_driver_version = igb_driver_version, + .isc_driver = &igbv_if_driver, + .isc_flags = + IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM | + IFLIB_IS_VF, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {IGB_MAX_RXD}, + .isc_ntxd_max = {IGB_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; + /***************************************************************** * * Dump Registers @@ -774,9 +891,19 @@ static int em_get_regs(SYSCTL_HANDLER_ARGS) struct sbuf *sb; u32 *regs_buff; int rc; + uint32_t rxqid, txqid; + + /* + * This sysctl is registered before iflib allocates the queue arrays, + * and remains registered while iflib tears them down. + */ + if (sc->rx_queues == NULL || sc->tx_queues == NULL) + return (ENXIO); regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK); memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); + rxqid = sc->rx_queues[0].rxr.me; + txqid = sc->tx_queues[0].txr.me; rc = sysctl_wire_old_buffer(req, 0); MPASS(rc == 0); @@ -798,19 +925,19 @@ static int em_get_regs(SYSCTL_HANDLER_ARGS) regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); - regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0)); - regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0)); - regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0)); - regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0)); - regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0)); - regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0)); + regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid)); + regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid)); + regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid)); + regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid)); + regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid)); + regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid)); regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); - regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0)); - regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0)); - regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0)); - regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0)); - regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0)); - regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0)); + regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid)); + regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid)); + regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid)); + regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid)); + regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid)); + regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid)); regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); @@ -897,6 +1024,65 @@ igb_register(device_t dev) return (&igb_sctx_init); } +static void * +igbv_register(device_t dev) +{ + return (&igbv_sctx_init); +} + +static int +igb_device_attach(device_t dev) +{ + struct e1000_softc *sc; + if_ctx_t ctx; + int error; + + error = iflib_device_attach(dev); + if (error != 0) + return (error); + + ctx = device_get_softc(dev); + sc = iflib_get_softc(ctx); + (void)igb_iov_attach(sc); + return (0); +} + +#ifdef PCI_IOV +static int +igb_device_iov_init(device_t dev, uint16_t num_vfs, + const nvlist_t *params) +{ + struct e1000_softc *sc; + if_ctx_t ctx; + int error; + + ctx = device_get_softc(dev); + sc = iflib_get_softc(ctx); + error = igb_iov_validate(sc, num_vfs); + if (error != 0) + return (error); + return (iflib_device_iov_init_restart(dev, num_vfs, params)); +} + +static void +igb_device_iov_uninit(device_t dev) +{ + struct e1000_softc *sc; + if_ctx_t ctx; + + ctx = device_get_softc(dev); + sc = iflib_get_softc(ctx); + /* + * pci_iov(4) has already detached the VF devices. Tell the stop + * half of iflib's restart transaction not to wait for acknowledgements + * from VFs which can no longer service their mailbox vectors. + */ + atomic_store_rel_32(&sc->iov_teardown, 1); + iflib_device_iov_uninit_restart(dev); +} + +#endif + static int em_set_num_queues(if_ctx_t ctx) { @@ -919,6 +1105,11 @@ em_set_num_queues(if_ctx_t ctx) case e1000_82574: maxqueues = 2; break; + case e1000_vfadapt: + /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */ + case e1000_vfadapt_i350: + maxqueues = 1; + break; default: maxqueues = 1; break; @@ -927,90 +1118,75 @@ em_set_num_queues(if_ctx_t ctx) return (maxqueues); } -#define LEM_CAPS \ +#define LEM_CAPS ( \ IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ - IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 + IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6) -#define EM_CAPS \ +#define EM_CAPS ( \ IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ - IFCAP_TSO6 + IFCAP_TSO6) -#define IGB_CAPS \ +#define IGB_CAPS ( \ IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ - IFCAP_TSO6 + IFCAP_TSO6) -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ -static int -em_if_attach_pre(if_ctx_t ctx) +/* + * VLAN filtering is an effective VF capability, but its policy is owned by + * the PF and cannot be disabled from the VF. vlan(4) registration callbacks + * are independent of this capability bit. + */ +#define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL) + +void +em_add_device_sysctls(struct e1000_softc *sc) { - struct e1000_softc *sc; - if_softc_ctx_t scctx; - device_t dev; struct e1000_hw *hw; struct sysctl_oid_list *child; struct sysctl_ctx_list *ctx_list; - int error = 0; - INIT_DEBUGOUT("em_if_attach_pre: begin"); - dev = iflib_get_dev(ctx); - sc = iflib_get_softc(ctx); - - sc->ctx = sc->osdep.ctx = ctx; - sc->dev = sc->osdep.dev = dev; - scctx = sc->shared = iflib_get_softc_ctx(ctx); - sc->media = iflib_get_media(ctx); hw = &sc->hw; - - /* Determine hardware and mac info */ - em_identify_hardware(ctx); - - /* SYSCTL stuff */ - ctx_list = device_get_sysctl_ctx(dev); - child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); - - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", - CTLTYPE_INT | CTLFLAG_RW, sc, 0, - em_sysctl_nvm_info, "I", "NVM Information"); + ctx_list = device_get_sysctl_ctx(sc->dev); + child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); sc->enable_aim = em_enable_aim; SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", CTLFLAG_RW, &sc->enable_aim, 0, "Interrupt Moderation (1=normal, 2=lowlatency)"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", - CTLTYPE_STRING | CTLFLAG_RD, sc, 0, - em_sysctl_print_fw_version, "A", - "Prints FW/NVM Versions"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, sc, 0, em_sysctl_debug_info, "I", "Debug Information"); + SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, + em_get_rs, "I", "Dump RS indexes"); + + if (sc->vf_ifp) { + SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, + igbv_get_regs, "A", "Dump VF registers"); + return; + } + + SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", + CTLTYPE_INT | CTLFLAG_RW, sc, 0, + em_sysctl_nvm_info, "I", "NVM Information"); + SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", + CTLTYPE_STRING | CTLFLAG_RD, sc, 0, + em_sysctl_print_fw_version, "A", + "Prints FW/NVM Versions"); SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, em_set_flowcntl, "I", "Flow Control"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, em_get_regs, "A", "Dump Registers"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, - em_get_rs, "I", "Dump RS indexes"); - if (hw->mac.type >= e1000_i350) { SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac", CTLTYPE_INT | CTLFLAG_RW, sc, 0, @@ -1022,18 +1198,67 @@ em_if_attach_pre(if_ctx_t ctx) CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, em_sysctl_tso_tcp_flags_mask, "IU", "TSO TCP flags mask for first segment"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tso_tcp_flags_mask_middle_segment", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 1, em_sysctl_tso_tcp_flags_mask, "IU", "TSO TCP flags mask for middle segment"); - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tso_tcp_flags_mask_last_segment", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 2, em_sysctl_tso_tcp_flags_mask, "IU", "TSO TCP flags mask for last segment"); +} + +/********************************************************************* + * Device initialization routine + * + * The attach entry point is called when the driver is being loaded. + * This routine identifies the type of hardware, allocates all resources + * and initializes the hardware. + * + * return 0 on success, positive on failure + *********************************************************************/ +int +em_if_attach_pre(if_ctx_t ctx) +{ + struct e1000_softc *sc; + if_softc_ctx_t scctx; + device_t dev; + struct e1000_hw *hw; + struct sysctl_oid_list *child; + struct sysctl_ctx_list *ctx_list; + int error = 0; + + INIT_DEBUGOUT("em_if_attach_pre: begin"); + dev = iflib_get_dev(ctx); + sc = iflib_get_softc(ctx); + + if (em_max_interrupt_rate <= 0) { + device_printf(dev, + "Invalid max_interrupt_rate %d; using default %d\n", + em_max_interrupt_rate, EM_INTS_DEFAULT); + em_max_interrupt_rate = EM_INTS_DEFAULT; + } + + sc->ctx = sc->osdep.ctx = ctx; + sc->dev = sc->osdep.dev = dev; + scctx = sc->shared = iflib_get_softc_ctx(ctx); + sc->media = iflib_get_media(ctx); + hw = &sc->hw; + sc->vf_ifp = + (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0; + sc->osdep.vf = sc->vf_ifp; + + /* Determine hardware and mac info */ + em_identify_hardware(ctx); + sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt; + + /* VF sysctls are deferred until attach-post confirms MSI-X. */ + ctx_list = device_get_sysctl_ctx(dev); + child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); + if (!sc->vf_ifp) + em_add_device_sysctls(sc); scctx->isc_tx_nsegments = EM_MAX_SCATTER; scctx->isc_nrxqsets_max = @@ -1053,7 +1278,8 @@ em_if_attach_pre(if_ctx_t ctx) scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; scctx->isc_tx_tso_size_max = EM_TSO_SIZE; scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; - scctx->isc_capabilities = scctx->isc_capenable = IGB_CAPS; + scctx->isc_capabilities = scctx->isc_capenable = + sc->vf_ifp ? IGBV_CAPS : IGB_CAPS; scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP; if (hw->mac.type != e1000_82575) @@ -1218,6 +1444,27 @@ em_if_attach_pre(if_ctx_t ctx) em_setup_msix(ctx); e1000_get_bus_info(hw); + /* + * Some conventional PCI systems hang when e1000 devices use + * DMA addresses above 4 GB. Keep PCI-mode DMA below that boundary + * by default; PCI-X and PCIe retain 64-bit DMA. + */ + if (hw->bus.type == e1000_bus_type_pci) { + SYSCTL_ADD_BOOL(ctx_list, child, OID_AUTO, "allow_64bit_dma", + CTLFLAG_RDTUN, &sc->allow_64bit_dma, 0, + "Allow 64-bit DMA in conventional PCI mode"); + if (sc->allow_64bit_dma) + device_printf(dev, "64-bit DMA in conventional PCI mode. " + "Some chipsets are unstable.\n"); + else { + scctx->isc_dma_width = 32; + device_printf(dev, "32-bit DMA in conventional PCI mode. " + "Set dev.%s.%d.allow_64bit_dma=1 at boot to enable " + "64-bit DMA if the chipset is stable with it.\n", + device_get_name(dev), device_get_unit(dev)); + } + } + /* Set up some sysctls for the tunable interrupt delays */ if (hw->mac.type < igb_mac_min) { em_add_int_delay_sysctl(sc, "rx_int_delay", @@ -1279,29 +1526,37 @@ em_if_attach_pre(if_ctx_t ctx) /* Clear the IFCAP_TSO auto mask */ sc->tso_automasked = 0; - /* Check SOL/IDER usage */ - if (e1000_check_reset_block(hw)) + /* Check SOL/IDER usage on physical functions. */ + if (!sc->vf_ifp && e1000_check_reset_block(hw)) device_printf(dev, "PHY reset is blocked due to SOL/IDER session.\n"); /* Sysctl for setting Energy Efficient Ethernet */ - if (hw->mac.type < igb_mac_min) - hw->dev_spec.ich8lan.eee_disable = eee_setting; - else - hw->dev_spec._82575.eee_disable = eee_setting; - SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, - em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); + if (!sc->vf_ifp) { + if (hw->mac.type < igb_mac_min) + hw->dev_spec.ich8lan.eee_disable = eee_setting; + else + hw->dev_spec._82575.eee_disable = eee_setting; + SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, + em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); + } /* ** Start from a known state, this is ** important in reading the nvm and ** mac from that. */ - e1000_reset_hw(hw); + error = e1000_reset_hw(hw); + if (sc->vf_ifp) { + atomic_store_rel_32(&sc->vf_mbx_ready, + error == E1000_SUCCESS); + if (error != E1000_SUCCESS) + igbv_log_reset_failure(sc, error, true); + } - /* Make sure we have a good EEPROM before we read from it */ - if (e1000_validate_nvm_checksum(hw) < 0) { + /* Make sure a PF has a good EEPROM before we read from it. */ + if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) { /* ** Some PCI-E parts fail the first check due to ** the link being in sleep state, call it again, @@ -1325,6 +1580,9 @@ em_if_attach_pre(if_ctx_t ctx) if (!em_is_valid_ether_addr(hw->mac.addr)) { if (sc->vf_ifp) { + device_printf(dev, + "PF did not assign a MAC address; using a " + "locally generated address\n"); ether_gen_addr(iflib_get_ifp(ctx), (struct ether_addr *)hw->mac.addr); } else { @@ -1334,20 +1592,23 @@ em_if_attach_pre(if_ctx_t ctx) } } - /* Save the EEPROM/NVM versions, must be done under IFLIB_CTX_LOCK */ - em_fw_version_locked(ctx); - - em_print_fw_version(sc); + if (!sc->vf_ifp) { + /* Save NVM versions while holding the IFLIB context lock. */ + em_fw_version_locked(ctx); + em_print_fw_version(sc); + } /* * Get Wake-on-Lan and Management info for later use */ - em_get_wakeup(ctx); + if (!sc->vf_ifp) { + em_get_wakeup(ctx); - /* Enable only WOL MAGIC by default */ - scctx->isc_capenable &= ~IFCAP_WOL; - if (sc->wol != 0) - scctx->isc_capenable |= IFCAP_WOL_MAGIC; + /* Enable only WOL MAGIC by default. */ + scctx->isc_capenable &= ~IFCAP_WOL; + if (sc->wol != 0) + scctx->isc_capenable |= IFCAP_WOL_MAGIC; + } iflib_set_mac(ctx, hw->mac.addr); @@ -1358,11 +1619,12 @@ err_late: err_pci: em_free_pci_resources(ctx); free(sc->mta, M_DEVBUF); + sc->mta = NULL; return (error); } -static int +int em_if_attach_post(if_ctx_t ctx) { struct e1000_softc *sc = iflib_get_softc(ctx); @@ -1376,17 +1638,24 @@ em_if_attach_post(if_ctx_t ctx) goto err_late; } - em_reset(ctx); + if (sc->vf_ifp) + (void)igbv_reset(ctx); + else + em_reset(ctx); /* Initialize statistics */ if (sc->vf_ifp) - sc->ustats.vf_stats = (struct e1000_vf_stats){}; + em_initialize_vf_stats(sc); else sc->ustats.stats = (struct e1000_hw_stats){}; em_update_stats_counters(sc); + atomic_readandclear_32(&sc->stats_pending); hw->mac.get_link_status = 1; - em_if_update_admin_status(ctx); + if (sc->vf_ifp) + igbv_if_update_admin_status(ctx); + else + em_if_update_admin_status(ctx); em_add_hw_stats(sc); /* Non-AMT based hardware can now take control from firmware */ @@ -1421,7 +1690,13 @@ em_if_detach(if_ctx_t ctx) INIT_DEBUGOUT("em_if_detach: begin"); - e1000_phy_hw_reset(&sc->hw); + igb_iov_detach(sc); + if (sc->vf_ifp) { + igbv_queue_retry_detach(sc); + igbv_mbx_retry_detach(sc); + } else { + e1000_phy_hw_reset(&sc->hw); + } em_release_manageability(sc); em_release_hw_control(sc); @@ -1452,6 +1727,10 @@ em_if_suspend(if_ctx_t ctx) { struct e1000_softc *sc = iflib_get_softc(ctx); + if (sc->vf_ifp) { + igbv_queue_retry_stop(sc); + igbv_mbx_retry_stop(sc); + } em_release_manageability(sc); em_release_hw_control(sc); em_enable_wakeup(ctx); @@ -1465,8 +1744,6 @@ em_if_resume(if_ctx_t ctx) if (sc->hw.mac.type == e1000_pch2lan) e1000_resume_workarounds_pchlan(&sc->hw); - em_if_init(ctx); - em_init_manageability(sc); return(0); } @@ -1509,7 +1786,7 @@ em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) break; default: if (sc->hw.mac.type >= igb_mac_min) - max_frame_size = 9234; + max_frame_size = IGB_MAX_FRAME_SIZE; else /* lem */ max_frame_size = MAX_JUMBO_FRAME_SIZE; } @@ -1541,12 +1818,21 @@ em_if_init(if_ctx_t ctx) int i; INIT_DEBUGOUT("em_if_init: begin"); + if (sc->vf_ifp) { + igbv_queue_retry_prepare(sc); + igbv_mbx_retry_prepare(sc); + sc->vf_reset_pending = true; + } /* Get the latest mac address, User can use a LAA */ bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); - /* Put the address into the Receive Address Array */ - e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); + /* + * A VF restores its address only after its reset handshake establishes + * CTS. The PF path programs RAR[0] directly here. + */ + if (!sc->vf_ifp) + e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); /* * With the 82571 adapter, RAR[0] may be overwritten @@ -1561,8 +1847,38 @@ em_if_init(if_ctx_t ctx) } /* Initialize the hardware */ - em_reset(ctx); - em_if_update_admin_status(ctx); + igb_iov_reset_prepare(sc); + if (sc->vf_ifp) { + (void)igbv_reset(ctx); + em_rebase_vf_stats(sc); + } else { + em_reset(ctx); + } + if (sc->vf_ifp && !sc->vf_queues_sanitized) { + /* + * Do not program or enable rings while retained queue state + * might still contain a previous VF owner's DMA address. A + * bounded callout retries initialization after iflib leaves the + * failed initialization stopped. + */ + igbv_queue_retry_failed(ctx); + return; + } + if (sc->vf_ifp && + atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { + igbv_mbx_retry_failed(ctx); + return; + } + if (sc->vf_ifp) + igbv_reconcile_mac(sc, ifp); + /* Re-arm a link-up transition deferred for this reset. */ + if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING || + sc->link_state == EM_LINK_STATE_UP_RESET_PENDING) + sc->link_state = EM_LINK_STATE_DOWN; + if (sc->vf_ifp) + igbv_if_update_admin_status(ctx); + else + em_if_update_admin_status(ctx); for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; i++, tx_que++) { @@ -1578,8 +1894,9 @@ em_if_init(if_ctx_t ctx) txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; } - /* Setup VLAN support, basic and offload if available */ - E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); + /* The VF VLAN EtherType is fixed and has no VET register. */ + if (!sc->vf_ifp) + E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); /* Clear bad data from Rx FIFOs */ if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp) @@ -1589,19 +1906,33 @@ em_if_init(if_ctx_t ctx) em_init_manageability(sc); /* Prepare transmit descriptors and buffers */ - em_initialize_transmit_unit(ctx); + if (sc->vf_ifp) + igbv_initialize_transmit_unit(ctx); + else + em_initialize_transmit_unit(ctx); - /* Setup Multicast table */ + /* + * A failed VF reset has no CTS channel on which to restore mailbox + * state. The reset detector schedules another complete init, which + * replays these interface-owned lists after the handshake succeeds. + */ em_if_multi_set(ctx); sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); - em_initialize_receive_unit(ctx); + if (sc->vf_ifp) + igbv_initialize_receive_unit(ctx); + else + em_initialize_receive_unit(ctx); - /* Set up VLAN support and filter */ + /* Set up VLAN support and filter. */ em_setup_vlan_hw_support(ctx); - /* Don't lose promiscuous settings */ - em_if_set_promisc(ctx, if_getflags(ifp)); + /* Don't lose promiscuous settings. */ + em_if_set_promisc_impl(ctx, if_getflags(ifp)); + atomic_readandclear_32(&sc->promisc_pending); + + /* Restore PF/VF pool configuration after the global reset. */ + igb_iov_initialize(sc); if (sc->hw.mac.ops.clear_hw_cntrs != NULL) sc->hw.mac.ops.clear_hw_cntrs(&sc->hw); @@ -1618,10 +1949,14 @@ em_if_init(if_ctx_t ctx) /* Set up queue routing */ igb_configure_queues(sc); } + if (sc->hw.mac.type >= igb_mac_min) + igb_initialize_interrupt_rate(sc); - /* this clears any pending interrupts */ - E1000_READ_REG(&sc->hw, E1000_ICR); - E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); + if (!sc->vf_ifp) { + /* Clear pending PF interrupts and request a link check. */ + E1000_READ_REG(&sc->hw, E1000_ICR); + E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); + } /* AMT based hardware can now take control from firmware */ if (sc->has_manage && sc->has_amt) @@ -1635,14 +1970,105 @@ em_if_init(if_ctx_t ctx) else e1000_set_eee_i350(&sc->hw, true, true); } + if (sc->vf_ifp) + sc->vf_reset_pending = false; +} + +/* + * RX publishes its byte and packet counters as one snapshot when iflib + * returns descriptors to hardware. This also covers watchdog-driven RX + * processing, which can run while the interrupt vector is unmasked. + */ +static __inline void +em_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) +{ + uint64_t snapshot; + u32 now_bytes, now_packets; + + snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); + now_bytes = snapshot >> 32; + now_packets = (u32)snapshot; + *bytes = now_bytes - rxr->rx_bytes_last; + *packets = now_packets - rxr->rx_packets_last; + rxr->rx_bytes_last = now_bytes; + rxr->rx_packets_last = now_packets; +} + +/* + * TX publishes its byte and packet counters as one snapshot at the doorbell, + * because encapsulation can overlap the interrupt filter. The two halves + * remain independent free running u32 counters, so their deltas are correct + * across wrap. + */ +static __inline void +em_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) +{ + uint64_t snapshot; + u32 now_bytes, now_packets; + + snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); + now_bytes = snapshot >> 32; + now_packets = (u32)snapshot; + *bytes = now_bytes - txr->tx_bytes_last; + *packets = now_packets - txr->tx_packets_last; + txr->tx_bytes_last = now_bytes; + txr->tx_packets_last = now_packets; +} + +/********************************************************************* + * + * Do Adaptive Interrupt Moderation: + * - Calculate based on average size over the last interval + * + * Returns interrupts per second rather than a register value, so that the + * caller's EM_INTS_TO_ITR()/IGB_INTS_TO_EITR() conversion applies, or zero + * if the interval carried no packet to measure. + * + *********************************************************************/ +static u32 +em_ring_itr(struct e1000_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, + u32 txpackets) +{ + u32 newitr = 0; + + if (txbytes && txpackets) + newitr = txbytes / txpackets; + if (rxbytes && rxpackets) + newitr = max(newitr, rxbytes / rxpackets); + + /* + * No packet was observed, so there is no size to work from. Report no + * observation and let the caller keep the rate it already has. + */ + if (newitr == 0) + return (0); + + newitr += 24; /* account for hardware frame, crc */ + /* set an upper boundary */ + newitr = min(newitr, 3000); + /* Be nice to the mid range */ + if ((newitr > 300) && (newitr < 1200)) + newitr = (newitr / 3); + else + newitr = (newitr / 2); + + /* The value above was written straight to EITR; make it a rate */ + newitr = EM_AIM_DIVIDEND / newitr; + + /* + * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts + * into the low latency end. The original was unbounded and would ask + * for ~95k ints/s on minimum sized frames. There is deliberately no + * floor, so jumbo traffic settles near 2.7k ints/s. + */ + if (sc->enable_aim == 1) + newitr = min(newitr, EM_INTS_20K); + else + newitr = min(newitr, EM_INTS_70K); + + return (newitr); } -enum itr_latency_target { - itr_latency_disabled = 0, - itr_latency_lowest = 1, - itr_latency_low = 2, - itr_latency_bulk = 3 -}; /********************************************************************* * * Helper to calculate next (E)ITR value for AIM @@ -1650,126 +2076,52 @@ enum itr_latency_target { *********************************************************************/ static void em_newitr(struct e1000_softc *sc, struct em_rx_queue *que, - struct tx_ring *txr, struct rx_ring *rxr) + struct rx_ring *rxr) { struct e1000_hw *hw = &sc->hw; - unsigned long bytes, bytes_per_packet, packets; - unsigned long rxbytes, rxpackets, txbytes, txpackets; + struct em_tx_queue *tx_que; + u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; u32 newitr; - u8 nextlatency; + int i; - rxbytes = atomic_load_long(&rxr->rx_bytes); - txbytes = atomic_load_long(&txr->tx_bytes); + em_aim_rx_delta(rxr, &rxbytes, &rxpackets); + + /* + * A vector can service more than one TX ring when iflib is configured + * with unequal RX and TX queue counts. Sample every ring routed to + * this vector rather than treating the vector as a TX queue index. + */ + txbytes = txpackets = 0; + for (i = 0; i < sc->tx_num_queues; i++) { + tx_que = &sc->tx_queues[i]; + if (tx_que->msix != que->msix) + continue; + em_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); + txbytes += ringbytes; + txpackets += ringpackets; + } /* Idle, do nothing */ if (txbytes == 0 && rxbytes == 0) return; - newitr = 0; - - if (sc->enable_aim) { - nextlatency = rxr->rx_nextlatency; - + if (sc->enable_aim == 0) { + newitr = em_max_interrupt_rate; + } else if (sc->link_speed < SPEED_1000) { /* Use half default (4K) ITR if sub-gig */ - if (sc->link_speed != 1000) { - newitr = EM_INTS_4K; - goto em_set_next_itr; - } - /* Want at least enough packet buffer for two frames to AIM */ - if (sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { - newitr = em_max_interrupt_rate; - sc->enable_aim = 0; - goto em_set_next_itr; - } - - bytes = bytes_per_packet = 0; - /* Get largest values from the associated tx and rx ring */ - txpackets = atomic_load_long(&txr->tx_packets); - if (txpackets != 0) { - bytes = txbytes; - bytes_per_packet = txbytes / txpackets; - packets = txpackets; - } - rxpackets = atomic_load_long(&rxr->rx_packets); - if (rxpackets != 0) { - bytes = lmax(bytes, rxbytes); - bytes_per_packet = - lmax(bytes_per_packet, rxbytes / rxpackets); - packets = lmax(packets, rxpackets); - } - - /* Latency state machine */ - switch (nextlatency) { - case itr_latency_disabled: /* Bootstrapping */ - nextlatency = itr_latency_low; - break; - case itr_latency_lowest: /* 70k ints/s */ - /* TSO and jumbo frames */ - if (bytes_per_packet > 8000) - nextlatency = itr_latency_bulk; - else if ((packets < 5) && (bytes > 512)) - nextlatency = itr_latency_low; - break; - case itr_latency_low: /* 20k ints/s */ - if (bytes > 10000) { - /* Handle TSO */ - if (bytes_per_packet > 8000) - nextlatency = itr_latency_bulk; - else if ((packets < 10) || - (bytes_per_packet > 1200)) - nextlatency = itr_latency_bulk; - else if (packets > 35) - nextlatency = itr_latency_lowest; - } else if (bytes_per_packet > 2000) { - nextlatency = itr_latency_bulk; - } else if (packets < 3 && bytes < 512) { - nextlatency = itr_latency_lowest; - } - break; - case itr_latency_bulk: /* 4k ints/s */ - if (bytes > 25000) { - if (packets > 35) - nextlatency = itr_latency_low; - } else if (bytes < 1500) - nextlatency = itr_latency_low; - break; - default: - nextlatency = itr_latency_low; - device_printf(sc->dev, - "Unexpected newitr transition %d\n", nextlatency); - break; - } - - /* Trim itr_latency_lowest for default AIM setting */ - if (sc->enable_aim == 1 && nextlatency == itr_latency_lowest) - nextlatency = itr_latency_low; - - /* Request new latency */ - rxr->rx_nextlatency = nextlatency; - } else { - /* We may have toggled to AIM disabled */ - nextlatency = itr_latency_disabled; - rxr->rx_nextlatency = nextlatency; - } - - /* ITR state machine */ - switch(nextlatency) { - case itr_latency_lowest: - newitr = EM_INTS_70K; - break; - case itr_latency_low: - newitr = EM_INTS_20K; - break; - case itr_latency_bulk: newitr = EM_INTS_4K; - break; - case itr_latency_disabled: - default: + } else if (!sc->vf_ifp && + sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { + /* Want at least enough packet buffer for two frames to AIM */ newitr = em_max_interrupt_rate; - break; + } else { + newitr = em_ring_itr(sc, rxbytes, rxpackets, txbytes, + txpackets); + /* No usable observation; leave the rate where it is */ + if (newitr == 0) + return; } -em_set_next_itr: if (hw->mac.type >= igb_mac_min) { newitr = IGB_INTS_TO_EITR(newitr); @@ -1788,7 +2140,8 @@ em_set_next_itr: if (newitr != que->itr_setting) { que->itr_setting = newitr; - if (hw->mac.type == e1000_82574 && que->msix) { + if (hw->mac.type == e1000_82574 && + sc->intr_type == IFLIB_INTR_MSIX) { E1000_WRITE_REG(hw, E1000_EITR_82574(que->msix), que->itr_setting); @@ -1811,7 +2164,6 @@ em_intr(void *arg) struct e1000_softc *sc = arg; struct e1000_hw *hw = &sc->hw; struct em_rx_queue *que = &sc->rx_queues[0]; - struct tx_ring *txr = &sc->tx_queues[0].txr; struct rx_ring *rxr = &que->rxr; if_ctx_t ctx = sc->ctx; u32 reg_icr; @@ -1850,13 +2202,7 @@ em_intr(void *arg) sc->rx_overruns++; if (hw->mac.type >= e1000_82540) - em_newitr(sc, que, txr, rxr); - - /* Reset state */ - txr->tx_bytes = 0; - txr->tx_packets = 0; - rxr->rx_bytes = 0; - rxr->rx_packets = 0; + em_newitr(sc, que, rxr); return (FILTER_SCHEDULE_THREAD); } @@ -1911,18 +2257,11 @@ em_msix_que(void *arg) { struct em_rx_queue *que = arg; struct e1000_softc *sc = que->sc; - struct tx_ring *txr = &sc->tx_queues[que->msix].txr; struct rx_ring *rxr = &que->rxr; ++que->irqs; - em_newitr(sc, que, txr, rxr); - - /* Reset state */ - txr->tx_bytes = 0; - txr->tx_packets = 0; - rxr->rx_bytes = 0; - rxr->rx_packets = 0; + em_newitr(sc, que, rxr); return (FILTER_SCHEDULE_THREAD); } @@ -1940,17 +2279,43 @@ em_msix_link(void *arg) ++sc->link_irq; MPASS(sc->hw.back != NULL); + /* + * The VF's admin vector represents mailbox and link activity. It has + * no PF ICR at E1000_ICR, so process every admin-vector interrupt, + * matching the igbvf misc-vector model. + */ + if (sc->vf_ifp) { + sc->hw.mac.get_link_status = true; + iflib_admin_intr_deferred(sc->ctx); + E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); + return (FILTER_HANDLED); + } + reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); + /* + * Enabling or disabling SR-IOV can briefly make PF MMIO reads return + * all ones. This is not an interrupt cause; in particular, do not + * turn it into a malicious-driver event. + */ + if (__predict_false(reg_icr == 0xffffffff)) + goto rearm; + if (reg_icr & E1000_ICR_RXO) sc->rx_overruns++; if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) em_handle_link(sc->ctx); + if (reg_icr & E1000_ICR_MDDET) + igb_iov_mdd_event(sc); + if (reg_icr & E1000_ICR_VMMB) + iflib_admin_intr_deferred(sc->ctx); +rearm: /* Re-arm unconditionally */ if (sc->hw.mac.type >= igb_mac_min) { - E1000_WRITE_REG(&sc->hw, E1000_IMS, E1000_IMS_LSC); + E1000_WRITE_REG(&sc->hw, E1000_IMS, + E1000_IMS_LSC | igb_iov_intr_mask(sc)); E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); } else if (sc->hw.mac.type == e1000_82574) { E1000_WRITE_REG(&sc->hw, E1000_IMS, @@ -1999,7 +2364,8 @@ em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; - if (!sc->link_active) { + if (sc->link_state == EM_LINK_STATE_DOWN || + sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING) { return; } @@ -2079,19 +2445,70 @@ em_if_media_change(if_ctx_t ctx) device_printf(sc->dev, "Unsupported media type\n"); } - em_if_init(ctx); - return (0); } static int em_if_set_promisc(if_ctx_t ctx, int flags) { + struct e1000_softc *sc; + + sc = iflib_get_softc(ctx); + if (em_if_defer_promisc(sc)) + return (0); + return (em_if_set_promisc_impl(ctx, flags)); +} + +static bool +em_if_defer_promisc(struct e1000_softc *sc) +{ + switch (sc->hw.mac.type) { + case e1000_82576: + case e1000_i350: + case e1000_vfadapt: + case e1000_vfadapt_i350: + break; + default: + return (false); + } + + /* + * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox + * and IOV register operations later from the locked admin task. + * A deferred VF mailbox rejection cannot be returned to ifconfig; the + * admin task logs it instead. + */ + atomic_set_32(&sc->promisc_pending, 1); + iflib_admin_intr_deferred(sc->ctx); + return (true); +} + +int +em_if_set_promisc_impl(if_ctx_t ctx, int flags) +{ struct e1000_softc *sc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); + enum e1000_promisc_type type; + s32 error; u32 reg_rctl; int mcnt = 0; + if (sc->vf_ifp) { + if (flags & IFF_PROMISC) + type = e1000_promisc_enabled; + else if (flags & IFF_ALLMULTI) + type = e1000_promisc_multicast; + else + type = e1000_promisc_disabled; + error = e1000_promisc_set_vf(&sc->hw, type); + if (error != E1000_SUCCESS) { + device_printf(sc->dev, + "VF promiscuous-mode request failed\n"); + return (EPERM); + } + return (0); + } + reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE); if (flags & IFF_ALLMULTI) @@ -2106,20 +2523,25 @@ em_if_set_promisc(if_ctx_t ctx, int flags) if (flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - em_if_vlan_filter_disable(sc); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); + if (igb_iov_enabled(sc)) + em_if_vlan_filter_enable(sc); + else + em_if_vlan_filter_disable(sc); } else { if (flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); } - if (em_if_vlan_filter_used(ctx)) + if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx)) em_if_vlan_filter_enable(sc); } + igb_iov_update_pf_vmolr(sc); + igb_iov_rebuild_vlan(sc); return (0); } @@ -2168,7 +2590,14 @@ em_if_multi_set(if_ctx_t ctx) mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta); - if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) + if (sc->vf_ifp) { + e1000_update_mc_addr_list(&sc->hw, mta, mcnt); + igbv_update_uc_addr_list(sc, ifp); + return; + } + + if (mcnt < MAX_NUM_MULTICAST_ADDRESSES && + !igb_iov_enabled(sc)) e1000_update_mc_addr_list(&sc->hw, mta, mcnt); reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); @@ -2193,6 +2622,8 @@ em_if_multi_set(if_ctx_t ctx) if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) e1000_pci_set_mwi(&sc->hw); } + igb_iov_rebuild_mta(sc); + igb_iov_update_pf_vmolr(sc); } /********************************************************************* @@ -2206,9 +2637,13 @@ em_if_multi_set(if_ctx_t ctx) static void em_if_timer(if_ctx_t ctx, uint16_t qid) { + struct e1000_softc *sc; + if (qid != 0) return; + sc = iflib_get_softc(ctx); + atomic_set_32(&sc->stats_pending, 1); iflib_admin_intr_deferred(ctx); } @@ -2219,7 +2654,15 @@ em_if_update_admin_status(if_ctx_t ctx) struct e1000_hw *hw = &sc->hw; device_t dev = iflib_get_dev(ctx); u32 link_check, thstat, ctrl; - bool automasked = false; + bool reset_requested = false; + + KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); + + if (atomic_readandclear_32(&sc->promisc_pending) != 0) + (void)em_if_set_promisc_impl(ctx, + if_getflags(iflib_get_ifp(ctx))); + igb_iov_handle_mdd(sc); + igb_iov_handle_mbx(sc); link_check = thstat = ctrl = 0; /* Get the cached link value or read phy for real */ @@ -2246,11 +2689,6 @@ em_if_update_admin_status(if_ctx_t ctx) e1000_check_for_link(hw); link_check = hw->mac.serdes_has_link; break; - /* VF device is type_unknown */ - case e1000_media_type_unknown: - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - /* FALLTHROUGH */ default: break; } @@ -2262,7 +2700,13 @@ em_if_update_admin_status(if_ctx_t ctx) } /* Now check for a transition */ - if (link_check && (sc->link_active == 0)) { + if (link_check && + (sc->link_state == EM_LINK_STATE_DOWN || + sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { + bool reset_pending; + + reset_pending = + sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING; e1000_get_speed_and_duplex(hw, &sc->link_speed, &sc->link_duplex); /* Check if we must disable SPEED_MODE bit on PCI-E */ @@ -2279,9 +2723,10 @@ em_if_update_admin_status(if_ctx_t ctx) sc->link_speed, ((sc->link_duplex == FULL_DUPLEX) ? "Full Duplex" : "Half Duplex")); - sc->link_active = 1; + sc->link_state = EM_LINK_STATE_UP; sc->smartspeed = 0; - if ((ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == + if (hw->mac.type == e1000_i350 && + (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == E1000_CTRL_EXT_LINK_MODE_GMII && (thstat & E1000_THSTAT_LINK_THROTTLE)) device_printf(dev, "Link: thermal downshift\n"); @@ -2295,23 +2740,54 @@ em_if_update_admin_status(if_ctx_t ctx) hw->mac.type >= igb_mac_min) { hw->dev_spec._82575.media_changed = false; sc->flags |= IGB_MEDIA_RESET; - em_reset(ctx); + if (igb_iov_enabled(sc)) { + iflib_request_reset(ctx); + iflib_admin_intr_deferred(ctx); + reset_requested = true; + } else + em_reset(ctx); } /* Only do TSO on gigabit for older chips due to errata */ if (hw->mac.type < igb_mac_min) - automasked = em_automask_tso(ctx); + reset_requested = em_automask_tso(ctx); - /* Automasking resets the interface so don't mark it up yet */ - if (!automasked) + if (reset_pending || reset_requested) { + /* + * The PHY is up, but publish it only after the TSO + * capability-change reset. + */ + sc->link_state = EM_LINK_STATE_UP_RESET_PENDING; + } else { iflib_link_state_change(ctx, LINK_STATE_UP, IF_Mbps(sc->link_speed)); - } else if (!link_check && (sc->link_active == 1)) { + } + igb_iov_ping_all_vfs(sc); + } else if (!link_check && + (sc->link_state == EM_LINK_STATE_UP || + sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { + bool link_was_published; + bool reset_pending; + + link_was_published = sc->link_state == EM_LINK_STATE_UP; + reset_pending = + sc->link_state == EM_LINK_STATE_UP_RESET_PENDING; sc->link_speed = 0; sc->link_duplex = 0; - sc->link_active = 0; - iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + sc->link_state = reset_pending ? + EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN; + if (link_was_published) + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + igb_iov_ping_all_vfs(sc); } - em_update_stats_counters(sc); + /* + * Mailbox, link, and timer events share this admin task. The PF + * statistics sweep performs 66 MMIO reads, so run it only when the + * ordinary iflib timer requests a sample rather than once per mailbox + * message. Exported counters can consequently trail hardware by the + * timer interval (normally 500 ms). + */ + if (atomic_readandclear_32(&sc->stats_pending) != 0) + em_update_stats_counters(sc); /* Reset LAA into RAR[0] on 82571 */ if (hw->mac.type == e1000_82571 && e1000_get_laa_state_82571(hw)) @@ -2321,18 +2797,6 @@ em_if_update_admin_status(if_ctx_t ctx) lem_smartspeed(sc); } -static void -em_if_watchdog_reset(if_ctx_t ctx) -{ - struct e1000_softc *sc = iflib_get_softc(ctx); - - /* - * Just count the event; iflib(4) will already trigger a - * sufficient reset of the controller. - */ - sc->watchdog_events++; -} - /********************************************************************* * * This routine disables all traffic on the adapter by issuing a @@ -2346,16 +2810,36 @@ em_if_stop(if_ctx_t ctx) INIT_DEBUGOUT("em_if_stop: begin"); + if (sc->vf_ifp) { + igbv_queue_retry_stop(sc); + igbv_mbx_retry_stop(sc); + } + /* I219 needs special flushing to avoid hangs */ if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) em_flush_desc_rings(sc); - e1000_reset_hw(&sc->hw); + igb_iov_reset_prepare(sc); + if (!sc->vf_ifp || + (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 && + (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) + e1000_reset_hw(&sc->hw); + if (sc->vf_ifp) + atomic_store_rel_32(&sc->vf_mbx_ready, 0); if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp) E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); - e1000_led_off(&sc->hw); - e1000_cleanup_led(&sc->hw); + if (!sc->vf_ifp) { + e1000_led_off(&sc->hw); + e1000_cleanup_led(&sc->hw); + } else { + sc->link_speed = 0; + sc->link_duplex = 0; + if (sc->link_state != EM_LINK_STATE_DOWN) { + sc->link_state = EM_LINK_STATE_DOWN; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + } + } } /********************************************************************* @@ -2385,12 +2869,15 @@ em_identify_hardware(if_ctx_t ctx) return; } - /* Are we a VF device? */ - if ((sc->hw.mac.type == e1000_vfadapt) || - (sc->hw.mac.type == e1000_vfadapt_i350)) - sc->vf_ifp = 1; - else - sc->vf_ifp = 0; + /* + * Function type comes from the selected iflib shared context, not from + * enum ordering. Keep the detected MAC type as an independent check + * that the igb/igbv probe tables selected the right policy. + */ + KASSERT(sc->vf_ifp == + (sc->hw.mac.type == e1000_vfadapt || + sc->hw.mac.type == e1000_vfadapt_i350), + ("%s: iflib function type and MAC type disagree", __func__)); } static int @@ -2558,10 +3045,18 @@ igb_configure_queues(struct e1000_softc *sc) struct e1000_hw *hw = &sc->hw; struct em_rx_queue *rx_que; struct em_tx_queue *tx_que; - u32 tmp, ivar = 0, newitr = 0; + u32 tmp, ivar = 0; - /* First turn on RSS capability */ - if (hw->mac.type != e1000_82575) + /* + * Queue ownership can change when SR-IOV is enabled or disabled. + * Rebuild the interrupt mask for the current layout instead of + * retaining vectors from a previous initialization. + */ + sc->que_mask = 0; + sc->link_mask = 0; + + /* GPIE controls the PF interrupt block and is not in the VF BAR. */ + if (!sc->vf_ifp && hw->mac.type != e1000_82575) E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | E1000_GPIE_PBA | E1000_GPIE_NSICR); @@ -2577,10 +3072,13 @@ igb_configure_queues(struct e1000_softc *sc) case e1000_vfadapt_i350: /* RX entries */ for (int i = 0; i < sc->rx_num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + uint32_t index, qid; + rx_que = &sc->rx_queues[i]; - if (i & 1) { + qid = rx_que->rxr.me; + index = qid >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + if (qid & 1) { ivar &= 0xFF00FFFF; ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; @@ -2589,13 +3087,17 @@ igb_configure_queues(struct e1000_softc *sc) ivar |= rx_que->msix | E1000_IVAR_VALID; } E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + sc->que_mask |= rx_que->eims; } /* TX entries */ for (int i = 0; i < sc->tx_num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + uint32_t index, qid; + tx_que = &sc->tx_queues[i]; - if (i & 1) { + qid = tx_que->txr.me; + index = qid >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + if (qid & 1) { ivar &= 0x00FFFFFF; ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; @@ -2609,17 +3111,27 @@ igb_configure_queues(struct e1000_softc *sc) } /* And for the link interrupt */ - ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; + if (sc->vf_ifp) { + /* + * VTIVAR_MISC maps the VF mailbox in bits 7:0. + * The PF IVAR_MISC maps other causes in bits 15:8. + */ + ivar = sc->linkvec | E1000_IVAR_VALID; + } else + ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; sc->link_mask = 1 << sc->linkvec; E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); break; case e1000_82576: /* RX entries */ for (int i = 0; i < sc->rx_num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + uint32_t index, qid; + rx_que = &sc->rx_queues[i]; - if (i < 8) { + qid = rx_que->rxr.me; + index = qid & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + if (qid < 8) { ivar &= 0xFFFFFF00; ivar |= rx_que->msix | E1000_IVAR_VALID; } else { @@ -2632,10 +3144,13 @@ igb_configure_queues(struct e1000_softc *sc) } /* TX entries */ for (int i = 0; i < sc->tx_num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + uint32_t index, qid; + tx_que = &sc->tx_queues[i]; - if (i < 8) { + qid = tx_que->txr.me; + index = qid & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + if (qid < 8) { ivar &= 0xFFFF00FF; ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; @@ -2682,22 +3197,30 @@ igb_configure_queues(struct e1000_softc *sc) break; } - /* Set the igb starting interrupt rate */ - if (em_max_interrupt_rate > 0) { - newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); + return; +} - if (hw->mac.type == e1000_82575) - newitr |= newitr << 16; - else - newitr |= E1000_EITR_CNT_IGNR; +static void +igb_initialize_interrupt_rate(struct e1000_softc *sc) +{ + struct e1000_hw *hw = &sc->hw; + struct em_rx_queue *rx_que; + u32 newitr; - for (int i = 0; i < sc->rx_num_queues; i++) { - rx_que = &sc->rx_queues[i]; - E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr); - } - } + newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); + if (hw->mac.type == e1000_82575) + newitr |= newitr << 16; + else + newitr |= E1000_EITR_CNT_IGNR; - return; + for (int i = 0; i < sc->rx_num_queues; i++) { + rx_que = &sc->rx_queues[i]; + rx_que->itr_setting = newitr; + E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), + rx_que->itr_setting); + } + if (sc->intr_type == IFLIB_INTR_MSIX) + E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr); } static void @@ -2758,7 +3281,9 @@ lem_smartspeed(struct e1000_softc *sc) { u16 phy_tmp; - if (sc->link_active || (sc->hw.phy.type != e1000_phy_igp) || + if (sc->link_state == EM_LINK_STATE_UP || + sc->link_state == EM_LINK_STATE_UP_RESET_PENDING || + (sc->hw.phy.type != e1000_phy_igp) || sc->hw.mac.autoneg == 0 || (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) return; @@ -2822,9 +3347,22 @@ igb_init_dmac(struct e1000_softc *sc, u32 pba) u16 hwm; u16 max_frame_size; + KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF", + __func__)); + if (hw->mac.type == e1000_i211) return; + /* + * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve + * the configured value so it can be restored after IOV is disabled. + */ + if (igb_iov_enabled(sc)) { + if (hw->mac.type > e1000_82580) + E1000_WRITE_REG(hw, E1000_DMACR, 0); + return; + } + max_frame_size = sc->shared->isc_max_frame_size; if (hw->mac.type > e1000_82580) { @@ -3042,6 +3580,8 @@ em_reset(if_ctx_t ctx) u32 pba; INIT_DEBUGOUT("em_reset: begin"); + KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); + /* Let the firmware know the OS is in control */ em_get_hw_control(sc); @@ -3114,14 +3654,12 @@ em_reset(if_ctx_t ctx) pba = E1000_PBA_32K; break; case e1000_82576: - case e1000_vfadapt: pba = E1000_READ_REG(hw, E1000_RXPBS); pba &= E1000_RXPBS_SIZE_MASK_82576; break; case e1000_82580: case e1000_i350: case e1000_i354: - case e1000_vfadapt_i350: pba = E1000_READ_REG(hw, E1000_RXPBS); pba = e1000_rxpbs_adjust_82580(pba); break; @@ -3166,7 +3704,7 @@ em_reset(if_ctx_t ctx) if (hw->mac.type < igb_mac_min) E1000_WRITE_REG(hw, E1000_PBA, pba); - INIT_DEBUGOUT1("em_reset: pba=%dK",pba); + INIT_DEBUGOUT1("em_reset: pba=%dK", pba); /* * These parameters control the automatic generation (Tx) and @@ -3225,13 +3763,14 @@ em_reset(if_ctx_t ctx) case e1000_pch_ptp: hw->fc.high_water = 0x5C20; hw->fc.low_water = 0x5048; - hw->fc.pause_time = 0x0650; - hw->fc.refresh_time = 0x0400; + hw->fc.pause_time = 0xFFFF; + hw->fc.refresh_time = 0xFFFF; /* Jumbos need adjusted PBA */ if (if_getmtu(ifp) > ETHERMTU) - E1000_WRITE_REG(hw, E1000_PBA, 12); + pba = E1000_PBA_12K; else - E1000_WRITE_REG(hw, E1000_PBA, 26); + pba = E1000_PBA_26K; + E1000_WRITE_REG(hw, E1000_PBA, pba); break; case e1000_82575: case e1000_82576: @@ -3243,8 +3782,6 @@ em_reset(if_ctx_t ctx) case e1000_i354: case e1000_i210: case e1000_i211: - case e1000_vfadapt: - case e1000_vfadapt_i350: /* 16-byte granularity */ hw->fc.low_water = hw->fc.high_water - 16; break; @@ -3268,13 +3805,11 @@ em_reset(if_ctx_t ctx) /* Issue a global reset */ e1000_reset_hw(hw); - if (!sc->vf_ifp) { - if (hw->mac.type >= igb_mac_min) { - E1000_WRITE_REG(hw, E1000_WUC, 0); - } else { - E1000_WRITE_REG(hw, E1000_WUFC, 0); - em_disable_aspm(sc); - } + if (hw->mac.type >= igb_mac_min) { + E1000_WRITE_REG(hw, E1000_WUC, 0); + } else { + E1000_WRITE_REG(hw, E1000_WUFC, 0); + em_disable_aspm(sc); } if (sc->flags & IGB_MEDIA_RESET) { e1000_setup_init_funcs(hw, true); @@ -3289,7 +3824,7 @@ em_reset(if_ctx_t ctx) if (hw->mac.type >= igb_mac_min) igb_init_dmac(sc, pba); - /* Save the final PBA off if it needs to be used elsewhere i.e. AIM */ + /* Save the receive packet-buffer allocation for AIM. */ sc->pba = pba; E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); @@ -3459,6 +3994,14 @@ em_setup_interface(if_ctx_t ctx) * Specify the media types supported by this adapter and register * callbacks to update media and link information */ + if (sc->vf_ifp) { + ifmedia_add(sc->media, + IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); + ifmedia_set(sc->media, + IFM_ETHER | IFM_1000_T | IFM_FDX); + return (0); + } + if (sc->hw.phy.media_type == e1000_media_type_fiber || sc->hw.phy.media_type == e1000_media_type_internal_serdes) { u_char fiber_type = IFM_1000_SX; /* default type */ @@ -3513,6 +4056,9 @@ em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, /* Set up some basics */ struct tx_ring *txr = &que->txr; + KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), + ("%s: misaligned TX AIM snapshot %p", __func__, + &txr->tx_aim_snapshot)); txr->sc = que->sc = sc; que->me = txr->me = i; @@ -3566,6 +4112,9 @@ em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { /* Set up some basics */ struct rx_ring *rxr = &que->rxr; + KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), + ("%s: misaligned RX AIM snapshot %p", __func__, + &rxr->rx_aim_snapshot)); rxr->sc = que->sc = sc; rxr->que = que; que->me = rxr->me = i; @@ -3612,59 +4161,174 @@ em_if_queues_free(if_ctx_t ctx) } } +static u32 +em_legacy_txdctl(struct e1000_hw *hw) +{ + u32 txdctl; + + /* + * Start with the established full-descriptor writeback policy. + * Several generations have descriptor-queue errata for which it is + * a documented workaround. The unsafe early controllers are + * overridden below. + */ + txdctl = EM_TX_PTHRESH | (EM_TX_HTHRESH << 8) | + (EM_TX_WTHRESH << 16) | E1000_TXDCTL_GRAN; + + switch (hw->mac.type) { + case e1000_82571: + case e1000_82572: + case e1000_82573: + case e1000_82574: + case e1000_82583: + case e1000_80003es2lan: + /* Match the Intel shared-code policy for these families. */ + txdctl |= E1000_TXDCTL_COUNT_DESC; + break; + case e1000_ich8lan: + case e1000_ich9lan: + case e1000_ich10lan: + case e1000_pchlan: + case e1000_pch2lan: + case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: + case e1000_pch_tgp: + case e1000_pch_adp: + case e1000_pch_mtp: + case e1000_pch_ptp: + /* Preserve the required bit set by the integrated shared code. */ + txdctl |= (1U << 22); + break; + case e1000_82542: + case e1000_82543: + case e1000_82544: + /* + * 82543 erratum 35 and 82544 erratum 20 require + * WTHRESH=0. Leave all descriptor-control thresholds at + * their reset values on these early controllers. + */ + txdctl = 0; + break; + case e1000_82540: + case e1000_82545: + case e1000_82545_rev_3: + case e1000_82546: + case e1000_82546_rev_3: + case e1000_82541: + case e1000_82541_rev_2: + case e1000_82547: + case e1000_82547_rev_2: + break; + default: + KASSERT(0, ("%s: unsupported MAC type %d", __func__, + hw->mac.type)); + break; + } + + return (txdctl); +} + +static u32 +igb_txdctl(struct e1000_hw *hw) +{ + u32 pthresh; + + switch (hw->mac.type) { + case e1000_i354: + pthresh = I354_TX_PTHRESH; + break; + case e1000_82575: + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + pthresh = IGB_TX_PTHRESH; + break; + default: + KASSERT(0, ("%s: unsupported MAC type %d", __func__, + hw->mac.type)); + pthresh = IGB_TX_PTHRESH; + break; + } + + return (pthresh | (IGB_TX_HTHRESH << 8) | + E1000_TXDCTL_QUEUE_ENABLE); +} + /********************************************************************* * * Enable transmit unit. * **********************************************************************/ -static void -em_initialize_transmit_unit(if_ctx_t ctx) +void +em_initialize_transmit_rings(if_ctx_t ctx) { struct e1000_softc *sc = iflib_get_softc(ctx); if_softc_ctx_t scctx = sc->shared; struct em_tx_queue *que; struct tx_ring *txr; struct e1000_hw *hw = &sc->hw; - u32 tctl, txdctl = 0, tarc, tipg = 0; + u32 txdctl; - INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); - - for (int i = 0; i < sc->tx_num_queues; i++, txr++) { + for (int i = 0; i < sc->tx_num_queues; i++) { u64 bus_addr; caddr_t offp, endp; + uint32_t qid; que = &sc->tx_queues[i]; txr = &que->txr; + qid = txr->me; bus_addr = txr->tx_paddr; /* Clear checksum offload context. */ offp = (caddr_t)txr + offsetof(struct tx_ring, csum_flags); endp = (caddr_t)(txr + 1); - bzero(offp, endp - offp); + memset(offp, 0, endp - offp); + + if (hw->mac.type >= igb_mac_min) { + txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid)); + E1000_WRITE_REG(hw, E1000_TXDCTL(qid), + txdctl & ~E1000_TXDCTL_QUEUE_ENABLE); + E1000_WRITE_FLUSH(hw); + } /* Base and Len of TX Ring */ - E1000_WRITE_REG(hw, E1000_TDLEN(i), + E1000_WRITE_REG(hw, E1000_TDLEN(qid), scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); - E1000_WRITE_REG(hw, E1000_TDBAH(i), (u32)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_TDBAL(i), (u32)bus_addr); + E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr); /* Init the HEAD/TAIL indices */ - E1000_WRITE_REG(hw, E1000_TDT(i), 0); - E1000_WRITE_REG(hw, E1000_TDH(i), 0); + E1000_WRITE_REG(hw, E1000_TDT(qid), 0); + E1000_WRITE_REG(hw, E1000_TDH(qid), 0); HW_DEBUGOUT2("Base = %x, Length = %x\n", - E1000_READ_REG(hw, E1000_TDBAL(i)), - E1000_READ_REG(hw, E1000_TDLEN(i))); + E1000_READ_REG(hw, E1000_TDBAL(qid)), + E1000_READ_REG(hw, E1000_TDLEN(qid))); - txdctl = 0; /* clear txdctl */ - txdctl |= 0x1f; /* PTHRESH */ - txdctl |= 1 << 8; /* HTHRESH */ - txdctl |= 1 << 16;/* WTHRESH */ - txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ - txdctl |= E1000_TXDCTL_GRAN; - txdctl |= 1 << 25; /* LWTHRESH */ + if (hw->mac.type < igb_mac_min) + txdctl = em_legacy_txdctl(hw); + else + txdctl = igb_txdctl(hw); - E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); + E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl); } +} + +static void +em_initialize_transmit_unit(if_ctx_t ctx) +{ + struct e1000_softc *sc = iflib_get_softc(ctx); + struct e1000_hw *hw = &sc->hw; + u32 tctl, tarc, tipg = 0; + + INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); + KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); + + em_initialize_transmit_rings(ctx); /* Set the default values for the Tx Inter Packet Gap timer */ switch (hw->mac.type) { @@ -3696,7 +4360,7 @@ em_initialize_transmit_unit(if_ctx_t ctx) sc->txd_cmd |= E1000_TXD_CMD_IDE; } - if (hw->mac.type >= e1000_82540) + if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) E1000_WRITE_REG(hw, E1000_TADV, sc->tx_abs_int_delay.value); if (hw->mac.type == e1000_82571 || hw->mac.type == e1000_82572) { @@ -3755,6 +4419,122 @@ em_initialize_transmit_unit(if_ctx_t ctx) **********************************************************************/ #define BSIZEPKT_ROUNDUP ((1<<E1000_SRRCTL_BSIZEPKT_SHIFT)-1) +static u32 +igb_rxdctl(struct e1000_softc *sc, u32 rxdctl) +{ + struct e1000_hw *hw; + u32 mask, pthresh, wthresh; + + hw = &sc->hw; + mask = IGB_RXDCTL_THRESH_MASK; + switch (hw->mac.type) { + case e1000_82575: + mask = IGB_82575_RXDCTL_THRESH_MASK; + pthresh = IGB_RX_PTHRESH; + wthresh = IGB_RX_WTHRESH; + break; + case e1000_82576: + pthresh = IGB_RX_PTHRESH; + wthresh = sc->intr_type == IFLIB_INTR_MSIX ? + IGB_82576_RX_WTHRESH : IGB_RX_WTHRESH; + break; + case e1000_vfadapt: + /* 82576 VFs always need the MSI-X writeback workaround. */ + pthresh = IGB_RX_PTHRESH; + wthresh = IGB_82576_RX_WTHRESH; + break; + case e1000_i354: + pthresh = I354_RX_PTHRESH; + wthresh = IGB_RX_WTHRESH; + break; + case e1000_82580: + case e1000_i350: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt_i350: + pthresh = IGB_RX_PTHRESH; + wthresh = IGB_RX_WTHRESH; + break; + default: + KASSERT(0, ("%s: unsupported MAC type %d", __func__, + hw->mac.type)); + pthresh = IGB_RX_PTHRESH; + wthresh = IGB_RX_WTHRESH; + break; + } + + rxdctl &= ~mask; + rxdctl |= pthresh | (IGB_RX_HTHRESH << 8) | + (wthresh << 16) | E1000_RXDCTL_QUEUE_ENABLE; + return (rxdctl); +} + +void +igb_initialize_receive_rings(if_ctx_t ctx, bool drop) +{ + struct e1000_softc *sc = iflib_get_softc(ctx); + if_softc_ctx_t scctx = sc->shared; + struct e1000_hw *hw = &sc->hw; + struct em_rx_queue *que; + u32 srrctl; + + srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> + E1000_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; + if (drop) + srrctl |= E1000_SRRCTL_DROP_EN; + + for (int i = 0; i < sc->rx_num_queues; i++) { + struct rx_ring *rxr; + u64 bus_addr; + u32 rxdctl; + uint32_t qid; + + que = &sc->rx_queues[i]; + rxr = &que->rxr; + bus_addr = rxr->rx_paddr; + qid = rxr->me; + + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid)); + E1000_WRITE_REG(hw, E1000_RXDCTL(qid), + rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE); + E1000_WRITE_FLUSH(hw); + + E1000_WRITE_REG(hw, E1000_RDLEN(qid), + scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); + E1000_WRITE_REG(hw, E1000_RDBAH(qid), + (uint32_t)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr); + E1000_WRITE_REG(hw, E1000_RDH(qid), 0); + E1000_WRITE_REG(hw, E1000_RDT(qid), 0); + E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl); + + rxdctl = igb_rxdctl(sc, rxdctl); + E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl); + } +} + +static bool +em_integrated_jumbo_rx(struct e1000_hw *hw) +{ + switch (hw->mac.type) { + case e1000_ich9lan: + case e1000_ich10lan: + case e1000_pchlan: + case e1000_pch2lan: + case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: + case e1000_pch_tgp: + case e1000_pch_adp: + case e1000_pch_mtp: + case e1000_pch_ptp: + return (true); + default: + return (false); + } +} + static void em_initialize_receive_unit(if_ctx_t ctx) { @@ -3767,32 +4547,29 @@ em_initialize_receive_unit(if_ctx_t ctx) uint32_t rctl, rxcsum; INIT_DEBUGOUT("em_initialize_receive_units: begin"); + KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); /* - * Make sure receives are disabled while setting - * up the descriptor ring + * Make sure receives are disabled while setting up the descriptor + * ring. */ rctl = E1000_READ_REG(hw, E1000_RCTL); - /* Do not disable if ever enabled on this hardware */ - if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583)) + /* Do not disable if ever enabled on this hardware. */ + if (hw->mac.type != e1000_82574 && + hw->mac.type != e1000_82583) E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - /* Setup the Receive Control Register */ + /* Setup the Receive Control Register. */ rctl &= ~(3 << E1000_RCTL_MO_SHIFT); rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - - /* Do not store bad packets */ rctl &= ~E1000_RCTL_SBP; - /* Enable Long Packet receive */ - if (if_getmtu(ifp) > ETHERMTU) + if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU) rctl |= E1000_RCTL_LPE; else rctl &= ~E1000_RCTL_LPE; - - /* Strip the CRC */ if (!em_disable_crc_stripping) rctl |= E1000_RCTL_SECRC; @@ -3805,6 +4582,19 @@ em_initialize_receive_unit(if_ctx_t ctx) /* Set the default interrupt throttling rate */ E1000_WRITE_REG(hw, E1000_ITR, EM_INTS_TO_ITR(em_max_interrupt_rate)); + + /* + * The 82574 MSI-X EITR registers are programmed + * with the same value further below. Either way + * the hardware now holds the default rate, so seed + * the software copy to match; otherwise a stale + * itr_setting left over from AIM makes em_newitr() + * skip the write that would restore it. + */ + for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; + i++, que++) + que->itr_setting = + EM_INTS_TO_ITR(em_max_interrupt_rate); } /* XXX TEMPORARY WORKAROUND: on some systems with 82573 @@ -3820,7 +4610,7 @@ em_initialize_receive_unit(if_ctx_t ctx) sc->rx_int_delay.value); } - if (hw->mac.type >= em_mac_min && !sc->vf_ifp) { + if (hw->mac.type >= em_mac_min) { uint32_t rfctl; /* Use extended rx descriptor formats */ rfctl = E1000_READ_REG(hw, E1000_RFCTL); @@ -3840,96 +4630,91 @@ em_initialize_receive_unit(if_ctx_t ctx) E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); } - /* - * Set up L3 and L4 csum Rx descriptor offloads only on Physical - * Functions. Virtual Functions have no access to this register. - */ - if (!sc->vf_ifp) { - rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (if_getcapenable(ifp) & IFCAP_RXCSUM) { - rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; - if (hw->mac.type > e1000_82575) - rxcsum |= E1000_RXCSUM_CRCOFL; - else if (hw->mac.type < em_mac_min && - if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) - rxcsum |= E1000_RXCSUM_IPV6OFL; - } else { - rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); - if (hw->mac.type > e1000_82575) - rxcsum &= ~E1000_RXCSUM_CRCOFL; - else if (hw->mac.type < em_mac_min) - rxcsum &= ~E1000_RXCSUM_IPV6OFL; - } + rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); + if (if_getcapenable(ifp) & IFCAP_RXCSUM) { + rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; + if (hw->mac.type > e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + else if (hw->mac.type < em_mac_min && + if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) + rxcsum |= E1000_RXCSUM_IPV6OFL; + } else { + rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); + if (hw->mac.type > e1000_82575) + rxcsum &= ~E1000_RXCSUM_CRCOFL; + else if (hw->mac.type < em_mac_min) + rxcsum &= ~E1000_RXCSUM_IPV6OFL; + } - if (sc->rx_num_queues > 1) { - /* RSS hash needed in the Rx descriptor */ - rxcsum |= E1000_RXCSUM_PCSD; + if (sc->rx_num_queues > 1) { + /* RSS hash needed in the Rx descriptor */ + rxcsum |= E1000_RXCSUM_PCSD; - if (hw->mac.type >= igb_mac_min) - igb_initialize_rss_mapping(sc); - else - em_initialize_rss_mapping(sc); - } - E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); + if (hw->mac.type >= igb_mac_min) + igb_initialize_rss_mapping(sc); + else + em_initialize_rss_mapping(sc); } + E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); - for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; i++, que++) { + for (i = 0, que = sc->rx_queues; + hw->mac.type < igb_mac_min && i < sc->rx_num_queues; + i++, que++) { struct rx_ring *rxr = &que->rxr; /* Setup the Base and Length of the Rx Descriptor Ring */ u64 bus_addr = rxr->rx_paddr; + uint32_t qid = rxr->me; #if 0 u32 rdt = sc->rx_num_queues -1; /* default */ #endif - E1000_WRITE_REG(hw, E1000_RDLEN(i), + E1000_WRITE_REG(hw, E1000_RDLEN(qid), scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended)); - E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); + E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr); /* Setup the Head and Tail Descriptor Pointers */ - E1000_WRITE_REG(hw, E1000_RDH(i), 0); - E1000_WRITE_REG(hw, E1000_RDT(i), 0); + E1000_WRITE_REG(hw, E1000_RDH(qid), 0); + E1000_WRITE_REG(hw, E1000_RDT(qid), 0); } - /* - * Set PTHRESH for improved jumbo performance - * According to 10.2.5.11 of Intel 82574 Datasheet, - * RXDCTL(1) is written whenever RXDCTL(0) is written. - * Only write to RXDCTL(1) if there is a need for different - * settings. - */ - if ((hw->mac.type == e1000_ich9lan || hw->mac.type == e1000_pch2lan || - hw->mac.type == e1000_ich10lan) && if_getmtu(ifp) > ETHERMTU) { + /* Increase receive-descriptor prefetching for integrated jumbo MACs. */ + if (em_integrated_jumbo_rx(hw) && if_getmtu(ifp) > ETHERMTU) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); - E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); + + rxdctl &= ~(EM_RXDCTL_PTHRESH_MASK | + EM_RXDCTL_HTHRESH_MASK); + rxdctl |= EM_JUMBO_RX_PTHRESH | + (EM_JUMBO_RX_HTHRESH << 8); + E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); } else if (hw->mac.type == e1000_82574) { + /* RXDCTL(0) writes are mirrored to RXDCTL(1) on 82574. */ for (int i = 0; i < sc->rx_num_queues; i++) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= 0x20; /* PTHRESH */ - rxdctl |= 4 << 8; /* HTHRESH */ - rxdctl |= 4 << 16;/* WTHRESH */ - rxdctl |= 1 << 24; /* Switch to granularity */ + + rxdctl &= ~EM_RXDCTL_THRESH_MASK; + rxdctl |= EM_82574_RX_PTHRESH | + (EM_82574_RX_HTHRESH << 8) | + (EM_82574_RX_WTHRESH << 16) | + E1000_RXDCTL_THRESH_UNIT_DESC; E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); } } else if (hw->mac.type >= igb_mac_min) { - u32 psize, srrctl = 0; + bool drop; + u32 psize; - if (if_getmtu(ifp) > ETHERMTU) { + if (igb_iov_enabled(sc)) { + E1000_WRITE_REG(hw, E1000_RLPML, + IGB_IOV_MAX_FRAME_SIZE); + } else if (if_getmtu(ifp) > ETHERMTU) { psize = scctx->isc_max_frame_size; /* are we on a vlan? */ if (if_vlantrunkinuse(ifp)) psize += VLAN_TAG_SIZE; - if (sc->vf_ifp) - e1000_rlpml_set_vf(hw, psize); - else - E1000_WRITE_REG(hw, E1000_RLPML, psize); + E1000_WRITE_REG(hw, E1000_RLPML, psize); } - /* Set maximum packet buffer len */ - srrctl |= (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> - E1000_SRRCTL_BSIZEPKT_SHIFT; - /* * If TX flow control is disabled and there's >1 queue * defined, enable DROP. @@ -3937,42 +4722,11 @@ em_initialize_receive_unit(if_ctx_t ctx) * This drops frames rather than hanging the RX MAC for all * queues. */ - if ((sc->rx_num_queues > 1) && + drop = igb_iov_enabled(sc) || + ((sc->rx_num_queues > 1) && (sc->fc == e1000_fc_none || - sc->fc == e1000_fc_rx_pause)) { - srrctl |= E1000_SRRCTL_DROP_EN; - } - /* Setup the Base and Length of the Rx Descriptor Rings */ - for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; - i++, que++) { - struct rx_ring *rxr = &que->rxr; - u64 bus_addr = rxr->rx_paddr; - u32 rxdctl; - -#ifdef notyet - /* Configure for header split? -- ignore for now */ - rxr->hdr_split = igb_header_split; -#else - srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; -#endif - - E1000_WRITE_REG(hw, E1000_RDLEN(i), - scctx->isc_nrxd[0] * - sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(hw, E1000_RDBAH(i), - (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_RDBAL(i), - (uint32_t)bus_addr); - E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); - /* Enable this Queue */ - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; - rxdctl &= 0xFFF00000; - rxdctl |= IGB_RX_PTHRESH; - rxdctl |= IGB_RX_HTHRESH << 8; - rxdctl |= IGB_RX_WTHRESH << 16; - E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); - } + sc->fc == e1000_fc_rx_pause)); + igb_initialize_receive_rings(ctx, drop); } else if (hw->mac.type >= e1000_pch2lan) { if (if_getmtu(ifp) > ETHERMTU) e1000_lv_jumbo_workaround_ich8lan(hw, true); @@ -4017,37 +4771,68 @@ static void em_if_vlan_register(if_ctx_t ctx, u16 vtag) { struct e1000_softc *sc = iflib_get_softc(ctx); - u32 index, bit; + bool present; + u32 index, mask; index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - sc->shadow_vfta[index] |= (1 << bit); - ++sc->num_vlans; - if (!sc->vf_ifp) - em_if_vlan_filter_write(sc); - else - /* - * Physical funtion may reject registering VLAN - * but we have no way to inform the stack - * about that. - */ - e1000_vfta_set_vf(&sc->hw, vtag, true); + mask = 1U << (vtag & 0x1F); + present = (sc->shadow_vfta[index] & mask) != 0; + /* + * On a VF, record registration intent for replay even if the PF is not + * ready to accept it yet. + */ + sc->shadow_vfta[index] |= mask; + sc->vf_vfta_stale[index] &= ~mask; + if (!present) + ++sc->num_vlans; + if (sc->vf_ifp && + e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { + igbv_vlan_retry_add(sc, vtag); + device_printf(sc->dev, + "VF VLAN %u add request failed\n", vtag); + } else if (sc->vf_ifp) + igbv_vlan_retry_clear(sc, vtag); + if (!sc->vf_ifp) { + if (igb_iov_enabled(sc)) + igb_iov_rebuild_vlan(sc); + else + em_if_vlan_filter_write(sc, index); + } } static void em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) { struct e1000_softc *sc = iflib_get_softc(ctx); - u32 index, bit; + bool present; + u32 index, mask; index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - sc->shadow_vfta[index] &= ~(1 << bit); - --sc->num_vlans; - if (!sc->vf_ifp) - em_if_vlan_filter_write(sc); - else - e1000_vfta_set_vf(&sc->hw, vtag, false); + mask = 1U << (vtag & 0x1F); + present = (sc->shadow_vfta[index] & mask) != 0; + if (sc->vf_ifp) + igbv_vlan_retry_clear(sc, vtag); + if (sc->vf_ifp && + e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { + device_printf(sc->dev, + "VF VLAN %u remove request failed\n", vtag); + /* + * Hardware might still admit this VID. Preserve its receive + * tag until a successful VF reset proves the stale filter gone. + */ + sc->vf_vfta_stale[index] |= mask; + } else { + sc->vf_vfta_stale[index] &= ~mask; + } + sc->shadow_vfta[index] &= ~mask; + if (present) + --sc->num_vlans; + if (!sc->vf_ifp) { + if (igb_iov_enabled(sc)) + igb_iov_rebuild_vlan(sc); + else + em_if_vlan_filter_write(sc, index); + } } static bool @@ -4101,7 +4886,7 @@ em_if_vlan_filter_disable(struct e1000_softc *sc) } static void -em_if_vlan_filter_write(struct e1000_softc *sc) +em_if_vlan_filter_write(struct e1000_softc *sc, int changed_index) { struct e1000_hw *hw = &sc->hw; @@ -4111,8 +4896,13 @@ em_if_vlan_filter_write(struct e1000_softc *sc) if (hw->mac.type < em_mac_min) em_if_intr_disable(sc->ctx); + /* + * Restore every retained VLAN after reset. Also write the changed + * word when its final VLAN was removed so stale hardware membership + * does not survive a zero shadow value. + */ for (int i = 0; i < EM_VFTA_SIZE; i++) - if (sc->shadow_vfta[i] != 0) + if (sc->shadow_vfta[i] != 0 || i == changed_index) e1000_write_vfta(hw, i, sc->shadow_vfta[i]); /* Re-enable interrupts for lem-class devices */ @@ -4126,15 +4916,42 @@ em_setup_vlan_hw_support(if_ctx_t ctx) struct e1000_softc *sc = iflib_get_softc(ctx); struct e1000_hw *hw = &sc->hw; if_t ifp = iflib_get_ifp(ctx); - u32 reg; + s32 error; + u32 max_frame_size, reg; + u16 vid; + int restore_failures; /* * Only PFs have control over VLAN HW filtering * configuration. VFs have to act as if it's always * enabled. */ - if (sc->vf_ifp) + if (sc->vf_ifp) { + max_frame_size = min(sc->shared->isc_max_frame_size + + VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); + e1000_rlpml_set_vf(hw, max_frame_size); + restore_failures = 0; + for (vid = 0; vid < 4096; vid++) { + if ((sc->shadow_vfta[vid >> 5] & + (1U << (vid & 0x1f))) == 0) + continue; + /* + * Desired state remains in shadow_vfta for the next + * replay if the PF mailbox is absent during reset. + */ + error = e1000_vfta_set_vf(hw, vid, true); + if (error != E1000_SUCCESS) { + igbv_vlan_retry_add(sc, vid); + restore_failures++; + } else + igbv_vlan_retry_clear(sc, vid); + } + if (restore_failures != 0) + device_printf(sc->dev, + "VF VLAN restore failed for %d VIDs; retrying\n", + restore_failures); return; + } if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && !em_disable_crc_stripping) { @@ -4147,11 +4964,25 @@ em_setup_vlan_hw_support(if_ctx_t ctx) E1000_WRITE_REG(hw, E1000_CTRL, reg); } - /* If we aren't doing HW filtering, we're done */ + /* + * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN + * filtering is disabled, the IOV VLAN rebuild instead makes the PF + * VLAN-promiscuous without disabling the global filter. + */ if (!em_if_vlan_filter_capable(ctx)) { - em_if_vlan_filter_disable(sc); + if (igb_iov_enabled(sc)) { +#ifdef PCI_IOV + sc->iov_pf_vlan_promisc = true; +#endif + em_if_vlan_filter_enable(sc); + } else + em_if_vlan_filter_disable(sc); return; } +#ifdef PCI_IOV + if (igb_iov_enabled(sc)) + sc->iov_pf_vlan_promisc = false; +#endif /* * A soft reset zero's out the VFTA, so @@ -4198,14 +5029,22 @@ igb_if_intr_enable(if_ctx_t ctx) { struct e1000_softc *sc = iflib_get_softc(ctx); struct e1000_hw *hw = &sc->hw; - u32 mask; + u32 mask, reg; if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { mask = (sc->que_mask | sc->link_mask); - E1000_WRITE_REG(hw, E1000_EIAC, mask); - E1000_WRITE_REG(hw, E1000_EIAM, mask); + /* + * VF interrupt controls are also mapped into these registers. + * Preserve them and change only the PF vectors we own. + */ + reg = E1000_READ_REG(hw, E1000_EIAC); + E1000_WRITE_REG(hw, E1000_EIAC, reg | mask); + reg = E1000_READ_REG(hw, E1000_EIAM); + E1000_WRITE_REG(hw, E1000_EIAM, reg | mask); + igb_iov_intr_drain_stale(sc); E1000_WRITE_REG(hw, E1000_EIMS, mask); - E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); + E1000_WRITE_REG(hw, E1000_IMS, + E1000_IMS_LSC | igb_iov_intr_mask(sc)); } else E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); E1000_WRITE_FLUSH(hw); @@ -4216,10 +5055,22 @@ igb_if_intr_disable(if_ctx_t ctx) { struct e1000_softc *sc = iflib_get_softc(ctx); struct e1000_hw *hw = &sc->hw; + u32 mask, reg; if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { - E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff); - E1000_WRITE_REG(hw, E1000_EIAC, 0); + /* + * Do not use a blanket EIMC write here. VF interrupt controls + * are mapped into the same PF register space, so clearing bits + * we do not own can leave running VFs with interrupts masked. + * Before initial queue configuration the owned mask is zero + * because this driver has not enabled a vector yet. + */ + mask = (sc->que_mask | sc->link_mask); + reg = E1000_READ_REG(hw, E1000_EIAM); + E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask); + E1000_WRITE_REG(hw, E1000_EIMC, mask); + reg = E1000_READ_REG(hw, E1000_EIAC); + E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask); } E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); E1000_WRITE_FLUSH(hw); @@ -4324,16 +5175,13 @@ em_release_hw_control(struct e1000_softc *sc) return; } -static int -em_is_valid_ether_addr(u8 *addr) +bool +em_is_valid_ether_addr(const u8 *addr) { - char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; + static const u8 zero_addr[ETHER_ADDR_LEN]; - if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { - return (false); - } - - return (true); + return (!ETHER_IS_MULTICAST(addr) && + memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0); } static bool @@ -4342,6 +5190,8 @@ em_automask_tso(if_ctx_t ctx) struct e1000_softc *sc = iflib_get_softc(ctx); if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); if_t ifp = iflib_get_ifp(ctx); + bool reset_needed; + int drvflags; if (!em_unsupported_tso && sc->link_speed && sc->link_speed != SPEED_1000 && @@ -4351,20 +5201,32 @@ em_automask_tso(if_ctx_t ctx) sc->tso_automasked = scctx->isc_capenable & IFCAP_TSO; scctx->isc_capenable &= ~IFCAP_TSO; if_setcapenablebit(ifp, 0, IFCAP_TSO); - /* iflib_init_locked handles ifnet hwassistbits */ - iflib_request_reset(ctx); - return true; } else if (sc->link_speed == SPEED_1000 && sc->tso_automasked) { device_printf(sc->dev, "Re-enabling TSO for GbE.\n"); scctx->isc_capenable |= sc->tso_automasked; if_setcapenablebit(ifp, sc->tso_automasked, 0); sc->tso_automasked = 0; - /* iflib_init_locked handles ifnet hwassistbits */ - iflib_request_reset(ctx); - return true; + } else { + return (false); } - return false; + /* + * Reset a running interface, or one being initialized while + * administratively up. OACTIVE remains set after iflib_stop(), so + * it alone cannot distinguish initialization from an interface that + * is down. In other states, the next initialization will apply the + * updated capabilities. + */ + drvflags = if_getdrvflags(ifp); + reset_needed = (drvflags & IFF_DRV_RUNNING) != 0 || + ((drvflags & IFF_DRV_OACTIVE) != 0 && + (if_getflags(ifp) & IFF_UP) != 0); + if (!reset_needed) + return (false); + + /* iflib_init_locked handles ifnet hwassistbits */ + iflib_request_reset(ctx); + return (true); } /* @@ -4385,8 +5247,6 @@ em_get_wakeup(if_ctx_t ctx) switch (sc->hw.mac.type) { case e1000_82542: case e1000_82543: - case e1000_vfadapt: - case e1000_vfadapt_i350: break; case e1000_82544: e1000_read_nvm(&sc->hw, @@ -4502,6 +5362,8 @@ em_enable_wakeup(if_ctx_t ctx) int error = 0; u32 ctrl, ctrl_ext, rctl; + if (sc->vf_ifp) + return; if (!pci_has_pm(dev)) return; @@ -4689,7 +5551,7 @@ em_disable_aspm(struct e1000_softc *sc) * Update the board statistics counters. * **********************************************************************/ -static void +void em_update_stats_counters(struct e1000_softc *sc) { struct e1000_hw_stats *stats; @@ -4801,15 +5663,60 @@ em_update_stats_counters(struct e1000_softc *sc) } static void -em_update_vf_stats_counters(struct e1000_softc *sc) +em_initialize_vf_stats(struct e1000_softc *sc) { struct e1000_vf_stats *stats; - if (sc->link_speed == 0) - return; + stats = &sc->ustats.vf_stats; + *stats = (struct e1000_vf_stats){}; + em_rebase_vf_stats(sc); +} +static void +em_rebase_vf_stats(struct e1000_softc *sc) +{ + struct e1000_vf_stats *stats; + + /* + * A PF reset starts a new VF counter epoch. Preserve the accumulated + * totals while establishing a new raw baseline so the reset is not + * mistaken for a 32-bit wrap. + */ stats = &sc->ustats.vf_stats; +#define INIT_VF_REG(reg, name) do { \ + stats->last_##name = E1000_READ_REG(&sc->hw, reg); \ +} while (0) + INIT_VF_REG(E1000_VFGPRC, gprc); + INIT_VF_REG(E1000_VFGORC, gorc); + INIT_VF_REG(E1000_VFGPTC, gptc); + INIT_VF_REG(E1000_VFGOTC, gotc); + /* + * I350 specification update erratum 31 says VFMPRC is not + * accessible from VF memory. The 0xf3c register remains valid on + * 82576 VFs, but must not be read on vfadapt_i350. + */ + if (sc->hw.mac.type == e1000_vfadapt) + INIT_VF_REG(E1000_VFMPRC, mprc); + else + stats->last_mprc = 0; + INIT_VF_REG(E1000_VFGOTLBC, gotlbc); + INIT_VF_REG(E1000_VFGPTLBC, gptlbc); + INIT_VF_REG(E1000_VFGORLBC, gorlbc); + INIT_VF_REG(E1000_VFGPRLBC, gprlbc); +#undef INIT_VF_REG +} +static void +em_update_vf_stats_counters(struct e1000_softc *sc) +{ + struct e1000_vf_stats *stats; + + stats = &sc->ustats.vf_stats; + + /* + * Internal VF loopback traffic can continue without physical link, + * so sample the counters regardless of link state. + */ UPDATE_VF_REG(E1000_VFGPRC, stats->last_gprc, stats->gprc); UPDATE_VF_REG(E1000_VFGORC, @@ -4818,8 +5725,17 @@ em_update_vf_stats_counters(struct e1000_softc *sc) stats->last_gptc, stats->gptc); UPDATE_VF_REG(E1000_VFGOTC, stats->last_gotc, stats->gotc); - UPDATE_VF_REG(E1000_VFMPRC, - stats->last_mprc, stats->mprc); + if (sc->hw.mac.type == e1000_vfadapt) + UPDATE_VF_REG(E1000_VFMPRC, + stats->last_mprc, stats->mprc); + UPDATE_VF_REG(E1000_VFGOTLBC, + stats->last_gotlbc, stats->gotlbc); + UPDATE_VF_REG(E1000_VFGPTLBC, + stats->last_gptlbc, stats->gptlbc); + UPDATE_VF_REG(E1000_VFGORLBC, + stats->last_gorlbc, stats->gorlbc); + UPDATE_VF_REG(E1000_VFGPRLBC, + stats->last_gprlbc, stats->gprlbc); } static uint64_t @@ -4831,9 +5747,6 @@ em_if_get_vf_counter(if_ctx_t ctx, ift_counter cnt) switch (cnt) { case IFCOUNTER_IERRORS: return sc->dropped_pkts; - case IFCOUNTER_OERRORS: - return (if_get_counter_default(ifp, cnt) + - sc->watchdog_events); default: return (if_get_counter_default(ifp, cnt)); } @@ -4861,7 +5774,7 @@ em_if_get_counter(if_ctx_t ctx, ift_counter cnt) stats->mpc + stats->cexterr); case IFCOUNTER_OERRORS: return (if_get_counter_default(ifp, cnt) + - stats->ecol + stats->latecol + sc->watchdog_events); + stats->ecol + stats->latecol); default: return (if_get_counter_default(ifp, cnt)); } @@ -4897,6 +5810,38 @@ em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) return (sysctl_handle_int(oidp, &val, 0, req)); } +enum em_ring_register { + EM_RING_HEAD, + EM_RING_TAIL, +}; + +/* Queue register addresses can change when the PF enters IOV mode. */ +static int +em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS) +{ + struct tx_ring *txr; + u_int reg, val; + + txr = oidp->oid_arg1; + reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) : + E1000_TDT(txr->me); + val = E1000_READ_REG(&txr->sc->hw, reg); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS) +{ + struct rx_ring *rxr; + u_int reg, val; + + rxr = oidp->oid_arg1; + reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) : + E1000_RDT(rxr->me); + val = E1000_READ_REG(&rxr->sc->hw, reg); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + /* Per queue holdoff interrupt rate handler */ static int em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) @@ -4913,9 +5858,10 @@ em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) tque = oidp->oid_arg1; hw = &tque->sc->hw; if (hw->mac.type >= igb_mac_min) - reg = E1000_READ_REG(hw, E1000_EITR(tque->me)); - else if (hw->mac.type == e1000_82574 && tque->msix) - reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->me)); + reg = E1000_READ_REG(hw, E1000_EITR(tque->msix)); + else if (hw->mac.type == e1000_82574 && + tque->sc->intr_type == IFLIB_INTR_MSIX) + reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->msix)); else reg = E1000_READ_REG(hw, E1000_ITR); } else { @@ -4923,7 +5869,8 @@ em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) hw = &rque->sc->hw; if (hw->mac.type >= igb_mac_min) reg = E1000_READ_REG(hw, E1000_EITR(rque->msix)); - else if (hw->mac.type == e1000_82574 && rque->msix) + else if (hw->mac.type == e1000_82574 && + rque->sc->intr_type == IFLIB_INTR_MSIX) reg = E1000_READ_REG(hw, E1000_EITR_82574(rque->msix)); else @@ -4938,7 +5885,7 @@ em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) } else { usec = (reg & IGB_QVECTOR_MASK); if (usec > 0) - rate = IGB_INTS_TO_EITR(usec); + rate = IGB_EITR_TO_INTS(usec); else rate = 0; } @@ -4980,23 +5927,22 @@ em_add_hw_stats(struct e1000_softc *sc) SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", CTLFLAG_RD, &sc->rx_overruns, "RX overruns"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", - CTLFLAG_RD, &sc->watchdog_events, - "Watchdog timeouts"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", - CTLTYPE_UINT | CTLFLAG_RD, - sc, E1000_CTRL, em_sysctl_reg_handler, "IU", - "Device Control Register"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", - CTLTYPE_UINT | CTLFLAG_RD, - sc, E1000_RCTL, em_sysctl_reg_handler, "IU", - "Receiver Control Register"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", - CTLFLAG_RD, &sc->hw.fc.high_water, 0, - "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", - CTLFLAG_RD, &sc->hw.fc.low_water, 0, - "Flow Control Low Watermark"); + if (!sc->vf_ifp) { + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", + CTLTYPE_UINT | CTLFLAG_RD, + sc, E1000_CTRL, em_sysctl_reg_handler, "IU", + "Device Control Register"); + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", + CTLTYPE_UINT | CTLFLAG_RD, + sc, E1000_RCTL, em_sysctl_reg_handler, "IU", + "Receiver Control Register"); + SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", + CTLFLAG_RD, &sc->hw.fc.high_water, 0, + "Flow Control High Watermark"); + SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", + CTLFLAG_RD, &sc->hw.fc.low_water, 0, + "Flow Control Low Watermark"); + } for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { struct tx_ring *txr = &tx_que->txr; @@ -5011,12 +5957,12 @@ em_add_hw_stats(struct e1000_softc *sc) "IU", "Interrupt Rate"); SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", - CTLTYPE_UINT | CTLFLAG_RD, sc, - E1000_TDH(txr->me), em_sysctl_reg_handler, "IU", + CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD, + em_sysctl_tx_ring_handler, "IU", "Transmit Descriptor Head"); SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", - CTLTYPE_UINT | CTLFLAG_RD, sc, - E1000_TDT(txr->me), em_sysctl_reg_handler, "IU", + CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL, + em_sysctl_tx_ring_handler, "IU", "Transmit Descriptor Tail"); SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", CTLFLAG_RD, &txr->tx_irq, @@ -5036,12 +5982,12 @@ em_add_hw_stats(struct e1000_softc *sc) "IU", "Interrupt Rate"); SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", - CTLTYPE_UINT | CTLFLAG_RD, sc, - E1000_RDH(rxr->me), em_sysctl_reg_handler, "IU", + CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD, + em_sysctl_rx_ring_handler, "IU", "Receive Descriptor Head"); SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", - CTLTYPE_UINT | CTLFLAG_RD, sc, - E1000_RDT(rxr->me), em_sysctl_reg_handler, "IU", + CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL, + em_sysctl_rx_ring_handler, "IU", "Receive Descriptor Tail"); SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", CTLFLAG_RD, &rxr->rx_irq, @@ -5072,9 +6018,35 @@ em_add_hw_stats(struct e1000_softc *sc) SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", CTLFLAG_RD, &vfstats->gotc, "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &vfstats->mprc, - "Multicast Packets Received"); + if (sc->hw.mac.type == e1000_vfadapt) { + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, + "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc, + "Multicast Packets Received"); + } + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, + "loopback_good_pkts_recvd", + CTLFLAG_RD, &vfstats->gprlbc, + "Good Loopback Packets Received"); + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, + "loopback_good_pkts_txd", + CTLFLAG_RD, &vfstats->gptlbc, + "Good Loopback Packets Transmitted"); + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, + "loopback_good_octets_recvd", + CTLFLAG_RD, &vfstats->gorlbc, + "Good Loopback Octets Received"); + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, + "loopback_good_octets_txd", + CTLFLAG_RD, &vfstats->gotlbc, + "Good Loopback Octets Transmitted"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, + "rx_csum_offload_good", + CTLFLAG_RD, &sc->rx_csum_good, + "Receive Checksum Offload Successes"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, + "rx_csum_offload_errors", + CTLFLAG_RD, &sc->rx_csum_errors, + "Receive Checksum Offload Errors"); return; } @@ -5449,7 +6421,12 @@ em_print_nvm_info(struct e1000_softc *sc) j = 0; ++row; printf("\n0x00%x0 ",row); } - e1000_read_nvm(hw, i, 1, &eeprom_data); + eeprom_data = 0; + if (e1000_read_nvm(hw, i, 1, &eeprom_data) != + E1000_SUCCESS) { + printf("\nNVM read failed at offset %#x\n", i); + break; + } printf("%04x ", eeprom_data); } sx_xunlock(iflib_ctx_lock); @@ -5587,6 +6564,16 @@ em_set_flowcntl(SYSCTL_HANDLER_ARGS) return (error); } +static void +em_sysctl_request_reinit(struct e1000_softc *sc) +{ + if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) + return; + + iflib_request_reset(sc->ctx); + iflib_admin_intr_deferred(sc->ctx); +} + /* * Manage DMA Coalesce: * Control values: @@ -5632,7 +6619,7 @@ igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) return (EINVAL); } /* Reinit the interface */ - em_if_init(sc->ctx); + em_sysctl_request_reinit(sc); return (error); } @@ -5658,7 +6645,7 @@ em_sysctl_eee(SYSCTL_HANDLER_ARGS) sc->hw.dev_spec.ich8lan.eee_disable = (value != 0); else sc->hw.dev_spec._82575.eee_disable = (value != 0); - em_if_init(sc->ctx); + em_sysctl_request_reinit(sc); return (0); } @@ -5716,8 +6703,15 @@ em_print_debug_info(struct e1000_softc *sc) { device_t dev = iflib_get_dev(sc->ctx); if_t ifp = iflib_get_ifp(sc->ctx); - struct tx_ring *txr = &sc->tx_queues->txr; - struct rx_ring *rxr = &sc->rx_queues->rxr; + struct tx_ring *txr; + struct rx_ring *rxr; + + if (sc->tx_queues == NULL || sc->rx_queues == NULL) { + device_printf(dev, "queue state is unavailable\n"); + return; + } + txr = &sc->tx_queues->txr; + rxr = &sc->rx_queues->rxr; if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) printf("Interface is RUNNING "); @@ -5732,15 +6726,15 @@ em_print_debug_info(struct e1000_softc *sc) for (int i = 0; i < sc->tx_num_queues; i++, txr++) { device_printf(dev, "TX Queue %d ------\n", i); device_printf(dev, "hw tdh = %d, hw tdt = %d\n", - E1000_READ_REG(&sc->hw, E1000_TDH(i)), - E1000_READ_REG(&sc->hw, E1000_TDT(i))); + E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)), + E1000_READ_REG(&sc->hw, E1000_TDT(txr->me))); } for (int j=0; j < sc->rx_num_queues; j++, rxr++) { device_printf(dev, "RX Queue %d ------\n", j); device_printf(dev, "hw rdh = %d, hw rdt = %d\n", - E1000_READ_REG(&sc->hw, E1000_RDH(j)), - E1000_READ_REG(&sc->hw, E1000_RDT(j))); + E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)), + E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me))); } } diff --git a/sys/dev/e1000/if_em.h b/sys/dev/e1000/if_em.h index 4c80c7696952..7116ff11c371 100644 --- a/sys/dev/e1000/if_em.h +++ b/sys/dev/e1000/if_em.h @@ -48,6 +48,7 @@ #endif #include <sys/buf_ring.h> #include <sys/bus.h> +#include <sys/callout.h> #include <sys/endian.h> #include <sys/kernel.h> #include <sys/kthread.h> @@ -95,6 +96,9 @@ #include "e1000_82571.h" #include "ifdi_if.h" +struct igb_vf; +struct igb_vf_mac_filter; + /* Tunables */ /* @@ -133,6 +137,7 @@ #define EM_DEFAULT_RXD 1024 #define EM_DEFAULT_MULTI_RXD 4096 #define IGB_MAX_RXD 4096 +#define IGB_MAX_FRAME_SIZE 9234 /* * EM_TIDV - Transmit Interrupt Delay Value @@ -253,8 +258,18 @@ #define IGB_EITR_DIVIDEND 1000000 #define IGB_EITR_SHIFT 2 #define IGB_QVECTOR_MASK 0x7FFC -#define IGB_INTS_TO_EITR(i) (((IGB_EITR_DIVIDEND/i) & IGB_QVECTOR_MASK) << \ - IGB_EITR_SHIFT) +#define IGB_INTS_TO_EITR(i) \ + (((IGB_EITR_DIVIDEND / (i)) << IGB_EITR_SHIFT) & IGB_QVECTOR_MASK) +#define IGB_EITR_TO_INTS(i) ((IGB_EITR_DIVIDEND << IGB_EITR_SHIFT) / \ + ((i) & IGB_QVECTOR_MASK)) + +/* + * The average packet size calculation in em_ring_itr() yields an EITR + * interval field value. That field is quarter microsecond granular (see + * IGB_EITR_SHIFT), so an interval of V is 1000000 / (V / 4) interrupts per + * second. + */ +#define EM_AIM_DIVIDEND (IGB_EITR_DIVIDEND << IGB_EITR_SHIFT) #define IGB_LINK_ITR 2000 #define I210_LINK_DELAY 1000 @@ -281,17 +296,40 @@ #define PCICFG_DESC_RING_STATUS 0xe4 #define FLUSH_DESC_REQUIRED 0x100 +#define EM_TX_PTHRESH 31 +#define EM_TX_HTHRESH 1 +#define EM_TX_WTHRESH 1 + +#define EM_RXDCTL_PTHRESH_MASK 0x0000003F +#define EM_RXDCTL_HTHRESH_MASK 0x00003F00 +#define EM_RXDCTL_WTHRESH_MASK 0x003F0000 +#define EM_RXDCTL_THRESH_MASK (EM_RXDCTL_PTHRESH_MASK | \ + EM_RXDCTL_HTHRESH_MASK | \ + EM_RXDCTL_WTHRESH_MASK) + +#define EM_JUMBO_RX_PTHRESH 3 +#define EM_JUMBO_RX_HTHRESH 1 +#define EM_82574_RX_PTHRESH 32 +#define EM_82574_RX_HTHRESH 4 +#define EM_82574_RX_WTHRESH 4 -#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ - ((hw->mac.type <= e1000_82576) ? 16 : 8)) -#define IGB_RX_HTHRESH 8 -#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ - (sc->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) +#define IGB_RXDCTL_PTHRESH_MASK 0x0000001F +#define IGB_RXDCTL_HTHRESH_MASK 0x00001F00 +#define IGB_RXDCTL_WTHRESH_MASK 0x001F0000 +#define IGB_RXDCTL_THRESH_MASK (IGB_RXDCTL_PTHRESH_MASK | \ + IGB_RXDCTL_HTHRESH_MASK | \ + IGB_RXDCTL_WTHRESH_MASK) +#define IGB_82575_RXDCTL_THRESH_MASK 0x003F3F3F -#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) -#define IGB_TX_HTHRESH 1 -#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ - sc->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) +#define IGB_RX_PTHRESH 8 +#define I354_RX_PTHRESH 12 +#define IGB_RX_HTHRESH 8 +#define IGB_RX_WTHRESH 4 +#define IGB_82576_RX_WTHRESH 1 + +#define IGB_TX_PTHRESH 8 +#define I354_TX_PTHRESH 20 +#define IGB_TX_HTHRESH 1 /* * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be @@ -374,11 +412,8 @@ #define UPDATE_VF_REG(reg, last, cur) \ do { \ u32 new = E1000_READ_REG(&sc->hw, reg); \ - if (new < last) \ - cur += 0x100000000LL; \ + cur += (u32)(new - last); \ last = new; \ - cur &= 0xFFFFFFFF00000000LL; \ - cur |= new; \ } while (0) struct e1000_softc; @@ -408,8 +443,18 @@ struct tx_ring { /* Soft stats */ unsigned long tx_irq; - unsigned long tx_packets; - unsigned long tx_bytes; + + /* + * Free running AIM counters. The producer updates these while + * encapsulating packets, then publishes both together at the TX + * doorbell. The interrupt handler samples only the published value, + * so it cannot observe one counter without the other. + */ + u32 tx_packets; + u32 tx_bytes; + uint64_t tx_aim_snapshot __aligned(8); + u32 tx_packets_last; + u32 tx_bytes_last; /* Saved csum offloading context information */ int csum_flags; @@ -424,6 +469,15 @@ struct tx_ring { uint32_t csum_txd_lower; /* last field */ }; +static __inline void +em_aim_publish(struct tx_ring *txr) +{ + uint64_t snapshot; + + snapshot = ((uint64_t)txr->tx_bytes << 32) | txr->tx_packets; + atomic_store_rel_64(&txr->tx_aim_snapshot, snapshot); +} + /* * The Receive ring, one per rx queue */ @@ -443,13 +497,29 @@ struct rx_ring { /* Soft stats */ unsigned long rx_irq; unsigned long rx_discarded; - unsigned long rx_packets; - unsigned long rx_bytes; - /* Next requested ITR latency */ - u8 rx_nextlatency; + /* + * Free running AIM counters. RX publishes both together when iflib + * returns descriptors to hardware. The interrupt handler samples only + * the published value, so watchdog-driven RX processing cannot expose + * one counter without the other. + */ + u32 rx_packets; + u32 rx_bytes; + uint64_t rx_aim_snapshot __aligned(8); + u32 rx_packets_last; + u32 rx_bytes_last; }; +static __inline void +em_aim_publish_rx(struct rx_ring *rxr) +{ + uint64_t snapshot; + + snapshot = ((uint64_t)rxr->rx_bytes << 32) | rxr->rx_packets; + atomic_store_rel_64(&rxr->rx_aim_snapshot, snapshot); +} + struct em_tx_queue { struct e1000_softc *sc; u32 msix; @@ -469,6 +539,14 @@ struct em_rx_queue { struct if_irq que_irq; }; +/* Driver-observed link state and its publication barrier. */ +enum em_link_state { + EM_LINK_STATE_DOWN = 0, + EM_LINK_STATE_DOWN_RESET_PENDING, + EM_LINK_STATE_UP, + EM_LINK_STATE_UP_RESET_PENDING, +}; + /* Our softc structure */ struct e1000_softc { struct e1000_hw hw; @@ -501,6 +579,7 @@ struct e1000_softc { int if_flags; int em_insert_vlan_header; u32 ims; + bool allow_64bit_dma; bool in_detach; u32 flags; @@ -528,9 +607,13 @@ struct e1000_softc { ** to repopulate it. */ u32 shadow_vfta[EM_VFTA_SIZE]; + u32 vf_vfta_stale[EM_VFTA_SIZE]; + u32 vf_vfta_retry[EM_VFTA_SIZE]; + sbintime_t vf_vlan_retry_deadline; + u16 vf_vlan_retry_cursor; /* Info about the interface */ - u16 link_active; + enum em_link_state link_state; u16 fc; u16 link_speed; u16 link_duplex; @@ -539,6 +622,31 @@ struct e1000_softc { u32 pba; int link_mask; int tso_automasked; + u32 promisc_pending; + u32 stats_pending; + +#ifdef PCI_IOV + struct igb_vf *vfs; + struct igb_vf_mac_filter *vf_mac_filters; + struct callout iov_mbx_retry; + u32 iov_vfta[EM_VFTA_SIZE]; + u32 iov_mdd_cause; + u32 iov_pending; + u32 iov_spoof_pending; + u32 iov_blocked_pending; + u32 iov_intr_drain_pending; + u32 iov_teardown; + struct timeval iov_last_mdd_log; + u16 num_vfs; + u16 num_vf_mac_filters; + u16 pool; + bool iov_hw_active; + bool iov_mta_valid; + bool iov_mbx_retry_initialized; + bool iov_pf_mdd_blocked; + bool iov_pf_vlan_promisc; + bool iov_vfta_valid; +#endif u64 que_mask; @@ -555,16 +663,71 @@ struct e1000_softc { unsigned long dropped_pkts; unsigned long link_irq; unsigned long rx_overruns; - unsigned long watchdog_events; + u64 rx_csum_good; + u64 rx_csum_errors; union { struct e1000_hw_stats stats; /* !sc->vf_ifp */ struct e1000_vf_stats vf_stats; /* sc->vf_ifp */ } ustats; + struct callout vf_queue_retry; + struct callout vf_mbx_retry; + struct timeval vf_last_queue_log; + struct timeval vf_last_mbx_log; + u32 vf_queue_retry_new_epoch; + u32 vf_queue_retry_pending; + u32 vf_mbx_ready; + u32 vf_mbx_retry_pending; u16 vf_ifp; + u8 vf_queue_failures; + u8 vf_mbx_retry_stage; + bool vf_queue_gave_up; + bool vf_queue_retry_initialized; + bool vf_mbx_retry_initialized; + bool vf_queues_sanitized; + bool vf_reset_pending; + /* A PF can retain auxiliary filters across a VF reset. */ + bool vf_uc_filters_set; }; +/* + * Shared PF/VF mechanisms and VF policy entry points. The latter live in + * if_igbv.c so the VF method table cannot accidentally select PF policy. + */ +int em_if_attach_pre(if_ctx_t); +int em_if_attach_post(if_ctx_t); +void em_add_device_sysctls(struct e1000_softc *); +int em_if_set_promisc_impl(if_ctx_t, int); +bool em_is_valid_ether_addr(const u8 *); +void em_initialize_transmit_rings(if_ctx_t); +void em_update_stats_counters(struct e1000_softc *); +void igb_initialize_receive_rings(if_ctx_t, bool); + +int igbv_get_regs(SYSCTL_HANDLER_ARGS); +int igbv_if_attach_pre(if_ctx_t); +int igbv_if_attach_post(if_ctx_t); +int igbv_if_media_change(if_ctx_t); +void igbv_if_intr_enable(if_ctx_t); +void igbv_if_intr_disable(if_ctx_t); +void igbv_if_update_admin_status(if_ctx_t); +void igbv_initialize_receive_unit(if_ctx_t); +void igbv_initialize_transmit_unit(if_ctx_t); +void igbv_mbx_retry_detach(struct e1000_softc *); +void igbv_mbx_retry_failed(if_ctx_t); +void igbv_mbx_retry_prepare(struct e1000_softc *); +void igbv_mbx_retry_stop(struct e1000_softc *); +void igbv_queue_retry_detach(struct e1000_softc *); +void igbv_queue_retry_failed(if_ctx_t); +void igbv_queue_retry_prepare(struct e1000_softc *); +void igbv_queue_retry_stop(struct e1000_softc *); +void igbv_reconcile_mac(struct e1000_softc *, if_t); +bool igbv_reset(if_ctx_t); +void igbv_log_reset_failure(struct e1000_softc *, s32, bool); +void igbv_update_uc_addr_list(struct e1000_softc *, if_t); +void igbv_vlan_retry_add(struct e1000_softc *, u16); +void igbv_vlan_retry_clear(struct e1000_softc *, u16); + /******************************************************************************** * vendor_info_array * diff --git a/sys/dev/e1000/if_igb_iov.c b/sys/dev/e1000/if_igb_iov.c new file mode 100644 index 000000000000..e62ca8aa029c --- /dev/null +++ b/sys/dev/e1000/if_igb_iov.c @@ -0,0 +1,2261 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2010-2016, Intel Corporation + * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org> + */ + +#include "if_em.h" +#include "if_igb_iov.h" + +#ifdef PCI_IOV + +#include <sys/iov.h> +#include <sys/sdt.h> +#include <sys/time.h> + +#define IGB_IOV_RAH_POOLSEL_SHIFT 18 +#define IGB_IOV_RAH_POOLSEL_MASK (0xffU << IGB_IOV_RAH_POOLSEL_SHIFT) +#define IGB_IOV_MAX_MAC_FILTERS 3 +#define IGB_IOV_MAX_MC_HASHES 30 +#define IGB_IOV_MBX_RETRY_COUNT 6 +/* Allow two complete 31-VID replays, then sustain eight additions/second. */ +#define IGB_IOV_VLAN_CHANGE_BURST 64 +#define IGB_IOV_VLAN_CHANGE_INTERVAL (SBT_1S / 8) +/* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */ +#define IGB_82576_VMOLR_RSV (1U << 31) +#define IGB_82576_LVMMC_BLOCK_MASK 0x1c +#define IGB_82576_NUM_QUEUES 16 +#define IGB_82576_QUEUE_MASK 0xffff +#define IGB_82576_STAGGERED_QUEUE_SHIFT 8 +#define IGB_82576_VF_QUEUE_STRIDE 8 +#define IGB_82576_VF_QUEUES 2 +#define IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE (1U << 2) +#define IGB_I350_LVMMC_MAC_VLAN_SPOOF (1U << 25) +#define IGB_I350_LVMMC_LAST_Q_SHIFT 29 +#define IGB_I350_LVMMC_LAST_Q_MASK 0x7 +#define IGB_I350_NUM_QUEUES 8 +#define IGB_I350_QUEUE_MASK 0xff +#define IGB_I350_RESET_ACK_TIMEOUT (100 * SBT_1MS) +#define IGB_I350_VF_QUEUES 1 +#define IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES 10 +#define IGB_IOV_QUEUE_DISABLE_DELAY_US 10 +#define IGB_IOV_QUEUE_DISABLE_PAUSE (100 * SBT_1US) +#define IGB_IOV_QUEUE_DISABLE_RETRIES 20 +#define IGB_IOV_VF_QUEUES_MAX 2 + +#define IGB_VF_CTS (1U << 0) +#define IGB_VF_CAP_MAC (1U << 1) +#define IGB_VF_ACTIVE (1U << 2) +#define IGB_VF_MAC_ANTI_SPOOF (1U << 3) +#define IGB_VF_ALLOW_PROMISC (1U << 4) +#define IGB_VF_UCAST_PROMISC (1U << 5) +#define IGB_VF_MCAST_PROMISC (1U << 6) +#define IGB_VF_MCAST_OVERFLOW (1U << 7) +#define IGB_VF_MCAST_OVERFLOW_WARNED (1U << 8) +#define IGB_VF_MDD_BLOCKED (1U << 9) +#define IGB_VF_MBX_PENDING (1U << 10) +/* + * After bounded PFU retries, suppress future or overlapping VF requests until + * RST/VFLR starts a new mailbox epoch. Intel VF drivers assert CTRL.RST + * before sending their mailbox reset request. + */ +#define IGB_VF_MBX_GAVE_UP (1U << 11) +#define IGB_VF_MDD_NOTIFY_PENDING (1U << 12) + +struct igb_vf { + u32 flags; + struct timeval last_nack; + struct timeval last_mbx_log; + struct timeval last_spoof_log; + struct timeval last_mdd_log; + struct timeval last_queue_log; + sbintime_t mbx_retry_at; + sbintime_t mdd_notify_at; + sbintime_t vlan_token_time; + u16 pool; + u16 rar_index; + u16 max_frame_size; + u16 mc_count; + u16 vlan_count; + u16 vlan_tokens; + u16 default_vlan; + u8 mbx_retry_count; + u8 mac[ETHER_ADDR_LEN]; + u16 mc_hashes[IGB_IOV_MAX_MC_HASHES]; + u32 vlans[EM_VFTA_SIZE]; +}; + +struct igb_vf_mac_filter { + bool active; + u16 pool; + u16 rar_index; + u8 mac[ETHER_ADDR_LEN]; +}; + +MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations"); + +/* + * These logical-write probes let hardware tests verify the elision policy. + * e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so + * the probes intentionally count calls made by the rebuild rather than MMIO + * transactions. The state probe exposes the final software images while the + * stack arrays are still live. + */ +SDT_PROVIDER_DEFINE(igb_iov); +SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear, + "struct e1000_softc *", "u_int", "uint32_t"); +SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write, + "struct e1000_softc *", "u_int", "uint32_t"); +SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set, + "struct e1000_softc *", "u_int", "uint32_t"); +SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state, + "struct e1000_softc *", "uint32_t *", "uint32_t *"); +SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr, + "struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t"); + +static const struct timeval igb_iov_nack_interval = { 2, 0 }; +static const struct timeval igb_iov_mbx_log_interval = { 2, 0 }; +static const struct timeval igb_iov_spoof_log_interval = { 2, 0 }; +static const struct timeval igb_iov_mdd_log_interval = { 2, 0 }; +static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2; +static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = { + SBT_1MS, + 2 * SBT_1MS, + 4 * SBT_1MS, + 8 * SBT_1MS, + 16 * SBT_1MS, + 32 * SBT_1MS, +}; + +static void igb_iov_clear_mac_filters(struct e1000_softc *, + const struct igb_vf *); +static bool igb_iov_mac_in_use(struct e1000_softc *, const u8 *, + const struct igb_vf *); +static bool igb_iov_vlan_present(struct e1000_softc *, u16, bool); +static int igb_iov_vlan_unique_count(struct e1000_softc *, bool); + +static void +igb_iov_mbx_retry_callout(void *arg) +{ + struct e1000_softc *sc; + + sc = arg; + /* + * Mailbox service is serialized by iflib's context lock. The + * callout only re-enters through the ordinary admin task. + */ + iflib_admin_intr_deferred(sc->ctx); +} + +static u_int +igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) +{ + u8 *mta; + + if (idx == MAX_NUM_MULTICAST_ADDRESSES) + return (0); + mta = arg; + memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN); + return (1); +} + +static bool +igb_iov_pf_vlan_promisc(struct e1000_softc *sc) +{ + if_t ifp; + + ifp = iflib_get_ifp(sc->ctx); + return (sc->iov_pf_vlan_promisc || + (if_getflags(ifp) & IFF_PROMISC) != 0); +} + +static bool +igb_iov_mac_valid(const u8 *mac) +{ + static const u8 zero[ETHER_ADDR_LEN]; + + return (!ETHER_IS_MULTICAST(mac) && + memcmp(mac, zero, ETHER_ADDR_LEN) != 0); +} + +static bool +igb_iov_nack_allowed(struct igb_vf *vf) +{ + return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0); +} + +static void +igb_iov_reset_vlan_rate(struct igb_vf *vf) +{ + + vf->vlan_token_time = getsbinuptime(); + vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST; +} + +static bool +igb_iov_vlan_add_allowed(struct igb_vf *vf) +{ + sbintime_t elapsed, now; + uint64_t refill; + + now = getsbinuptime(); + elapsed = now - vf->vlan_token_time; + if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) { + refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL; + vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST, + vf->vlan_tokens + refill); + vf->vlan_token_time = now; + } + if (vf->vlan_tokens == 0) + return (false); + vf->vlan_tokens--; + return (true); +} + +static u32 +igb_iov_reply_header(u32 request, bool cts, bool ack) +{ + u32 reply, type; + + type = request & 0xffff; + if (type == E1000_VF_SET_MAC_ADDR && + (request & E1000_VT_MSGINFO_MASK) != 0) + reply = request; + else + reply = type; + reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK | + E1000_VT_MSGTYPE_CTS); + if (cts) + reply |= E1000_VT_MSGTYPE_CTS; + reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK; + return (reply); +} + +bool +igb_iov_supported(const struct e1000_softc *sc) +{ + switch (sc->hw.mac.type) { + case e1000_82576: + case e1000_i350: + return (true); + default: + return (false); + } +} + +bool +igb_iov_enabled(const struct e1000_softc *sc) +{ + return (sc->num_vfs != 0); +} + +int +igb_iov_attach(struct e1000_softc *sc) +{ + nvlist_t *pf_schema, *vf_schema; + int error, iov_pos; + + if (!igb_iov_supported(sc)) + return (0); + if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0) + return (0); + + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", + IOV_SCHEMA_HASDEFAULT, true); + pci_iov_schema_add_bool(vf_schema, "allow-set-mac", + IOV_SCHEMA_HASDEFAULT, false); + pci_iov_schema_add_bool(vf_schema, "allow-promisc", + IOV_SCHEMA_HASDEFAULT, false); + pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT, + VF_VLAN_TRUNK); + + error = pci_iov_attach(sc->dev, pf_schema, vf_schema); + if (error != 0) + device_printf(sc->dev, + "failed to attach SR-IOV configuration interface: %d\n", + error); + else { + callout_init(&sc->iov_mbx_retry, 1); + sc->iov_mbx_retry_initialized = true; + } + return (error); +} + +void +igb_iov_detach(struct e1000_softc *sc) +{ + + if (!sc->iov_mbx_retry_initialized) + return; + callout_drain(&sc->iov_mbx_retry); + sc->iov_mbx_retry_initialized = false; +} + +static u32 +igb_iov_active_mask(struct e1000_softc *sc) +{ + u32 mask; + int i; + + mask = 0; + for (i = 0; i < sc->num_vfs; i++) + if (sc->vfs[i].flags & IGB_VF_ACTIVE) + mask |= 1U << i; + return (mask); +} + +static void +igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool) +{ + struct e1000_hw *hw; + u32 rah; + + hw = &sc->hw; + e1000_rar_set(hw, __DECONST(u8 *, mac), rar); + rah = E1000_READ_REG(hw, E1000_RAH(rar)); + rah &= ~IGB_IOV_RAH_POOLSEL_MASK; + rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool); + E1000_WRITE_REG(hw, E1000_RAH(rar), rah); +} + +static void +igb_iov_clear_rar(struct e1000_softc *sc, u16 rar) +{ + u8 zero[ETHER_ADDR_LEN] = {}; + + e1000_rar_set(&sc->hw, zero, rar); +} + +static void +igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf) +{ + struct igb_vf_mac_filter *filter; + int i; + + for (i = 0; i < sc->num_vf_mac_filters; i++) { + filter = &sc->vf_mac_filters[i]; + if (!filter->active || filter->pool != vf->pool) + continue; + igb_iov_clear_rar(sc, filter->rar_index); + filter->active = false; + memset(filter->mac, 0, sizeof(filter->mac)); + } +} + +static u32 +igb_iov_switch_reg(struct e1000_softc *sc) +{ + return (sc->hw.mac.type == e1000_82576 ? + E1000_DTXSWC : E1000_TXSWC); +} + +static void +igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf) +{ + struct e1000_hw *hw; + u32 reg, value; + + hw = &sc->hw; + reg = igb_iov_switch_reg(sc); + value = E1000_READ_REG(hw, reg); + value &= ~((1U << vf->pool) | + (1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT))); + if (vf->flags & IGB_VF_MAC_ANTI_SPOOF) + value |= 1U << vf->pool; + if (vf->flags & IGB_VF_ACTIVE) + value |= 1U << + (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT); + E1000_WRITE_REG(hw, reg, value); +} + +static void +igb_iov_set_uta(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + bool enable; + int i; + + if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576) + return; + + hw = &sc->hw; + enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) & + E1000_VMOLR_ROPE) != 0; + for (i = 0; i < sc->num_vfs; i++) + if ((sc->vfs[i].flags & + (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) == + (IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) { + enable = true; + break; + } + + for (i = 0; i < MAX_MTA_REG; i++) + E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, + enable ? 0xffffffffU : 0); +} + +static void +igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool, + bool strip_vlan, bool hide_vlan, bool vf_pool) +{ + struct e1000_hw *hw; + u32 dvmolr; + + hw = &sc->hw; + if (hw->mac.type != e1000_i350) + return; + + dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool)); + dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN | + E1000_DVMOLR_STRCRC); + if (hide_vlan) + dvmolr |= E1000_DVMOLR_HIDVLAN; + if (strip_vlan) + dvmolr |= E1000_DVMOLR_STRVLAN; + if (vf_pool || strip_vlan || + (E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0) + dvmolr |= E1000_DVMOLR_STRCRC; + E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr); +} + +static void +igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf) +{ + struct e1000_hw *hw; + u32 max_frame_size, vmolr, vmvir; + + hw = &sc->hw; + max_frame_size = vf->max_frame_size; + if (vf->vlan_count != 0) + max_frame_size = min(max_frame_size + VLAN_TAG_SIZE, + IGB_IOV_MAX_FRAME_SIZE); + vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool)); + vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE | + E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE | + E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN); + vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE | + (max_frame_size & E1000_VMOLR_RLPML_MASK); + if (vf->default_vlan == 0) + vmolr |= E1000_VMOLR_AUPE; + if (vf->mc_count != 0 && + (vf->flags & (IGB_VF_MCAST_PROMISC | + IGB_VF_MCAST_OVERFLOW)) == 0) + vmolr |= E1000_VMOLR_ROMPE; + if (hw->mac.type == e1000_82576) + vmolr |= IGB_82576_VMOLR_RSV; + + if (vf->flags & IGB_VF_UCAST_PROMISC) { + if (hw->mac.type == e1000_82576) + vmolr |= E1000_VMOLR_ROPE; + else + vmolr |= E1000_VMOLR_UPE; + } + /* + * The mailbox can describe only 30 hashes. Fall back to receiving all + * multicast within the VF's VLAN membership when that list overflows. + */ + if ((vf->flags & (IGB_VF_MCAST_PROMISC | + IGB_VF_MCAST_OVERFLOW)) != 0) + vmolr |= E1000_VMOLR_MPME; + if (hw->mac.type == e1000_82576 && vf->vlan_count != 0) + vmolr |= E1000_VMOLR_STRVLAN; + /* A nonzero default VLAN makes this VF an untagged access port. */ + if (vf->default_vlan == 0) + vmvir = 0; + else + vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT; + + E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr); + E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir); + igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0, + vf->default_vlan != 0, true); +} + +static void +igb_iov_configure_pf_vmolr(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + if_t ifp; + bool strip_vlan; + u32 max_frame_size; + u32 old_vmolr, vmolr; + + hw = &sc->hw; + ifp = iflib_get_ifp(sc->ctx); + max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE, + IGB_IOV_MAX_FRAME_SIZE); + strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0; + old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool)); + vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE | + E1000_VMOLR_LPE | + (max_frame_size & E1000_VMOLR_RLPML_MASK); + if (hw->mac.type == e1000_82576) { + vmolr |= IGB_82576_VMOLR_RSV; + if (strip_vlan) + vmolr |= E1000_VMOLR_STRVLAN; + } else + vmolr |= old_vmolr & E1000_VMOLR_VPE; + + if (if_getflags(ifp) & IFF_PROMISC) { + if (hw->mac.type == e1000_82576) + vmolr |= E1000_VMOLR_ROPE; + else + vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE; + vmolr |= E1000_VMOLR_MPME; + } else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 || + if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES) + vmolr |= E1000_VMOLR_MPME; + else if (if_llmaddr_count(ifp) != 0) + vmolr |= E1000_VMOLR_ROMPE; + + E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr); + igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false); +} + +void +igb_iov_update_pf_vmolr(struct e1000_softc *sc) +{ + if (!igb_iov_enabled(sc)) + return; + + igb_iov_configure_pf_vmolr(sc); + igb_iov_set_uta(sc); +} + +u32 +igb_iov_intr_mask(const struct e1000_softc *sc) +{ + if (!sc->iov_hw_active) + return (0); + return (E1000_IMS_VMMB | E1000_IMS_MDDET); +} + +void +igb_iov_intr_drain_stale(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + u32 icr; + + if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0) + return; + hw = &sc->hw; + /* + * Consume setup-time diagnostic state at the actual transition from + * masked to armed. Read ICR last so an event arriving after the drain + * remains pending and is delivered when the caller enables MDDET. + */ + (void)E1000_READ_REG(hw, E1000_LVMMC); + if (hw->mac.type == e1000_82576) + (void)E1000_READ_REG(hw, E1000_WVBR); + icr = E1000_READ_REG(hw, E1000_ICR); + /* + * em_if_init() injects LSC after IOV setup to close the post-reset + * link race. Preserve that cause across this MDDET-specific drain. + */ + if (__predict_true(icr != 0xffffffff) && + (icr & E1000_ICR_LSC) != 0) + E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC); +} + +static void +igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc) +{ + + /* + * I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or + * anti-spoofing is active. The shadow is therefore authoritative + * until a reset or another independent hardware writer invalidates + * it. Readback cannot reliably audit a stale-but-valid shadow on + * this part, so keep all shadow mutation in these two helpers. + */ + memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta)); + sc->iov_vfta_valid = false; +} + +static void +igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta) +{ + + memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta)); + sc->iov_vfta_valid = true; +} + +static void +igb_iov_notify_vfs_reset(struct e1000_softc *sc) +{ + struct igb_vf *vf; + struct e1000_hw *hw; + sbintime_t deadline; + u32 msg, pending, undelivered; + int i; + + hw = &sc->hw; + /* + * Process VFLRs first and wait only for VFs that completed their + * mailbox handshake. An unattached VF has nobody who can acknowledge. + */ + igb_iov_handle_mbx(sc); + pending = 0; + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) == + (IGB_VF_ACTIVE | IGB_VF_CTS)) + pending |= 1U << i; + } + if (pending == 0) + return; + + /* + * I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a + * mailbox warning before the PF asserts CTRL.RST. + * + * The mailbox pass above drained requests and stale acknowledgements. + * A VF read of the new notification sets its ACK bit. + */ + undelivered = 0; + for (i = 0; i < sc->num_vfs; i++) { + if ((pending & (1U << i)) == 0) + continue; + msg = E1000_PF_CONTROL_MSG; + if (e1000_write_mbx(hw, &msg, 1, i) != 0) { + undelivered |= 1U << i; + pending &= ~(1U << i); + } + } + if (undelivered != 0) + device_printf(sc->dev, + "could not deliver reset warning to VF mask %#x\n", + undelivered); + + deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT; + while (pending != 0 && getsbinuptime() < deadline) { + for (i = 0; i < sc->num_vfs; i++) { + if ((pending & (1U << i)) != 0 && + e1000_check_for_ack(hw, i) == 0) + pending &= ~(1U << i); + } + if (pending != 0) + pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK); + } + if (pending != 0) + device_printf(sc->dev, + "VF reset acknowledgement timed out for mask %#x\n", + pending); +} + +void +igb_iov_reset_prepare(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + u32 mask; + + if (sc->iov_hw_active) { + hw = &sc->hw; + if (atomic_load_acq_32(&sc->iov_teardown) == 0) { + if (hw->mac.type == e1000_i350) + igb_iov_notify_vfs_reset(sc); + else + igb_iov_ping_all_vfs(sc); + } + + /* Stop VF DMA before the PF asserts CTRL.RST. */ + mask = 1U << sc->pool; + E1000_WRITE_REG(hw, E1000_VFRE, mask); + E1000_WRITE_REG(hw, E1000_VFTE, mask); + E1000_WRITE_FLUSH(hw); + } + sc->iov_hw_active = false; + if (sc->iov_mbx_retry_initialized) + callout_stop(&sc->iov_mbx_retry); + sc->iov_mta_valid = false; + igb_iov_vfta_shadow_invalidate(sc); + atomic_readandclear_32(&sc->iov_mdd_cause); + atomic_readandclear_32(&sc->iov_pending); + atomic_readandclear_32(&sc->iov_spoof_pending); + atomic_readandclear_32(&sc->iov_blocked_pending); + /* + * Normal iflib initialization prepares the reset before + * igb_iov_initialize() requests this drain. Preserve a still-pending + * I350 request across a later stop or repeated preparation so the next + * interrupt arm consumes it. Other families retain the ordinary + * stop-time cleanup. + */ + if (sc->hw.mac.type != e1000_i350) + atomic_readandclear_32(&sc->iov_intr_drain_pending); +} + +void +igb_iov_rebuild_mta(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + struct igb_vf *vf; + u32 hash_bit, hash_reg, hash_value; + u32 mta[MAX_MTA_REG] = {}; + u16 hash; + bool changed; + int i, j, mcnt; + + if (!igb_iov_enabled(sc)) + return; + + hw = &sc->hw; + memset(sc->mta, 0, + ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); + mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx), + igb_iov_copy_maddr, sc->mta); + mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES); + for (i = 0; i < mcnt; i++) { + hash_value = e1000_hash_mc_addr(hw, + &sc->mta[i * ETHER_ADDR_LEN]); + hash_reg = (hash_value >> 5) & + (hw->mac.mta_reg_count - 1); + hash_bit = hash_value & 0x1f; + mta[hash_reg] |= 1U << hash_bit; + } + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + for (j = 0; j < vf->mc_count; j++) { + hash = vf->mc_hashes[j] & 0xfff; + mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |= + 1U << (hash & 0x1f); + } + } + + changed = false; + for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) { + if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i]) + continue; + hw->mac.mta_shadow[i] = mta[i]; + E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]); + changed = true; + } + if (changed) + E1000_WRITE_FLUSH(hw); + sc->iov_mta_valid = true; +} + +static int +igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool, + bool preserve_only) +{ + int free_slot, i; + + free_slot = -1; + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { + if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 && + (vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) { + vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool); + return (0); + } + if (free_slot == -1 && + (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) + free_slot = i; + } + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) + if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 && + (old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid && + (vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) { + free_slot = i; + break; + } + if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE)) + return (ENOENT); + if (free_slot == -1) + return (ENOSPC); + + vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid | + (1U << (E1000_VLVF_POOLSEL_SHIFT + pool)); + return (0); +} + +void +igb_iov_rebuild_vlan(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + struct igb_vf *vf; + u32 old_vlvf[E1000_VLVF_ARRAY_SIZE]; + u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE]; + u32 vlvf[E1000_VLVF_ARRAY_SIZE]; + u32 old_vfta, rctl, vmolr; + bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf; + bool vfta_changed, vlvf_changed; + int i, vid; + + if (!igb_iov_enabled(sc)) + return; + + hw = &sc->hw; + rctl = E1000_READ_REG(hw, E1000_RCTL); + rctl &= ~E1000_RCTL_CFIEN; + rctl |= E1000_RCTL_VFE; + E1000_WRITE_REG(hw, E1000_RCTL, rctl); + memcpy(vfta, sc->shadow_vfta, sizeof(vfta)); + memset(vlvf, 0, sizeof(vlvf)); + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) + old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i)); + + pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc); + pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 && + igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE; + preserve_pf = !pf_vlan_promisc && !pf_overflow; + + /* First keep every surviving VF mapping in its current slot. */ + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + for (vid = 0; vid < 4096; vid++) { + if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) == + 0) + continue; + (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid, + vf->pool, true); + } + } + + /* + * Preserve PF mappings unless I350 needs their slots for VFs. + * PF-only VLANs on 82576 intentionally have no VLVF mapping and + * reach the default PF pool after passing the global VFTA. + */ + if (preserve_pf) + for (vid = 0; vid < 4096; vid++) { + if ((sc->shadow_vfta[vid >> 5] & + (1U << (vid & 0x1f))) == 0) + continue; + if (hw->mac.type == e1000_82576 && + !igb_iov_vlan_present(sc, vid, false)) + continue; + (void)igb_iov_vlvf_add(vlvf, old_vlvf, vid, + sc->pool, true); + } + + /* Allocate new VF mappings before PF mappings. */ + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + for (vid = 0; vid < 4096; vid++) { + if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) == + 0) + continue; + if (igb_iov_vlvf_add(vlvf, old_vlvf, vid, + vf->pool, false) == 0) + vfta[vid >> 5] |= 1U << (vid & 0x1f); + } + igb_iov_configure_vmolr(sc, vf); + } + if (!pf_vlan_promisc) + for (vid = 0; vid < 4096; vid++) { + if ((sc->shadow_vfta[vid >> 5] & + (1U << (vid & 0x1f))) == 0) + continue; + /* + * With no VLVF match, 82576 sends a globally admitted + * VLAN to the default PF pool. A VLVF entry is needed + * only when this VLAN is also assigned to a VF. + */ + if (hw->mac.type == e1000_82576 && + !igb_iov_vlan_present(sc, vid, false)) + continue; + if (igb_iov_vlvf_add(vlvf, old_vlvf, vid, + sc->pool, false) != 0) + pf_overflow = true; + } + + if (pf_vlan_promisc) { + memset(vfta, 0xff, sizeof(vfta)); + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) + if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0) + vlvf[i] |= 1U << + (E1000_VLVF_POOLSEL_SHIFT + sc->pool); + } + + /* + * Establish the PF fallback before an overflowing I350 rebuild can + * displace one of its old VLVF mappings. + */ + vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool)); + vmolr &= ~E1000_VMOLR_VPE; + if (hw->mac.type == e1000_i350 && + (pf_overflow || pf_vlan_promisc)) + vmolr |= E1000_VMOLR_VPE; + E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr); + + /* + * Remove global VFTA membership before removing a VLAN entirely, and + * add a VLVF mapping before globally admitting a new VF VLAN. A + * transition to a PF-only VLAN deliberately retains VFTA membership + * and falls through to the default PF pool. + */ + force_vfta = hw->mac.type == e1000_i350 && + !sc->iov_vfta_valid; + vfta_changed = false; + for (i = 0; i < EM_VFTA_SIZE; i++) { + /* + * I350 erratum 20 makes VFTA reads unreliable while VMDq + * loopback or anti-spoofing is active. Its ten-write + * workaround is already in e1000_write_vfta_i350(). Force a + * complete clear when the authoritative shadow is invalid; + * 82576 can safely diff against its live register contents. + */ + if (hw->mac.type == e1000_i350) + old_vfta = force_vfta ? 0 : sc->iov_vfta[i]; + else + old_vfta = + E1000_READ_REG_ARRAY(hw, E1000_VFTA, i); + effective_vfta[i] = old_vfta & vfta[i]; + if (force_vfta || effective_vfta[i] != old_vfta) { + SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear, + sc, i, effective_vfta[i]); + e1000_write_vfta(hw, i, effective_vfta[i]); + vfta_changed = true; + } + } + if (vfta_changed) + E1000_WRITE_FLUSH(hw); + vlvf_changed = false; + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) + if (vlvf[i] != old_vlvf[i]) { + SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write, + sc, i, vlvf[i]); + E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]); + vlvf_changed = true; + } + if (vlvf_changed) + E1000_WRITE_FLUSH(hw); + vfta_changed = false; + for (i = 0; i < EM_VFTA_SIZE; i++) + if (vfta[i] != effective_vfta[i]) { + SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set, + sc, i, vfta[i]); + e1000_write_vfta(hw, i, vfta[i]); + vfta_changed = true; + } + if (vfta_changed) + E1000_WRITE_FLUSH(hw); + SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf); + igb_iov_vfta_shadow_store(sc, vfta); +} + +static bool +igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf) +{ + int i; + + if (include_pf && + (sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0) + return (true); + for (i = 0; i < sc->num_vfs; i++) + if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 && + (sc->vfs[i].vlans[vid >> 5] & + (1U << (vid & 0x1f))) != 0) + return (true); + return (false); +} + +static int +igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf) +{ + u32 vlans; + int count, i, word; + + count = 0; + for (word = 0; word < EM_VFTA_SIZE; word++) { + vlans = include_pf ? sc->shadow_vfta[word] : 0; + for (i = 0; i < sc->num_vfs; i++) + if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0) + vlans |= sc->vfs[i].vlans[word]; + count += bitcount32(vlans); + } + return (count); +} + +static int +igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid, + bool add) +{ + u32 bit; + bool present; + + bit = 1U << (vid & 0x1f); + present = (vf->vlans[vid >> 5] & bit) != 0; + if (vid == 0) { + if (!present) { + vf->vlans[0] |= 1U; + igb_iov_rebuild_vlan(sc); + } + return (0); + } + if (add == present) + return (0); + + /* + * Removals always reduce privilege and remain available. Charge only + * additions, which a hostile VF must alternate with removals to force + * repeated global VLAN rebuilds. + */ + if (add && !igb_iov_vlan_present(sc, vid, false) && + igb_iov_vlan_unique_count(sc, false) >= + E1000_VLVF_ARRAY_SIZE) + return (ENOSPC); + if (add && !igb_iov_vlan_add_allowed(vf)) + return (EBUSY); + + if (add) { + vf->vlans[vid >> 5] |= bit; + vf->vlan_count++; + } else { + vf->vlans[vid >> 5] &= ~bit; + vf->vlan_count--; + } + igb_iov_rebuild_vlan(sc); + return (0); +} + +static void +igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf) +{ + bool update_uta; + + update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0; + vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC | + IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW | + IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP | + IGB_VF_MDD_NOTIFY_PENDING); + vf->mbx_retry_at = 0; + vf->mdd_notify_at = 0; + vf->mbx_retry_count = 0; + /* + * A reset starts a new mailbox epoch. Permit one immediate NACK so a + * premature non-reset request does not wait for its posted-read + * timeout. + */ + memset(&vf->last_nack, 0, sizeof(vf->last_nack)); + vf->max_frame_size = ETHER_MAX_LEN; + vf->mc_count = 0; + vf->vlan_count = 0; + memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes)); + memset(vf->vlans, 0, sizeof(vf->vlans)); + /* Preserve the administrative access VLAN across VF and PF resets. */ + if (vf->default_vlan == 0) + vf->vlans[0] = 1U; + else { + vf->vlans[vf->default_vlan >> 5] = + 1U << (vf->default_vlan & 0x1f); + vf->vlan_count = 1; + } + igb_iov_configure_vmolr(sc, vf); + if (update_uta) + igb_iov_set_uta(sc); +} + +static bool +igb_iov_vf_vlan_is_default(const struct igb_vf *vf) +{ + u32 expected; + int i; + + for (i = 0; i < EM_VFTA_SIZE; i++) { + expected = 0; + if (i == vf->default_vlan >> 5) + expected = 1U << (vf->default_vlan & 0x1f); + if (vf->vlans[i] != expected) + return (false); + } + return (true); +} + +static bool +igb_iov_sanitize_vf_queues(struct e1000_softc *sc, + struct igb_vf *vf) +{ + struct e1000_hw *hw; + u16 qid[IGB_IOV_VF_QUEUES_MAX]; + u32 rxdctl, txdctl; + int i, nqueues, retry; + + hw = &sc->hw; + switch (hw->mac.type) { + case e1000_82576: + nqueues = IGB_82576_VF_QUEUES; + qid[0] = vf->pool; + qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE; + break; + case e1000_i350: + nqueues = IGB_I350_VF_QUEUES; + qid[0] = vf->pool; + break; + default: + return (true); + } + + /* + * I350 maps pool n to queue n. 82576 gives VF n physical queues n + * and n + 8, so both retained queue configurations must be cleared. + */ + for (i = 0; i < nqueues; i++) + KASSERT(qid[i] < (hw->mac.type == e1000_82576 ? + IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES), + ("%s: invalid VF queue %u", __func__, qid[i])); + + /* + * The 82576 and I350 specification updates, Software Clarification 3, + * note that VFLR does not reset the VF queue configuration. Clear the + * PF-programmable state before acknowledging the reset so a new VF + * owner cannot inherit it, particularly a descriptor-head write-back + * DMA address. The new VF driver initializes its active ring pointers + * during queue setup. + * + * Disable every queue first, then wait for outstanding DMA activity to + * stop before clearing TDWBAL/H and the remaining retained state. + * Spin only for the normal fast transition, then sleep so a VF that + * keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms. + */ + for (i = 0; i < nqueues; i++) { + E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0); + E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0); + } + E1000_WRITE_FLUSH(hw); + for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) { + for (i = 0; i < nqueues; i++) { + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i])); + txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i])); + if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 || + (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0) + break; + } + if (i == nqueues) + break; + if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) { + if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES) + DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US); + else + pause_sbt("igbqds", + IGB_IOV_QUEUE_DISABLE_PAUSE, 0, + C_PREL(1)); + } + } + if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) { + if (ratecheck(&vf->last_queue_log, + &igb_iov_mbx_log_interval)) + device_printf(sc->dev, + "could not disable queues for VF %u; " + "reset deferred\n", vf->pool); + return (false); + } + + for (i = 0; i < nqueues; i++) { + E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0); + E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0); + E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0); + E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0); + E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0); + } + E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0); + E1000_WRITE_FLUSH(hw); + return (true); +} + +static bool +igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf, + bool reset_intrs) +{ + struct e1000_hw *hw; + bool rebuild_mta, rebuild_vlan, sanitized; + u32 reg; + + hw = &sc->hw; + rebuild_mta = vf->mc_count != 0; + rebuild_vlan = !igb_iov_vf_vlan_is_default(vf); + reg = E1000_READ_REG(hw, E1000_VFTE); + E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool)); + reg = E1000_READ_REG(hw, E1000_VFRE); + E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool)); + if (reset_intrs) + E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool), + E1000_VTCTRL_RST); + sanitized = igb_iov_sanitize_vf_queues(sc, vf); + E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0); + igb_iov_clear_mac_filters(sc, vf); + igb_iov_clear_rar(sc, vf->rar_index); + igb_iov_reset_vf_state(sc, vf); + if (rebuild_mta) + igb_iov_rebuild_mta(sc); + if (rebuild_vlan) + igb_iov_rebuild_vlan(sc); + return (sanitized); +} + +static bool +igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf) +{ + return (igb_iov_reset_event_common(sc, vf, true)); +} + +static void +igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf) +{ + /* + * VTCTRL.RST clears the VF's queue-enable and interrupt registers + * (I350 section 8.28.1). It therefore also removes the admin-vector + * route needed to deliver the reset notification below. MDD recovery + * explicitly permits toggling VFTE instead (section 7.8.3.8.3). + * + * Leave the interrupt registers intact, keep VFTE/VFRE disabled until + * the VF completes a new reset handshake, and use the no-CTS control + * message to make the guest reinitialize. FreeBSD and DPDK consume + * that message directly; Linux ACKs it and the PF's non-CTS ACK path + * replies with the NACK that schedules igbvf's reset task. + * + * Sanitization failure leaves the pool disabled. The VF reset + * handshake retries it and is NACKed while a queue remains active. + */ + (void)igb_iov_reset_event_common(sc, vf, false); +} + +static void +igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf) +{ + struct e1000_hw *hw; + u32 msg[3], reg; + + hw = &sc->hw; + if (!igb_iov_reset_event(sc, vf)) { + msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK; + e1000_write_mbx(hw, msg, 1, vf->pool); + return; + } + igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool); + igb_iov_set_anti_spoof(sc, vf); + + reg = E1000_READ_REG(hw, E1000_VFTE); + E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool)); + reg = E1000_READ_REG(hw, E1000_VFRE); + E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool)); + /* + * 82576's WVBR blocked bitmap is read-clear, so the reset handshake + * completes that event's lifetime. I350 MDFB might be read-only; + * re-arm its edge latch only after a valid MDFB sample reads clear. + */ + if (hw->mac.type == e1000_82576) + vf->flags &= ~IGB_VF_MDD_BLOCKED; + vf->flags |= IGB_VF_CTS; + + memset(msg, 0, sizeof(msg)); + msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK; + memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN); + e1000_write_mbx(hw, msg, 3, vf->pool); +} + +static int +igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg) +{ + struct igb_vf_mac_filter *filter, *free_filter; + const u8 *mac; + u32 info; + int count, i; + + info = msg[0] & E1000_VT_MSGINFO_MASK; + if (info == E1000_VF_MAC_FILTER_CLR) { + igb_iov_clear_mac_filters(sc, vf); + return (0); + } + if (info != E1000_VF_MAC_FILTER_ADD) + return (EINVAL); + if ((vf->flags & IGB_VF_CAP_MAC) == 0) + return (EPERM); + + mac = (const u8 *)&msg[1]; + if (!igb_iov_mac_valid(mac)) + return (EINVAL); + if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0) + return (0); + + count = 0; + free_filter = NULL; + for (i = 0; i < sc->num_vf_mac_filters; i++) { + filter = &sc->vf_mac_filters[i]; + if (!filter->active) { + if (free_filter == NULL) + free_filter = filter; + continue; + } + if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0) + continue; + return (filter->pool == vf->pool ? 0 : EADDRINUSE); + } + for (i = 0; i < sc->num_vf_mac_filters; i++) + if (sc->vf_mac_filters[i].active && + sc->vf_mac_filters[i].pool == vf->pool) + count++; + if (igb_iov_mac_in_use(sc, mac, vf)) + return (EADDRINUSE); + if (count >= IGB_IOV_MAX_MAC_FILTERS) + return (ENOSPC); + if (free_filter == NULL) + return (ENOSPC); + + free_filter->active = true; + free_filter->pool = vf->pool; + memcpy(free_filter->mac, mac, ETHER_ADDR_LEN); + igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool); + return (0); +} + +static int +igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg) +{ + u8 *mac; + + if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0) + return (igb_iov_set_mac_filter(sc, vf, msg)); + + mac = (u8 *)&msg[1]; + if (!igb_iov_mac_valid(mac)) + return (EINVAL); + if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 && + !(vf->flags & IGB_VF_CAP_MAC)) + return (EPERM); + if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 && + igb_iov_mac_in_use(sc, mac, vf)) + return (EADDRINUSE); + + memcpy(vf->mac, mac, ETHER_ADDR_LEN); + igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool); + return (0); +} + +static int +igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg) +{ + u16 hashes[IGB_IOV_MAX_MC_HASHES] = {}; + bool overflow; + int count, i; + + count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >> + E1000_VT_MSGINFO_SHIFT; + overflow = count > IGB_IOV_MAX_MC_HASHES || + (msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0; + count = min(count, IGB_IOV_MAX_MC_HASHES); + for (i = 0; i < count; i++) + hashes[i] = + (msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff; + if (vf->mc_count == count && + ((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow && + memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0) + return (0); + memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes)); + vf->mc_count = count; + if (overflow) + vf->flags |= IGB_VF_MCAST_OVERFLOW; + else + vf->flags &= ~IGB_VF_MCAST_OVERFLOW; + if (overflow && + (vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) { + vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED; + device_printf(sc->dev, + "VF %u multicast list exceeds 30 entries; " + "enabling all-multicast reception\n", vf->pool); + } + igb_iov_configure_vmolr(sc, vf); + igb_iov_rebuild_mta(sc); + return (0); +} + +static int +igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg) +{ + u32 size; + + size = msg[1]; + if (size < ETHER_MIN_LEN) + return (EINVAL); + vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE); + igb_iov_configure_vmolr(sc, vf); + return (0); +} + +static int +igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg) +{ + u32 mode; + + mode = msg & E1000_VT_MSGINFO_MASK; + if (mode & ~(E1000_VF_SET_PROMISC_UNICAST | + E1000_VF_SET_PROMISC_MULTICAST)) + return (EINVAL); + if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC)) + return (EPERM); + + vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC); + if (mode & E1000_VF_SET_PROMISC_UNICAST) + vf->flags |= IGB_VF_UCAST_PROMISC; + if (mode & E1000_VF_SET_PROMISC_MULTICAST) + vf->flags |= IGB_VF_MCAST_PROMISC; + igb_iov_configure_vmolr(sc, vf); + igb_iov_set_uta(sc); + return (0); +} + +static bool +igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf) +{ + struct e1000_hw *hw; + u32 msg[E1000_VFMAILBOX_SIZE], type; + int error; + + hw = &sc->hw; + memset(msg, 0, sizeof(msg)); + if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0) + return (false); + vf->flags &= ~IGB_VF_MBX_PENDING; + vf->mbx_retry_at = 0; + vf->mbx_retry_count = 0; + + if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) { + e1000_unlock_mbx(hw, vf->pool); + return (true); + } + if (msg[0] == E1000_VF_RESET) { + igb_iov_reset_msg(sc, vf); + return (true); + } + if (!(vf->flags & IGB_VF_CTS)) { + if (igb_iov_nack_allowed(vf)) { + msg[0] = igb_iov_reply_header(msg[0], false, false); + e1000_write_mbx(hw, msg, 1, vf->pool); + } else + e1000_unlock_mbx(hw, vf->pool); + return (true); + } + + type = msg[0] & 0xffff; + switch (type) { + case E1000_VF_SET_MAC_ADDR: + error = igb_iov_set_mac(sc, vf, msg); + break; + case E1000_VF_SET_MULTICAST: + error = igb_iov_set_multicast(sc, vf, msg); + break; + case E1000_VF_SET_VLAN: + if (vf->default_vlan != 0) + error = EPERM; + else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0) + error = EINVAL; + else + error = igb_iov_set_vlan(sc, vf, + msg[1] & E1000_VLVF_VLANID_MASK, + (msg[0] & E1000_VF_SET_VLAN_ADD) != 0); + break; + case E1000_VF_SET_LPE: + error = igb_iov_set_lpe(sc, vf, msg); + break; + case E1000_VF_SET_PROMISC: + error = igb_iov_set_promisc(sc, vf, msg[0]); + break; + default: + error = EOPNOTSUPP; + break; + } + + msg[0] = igb_iov_reply_header(msg[0], true, error == 0); + e1000_write_mbx(hw, msg, 1, vf->pool); + return (true); +} + +static sbintime_t +igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf, + sbintime_t now) +{ + sbintime_t delay; + + if ((vf->flags & IGB_VF_MBX_PENDING) == 0) + return (0); + if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at) + return (vf->mbx_retry_at); + if (igb_iov_process_msg(sc, vf)) + return (0); + + now = getsbinuptime(); + if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) { + delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++]; + vf->mbx_retry_at = now + delay; + return (vf->mbx_retry_at); + } + + vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING); + vf->flags |= IGB_VF_MBX_GAVE_UP; + vf->mbx_retry_at = 0; + if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval)) + device_printf(sc->dev, + "mailbox remained busy for VF %u; CTS revoked\n", + vf->pool); + return (0); +} + +void +igb_iov_handle_mbx(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + struct igb_vf *vf; + sbintime_t delay, next_retry_at, now, retry_at; + u32 msg; + int i; + + if (!sc->iov_hw_active) + return; + + hw = &sc->hw; + next_retry_at = 0; + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + now = getsbinuptime(); + if (e1000_check_for_rst(hw, vf->pool) == 0) { + /* + * The old VF is gone. A new owner's reset handshake + * reruns sanitization before enabling its pool. + */ + (void)igb_iov_reset_event(sc, vf); + } + if ((vf->flags & + (IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 && + e1000_check_for_msg(hw, vf->pool) == 0) { + vf->flags |= IGB_VF_MBX_PENDING; + vf->mbx_retry_at = 0; + vf->mbx_retry_count = 0; + } + retry_at = igb_iov_service_pending_mbx(sc, vf, now); + if (retry_at != 0 && + (next_retry_at == 0 || retry_at < next_retry_at)) + next_retry_at = retry_at; + if (e1000_check_for_ack(hw, vf->pool) == 0 && + !(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) { + msg = E1000_VT_MSGTYPE_NACK; + e1000_write_mbx(hw, &msg, 1, vf->pool); + } + } + if (next_retry_at != 0) { + delay = next_retry_at - getsbinuptime(); + if (delay <= 0) + delay = SBT_1MS; + callout_reset_sbt(&sc->iov_mbx_retry, delay, 0, + igb_iov_mbx_retry_callout, sc, C_PREL(1)); + } +} + +static bool +igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf) +{ + u32 msg; + + /* + * MDD recovery preserves the VF's admin-vector configuration. Send + * the same no-CTS control message used for PF reset notification so + * the VF discards its state and completes a new reset handshake. + * A failed write is retried from the timer-driven admin pass; the VF's + * transmit watchdog remains the final fallback when traffic is still + * queued and notification never succeeds. + */ + msg = E1000_PF_CONTROL_MSG; + if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) { + vf->mdd_notify_at = + getsbinuptime() + igb_iov_mdd_notify_retry; + if (ratecheck(&vf->last_mbx_log, + &igb_iov_mbx_log_interval)) + device_printf(sc->dev, + "could not notify VF %u of malicious-driver " + "reset; will retry\n", vf->pool); + return (false); + } + vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING; + vf->mdd_notify_at = 0; + return (true); +} + +void +igb_iov_handle_mdd(struct e1000_softc *sc) +{ + struct igb_vf *vf; + u32 blocked, cleared, handled, lvmmc; + u32 readback, spoofed; + bool mdfb_valid, pending; + int i; + + pending = atomic_readandclear_32(&sc->iov_pending) != 0; + lvmmc = pending ? + atomic_readandclear_32(&sc->iov_mdd_cause) : 0; + if (!sc->iov_hw_active) { + atomic_readandclear_32(&sc->iov_spoof_pending); + atomic_readandclear_32(&sc->iov_blocked_pending); + return; + } + + blocked = 0; + handled = 0; + mdfb_valid = false; + if (sc->hw.mac.type == e1000_i350) { + u32 mdfb; + + spoofed = atomic_readandclear_32(&sc->iov_spoof_pending); + /* + * I350 reports ordinary MAC/VLAN spoofing through the + * interrupt-time LVMMC snapshot rather than WVBR. The + * filter accumulates Last_Q into iov_spoof_pending so events + * from different VFs coalesce safely until this timer-driven + * admin pass. + */ + spoofed &= IGB_I350_QUEUE_MASK; + /* + * Sample MDFB on every admin pass so a blocked queue is not + * mislabeled as an ordinary spoof when no MDDET observation + * is pending. + */ + mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB); + if (__predict_false(mdfb == 0xffffffff)) + mdfb = 0; + else { + mdfb &= IGB_I350_QUEUE_MASK; + mdfb_valid = true; + } + /* + * I350 SDM sections 8.14.10 and 8.14.11: WVBR reports + * spoof and malicious-driver events, while MDFB identifies + * the queues actually blocked for malicious behavior. + */ + spoofed &= ~mdfb; + blocked = mdfb; + if (blocked != 0 && lvmmc == 0) + lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC); + /* + * A failed diagnostic read does not invalidate the + * blocked-queue bitmap that was read successfully above. + */ + if (__predict_false(lvmmc == 0xffffffff)) + lvmmc = 0; + /* + * MDFB is authoritative for queues stopped by malicious-driver + * detection. LVMMC reports causes such as VLAN IERR and + * Mal_PF, but its Last_Q field does not establish that a queue + * was blocked. Do not manufacture a blocked bit when MDFB is + * clear. + */ + } else { + if (!pending) + return; + /* + * WVBR is read-clear and does not preserve every queue across + * multiple MDDET interrupts. The interrupt filter snapshots and + * accumulates its pool bitmaps before this deferred admin pass. + */ + spoofed = atomic_readandclear_32(&sc->iov_spoof_pending); + blocked = atomic_readandclear_32(&sc->iov_blocked_pending); + /* A blocked-queue classification dominates its WVBR low bit. */ + spoofed &= ~blocked; + } + + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 && + getsbinuptime() >= vf->mdd_notify_at) + (void)igb_iov_notify_vf_mdd_reset(sc, vf); + /* + * An invalid MDFB sample must neither report a new edge nor + * masquerade as evidence that an old edge has cleared. + */ + if (sc->hw.mac.type == e1000_i350 && mdfb_valid && + (blocked & (1U << i)) == 0) + vf->flags &= ~IGB_VF_MDD_BLOCKED; + if ((spoofed & (1U << i)) != 0 && + ratecheck(&vf->last_spoof_log, + &igb_iov_spoof_log_interval)) + device_printf(sc->dev, + "spoof event detected from VF %u; packet dropped\n", + vf->pool); + if ((blocked & (1U << i)) == 0) + continue; + if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0) + continue; + vf->flags |= IGB_VF_MDD_BLOCKED; + if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval)) + device_printf(sc->dev, + "malicious-driver event 0x%08x from VF %u; " + "resetting VF\n", lvmmc, vf->pool); + igb_iov_mdd_reset_event(sc, vf); + vf->flags |= IGB_VF_MDD_NOTIFY_PENDING; + (void)igb_iov_notify_vf_mdd_reset(sc, vf); + handled |= 1U << i; + } + if (sc->hw.mac.type == e1000_i350 && mdfb_valid && + (blocked & (1U << sc->pool)) == 0) + sc->iov_pf_mdd_blocked = false; + if ((blocked & (1U << sc->pool)) != 0 && + (sc->hw.mac.type != e1000_i350 || + !sc->iov_pf_mdd_blocked)) { + if (sc->hw.mac.type == e1000_i350) + sc->iov_pf_mdd_blocked = true; + if (ratecheck(&sc->iov_last_mdd_log, + &igb_iov_mdd_log_interval)) + device_printf(sc->dev, + "malicious-driver event 0x%08x from PF queue; " + "resetting PF\n", lvmmc); + iflib_request_reset(sc->ctx); + iflib_admin_intr_deferred(sc->ctx); + handled |= 1U << sc->pool; + } + if (sc->hw.mac.type == e1000_i350 && handled != 0) { + /* + * I350 documentation conflicts: the register summary calls + * MDFB RWS while the detailed field table calls it RO. I350 + * silicon clears a blocked bit when software writes it back. + * Write only bits whose recovery was initiated. If a revision + * instead implements MDFB as RO, the edge latch above prevents + * a reset loop and this one transition-time write is harmless. + */ + E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled); + E1000_WRITE_FLUSH(&sc->hw); + /* + * Rearm from observed hardware state instead of waiting for + * the next admin pass. The PF context lock prevents a reset + * handshake from re-enabling the VF before this readback. A + * write-to-clear part reports zero; a read-only part retains + * the bit and therefore retains the one-shot edge latch. + */ + readback = E1000_READ_REG(&sc->hw, E1000_MDFB); + if (__predict_false(readback == 0xffffffff)) + cleared = 0; + else + cleared = handled & + ~(readback & IGB_I350_QUEUE_MASK); + for (i = 0; i < sc->num_vfs; i++) + if ((cleared & (1U << i)) != 0) + sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED; + if ((cleared & (1U << sc->pool)) != 0) + sc->iov_pf_mdd_blocked = false; + } + if (sc->hw.mac.type == e1000_i350) { + /* + * I350 can retain EICR.OTHER without delivering the admin MSI-X + * even though its EIMS and legacy IMS bits remain enabled. Kick + * the already-enabled vector on each admin pass so its filter + * consumes any retained ICR/LVMMC cause. A synthetic interrupt + * with no legacy cause is handled entirely by the filter. + */ + E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask); + E1000_WRITE_FLUSH(&sc->hw); + } +} + +void +igb_iov_mdd_event(struct e1000_softc *sc) +{ + u32 blocked, cause, queues, queue, spoofed, wvbr; + + /* + * LVMMC is clear-on-read. Preserve it in the interrupt filter, as + * Intel's igb driver does, rather than deferring the only copy. + */ + cause = E1000_READ_REG(&sc->hw, E1000_LVMMC); + if (__predict_false(cause == 0xffffffff)) + return; + if (sc->hw.mac.type == e1000_82576) { + /* + * Snapshot WVBR in the interrupt filter. Waiting for the admin + * task loses all but the last of back-to-back VF MDD events on + * 82576. Convert the staggered queue map into pool bits and OR + * each observation into software latches for deferred recovery. + */ + wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR); + if (__predict_false(wvbr == 0xffffffff)) { + spoofed = 0; + blocked = 0; + } else { + queues = wvbr & IGB_82576_QUEUE_MASK; + spoofed = (queues & 0xff) | + (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT); + queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK; + blocked = (queues & 0xff) | + (queues >> IGB_82576_STAGGERED_QUEUE_SHIFT); + } + SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed, + blocked); + /* + * 82576 can report a coalesced block-class event with all affected + * queues in WVBR's low half and no high-half blocked bits. If an + * ordinary spoof shares that snapshot, the register has no per-queue + * cause information. Deliberately fail closed by recovering every + * low-half queue; this can reset a spoof-only sibling, but avoids + * stranding a blocked VF. LVMMC.Last_Q identifies only the final + * event and lost simultaneous blocked VFs on tested silicon. + */ + if (blocked == 0 && + (cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) { + blocked = spoofed; + if (blocked == 0) { + queue = (cause >> 16) & 0xf; + blocked = 1U << (queue & 0x7); + } + } + if (spoofed != 0) + atomic_set_32(&sc->iov_spoof_pending, spoofed); + if (blocked != 0) + atomic_set_32(&sc->iov_blocked_pending, blocked); + } + if (sc->hw.mac.type == e1000_i350 && + (cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) { + queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) & + IGB_I350_LVMMC_LAST_Q_MASK; + /* + * FreeBSD assigns one queue to each VF pool, so Last_Q is + * also the VF number. Preserve all VFs observed before the + * timer pass, and do not overwrite an unrelated blocked + * queue's diagnostic with this non-blocking spoof event. + */ + atomic_set_32(&sc->iov_spoof_pending, 1U << queue); + return; + } + atomic_store_rel_32(&sc->iov_mdd_cause, cause); + atomic_set_32(&sc->iov_pending, 1); +} + +void +igb_iov_ping_all_vfs(struct e1000_softc *sc) +{ + struct igb_vf *vf; + u32 msg; + int i; + + if (!sc->iov_hw_active) + return; + + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + msg = E1000_PF_CONTROL_MSG; + if (vf->flags & IGB_VF_CTS) + msg |= E1000_VT_MSGTYPE_CTS; + e1000_write_mbx(&sc->hw, &msg, 1, vf->pool); + } +} + +void +igb_iov_initialize(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + struct igb_vf *vf; + u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl; + int i; + + if (sc->num_vfs == 0) + return; + + hw = &sc->hw; + atomic_readandclear_32(&sc->iov_mdd_cause); + atomic_readandclear_32(&sc->iov_pending); + atomic_readandclear_32(&sc->iov_spoof_pending); + atomic_readandclear_32(&sc->iov_blocked_pending); + /* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */ + E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ); + + vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL); + vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK | + E1000_VT_CTL_DISABLE_DEF_POOL); + vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT; + vt_ctl |= E1000_VT_CTL_VM_REPL_EN; + E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl); + + mask = 1U << sc->pool; + E1000_WRITE_REG(hw, E1000_VFRE, mask); + E1000_WRITE_REG(hw, E1000_VFTE, mask); + /* A VF without RX descriptors must not block any other pool. */ + E1000_WRITE_REG(hw, E1000_QDE, + hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES); + e1000_vmdq_set_loopback_pf(hw, true); + dtxctl = E1000_READ_REG(hw, E1000_DTXCTL); + dtxctl |= E1000_DTXCTL_MDP_EN; + if (hw->mac.type == e1000_82576) { + dtxctl |= E1000_DTXCTL_VLAN_ADDED | + E1000_DTXCTL_SPOOF_INT; + rplolr = E1000_READ_REG(hw, E1000_RPLOLR); + rplolr |= E1000_RPLOLR_STRVLAN; + E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr); + } else { + /* + * I350 SDM section 8.12.5 defines this field with inverted + * polarity: setting it keeps an ordinary spoof from disabling + * the VF queue. Enable its notification as well. I350 + * hardware reports the VF in LVMMC.Last_Q (WVBR remains zero); + * the moderated admin vector captures that value, while + * timer-driven administration and per-VF ratecheck bound the + * work and console output. + */ + dtxctl |= E1000_DTXCTL_SPOOF_INT | + IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE; + } + E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl); + + igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool); + igb_iov_configure_pf_vmolr(sc); + igb_iov_set_uta(sc); + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IGB_VF_ACTIVE)) + continue; + /* + * A PF-wide reset is trusted and can require a complete guest + * replay. Guest-controlled RESET and VFLR do not refill this + * allowance. + */ + igb_iov_reset_vlan_rate(vf); + igb_iov_clear_mac_filters(sc, vf); + igb_iov_reset_vf_state(sc, vf); + igb_iov_clear_rar(sc, vf->rar_index); + igb_iov_set_anti_spoof(sc, vf); + } + igb_iov_rebuild_mta(sc); + igb_iov_rebuild_vlan(sc); + + rctl = E1000_READ_REG(hw, E1000_RCTL); + E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE); + E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc)); + if (hw->mac.type == e1000_i350) + E1000_WRITE_REG(hw, E1000_DMACR, 0); + + ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); + E1000_WRITE_REG(hw, E1000_CTRL_EXT, + ctrl_ext | E1000_CTRL_EXT_PFRSTD); + E1000_WRITE_FLUSH(hw); + /* + * MDDET remains masked until iov_hw_active is published and iflib + * rearms the admin vector. Programming the per-pool policy above can + * leave a setup-time MDDET observation in the read-clear registers. + * If that stale cause is carried across the unmask, a later ordinary + * spoof can update LVMMC without generating a new interrupt edge. + * + * Drain only after all IOV policy is installed and before exposing the + * active state. Mailbox requests are also serviced by the periodic + * admin pass, and ping_all_vfs() below supplies a fresh notification. + */ + /* + * Clear the setup-time interrupt latch before its diagnostic state. + * I350 does not reliably generate the next MDDET edge when LVMMC is + * consumed while ICR.MDDET remains latched. This differs deliberately + * from the final arm-time drain, where ICR is read last so a later event + * remains pending for the unmask. + */ + if (hw->mac.type == e1000_i350) + (void)E1000_READ_REG(hw, E1000_ICR); + (void)E1000_READ_REG(hw, E1000_LVMMC); + if (hw->mac.type == e1000_82576) + (void)E1000_READ_REG(hw, E1000_WVBR); + if (hw->mac.type != e1000_i350) + (void)E1000_READ_REG(hw, E1000_ICR); + atomic_readandclear_32(&sc->iov_mdd_cause); + atomic_readandclear_32(&sc->iov_pending); + atomic_readandclear_32(&sc->iov_spoof_pending); + atomic_readandclear_32(&sc->iov_blocked_pending); + atomic_store_rel_32(&sc->iov_intr_drain_pending, 1); + sc->iov_hw_active = true; + igb_iov_ping_all_vfs(sc); +} + +int +igb_iov_validate(struct e1000_softc *sc, u16 num_vfs) +{ + if (!igb_iov_supported(sc)) + return (ENXIO); + /* One of the eight hardware pools is reserved for the PF. */ + if (num_vfs == 0 || num_vfs > MAX_NUM_VFS) + return (EINVAL); + if (sc->vfs != NULL) + return (EBUSY); + if (sc->intr_type != IFLIB_INTR_MSIX) { + device_printf(sc->dev, "SR-IOV requires MSI-X\n"); + return (ENOTSUP); + } + if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) { + device_printf(sc->dev, + "SR-IOV requires one PF TX and RX queue; set " + "dev.igb.%d.iflib.override_ntxqs=1 and " + "dev.igb.%d.iflib.override_nrxqs=1 before attach\n", + device_get_unit(sc->dev), device_get_unit(sc->dev)); + return (EINVAL); + } + return (0); +} + +int +igb_if_vf_status(if_ctx_t ctx, nvlist_t *status) +{ + struct e1000_softc *sc; + struct igb_vf *vf; + nvlist_t **vfs; + u_int num_queues; + int error, i; + + sc = iflib_get_softc(ctx); + if (sc->num_vfs == 0) + return (ENXIO); + num_queues = sc->hw.mac.type == e1000_82576 ? + IGB_82576_VF_QUEUES : IGB_I350_VF_QUEUES; + vfs = mallocarray(sc->num_vfs, sizeof(*vfs), M_IGB_IOV, + M_WAITOK | M_ZERO); + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + vfs[i] = nvlist_create(0); + nvlist_add_number(vfs[i], IFVF_STATUS_INDEX, i); + nvlist_add_bool(vfs[i], IFVF_STATUS_CONFIGURED, + (vf->flags & IGB_VF_ACTIVE) != 0); + nvlist_add_bool(vfs[i], IFVF_STATUS_INITIALIZED, + (vf->flags & IGB_VF_CTS) != 0); + nvlist_add_binary(vfs[i], IFVF_STATUS_MAC, vf->mac, + ETHER_ADDR_LEN); + if (vf->default_vlan == 0) + nvlist_add_string(vfs[i], IFVF_STATUS_VLAN_MODE, + IFVF_VLAN_MODE_TRUNK); + else { + nvlist_add_string(vfs[i], IFVF_STATUS_VLAN_MODE, + IFVF_VLAN_MODE_ACCESS); + nvlist_add_number(vfs[i], IFVF_STATUS_VLAN, + vf->default_vlan); + } + nvlist_add_number(vfs[i], IFVF_STATUS_VLAN_COUNT, + vf->vlan_count); + nvlist_add_number(vfs[i], IFVF_STATUS_NUM_QUEUES, num_queues); + nvlist_add_bool(vfs[i], IFVF_STATUS_ALLOW_SET_MAC, + (vf->flags & IGB_VF_CAP_MAC) != 0); + nvlist_add_bool(vfs[i], IFVF_STATUS_ALLOW_SET_VLAN, + vf->default_vlan == 0); + nvlist_add_bool(vfs[i], IFVF_STATUS_MAC_ANTI_SPOOF, + (vf->flags & IGB_VF_MAC_ANTI_SPOOF) != 0); + nvlist_add_bool(vfs[i], IFVF_STATUS_ALLOW_PROMISC, + (vf->flags & IGB_VF_ALLOW_PROMISC) != 0); + nvlist_add_bool(vfs[i], IFVF_STATUS_MDD_BLOCKED, + (vf->flags & IGB_VF_MDD_BLOCKED) != 0); + } + nvlist_add_nvlist_array(status, IFVF_STATUS_VFS, + (const nvlist_t * const *)vfs, sc->num_vfs); + error = nvlist_error(status); + for (i = 0; i < sc->num_vfs; i++) + nvlist_destroy(vfs[i]); + free(vfs, M_IGB_IOV); + return (error); +} + +int +igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config) +{ + struct e1000_softc *sc; + int error, i; + + sc = iflib_get_softc(ctx); + (void)config; + /* + * This callback may run while the PF is down. Record the software + * layout here; igb_iov_initialize() programs it during interface init. + */ + atomic_store_rel_32(&sc->iov_teardown, 0); + error = igb_iov_validate(sc, num_vfs); + if (error != 0) + return (error); + + sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV, + M_WAITOK | M_ZERO); + sc->num_vf_mac_filters = + sc->hw.mac.rar_entry_count - num_vfs - 1; + sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters, + sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO); + for (i = 0; i < sc->num_vf_mac_filters; i++) + sc->vf_mac_filters[i].rar_index = i + 1; + sc->pool = num_vfs; + sc->iov_mta_valid = false; + sc->iov_pf_mdd_blocked = false; + sc->tx_queues[0].txr.me = sc->pool; + sc->rx_queues[0].rxr.me = sc->pool; + e1000_init_mbx_params_pf(&sc->hw); + sc->num_vfs = num_vfs; + return (0); +} + +void +igb_if_iov_uninit(if_ctx_t ctx) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + u32 mask, rah; + int error, i, iov_pos; + u16 iov_ctl; + + sc = iflib_get_softc(ctx); + if (sc->vfs == NULL) + return; + hw = &sc->hw; + sc->iov_hw_active = false; + if (sc->iov_mbx_retry_initialized) + callout_drain(&sc->iov_mbx_retry); + + E1000_WRITE_REG(hw, E1000_MBVFIMR, 0); + mask = 1U << sc->pool; + E1000_WRITE_REG(hw, E1000_VFRE, mask); + E1000_WRITE_REG(hw, E1000_VFTE, mask); + + /* + * pci_iov(4) invokes the driver before it clears VF Enable. Quiesce + * the VFs and clear it here so that 82576's queue-reuse interval is + * measured from the actual IOV-disable event. + */ + error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos); + if (error == 0) { + iov_ctl = pci_read_config(sc->dev, + iov_pos + PCIR_SRIOV_CTL, 2); + iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE); + pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL, + iov_ctl, 2); + if (hw->mac.type == e1000_82576) { + pause("igbiov", MAX(1, howmany(hz, 10))); + E1000_WRITE_REG(hw, E1000_IOVCTL, + E1000_IOVCTL_REUSE_VFQ); + E1000_WRITE_FLUSH(hw); + pause("igbiov", MAX(1, howmany(hz, 10))); + } + } else + device_printf(sc->dev, + "could not disable PCI SR-IOV before queue reuse: %d\n", + error); + + E1000_WRITE_REG(hw, E1000_VT_CTL, 0); + e1000_vmdq_set_loopback_pf(hw, false); + e1000_vmdq_set_anti_spoofing_pf(hw, false, 0); + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) + E1000_WRITE_REG(hw, E1000_VLVF(i), 0); + for (i = 0; i < sc->num_vfs; i++) + if (sc->vfs[i].flags & IGB_VF_ACTIVE) + igb_iov_clear_rar(sc, sc->vfs[i].rar_index); + for (i = 0; i < sc->num_vf_mac_filters; i++) + if (sc->vf_mac_filters[i].active) + igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index); + rah = E1000_READ_REG(hw, E1000_RAH(0)); + rah &= ~IGB_IOV_RAH_POOLSEL_MASK; + E1000_WRITE_REG(hw, E1000_RAH(0), rah); + + free(sc->vfs, M_IGB_IOV); + free(sc->vf_mac_filters, M_IGB_IOV); + sc->vfs = NULL; + sc->vf_mac_filters = NULL; + sc->num_vfs = 0; + sc->num_vf_mac_filters = 0; + sc->pool = 0; + sc->iov_mta_valid = false; + sc->iov_pf_mdd_blocked = false; + sc->iov_pf_vlan_promisc = false; + igb_iov_vfta_shadow_invalidate(sc); + sc->tx_queues[0].txr.me = 0; + sc->rx_queues[0].rxr.me = 0; + atomic_readandclear_32(&sc->iov_mdd_cause); + atomic_readandclear_32(&sc->iov_pending); + atomic_readandclear_32(&sc->iov_spoof_pending); + atomic_readandclear_32(&sc->iov_blocked_pending); + atomic_readandclear_32(&sc->iov_intr_drain_pending); + atomic_store_rel_32(&sc->iov_teardown, 0); +} + +static bool +igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac, + const struct igb_vf *skip) +{ + int i; + + if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0) + return (true); + for (i = 0; i < sc->num_vfs; i++) + if (&sc->vfs[i] != skip && + (sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 && + memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0) + return (true); + for (i = 0; i < sc->num_vf_mac_filters; i++) + if (sc->vf_mac_filters[i].active && + memcmp(sc->vf_mac_filters[i].mac, mac, + ETHER_ADDR_LEN) == 0) + return (true); + return (false); +} + +int +igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config) +{ + struct e1000_softc *sc; + struct igb_vf *vf; + struct ether_addr generated; + const void *mac; + char nameunit[IFNAMSIZ + sizeof("-vf65535")]; + size_t mac_size; + uint64_t configured_vlan; + u16 vlan; + + sc = iflib_get_softc(ctx); + if (vfnum >= sc->num_vfs) + return (EINVAL); + vf = &sc->vfs[vfnum]; + if (vf->flags & IGB_VF_ACTIVE) + return (EBUSY); + + configured_vlan = nvlist_get_number(config, "vlan"); + if (configured_vlan > VF_VLAN_TRUNK) + return (EINVAL); + vlan = configured_vlan; + if (vlan == 0) + return (ENOTSUP); + if (vlan == VF_VLAN_TRUNK) + vlan = 0; + if (!igb_iov_vlan_present(sc, vlan, false) && + igb_iov_vlan_unique_count(sc, false) >= + E1000_VLVF_ARRAY_SIZE) + return (ENOSPC); + + vf->pool = vfnum; + vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1); + vf->max_frame_size = ETHER_MAX_LEN; + vf->default_vlan = vlan; + igb_iov_reset_vlan_rate(vf); + if (nvlist_exists_binary(config, "mac-addr")) { + mac = nvlist_get_binary(config, "mac-addr", &mac_size); + if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac)) + return (EINVAL); + if (igb_iov_mac_in_use(sc, mac, vf)) + return (EADDRINUSE); + memcpy(vf->mac, mac, ETHER_ADDR_LEN); + } else { + snprintf(nameunit, sizeof(nameunit), "%s-vf%u", + device_get_nameunit(sc->dev), vfnum); + ether_gen_addr_byname(nameunit, &generated); + memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN); + if (igb_iov_mac_in_use(sc, vf->mac, vf)) + return (EADDRINUSE); + } + if (nvlist_get_bool(config, "allow-set-mac")) + vf->flags |= IGB_VF_CAP_MAC; + if (nvlist_get_bool(config, "mac-anti-spoof")) + vf->flags |= IGB_VF_MAC_ANTI_SPOOF; + if (nvlist_get_bool(config, "allow-promisc")) + vf->flags |= IGB_VF_ALLOW_PROMISC; + vf->flags |= IGB_VF_ACTIVE; + + igb_iov_reset_vf_state(sc, vf); + igb_iov_set_anti_spoof(sc, vf); + igb_iov_rebuild_vlan(sc); + E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc)); + return (0); +} + +#endif /* PCI_IOV */ diff --git a/sys/dev/e1000/if_igb_iov.h b/sys/dev/e1000/if_igb_iov.h new file mode 100644 index 000000000000..390e63175c7a --- /dev/null +++ b/sys/dev/e1000/if_igb_iov.h @@ -0,0 +1,60 @@ +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2010-2016, Intel Corporation + * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org> + */ + +#ifndef _IF_IGB_IOV_H_ +#define _IF_IGB_IOV_H_ + +#define IGB_IOV_MAX_FRAME_SIZE 0x2600 + +#ifdef PCI_IOV + +#include <sys/nv.h> +#include <sys/iov_schema.h> +#include <dev/pci/pci_iov.h> + +int igb_iov_attach(struct e1000_softc *); +void igb_iov_detach(struct e1000_softc *); +bool igb_iov_supported(const struct e1000_softc *); +bool igb_iov_enabled(const struct e1000_softc *); +int igb_iov_validate(struct e1000_softc *, u16); +int igb_if_iov_init(if_ctx_t, u16, const nvlist_t *); +void igb_if_iov_uninit(if_ctx_t); +int igb_if_iov_vf_add(if_ctx_t, u16, const nvlist_t *); +int igb_if_vf_status(if_ctx_t, nvlist_t *); +void igb_iov_initialize(struct e1000_softc *); +void igb_iov_handle_mbx(struct e1000_softc *); +void igb_iov_handle_mdd(struct e1000_softc *); +void igb_iov_mdd_event(struct e1000_softc *); +void igb_iov_ping_all_vfs(struct e1000_softc *); +void igb_iov_reset_prepare(struct e1000_softc *); +u32 igb_iov_intr_mask(const struct e1000_softc *); +void igb_iov_intr_drain_stale(struct e1000_softc *); +void igb_iov_rebuild_mta(struct e1000_softc *); +void igb_iov_rebuild_vlan(struct e1000_softc *); +void igb_iov_update_pf_vmolr(struct e1000_softc *); + +#else + +#define igb_iov_attach(_sc) ((void)(_sc), 0) +#define igb_iov_detach(_sc) ((void)(_sc)) +#define igb_iov_supported(_sc) (false) +#define igb_iov_enabled(_sc) (false) +#define igb_iov_initialize(_sc) +#define igb_iov_handle_mbx(_sc) +#define igb_iov_handle_mdd(_sc) +#define igb_iov_mdd_event(_sc) +#define igb_iov_ping_all_vfs(_sc) +#define igb_iov_reset_prepare(_sc) +#define igb_iov_intr_mask(_sc) (0) +#define igb_iov_intr_drain_stale(_sc) ((void)(_sc)) +#define igb_iov_rebuild_mta(_sc) +#define igb_iov_rebuild_vlan(_sc) +#define igb_iov_update_pf_vmolr(_sc) + +#endif + +#endif /* _IF_IGB_IOV_H_ */ diff --git a/sys/dev/e1000/if_igbv.c b/sys/dev/e1000/if_igbv.c new file mode 100644 index 000000000000..2d57a39f934e --- /dev/null +++ b/sys/dev/e1000/if_igbv.c @@ -0,0 +1,836 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2001-2024, Intel Corporation + * Copyright (c) 2026 Kevin Bowling <kbowling@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "if_em.h" + +#include <sys/sbuf.h> + +#define IGBV_82576_QUEUES 2 +#define IGBV_I350_QUEUES 1 +#define IGBV_MAX_MAC_FILTERS 3 +#define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10 +#define IGBV_QUEUE_DISABLE_DELAY_US 10 +#define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US) +#define IGBV_QUEUE_DISABLE_RETRIES 20 +#define IGBV_QUEUE_SANITIZE_ATTEMPTS 3 +#define IGBV_VLAN_RETRY_BATCH 4 +#define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S) + +static const struct timeval igbv_queue_log_interval = { 2, 0 }; +static const struct timeval igbv_mbx_log_interval = { 60, 0 }; +static const sbintime_t igbv_queue_retry_delay[] = { + 100 * SBT_1MS, + 500 * SBT_1MS, +}; +static const sbintime_t igbv_mbx_retry_delay[] = { + 250 * SBT_1MS, + 1 * SBT_1S, + 4 * SBT_1S, + 8 * SBT_1S, +}; +_Static_assert(nitems(igbv_queue_retry_delay) + 1 == + IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay"); + +struct igb_vf_uc_addr_list { + struct e1000_softc *sc; + u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN]; +}; + +static bool igbv_tx_pending(struct e1000_softc *); +static bool igbv_vlan_retry_pending(const struct e1000_softc *); +static void igbv_vlan_retry_tick(struct e1000_softc *); + +static void +igbv_queue_retry_callout(void *arg) +{ + struct e1000_softc *sc; + if_t ifp; + + sc = arg; + if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0) + return; + ifp = iflib_get_ifp(sc->ctx); + if ((if_getflags(ifp) & IFF_UP) == 0) { + atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); + return; + } + iflib_request_reset_if_up(sc->ctx); + iflib_admin_intr_deferred(sc->ctx); +} + +/* + * A missing PF can make the posted reset handshake wait for a full mailbox + * timeout. Keep that work out of stopped status paths. An administratively + * up VF retries complete initialization with an exponential delay capped at + * eight seconds, so it recovers without creating a tight mailbox poller. + */ +static void +igbv_mbx_retry_callout(void *arg) +{ + struct e1000_softc *sc; + if_t ifp; + + sc = arg; + if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 || + atomic_load_acq_32(&sc->vf_mbx_ready) != 0 || + iflib_in_detach(sc->ctx)) + return; + ifp = iflib_get_ifp(sc->ctx); + if ((if_getflags(ifp) & IFF_UP) == 0) + return; + + iflib_request_reset_if_up(sc->ctx); + iflib_admin_intr_deferred(sc->ctx); +} + +static const char * +igbv_reset_error_desc(s32 error) +{ + + switch (error) { + case -E1000_ERR_RESET: + return ("PF reset acknowledgement timed out"); + case -E1000_ERR_MAC_INIT: + return ("PF returned an invalid VF reset response"); + case -E1000_ERR_MBX: + return ("PF mailbox reset exchange failed"); + default: + return ("VF reset handshake failed"); + } +} + +void +igbv_log_reset_failure(struct e1000_softc *sc, s32 error, bool attaching) +{ + + /* Report each backoff stage, then limit the steady eight-second retry. */ + if (sc->vf_mbx_retry_stage == nitems(igbv_mbx_retry_delay) - 1 && + !ratecheck(&sc->vf_last_mbx_log, &igbv_mbx_log_interval)) + return; + device_printf(sc->dev, "%s (%d)%s\n", igbv_reset_error_desc(error), + error, attaching ? "; continuing attach" : ""); +} + +void +igbv_mbx_retry_detach(struct e1000_softc *sc) +{ + + if (!sc->vf_mbx_retry_initialized) + return; + atomic_readandclear_32(&sc->vf_mbx_retry_pending); + callout_drain(&sc->vf_mbx_retry); + sc->vf_mbx_retry_initialized = false; +} + +void +igbv_mbx_retry_prepare(struct e1000_softc *sc) +{ + + if (!sc->vf_mbx_retry_initialized) + return; + atomic_readandclear_32(&sc->vf_mbx_retry_pending); + callout_drain(&sc->vf_mbx_retry); +} + +void +igbv_mbx_retry_stop(struct e1000_softc *sc) +{ + if_t ifp; + + if (!sc->vf_mbx_retry_initialized) + return; + atomic_readandclear_32(&sc->vf_mbx_retry_pending); + callout_drain(&sc->vf_mbx_retry); + ifp = iflib_get_ifp(sc->ctx); + if ((if_getflags(ifp) & IFF_UP) == 0) + sc->vf_mbx_retry_stage = 0; +} + +void +igbv_mbx_retry_failed(if_ctx_t ctx) +{ + struct e1000_softc *sc; + if_t ifp; + sbintime_t delay; + u_int stage; + + sc = iflib_get_softc(ctx); + atomic_store_rel_32(&sc->vf_mbx_ready, 0); + sc->link_speed = 0; + sc->link_duplex = 0; + if (sc->link_state != EM_LINK_STATE_DOWN) { + sc->link_state = EM_LINK_STATE_DOWN; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + } + iflib_init_failed(ctx); + + ifp = iflib_get_ifp(ctx); + if (!sc->vf_mbx_retry_initialized || + (if_getflags(ifp) & IFF_UP) == 0) + return; + stage = sc->vf_mbx_retry_stage; + if (stage >= nitems(igbv_mbx_retry_delay)) + stage = nitems(igbv_mbx_retry_delay) - 1; + delay = igbv_mbx_retry_delay[stage]; + if (sc->vf_mbx_retry_stage + 1 < nitems(igbv_mbx_retry_delay)) + sc->vf_mbx_retry_stage++; + atomic_set_32(&sc->vf_mbx_retry_pending, 1); + callout_reset_sbt(&sc->vf_mbx_retry, delay, 0, + igbv_mbx_retry_callout, sc, C_PREL(1)); +} + +static void +igbv_mbx_retry_succeeded(struct e1000_softc *sc) +{ + + atomic_store_rel_32(&sc->vf_mbx_ready, 1); + atomic_readandclear_32(&sc->vf_mbx_retry_pending); + if (sc->vf_mbx_retry_initialized) + callout_stop(&sc->vf_mbx_retry); + sc->vf_mbx_retry_stage = 0; + sc->vf_last_mbx_log.tv_sec = 0; + sc->vf_last_mbx_log.tv_usec = 0; +} + +void +igbv_queue_retry_detach(struct e1000_softc *sc) +{ + + if (!sc->vf_queue_retry_initialized) + return; + atomic_readandclear_32(&sc->vf_queue_retry_pending); + callout_drain(&sc->vf_queue_retry); + sc->vf_queue_retry_initialized = false; +} + +void +igbv_queue_retry_stop(struct e1000_softc *sc) +{ + + if (!sc->vf_queue_retry_initialized) + return; + if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0) + atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); + callout_stop(&sc->vf_queue_retry); +} + +void +igbv_queue_retry_prepare(struct e1000_softc *sc) +{ + bool new_epoch; + + new_epoch = + atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0; + if (!sc->vf_queue_gave_up && !new_epoch) + return; + sc->vf_queue_failures = 0; + sc->vf_queue_gave_up = false; +} + +static void +igbv_queue_retry_succeeded(struct e1000_softc *sc) +{ + + atomic_readandclear_32(&sc->vf_queue_retry_pending); + atomic_readandclear_32(&sc->vf_queue_retry_new_epoch); + if (sc->vf_queue_retry_initialized) + callout_stop(&sc->vf_queue_retry); + sc->vf_queue_failures = 0; + sc->vf_queue_gave_up = false; +} + +void +igbv_queue_retry_failed(if_ctx_t ctx) +{ + struct e1000_softc *sc; + sbintime_t delay; + + sc = iflib_get_softc(ctx); + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + + if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) + sc->vf_queue_failures++; + if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) { + delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1]; + atomic_set_32(&sc->vf_queue_retry_pending, 1); + callout_reset_sbt(&sc->vf_queue_retry, delay, 0, + igbv_queue_retry_callout, sc, C_PREL(1)); + } else if (!sc->vf_queue_gave_up) { + atomic_readandclear_32(&sc->vf_queue_retry_pending); + callout_stop(&sc->vf_queue_retry); + sc->vf_queue_gave_up = true; + device_printf(sc->dev, + "retained VF queues remained active after %u attempts; " + "interface left down; toggle it down/up to retry\n", + sc->vf_queue_failures); + } + + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + iflib_init_failed(ctx); +} + +void +igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid) +{ + bool pending; + + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + pending = igbv_vlan_retry_pending(sc); + sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); + /* Bound the whole batch from its first failure, not each new VID. */ + if (!pending) + sc->vf_vlan_retry_deadline = + getsbinuptime() + IGBV_VLAN_RETRY_WINDOW; +} + +void +igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid) +{ + + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f)); +} + +static bool +igbv_vlan_retry_pending(const struct e1000_softc *sc) +{ + int i; + + for (i = 0; i < EM_VFTA_SIZE; i++) + if (sc->vf_vfta_retry[i] != 0) + return (true); + return (false); +} + +static void +igbv_vlan_retry_tick(struct e1000_softc *sc) +{ + u32 bit; + u16 vid; + int attempts, i, remaining; + + if (!igbv_vlan_retry_pending(sc)) { + sc->vf_vlan_retry_deadline = 0; + return; + } + if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { + remaining = 0; + for (i = 0; i < EM_VFTA_SIZE; i++) + remaining += bitcount32(sc->vf_vfta_retry[i]); + memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); + sc->vf_vlan_retry_deadline = 0; + device_printf(sc->dev, + "VF VLAN restore retries exhausted for %d VIDs\n", + remaining); + return; + } + + /* + * The mailbox NACK does not distinguish a transient PF rate limit + * from permanent VLVF exhaustion. Retry at the PF's sustained + * allowance, but bound the entire recovery window so ENOSPC cannot + * create a permanent mailbox poller. + */ + for (attempts = 0, i = 0; + attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) { + vid = sc->vf_vlan_retry_cursor; + sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; + bit = 1U << (vid & 0x1f); + if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) + continue; + attempts++; + if ((sc->shadow_vfta[vid >> 5] & bit) == 0 || + e1000_vfta_set_vf(&sc->hw, vid, true) == + E1000_SUCCESS) + sc->vf_vfta_retry[vid >> 5] &= ~bit; + } + if (!igbv_vlan_retry_pending(sc)) + sc->vf_vlan_retry_deadline = 0; +} + +int +igbv_if_attach_pre(if_ctx_t ctx) +{ + struct e1000_softc *sc; + device_t dev; + int error; + + dev = iflib_get_dev(ctx); + if (pci_msix_count(dev) < 2) { + device_printf(dev, "VF operation requires two MSI-X vectors\n"); + return (ENXIO); + } + error = em_if_attach_pre(ctx); + if (error != 0) + return (error); + + sc = iflib_get_softc(ctx); + callout_init(&sc->vf_queue_retry, 1); + sc->vf_queue_retry_initialized = true; + callout_init(&sc->vf_mbx_retry, 1); + sc->vf_mbx_retry_initialized = true; + + KASSERT(sc->vf_ifp && + (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0, + ("%s: igbv attached without VF policy", __func__)); + return (0); +} + +int +igbv_if_attach_post(if_ctx_t ctx) +{ + struct e1000_softc *sc; + int error; + + sc = iflib_get_softc(ctx); + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + if (sc->intr_type != IFLIB_INTR_MSIX) { + device_printf(sc->dev, "VF operation requires MSI-X\n"); + return (ENXIO); + } + error = em_if_attach_post(ctx); + if (error != 0) + return (error); + + /* + * Attach failures can leave the device sysctl tree registered when + * hw.bus.disable_failed_devices is set. Do not publish handlers with + * softc arguments until iflib has successfully allocated MSI-X. + */ + em_add_device_sysctls(sc); + return (0); +} + +int +igbv_if_media_change(if_ctx_t ctx __unused) +{ + + return (EOPNOTSUPP); +} + +void +igbv_if_update_admin_status(if_ctx_t ctx) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + device_t dev; + bool link_check, timer_tick; + + sc = iflib_get_softc(ctx); + hw = &sc->hw; + dev = iflib_get_dev(ctx); + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + + if ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) == 0 || + !sc->vf_queues_sanitized || + atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { + if (sc->link_state != EM_LINK_STATE_DOWN) { + sc->link_speed = 0; + sc->link_duplex = 0; + sc->link_state = EM_LINK_STATE_DOWN; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + } + return; + } + + if (!sc->vf_reset_pending && + atomic_readandclear_32(&sc->promisc_pending) != 0) + (void)em_if_set_promisc_impl(ctx, + if_getflags(iflib_get_ifp(ctx))); + + if (e1000_check_for_link(hw) != E1000_SUCCESS && + !sc->vf_reset_pending) { + sc->vf_reset_pending = true; + iflib_request_reset(ctx); + iflib_admin_intr_deferred(ctx); + } + link_check = !hw->mac.get_link_status; + + if (link_check && + (sc->link_state == EM_LINK_STATE_DOWN || + sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { + e1000_get_speed_and_duplex(hw, &sc->link_speed, + &sc->link_duplex); + if (bootverbose) + device_printf(dev, "Link is up %d Mbps %s\n", + sc->link_speed, + sc->link_duplex == FULL_DUPLEX ? + "Full Duplex" : "Half Duplex"); + sc->link_state = EM_LINK_STATE_UP; + iflib_link_state_change(ctx, LINK_STATE_UP, + IF_Mbps(sc->link_speed)); + } else if (!link_check && + (sc->link_state == EM_LINK_STATE_UP || + sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { + sc->link_speed = 0; + sc->link_duplex = 0; + sc->link_state = EM_LINK_STATE_DOWN; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + } + + /* + * A VF stops transmit DMA when its PF reports link down. Reset if + * descriptors remain queued so they cannot be sent stale when carrier + * returns, matching the periodic check in Linux igbvf. + */ + if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) { + sc->vf_reset_pending = true; + iflib_request_reset(ctx); + iflib_admin_intr_deferred(ctx); + } + /* em_if_init() establishes a new counter baseline after the reset. */ + timer_tick = !sc->vf_reset_pending && + atomic_readandclear_32(&sc->stats_pending) != 0; + if (timer_tick) { + em_update_stats_counters(sc); + /* iflib clears RUNNING before stop; do not replay after reset. */ + if ((if_getdrvflags(iflib_get_ifp(ctx)) & + IFF_DRV_RUNNING) != 0) + igbv_vlan_retry_tick(sc); + } +} + +static bool +igbv_tx_pending(struct e1000_softc *sc) +{ + struct tx_ring *txr; + u32 head, tail; + + for (int i = 0; i < sc->tx_num_queues; i++) { + txr = &sc->tx_queues[i].txr; + head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)); + tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me)); + if (head != tail) + return (true); + } + return (false); +} + +static bool +igbv_sanitize_queues(struct e1000_softc *sc) +{ + struct e1000_hw *hw; + u32 rxdctl, txdctl; + int i, nqueues, retry; + + hw = &sc->hw; + switch (hw->mac.type) { + case e1000_vfadapt: + nqueues = IGBV_82576_QUEUES; + break; + case e1000_vfadapt_i350: + nqueues = IGBV_I350_QUEUES; + break; + default: + return (true); + } + + /* + * The 82576 and I350 specification updates, Software Clarification 3, + * note that VFLR leaves this queue configuration intact. Clear it + * before programming the new rings so igbv does not depend on its PF + * to sanitize state left by a previous VF owner. igbv uses only queue + * zero, but must also clear the unused second 82576 queue. + * + * Disable every queue first and wait for outstanding DMA activity to + * stop before programming TDWBAL/H. Spin only for the normal fast + * transition, then sleep until the bounded deadline. + */ + for (i = 0; i < nqueues; i++) { + E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0); + E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0); + } + E1000_WRITE_FLUSH(hw); + for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) { + for (i = 0; i < nqueues; i++) { + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); + if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 || + (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0) + break; + } + if (i == nqueues) + break; + if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) { + if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES) + DELAY(IGBV_QUEUE_DISABLE_DELAY_US); + else + pause_sbt("igbvqds", + IGBV_QUEUE_DISABLE_PAUSE, 0, + C_PREL(1)); + } + } + if (retry == IGBV_QUEUE_DISABLE_RETRIES) { + if (ratecheck(&sc->vf_last_queue_log, + &igbv_queue_log_interval)) + device_printf(sc->dev, + "could not disable retained VF queues; " + "reset deferred\n"); + return (false); + } + + for (i = 0; i < nqueues; i++) { + E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0); + E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0); + E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0); + E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0); + E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0); + } + E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0); + E1000_WRITE_FLUSH(hw); + return (true); +} + +bool +igbv_reset(if_ctx_t ctx) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + s32 error; + + sc = iflib_get_softc(ctx); + hw = &sc->hw; + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + + /* + * Receive-buffer allocation and flow control are port resources owned + * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size. + */ + sc->pba = 0; + hw->fc = (struct e1000_fc_info){ + .current_mode = e1000_fc_none, + .requested_mode = e1000_fc_none, + }; + + error = e1000_reset_hw(hw); + atomic_store_rel_32(&sc->vf_mbx_ready, 0); + sc->vf_queues_sanitized = igbv_sanitize_queues(sc); + if (!sc->vf_queues_sanitized) { + return (false); + } + igbv_queue_retry_succeeded(sc); + if (error != E1000_SUCCESS) { + igbv_log_reset_failure(sc, error, false); + return (false); + } + memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale)); + memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); + sc->vf_vlan_retry_deadline = 0; + sc->vf_vlan_retry_cursor = 0; + if (e1000_init_hw(hw) < 0) { + device_printf(sc->dev, "Hardware Initialization Failed\n"); + return (false); + } + e1000_check_for_link(hw); + igbv_mbx_retry_succeeded(sc); + return (true); +} + +void +igbv_initialize_transmit_unit(if_ctx_t ctx) +{ + + KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, + ("%s called for a PF", __func__)); + em_initialize_transmit_rings(ctx); +} + +void +igbv_initialize_receive_unit(if_ctx_t ctx) +{ + + KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, + ("%s called for a PF", __func__)); + igb_initialize_receive_rings(ctx, true); +} + +void +igbv_if_intr_enable(if_ctx_t ctx) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + u32 mask; + + sc = iflib_get_softc(ctx); + hw = &sc->hw; + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + if (!sc->vf_queues_sanitized || + atomic_load_acq_32(&sc->vf_mbx_ready) == 0) + return; + mask = sc->que_mask | sc->link_mask; + + E1000_WRITE_REG(hw, E1000_EIAC, mask); + E1000_WRITE_REG(hw, E1000_EIAM, mask); + E1000_WRITE_REG(hw, E1000_EIMS, mask); + E1000_WRITE_FLUSH(hw); +} + +void +igbv_if_intr_disable(if_ctx_t ctx) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + + sc = iflib_get_softc(ctx); + hw = &sc->hw; + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + + E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff); + E1000_WRITE_REG(hw, E1000_EIAC, 0); + E1000_WRITE_FLUSH(hw); +} + +int +igbv_get_regs(SYSCTL_HANDLER_ARGS) +{ + struct e1000_softc *sc; + struct e1000_hw *hw; + struct sbuf *sb; + int error; + + sc = (struct e1000_softc *)arg1; + hw = &sc->hw; + KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); + + sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); + if (sb == NULL) + return (ENOMEM); + + /* + * Limited VF register set: + * Don't read EICR here because it is clear-on-read. The VF register + * file exposes its queue pair at index zero, so this diagnostic does + * not depend on the narrower lifetime of iflib's queue arrays. + */ + sbuf_printf(sb, "VF Registers\n"); + sbuf_printf(sb, "\tVTCTRL\t %08x\n", + E1000_READ_REG(hw, E1000_CTRL)); + sbuf_printf(sb, "\tSTATUS\t %08x\n", + E1000_READ_REG(hw, E1000_STATUS)); + sbuf_printf(sb, "\tRDLEN\t %08x\n", + E1000_READ_REG(hw, E1000_RDLEN(0))); + sbuf_printf(sb, "\tRDH\t %08x\n", + E1000_READ_REG(hw, E1000_RDH(0))); + sbuf_printf(sb, "\tRDT\t %08x\n", + E1000_READ_REG(hw, E1000_RDT(0))); + sbuf_printf(sb, "\tTDLEN\t %08x\n", + E1000_READ_REG(hw, E1000_TDLEN(0))); + sbuf_printf(sb, "\tTDH\t %08x\n", + E1000_READ_REG(hw, E1000_TDH(0))); + sbuf_printf(sb, "\tTDT\t %08x\n", + E1000_READ_REG(hw, E1000_TDT(0))); + + error = sbuf_finish(sb); + sbuf_delete(sb); + return (error); +} + +static u_int +igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx) +{ + struct igb_vf_uc_addr_list *list; + const u8 *addr; + + list = arg; + addr = (const u8 *)LLADDR(sdl); + if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) + return (0); + if (idx < IGBV_MAX_MAC_FILTERS) + memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN); + return (1); +} + +void +igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp) +{ + struct igb_vf_uc_addr_list list = { + .sc = sc, + }; + u_int count; + + count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list); + if (count > IGBV_MAX_MAC_FILTERS) { + device_printf(sc->dev, + "too many secondary unicast addresses; maximum is %u\n", + IGBV_MAX_MAC_FILTERS); + } + if (count == 0 && !sc->vf_uc_filters_set) + return; + /* + * Linux igb PFs validate the address field before dispatching the CLR + * subcommand. Supply the primary address rather than the zero payload + * used by igbvf so those PFs actually remove the old filters. FreeBSD + * PFs dispatch CLR before inspecting the otherwise-ignored address. + */ + if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR, + sc->hw.mac.addr) != E1000_SUCCESS) { + device_printf(sc->dev, + "VF secondary unicast filter clear request failed\n"); + return; + } + sc->vf_uc_filters_set = false; + if (count > IGBV_MAX_MAC_FILTERS) + return; + + for (u_int i = 0; i < count; i++) { + if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD, + list.addrs[i]) != E1000_SUCCESS) { + device_printf(sc->dev, + "VF secondary unicast filter add request failed " + "for %6D\n", list.addrs[i], ":"); + } else + sc->vf_uc_filters_set = true; + usec_delay(200); + } +} + +void +igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp) +{ + u8 *lladdr; + + if (!em_is_valid_ether_addr(sc->hw.mac.addr)) + return; + lladdr = (u8 *)if_getlladdr(ifp); + if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) + return; + + device_printf(sc->dev, + "PF rejected or replaced the requested MAC; using %6D\n", + sc->hw.mac.addr, ":"); + /* + * if_setlladdr() would re-enter the driver's address-change path. + * Initialization already holds the context lock, so update the + * storage directly and issue the notification it would have sent. + */ + memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN); + + CURVNET_SET_QUIET(if_getvnet(ifp)); + EVENTHANDLER_INVOKE(iflladdr_event, ifp); + CURVNET_RESTORE(); +} diff --git a/sys/dev/e1000/igb_txrx.c b/sys/dev/e1000/igb_txrx.c index 46fe5c741055..dd4d1e5b5974 100644 --- a/sys/dev/e1000/igb_txrx.c +++ b/sys/dev/e1000/igb_txrx.c @@ -56,7 +56,14 @@ static int igb_tx_ctx_setup(struct tx_ring *, if_pkt_info_t, uint32_t *, static int igb_tso_setup(struct tx_ring *, if_pkt_info_t, uint32_t *, uint32_t *); -static void igb_rx_checksum(uint32_t, if_rxd_info_t, uint32_t); +enum igb_rx_csum_status { + IGB_RX_CSUM_NONE, + IGB_RX_CSUM_GOOD, + IGB_RX_CSUM_ERROR, +}; + +static enum igb_rx_csum_status igb_rx_checksum(uint32_t, if_rxd_info_t, + uint32_t); static int igb_determine_rsstype(uint16_t); extern void igb_if_enable_intr(if_ctx_t); @@ -73,6 +80,29 @@ struct if_txrx igb_txrx = { .ift_legacy_intr = em_intr }; +static bool +igb_vf_vlan_registered(const struct e1000_softc *sc, u16 vtag) +{ + u32 vlans; + u16 vid; + + /* + * 82576 strips an administrative access VLAN but still reports it in + * the descriptor. Like Linux igbvf, expose tags only when the VF + * requested that VID from the PF. + */ + if (sc->hw.mac.type != e1000_vfadapt && + sc->hw.mac.type != e1000_vfadapt_i350) + return (true); + vid = EVL_VLANOFTAG(vtag); + /* + * A trunk VF is an implicit member of VID 0, so retain priority-tag + * metadata without requiring vlan(4) to register a VID-0 interface. + */ + vlans = sc->shadow_vfta[vid >> 5] | sc->vf_vfta_stale[vid >> 5]; + return (vid == 0 || (vlans & (1U << (vid & 0x1f))) != 0); +} + /********************************************************************** * * Setup work for hardware segmentation offload (TSO) on @@ -289,10 +319,27 @@ igb_isc_txd_encap(void *arg, if_pkt_info_t pi) txd->read.cmd_type_len |= htole32(E1000_TXD_CMD_EOP | txd_flags); pi->ipi_new_pidx = i; - /* Sent data accounting for AIM */ + /* + * Sent data accounting for AIM. For TSO, ipi_len is the whole + * unsegmented payload, which is not a size the moderation calculation + * can use. Count the segments the hardware will put on the wire and + * the header each of them carries, so that the average it sees is a + * wire packet. + */ + if ((pi->ipi_csum_flags & CSUM_TSO) && pi->ipi_tso_segsz != 0) { + u32 hdrlen, segs; + + hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; + if (pi->ipi_len > hdrlen) { + segs = howmany(pi->ipi_len - hdrlen, pi->ipi_tso_segsz); + txr->tx_bytes += pi->ipi_len + (segs - 1) * hdrlen; + txr->tx_packets += segs; + return (0); + } + } + txr->tx_bytes += pi->ipi_len; ++txr->tx_packets; - return (0); } @@ -304,6 +351,7 @@ igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) struct tx_ring *txr = &que->txr; E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); + em_aim_publish(txr); } static int @@ -395,6 +443,7 @@ igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, struct rx_ring *rxr = &que->rxr; E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); + em_aim_publish_rx(rxr); } static int @@ -440,6 +489,7 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) uint16_t pkt_info, len; uint32_t ptype, staterr; + enum igb_rx_csum_status csum_status; int i, cidx; bool eop; @@ -458,7 +508,6 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; ri->iri_len += len; - rxr->rx_bytes += ri->iri_len; rxd->wb.upper.status_error = 0; eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); @@ -487,19 +536,29 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) i++; } while (!eop); + rxr->rx_bytes += ri->iri_len; rxr->rx_packets++; - if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) - igb_rx_checksum(staterr, ri, ptype); + if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) { + csum_status = igb_rx_checksum(staterr, ri, ptype); + if (sc->vf_ifp) { + if (csum_status == IGB_RX_CSUM_GOOD) + sc->rx_csum_good++; + else if (csum_status == IGB_RX_CSUM_ERROR) + sc->rx_csum_errors++; + } + } if (staterr & E1000_RXD_STAT_VP) { if (((sc->hw.mac.type == e1000_i350) || - (sc->hw.mac.type == e1000_i354)) && + (sc->hw.mac.type == e1000_i354) || + (sc->hw.mac.type == e1000_vfadapt_i350)) && (staterr & E1000_RXDEXT_STATERR_LB)) ri->iri_vtag = be16toh(rxd->wb.upper.vlan); else ri->iri_vtag = le16toh(rxd->wb.upper.vlan); - ri->iri_flags |= M_VLANTAG; + if (igb_vf_vlan_registered(sc, ri->iri_vtag)) + ri->iri_flags |= M_VLANTAG; } ri->iri_flowid = @@ -517,19 +576,19 @@ igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) * doesn't spend time verifying the checksum. * *********************************************************************/ -static void +static enum igb_rx_csum_status igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype) { uint16_t status = (uint16_t)staterr; uint8_t errors = (uint8_t)(staterr >> 24); if (__predict_false(status & E1000_RXD_STAT_IXSM)) - return; + return (IGB_RX_CSUM_NONE); /* If there is a layer 3 or 4 error we are done */ if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))) - return; + return (IGB_RX_CSUM_ERROR); /* IP Checksum Good */ if (status & E1000_RXD_STAT_IPCS) @@ -549,6 +608,8 @@ igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype) ri->iri_csum_data = htons(0xffff); } } + + return (IGB_RX_CSUM_GOOD); } /******************************************************************** @@ -572,7 +633,13 @@ igb_determine_rsstype(uint16_t pkt_info) return M_HASHTYPE_RSS_IPV6; case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: return M_HASHTYPE_RSS_TCP_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV4_UDP: + return M_HASHTYPE_RSS_UDP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_UDP: + return M_HASHTYPE_RSS_UDP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_UDP_EX: + return M_HASHTYPE_RSS_UDP_IPV6_EX; default: - return M_HASHTYPE_OPAQUE; + return M_HASHTYPE_NONE; } } |