ice: Update to 1.43.2-k

- Add extra time while waiting for an EMPR to finish on E830 and E825C
  devices
- Fix debug mask sysctl help message
- Handle EAGAIN error messages from the Get Link Status AQ command by
  retrying them several times
- Handle the condition where E830 extended temperature SKUs can have the
  chip initialize without the PHY subsystem being ready due to very low
  temperatures; the driver will print out a message when this happens
- Fix inability to set link speed using the advertise_speed sysctl when
  link is down and the link_active_on_if_down sysctl is set to 0
- Fix issue where VLANs would not properly be set back up after a
  hardware reset (e.g. PF reset) (Thanks jacob.e.keller@intel.com)

Signed-off-by: Eric Joyner <erj@FreeBSD.org>

Tested by:	Jeffrey Pieper <jeffrey.e.pieper@intel.com>
MFC after:	2 days
Relnotes:	yes
Sponsored by:	Intel Corporation
Differential Revision:	https://reviews.freebsd.org/D46951

7 files changed, 128 insertions, 36 deletions

diff --git a/sys/dev/ice/ice_controlq.c b/sys/dev/ice/ice_controlq.c
index e96c7e230310..e1a6b0fb5662 100644
--- a/sys/dev/ice/ice_controlq.c
+++ b/sys/dev/ice/ice_controlq.c
@@ -879,16 +879,35 @@ static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 	struct ice_ctl_q_ring *sq = &cq->sq;
 	u16 ntc = sq->next_to_clean;
 	struct ice_aq_desc *desc;
+	u32 head;
 
 	desc = ICE_CTL_Q_DESC(*sq, ntc);
 
-	while (rd32(hw, cq->sq.head) != ntc) {
-		ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));
+	head = rd32(hw, sq->head);
+	if (head >= sq->count) {
+		ice_debug(hw, ICE_DBG_AQ_MSG,
+			  "Read head value (%d) exceeds allowed range.\n",
+			  head);
+		return 0;
+	}
+
+	while (head != ntc) {
+		ice_debug(hw, ICE_DBG_AQ_MSG,
+			  "ntc %d head %d.\n",
+			  ntc, head);
 		ice_memset(desc, 0, sizeof(*desc), ICE_DMA_MEM);
 		ntc++;
 		if (ntc == sq->count)
 			ntc = 0;
 		desc = ICE_CTL_Q_DESC(*sq, ntc);
+
+		head = rd32(hw, sq->head);
+		if (head >= sq->count) {
+			ice_debug(hw, ICE_DBG_AQ_MSG,
+				  "Read head value (%d) exceeds allowed range.\n",
+				  head);
+			return 0;
+		}
 	}
 
 	sq->next_to_clean = ntc;
diff --git a/sys/dev/ice/ice_drv_info.h b/sys/dev/ice/ice_drv_info.h
index 22e23ee53491..80b517bb5b08 100644
--- a/sys/dev/ice/ice_drv_info.h
+++ b/sys/dev/ice/ice_drv_info.h
@@ -62,16 +62,16 @@
  * @var ice_rc_version
  * @brief driver release candidate version number
  */
-const char ice_driver_version[] = "1.42.5-k";
+const char ice_driver_version[] = "1.43.2-k";
 const uint8_t ice_major_version = 1;
-const uint8_t ice_minor_version = 42;
-const uint8_t ice_patch_version = 5;
+const uint8_t ice_minor_version = 43;
+const uint8_t ice_patch_version = 2;
 const uint8_t ice_rc_version = 0;
 
 #define PVIDV(vendor, devid, name) \
-	PVID(vendor, devid, name " - 1.42.5-k")
+	PVID(vendor, devid, name " - 1.43.2-k")
 #define PVIDV_OEM(vendor, devid, svid, sdevid, revid, name) \
-	PVID_OEM(vendor, devid, svid, sdevid, revid, name " - 1.42.5-k")
+	PVID_OEM(vendor, devid, svid, sdevid, revid, name " - 1.43.2-k")
 
 /**
  * @var ice_vendor_info_array
diff --git a/sys/dev/ice/ice_lib.c b/sys/dev/ice/ice_lib.c
index d2d13cd2db60..d44ae5f37750 100644
--- a/sys/dev/ice/ice_lib.c
+++ b/sys/dev/ice/ice_lib.c
@@ -3308,7 +3308,8 @@ ice_sysctl_advertise_speed(SYSCTL_HANDLER_ARGS)
 
 	pi->phy.curr_user_speed_req = sysctl_speeds;
 
-	if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && !sc->link_up)
+	if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) &&
+	    !sc->link_up && !(if_getflags(sc->ifp) & IFF_UP))
 		return 0;
 
 	/* Apply settings requested by user */
@@ -6246,11 +6247,12 @@ ice_sysctl_dump_state_flags(SYSCTL_HANDLER_ARGS)
 "\n\t     0x20000 - Resources"					\
 "\n\t     0x40000 - ACL"					\
 "\n\t     0x80000 - PTP"					\
-"\n\t    0x100000 - Admin Queue messages"			\
-"\n\t    0x200000 - Admin Queue descriptors"			\
-"\n\t    0x400000 - Admin Queue descriptor buffers"		\
-"\n\t    0x800000 - Admin Queue commands"			\
-"\n\t   0x1000000 - Parser"					\
+"\n\t   ..."							\
+"\n\t   0x1000000 - Admin Queue messages"			\
+"\n\t   0x2000000 - Admin Queue descriptors"			\
+"\n\t   0x4000000 - Admin Queue descriptor buffers"		\
+"\n\t   0x8000000 - Admin Queue commands"			\
+"\n\t  0x10000000 - Parser"					\
 "\n\t   ..."							\
 "\n\t  0x80000000 - (Reserved for user)"			\
 "\n\t"								\
@@ -9493,15 +9495,28 @@ ice_init_link_configuration(struct ice_softc *sc)
 	struct ice_port_info *pi = sc->hw.port_info;
 	struct ice_hw *hw = &sc->hw;
 	device_t dev = sc->dev;
-	int status;
+	int status, retry_count = 0;
 
+retry:
 	pi->phy.get_link_info = true;
 	status = ice_get_link_status(pi, &sc->link_up);
+
 	if (status) {
-		device_printf(dev,
-		    "%s: ice_get_link_status failed; status %s, aq_err %s\n",
-		    __func__, ice_status_str(status),
-		    ice_aq_str(hw->adminq.sq_last_status));
+		if (hw->adminq.sq_last_status == ICE_AQ_RC_EAGAIN) {
+			retry_count++;
+			ice_debug(hw, ICE_DBG_LINK,
+			    "%s: ice_get_link_status failed with EAGAIN, attempt %d\n",
+			    __func__, retry_count);
+			if (retry_count < ICE_LINK_AQ_MAX_RETRIES) {
+				ice_msec_pause(ICE_LINK_RETRY_DELAY);
+				goto retry;
+			}
+		} else {
+			device_printf(dev,
+			    "%s: ice_get_link_status failed; status %s, aq_err %s\n",
+			    __func__, ice_status_str(status),
+			    ice_aq_str(hw->adminq.sq_last_status));
+		}
 		return;
 	}
 
diff --git a/sys/dev/ice/ice_lib.h b/sys/dev/ice/ice_lib.h
index afc03ebd3b51..466cb8701b79 100644
--- a/sys/dev/ice/ice_lib.h
+++ b/sys/dev/ice/ice_lib.h
@@ -396,6 +396,16 @@ enum ice_rx_dtype {
 #define ICE_I2C_MAX_RETRIES		10
 
 /*
+ * The Get Link Status AQ command and other link commands can return
+ * EAGAIN, indicating that the FW Link Management engine is busy.
+ * Define the number of times that the driver should retry sending these
+ * commands and the amount of time it should wait between those retries
+ * (in milliseconds) here.
+ */
+#define ICE_LINK_AQ_MAX_RETRIES		10
+#define ICE_LINK_RETRY_DELAY		17
+
+/*
  * The Start LLDP Agent AQ command will fail if it's sent too soon after
  * the LLDP agent is stopped. The period between the stop and start
  * commands must currently be at least 2 seconds.
@@ -698,6 +708,7 @@ enum ice_state {
 	ICE_STATE_FIRST_INIT_LINK,
 	ICE_STATE_DO_CREATE_MIRR_INTFC,
 	ICE_STATE_DO_DESTROY_MIRR_INTFC,
+	ICE_STATE_PHY_FW_INIT_PENDING,
 	/* This entry must be last */
 	ICE_STATE_LAST,
 };
diff --git a/sys/dev/ice/ice_strings.c b/sys/dev/ice/ice_strings.c
index 5b5da737cadb..1b377a1bf518 100644
--- a/sys/dev/ice/ice_strings.c
+++ b/sys/dev/ice/ice_strings.c
@@ -1052,6 +1052,8 @@ ice_state_to_str(enum ice_state state)
 		return "DO_CREATE_MIRR_INTFC";
 	case ICE_STATE_DO_DESTROY_MIRR_INTFC:
 		return "DO_DESTROY_MIRR_INTFC";
+	case ICE_STATE_PHY_FW_INIT_PENDING:
+		return "PHY_FW_INIT_PENDING";
 	case ICE_STATE_LAST:
 		return NULL;
 	}
diff --git a/sys/dev/ice/ice_switch.c b/sys/dev/ice/ice_switch.c
index 1edd39497ab5..1880d6abdd26 100644
--- a/sys/dev/ice/ice_switch.c
+++ b/sys/dev/ice/ice_switch.c
@@ -4292,8 +4292,6 @@ ice_replay_vsi_fltr(struct ice_hw *hw, struct ice_port_info *pi,
 		if (!itr->vsi_list_info ||
 		    !ice_is_bit_set(itr->vsi_list_info->vsi_map, vsi_handle))
 			continue;
-		/* Clearing it so that the logic can add it back */
-		ice_clear_bit(vsi_handle, itr->vsi_list_info->vsi_map);
 		f_entry.fltr_info.vsi_handle = vsi_handle;
 		f_entry.fltr_info.fltr_act = ICE_FWD_TO_VSI;
 		/* update the src in case it is VSI num */
diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c
index 0fb7faecb2d7..9b3f38f885b9 100644
--- a/sys/dev/ice/if_ice_iflib.c
+++ b/sys/dev/ice/if_ice_iflib.c
@@ -84,6 +84,7 @@ static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
 static int ice_if_suspend(if_ctx_t ctx);
 static int ice_if_resume(if_ctx_t ctx);
 static bool ice_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event);
+static void ice_init_link(struct ice_softc *sc);
 static int ice_setup_mirror_vsi(struct ice_mirr_if *mif);
 static int ice_wire_mirror_intrs(struct ice_mirr_if *mif);
 static void ice_free_irqvs_subif(struct ice_mirr_if *mif);
@@ -833,9 +834,8 @@ ice_if_attach_post(if_ctx_t ctx)
 	/* Get DCBX/LLDP state and start DCBX agent */
 	ice_init_dcb_setup(sc);
 
-	/* Setup link configuration parameters */
-	ice_init_link_configuration(sc);
-	ice_update_link_status(sc, true);
+	/* Setup link, if PHY FW is ready */
+	ice_init_link(sc);
 
 	/* Configure interrupt causes for the administrative interrupt */
 	ice_configure_misc_interrupts(sc);
@@ -2136,6 +2136,18 @@ ice_poll_for_media_avail(struct ice_softc *sc)
 	struct ice_hw *hw = &sc->hw;
 	struct ice_port_info *pi = hw->port_info;
 
+	/* E830 only: There's no interrupt for when the PHY FW has finished loading,
+	 * so poll for the status in the media task here if it's previously
+	 * been detected that it's still loading.
+	 */
+	if (ice_is_e830(hw) &&
+	    ice_test_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING)) {
+		if (rd32(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_LOADING_M)
+			ice_clear_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING);
+		else
+			return;
+	}
+
 	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
 		pi->phy.get_link_info = true;
 		ice_get_link_status(pi, &sc->link_up);
@@ -2705,11 +2717,11 @@ ice_rebuild(struct ice_softc *sc)
 	if (hw->port_info->qos_cfg.is_sw_lldp)
 		ice_add_rx_lldp_filter(sc);
 
-	/* Refresh link status */
+	/* Apply previous link settings and refresh link status, if PHY
+	 * FW is ready.
+	 */
 	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
-	sc->hw.port_info->phy.get_link_info = true;
-	ice_get_link_status(sc->hw.port_info, &sc->link_up);
-	ice_update_link_status(sc, true);
+	ice_init_link(sc);
 
 	/* RDMA interface will be restarted by the stack re-init */
 
@@ -2795,12 +2807,23 @@ ice_handle_reset_event(struct ice_softc *sc)
 	 * resetting.
 	 */
 	IFLIB_CTX_UNLOCK(sc);
+
+#define ICE_EMPR_ADDL_WAIT_MSEC_SLOW		20000
+	if ((ice_is_e830(hw) || ice_is_e825c(hw)) &&
+	    (((rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
+	         GLGEN_RSTAT_RESET_TYPE_S) == ICE_RESET_EMPR))
+			ice_msec_pause(ICE_EMPR_ADDL_WAIT_MSEC_SLOW);
+
 	status = ice_check_reset(hw);
 	IFLIB_CTX_LOCK(sc);
 	if (status) {
 		device_printf(dev, "Device never came out of reset, err %s\n",
 			      ice_status_str(status));
+
 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
+		ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
+		ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
+		device_printf(dev, "Reset failed; please reload the device driver\n");
 		return;
 	}
 
@@ -2888,16 +2911,8 @@ ice_init_device_features(struct ice_softc *sc)
 	if (ice_is_e810(hw))
 		ice_set_bit(ICE_FEATURE_PHY_STATISTICS, sc->feat_en);
 
-	/* Set capabilities based on device */
-	switch (hw->device_id) {
-	case ICE_DEV_ID_E825C_BACKPLANE:
-	case ICE_DEV_ID_E825C_QSFP:
-	case ICE_DEV_ID_E825C_SFP:
+	if (ice_is_e825c(hw))
 		ice_set_bit(ICE_FEATURE_DUAL_NAC, sc->feat_cap);
-		break;
-	default:
-		break;
-	}
 	/* Disable features due to hardware limitations... */
 	if (!hw->func_caps.common_cap.rss_table_size)
 		ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
@@ -3303,6 +3318,38 @@ ice_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event)
 	}
 }
 
+/**
+ * ice_init_link - Do link configuration and link status reporting
+ * @sc: driver private structure
+ *
+ * Contains an extra check that skips link config when an E830 device
+ * does not have the "FW_LOADING"/"PHYBUSY" bit set in GL_MNG_FWSM set.
+ */
+static void
+ice_init_link(struct ice_softc *sc)
+{
+	struct ice_hw *hw = &sc->hw;
+	device_t dev = sc->dev;
+
+	/* Check if FW is ready before setting up link; defer setup to the
+	 * admin task if it isn't.
+	 */
+	if (ice_is_e830(hw) &&
+	    (rd32(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_LOADING_M)) {
+		ice_set_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING);
+		device_printf(dev,
+		    "Link initialization is blocked by PHY FW initialization.\n");
+		device_printf(dev,
+		    "Link initialization will continue after PHY FW initialization completes.\n");
+		/* Do not access PHY config while PHY FW is busy initializing */
+	} else {
+		ice_clear_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING);
+		ice_init_link_configuration(sc);
+		ice_update_link_status(sc, true);
+	}
+
+}
+
 extern struct if_txrx ice_subif_txrx;
 
 /**