ixl: Track and recover MDD-blocked VFs

The hardware identifies each VF with TX and RX malicious-driver
status latches, but the driver combined all events into one counter
and reported only the last VF found.  It also did not record that
hardware had blocked the VF, leaving the condition invisible to
management tools.

Consume every PF and VF latch, keep per-direction VF counters,
rate-limit per-VF diagnostics, and report the blocked and
traffic-enabled state via the VF status interface.  Clear the
software block only after a successful VF or PF reset reconstructs
its resources.

Match Linux i40e policy by leaving a detected VF blocked by default.
Add an opt-in hw.ixl.mdd_auto_reset_vf tunable that notifies and
resets the VF for installations that prefer availability.  DPDK
provides the register clear and per-VF attribution precedent; Linux
provides the recovery policy.

MFC after:	2 weeks

5 files changed, 119 insertions, 69 deletions

diff --git a/share/man/man4/ixl.4 b/share/man/man4/ixl.4
index ef52b9f8b456..37d8c0cf5db8 100644
--- a/share/man/man4/ixl.4
+++ b/share/man/man4/ixl.4
@@ -213,6 +213,12 @@ use a value written to memory by the hardware instead of scanning the
 descriptor ring for completed descriptors.
 Enabled by default; disable to mimic the TX behavior found in
 .Xr ix 4 .
+.It Va hw.ixl.mdd_auto_reset_vf
+Automatically reset a VF after malicious-driver detection blocks it.
+This is disabled by default so a non-cooperative VF remains unable to issue
+traffic until it performs an FLR or the SR-IOV configuration is recreated.
+Enabling this tunable favors availability but allows a persistently faulty or
+hostile VF to resume after each reset.
 .El
 .Sh SYSCTL PROCEDURES
 .Bl -tag -width indent
diff --git a/sys/dev/ixl/if_ixl.c b/sys/dev/ixl/if_ixl.c
index b7234cf89bef..5a5d361a12c6 100644
--- a/sys/dev/ixl/if_ixl.c
+++ b/sys/dev/ixl/if_ixl.c
@@ -253,6 +253,12 @@ SYSCTL_INT(_hw_ixl, OID_AUTO, enable_vf_loopback, CTLFLAG_RDTUN,
     &ixl_enable_vf_loopback, 0,
     IXL_SYSCTL_HELP_VF_LOOPBACK);
 
+static int ixl_mdd_auto_reset_vf;
+TUNABLE_INT("hw.ixl.mdd_auto_reset_vf", &ixl_mdd_auto_reset_vf);
+SYSCTL_INT(_hw_ixl, OID_AUTO, mdd_auto_reset_vf, CTLFLAG_RDTUN,
+    &ixl_mdd_auto_reset_vf, 0,
+    "Automatically reset VFs blocked by malicious-driver detection");
+
 /*
  * Different method for processing TX descriptor
  * completion.
@@ -1969,6 +1975,10 @@ ixl_if_vf_status(if_ctx_t ctx, nvlist_t *status)
 		    (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) != 0);
 		nvlist_add_bool(vfs[i], IFVF_STATUS_ALLOW_PROMISC,
 		    (vf->vf_flags & VF_FLAG_PROMISC_CAP) != 0);
+		nvlist_add_bool(vfs[i], IFVF_STATUS_TRAFFIC_ENABLED,
+		    !vf->mdd_blocked);
+		nvlist_add_bool(vfs[i], IFVF_STATUS_MDD_BLOCKED,
+		    vf->mdd_blocked);
 	}
 	nvlist_add_nvlist_array(status, IFVF_STATUS_VFS,
 	    (const nvlist_t * const *)vfs, pf->num_vfs);
@@ -1998,6 +2008,7 @@ ixl_save_pf_tunables(struct ixl_pf *pf)
 	pf->hw.debug_mask = ixl_shared_debug_mask;
 	pf->vsi.enable_head_writeback = !!(ixl_enable_head_writeback);
 	pf->enable_vf_loopback = !!(ixl_enable_vf_loopback);
+	pf->mdd_auto_reset_vf = !!(ixl_mdd_auto_reset_vf);
 #if 0
 	pf->dynamic_rx_itr = ixl_dynamic_rx_itr;
 	pf->dynamic_tx_itr = ixl_dynamic_tx_itr;
diff --git a/sys/dev/ixl/ixl_pf.h b/sys/dev/ixl/ixl_pf.h
index 8d06738034a4..80631e861310 100644
--- a/sys/dev/ixl/ixl_pf.h
+++ b/sys/dev/ixl/ixl_pf.h
@@ -104,7 +104,12 @@ enum ixl_state {
 struct ixl_vf {
 	struct ixl_vsi		vsi;
 	u32			vf_flags;
-	u32			num_mdd_events;
+	u64			mdd_tx_events;
+	u64			mdd_rx_events;
+	struct timeval		last_mdd_log;
+	bool			mdd_blocked;
+	bool			mdd_event_pending;
+	bool			mdd_reset_pending;
 
 	u8			mac[ETHER_ADDR_LEN];
 	u8			mac_filters[IXL_VF_MAX_MAC_FILTERS][ETHER_ADDR_LEN];
@@ -152,6 +157,7 @@ struct ixl_pf {
 	int			tx_itr;
 	int			rx_itr;
 	int			enable_vf_loopback;
+	int			mdd_auto_reset_vf;
 
 	bool			link_up;
 	int			advertised_speed;
@@ -387,6 +393,7 @@ int	ixl_pf_reset(struct ixl_pf *);
 #ifdef PCI_IOV
 void	ixl_notify_vfs_reset(struct ixl_pf *);
 int	ixl_rebuild_vfs_after_reset(struct ixl_pf *);
+int	ixl_reset_vf_on_mdd(struct ixl_pf *, uint16_t);
 #endif
 
 void	ixl_set_queue_rx_itr(struct ixl_rx_queue *);
diff --git a/sys/dev/ixl/ixl_pf_iov.c b/sys/dev/ixl/ixl_pf_iov.c
index 358210f0cc2f..5254a59df3ab 100644
--- a/sys/dev/ixl/ixl_pf_iov.c
+++ b/sys/dev/ixl/ixl_pf_iov.c
@@ -498,6 +498,9 @@ ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
 
 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), VIRTCHNL_VFR_VFACTIVE);
 	ixl_flush(hw);
+	vf->mdd_blocked = false;
+	vf->mdd_event_pending = false;
+	vf->mdd_reset_pending = false;
 	return (0);
 }
 
@@ -1768,6 +1771,10 @@ ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
 	    (vf->vf_flags & VF_FLAG_ENABLED) ? " " : " disabled ",
 	    vf_num, msg_size);
 
+	/* Only a reset outside the virtchnl dispatcher may unblock the VF. */
+	if (vf->mdd_blocked)
+		return;
+
 	/* Perform basic checks on the msg */
 	err = virtchnl_vc_validate_vf_msg(&vf->version, opcode, msg, msg_size);
 	if (err) {
@@ -2039,6 +2046,26 @@ ixl_notify_vfs_reset(struct ixl_pf *pf)
 }
 
 int
+ixl_reset_vf_on_mdd(struct ixl_pf *pf, uint16_t vfnum)
+{
+	struct virtchnl_pf_event event;
+	struct ixl_vf *vf;
+
+	if (vfnum >= pf->num_vfs)
+		return (EINVAL);
+	vf = &pf->vfs[vfnum];
+	if (!(vf->vf_flags & VF_FLAG_ENABLED))
+		return (ENXIO);
+
+	bzero(&event, sizeof(event));
+	event.event = VIRTCHNL_EVENT_RESET_IMPENDING;
+	event.severity = PF_EVENT_SEVERITY_CERTAIN_DOOM;
+	ixl_send_vf_msg(pf, vf, VIRTCHNL_OP_EVENT, I40E_SUCCESS,
+	    &event, sizeof(event));
+	return (ixl_reset_vf(pf, vf));
+}
+
+int
 ixl_rebuild_vfs_after_reset(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw;
@@ -2067,6 +2094,9 @@ ixl_rebuild_vfs_after_reset(struct ixl_pf *pf)
 		bit_nclear(vf->vsi.vlans_map, 0,
 		    IXL_VLANS_MAP_LEN - 1);
 		vf->num_mac_filters = 0;
+		vf->mdd_blocked = false;
+		vf->mdd_event_pending = false;
+		vf->mdd_reset_pending = false;
 	}
 
 	error = ixl_setup_iov_switch(pf);
diff --git a/sys/dev/ixl/ixl_pf_main.c b/sys/dev/ixl/ixl_pf_main.c
index cb6f64f66209..3dea1c14ddd6 100644
--- a/sys/dev/ixl/ixl_pf_main.c
+++ b/sys/dev/ixl/ixl_pf_main.c
@@ -1843,12 +1843,8 @@ ixl_handle_tx_mdd_event(struct ixl_pf *pf)
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct ixl_vf *vf;
-	bool mdd_detected = false;
-	bool pf_mdd_detected = false;
-	bool vf_mdd_detected = false;
 	u16 vf_num, queue;
 	u8 pf_num, event;
-	u8 pf_mdet_num, vp_mdet_num;
 	u32 reg;
 
 	/* find what triggered the MDD event */
@@ -1862,18 +1858,20 @@ ixl_handle_tx_mdd_event(struct ixl_pf *pf)
 		    I40E_GL_MDET_TX_EVENT_SHIFT;
 		queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
 		    I40E_GL_MDET_TX_QUEUE_SHIFT;
+		if (queue >= hw->func_caps.base_queue)
+			queue -= hw->func_caps.base_queue;
+		device_printf(dev,
+		    "last TX malicious-driver cause %#x on queue %u, "
+		    "PF %#x, VF %#x\n", event, queue, pf_num, vf_num);
 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
-		mdd_detected = true;
 	}
 
-	if (!mdd_detected)
-		return;
-
 	reg = rd32(hw, I40E_PF_MDET_TX);
 	if (reg & I40E_PF_MDET_TX_VALID_MASK) {
 		wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
-		pf_mdet_num = hw->pf_id;
-		pf_mdd_detected = true;
+		device_printf(dev,
+		    "TX malicious-driver issue detected on PF-%u\n",
+		    hw->pf_id);
 	}
 
 	/* Check if MDD was caused by a VF */
@@ -1882,32 +1880,16 @@ ixl_handle_tx_mdd_event(struct ixl_pf *pf)
 		reg = rd32(hw, I40E_VP_MDET_TX(i));
 		if (reg & I40E_VP_MDET_TX_VALID_MASK) {
 			wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
-			vp_mdet_num = i;
-			vf->num_mdd_events++;
-			vf_mdd_detected = true;
+			if (!(vf->vf_flags & VF_FLAG_ENABLED))
+				continue;
+			vf->mdd_tx_events++;
+			vf->mdd_event_pending = true;
+			if (!vf->mdd_blocked) {
+				vf->mdd_blocked = true;
+				vf->mdd_reset_pending = true;
+			}
 		}
 	}
-
-	/* Print out an error message */
-	if (vf_mdd_detected && pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on TX queue %d, pf number %d (PF-%d), vf number %d (VF-%d)\n",
-		    event, queue, pf_num, pf_mdet_num, vf_num, vp_mdet_num);
-	else if (vf_mdd_detected && !pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on TX queue %d, pf number %d, vf number %d (VF-%d)\n",
-		    event, queue, pf_num, vf_num, vp_mdet_num);
-	else if (!vf_mdd_detected && pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on TX queue %d, pf number %d (PF-%d)\n",
-		    event, queue, pf_num, pf_mdet_num);
-	/* Theoretically shouldn't happen */
-	else
-		device_printf(dev,
-		    "TX Malicious Driver Detection event (unknown)\n");
 }
 
 static void
@@ -1916,12 +1898,8 @@ ixl_handle_rx_mdd_event(struct ixl_pf *pf)
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct ixl_vf *vf;
-	bool mdd_detected = false;
-	bool pf_mdd_detected = false;
-	bool vf_mdd_detected = false;
 	u16 queue;
 	u8 pf_num, event;
-	u8 pf_mdet_num, vp_mdet_num;
 	u32 reg;
 
 	/*
@@ -1936,18 +1914,20 @@ ixl_handle_rx_mdd_event(struct ixl_pf *pf)
 		    I40E_GL_MDET_RX_EVENT_SHIFT;
 		queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
 		    I40E_GL_MDET_RX_QUEUE_SHIFT;
+		if (queue >= hw->func_caps.base_queue)
+			queue -= hw->func_caps.base_queue;
+		device_printf(dev,
+		    "last RX malicious-driver cause %#x on queue %u, "
+		    "function %#x\n", event, queue, pf_num);
 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
-		mdd_detected = true;
 	}
 
-	if (!mdd_detected)
-		return;
-
 	reg = rd32(hw, I40E_PF_MDET_RX);
 	if (reg & I40E_PF_MDET_RX_VALID_MASK) {
 		wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
-		pf_mdet_num = hw->pf_id;
-		pf_mdd_detected = true;
+		device_printf(dev,
+		    "RX malicious-driver issue detected on PF-%u\n",
+		    hw->pf_id);
 	}
 
 	/* Check if MDD was caused by a VF */
@@ -1956,32 +1936,16 @@ ixl_handle_rx_mdd_event(struct ixl_pf *pf)
 		reg = rd32(hw, I40E_VP_MDET_RX(i));
 		if (reg & I40E_VP_MDET_RX_VALID_MASK) {
 			wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
-			vp_mdet_num = i;
-			vf->num_mdd_events++;
-			vf_mdd_detected = true;
+			if (!(vf->vf_flags & VF_FLAG_ENABLED))
+				continue;
+			vf->mdd_rx_events++;
+			vf->mdd_event_pending = true;
+			if (!vf->mdd_blocked) {
+				vf->mdd_blocked = true;
+				vf->mdd_reset_pending = true;
+			}
 		}
 	}
-
-	/* Print out an error message */
-	if (vf_mdd_detected && pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on RX queue %d, pf number %d (PF-%d), (VF-%d)\n",
-		    event, queue, pf_num, pf_mdet_num, vp_mdet_num);
-	else if (vf_mdd_detected && !pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on RX queue %d, pf number %d, (VF-%d)\n",
-		    event, queue, pf_num, vp_mdet_num);
-	else if (!vf_mdd_detected && pf_mdd_detected)
-		device_printf(dev,
-		    "Malicious Driver Detection event %d"
-		    " on RX queue %d, pf number %d (PF-%d)\n",
-		    event, queue, pf_num, pf_mdet_num);
-	/* Theoretically shouldn't happen */
-	else
-		device_printf(dev,
-		    "RX Malicious Driver Detection event (unknown)\n");
 }
 
 /**
@@ -1994,6 +1958,10 @@ void
 ixl_handle_mdd_event(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
+	struct ixl_vf *vf;
+	static const struct timeval log_interval = { 2, 0 };
+	bool reset;
+	int i;
 	u32 reg;
 
 	/*
@@ -2002,6 +1970,34 @@ ixl_handle_mdd_event(struct ixl_pf *pf)
 	 */
 	ixl_handle_tx_mdd_event(pf);
 	ixl_handle_rx_mdd_event(pf);
+	for (i = 0; i < pf->num_vfs; i++) {
+		vf = &pf->vfs[i];
+		if (!vf->mdd_event_pending)
+			continue;
+		vf->mdd_event_pending = false;
+		reset = vf->mdd_reset_pending;
+		vf->mdd_reset_pending = false;
+		if (ratecheck(&vf->last_mdd_log, &log_interval)) {
+			device_printf(pf->dev,
+			    "malicious-driver event from VF-%d "
+			    "(tx %ju, rx %ju); %s\n", i,
+			    (uintmax_t)vf->mdd_tx_events,
+			    (uintmax_t)vf->mdd_rx_events,
+			    pf->mdd_auto_reset_vf && reset ?
+			    "resetting VF" : "VF remains blocked");
+		}
+#ifdef PCI_IOV
+		if (pf->mdd_auto_reset_vf && reset) {
+			int error;
+
+			error = ixl_reset_vf_on_mdd(pf, i);
+			if (error != 0)
+				device_printf(pf->dev,
+				    "failed to reset MDD-blocked VF-%d: %d\n",
+				    i, error);
+		}
+#endif
+	}
 
 	ixl_clear_state(&pf->state, IXL_STATE_MDD_PENDING);