iflib: Use the framework run-state accessor in drivers

Replace direct IFF_DRV_RUNNING reads in igbv, ixgbe, ixv, iavf, ixl,
ice, bnxt Ethernet, aq, enic and the axgbe PCI frontend with
iflib_is_running().  Keep each existing mailbox-ready, fault-state,
link-state and administrative-up condition: framework admission is not
a substitute for device specific readiness.

This covers interrupt admission, operational link publication, VF
mailbox and VLAN replay, and live-configuration decisions.  Remove
ifnet temporaries used only to read the driver flags.

Replace the em and igc debug dumps' RUNNING/OACTIVE text with one
framework admission snapshot.  OACTIVE does not mean that the interface
or hardware is inactive.  Label the result as software admission rather
than link state or proof that DMA has stopped, and retain the queue
head/tail dumps and separate RS diagnostics.  Keep the filter register
and requested interface flags in the axgbe PCI promiscuous-mode trace,
but remove its unrelated legacy driver flags.

Use the parent Ethernet PF context for the bnxt RDMA running check,
retaining the link-state check and the existing IB_PORT_ACTIVE policy.
Both current RDMA construction paths use the PF netdev, and Ethernet
detach synchronously removes the RDMA auxiliary child before iflib frees
that context.  Declare the direct iflib module dependency.  This does
not transfer ownership of RDMA queues to iflib.

Leave the independent, non-iflib axgbe ARM frontend unchanged.

Reviewed by:	iflib (gallatin)
MFC after:	2 weeks
Sponsored by:	BBOX.io
Differential Revision:	https://reviews.freebsd.org/D59601

15 files changed, 34 insertions, 61 deletions

diff --git a/sys/dev/aq/aq_irq.c b/sys/dev/aq/aq_irq.c
index 374a638289a9..d3c1d5043969 100644
--- a/sys/dev/aq/aq_irq.c
+++ b/sys/dev/aq/aq_irq.c
@@ -357,7 +357,7 @@ aq_if_update_admin_status(if_ctx_t ctx)
 	}
 
 	/* A stopped or half-initialized interface has no link. */
-	running = (if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) != 0;
+	running = iflib_is_running(ctx);
 	if (!running || aq_dev->init_failed)
 		link_speed = 0;
 
diff --git a/sys/dev/axgbe/if_axgbe_pci.c b/sys/dev/axgbe/if_axgbe_pci.c
index 00b4c6c1d451..0153247495e3 100644
--- a/sys/dev/axgbe/if_axgbe_pci.c
+++ b/sys/dev/axgbe/if_axgbe_pci.c
@@ -357,7 +357,7 @@ axgbe_miibus_statchg(device_t dev)
 	    pdata->phy_link);
 
 	if (mii == NULL || ifp == NULL ||
-	    (if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
+	    !iflib_is_running(sc->ctx))
 		return;
 
 	if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
@@ -2357,11 +2357,9 @@ axgbe_if_promisc_set(if_ctx_t ctx, int flags)
 {
 	struct axgbe_if_softc *sc = iflib_get_softc(ctx);
 	struct xgbe_prv_data *pdata = &sc->pdata;
-	if_t ifp = pdata->netdev;
 
-	axgbe_printf(1, "%s: MAC_PFR 0x%x drv_flags 0x%x if_flags 0x%x\n",
-	    __func__, XGMAC_IOREAD(pdata, MAC_PFR), if_getdrvflags(ifp),
-	    flags);
+	axgbe_printf(1, "%s: MAC_PFR 0x%x if_flags 0x%x\n",
+	    __func__, XGMAC_IOREAD(pdata, MAC_PFR), flags);
 
 	if (flags & IFF_PROMISC) {
 
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sriov.c b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c
index 071feffbadfd..378cb2c77dfa 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_sriov.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c
@@ -899,7 +899,7 @@ bnxt_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params)
 	if_t ifp = iflib_get_ifp(ctx);
 	struct bnxt_softc *softc = iflib_get_softc(ctx);
 	bool admin_up = !!(if_getflags(ifp) & IFF_UP);
-	bool running  = !!(if_getdrvflags(ifp) & IFF_DRV_RUNNING);
+	bool running = iflib_is_running(ctx);
 
 	if (!admin_up || !running) {
 		device_printf(softc->dev, "PF is down, rejecting VF creation\n");
diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sysctl.c b/sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
index 741cba4588aa..88afc83edd9d 100644
--- a/sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
+++ b/sys/dev/bnxt/bnxt_en/bnxt_sysctl.c
@@ -1283,7 +1283,7 @@ bnxt_rss_key_sysctl(SYSCTL_HANDLER_ARGS)
 		p += 2;
 	}
 
-	if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)
+	if (iflib_is_running(softc->ctx))
 		bnxt_hwrm_rss_cfg(softc, &softc->vnic_info,
 		    softc->vnic_info.rss_hash_type);
 
@@ -1338,7 +1338,7 @@ bnxt_rss_type_sysctl(SYSCTL_HANDLER_ARGS)
 	}
 	if (type != softc->vnic_info.rss_hash_type) {
 		softc->vnic_info.rss_hash_type = type;
-		if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)
+		if (iflib_is_running(softc->ctx))
 			bnxt_hwrm_rss_cfg(softc, &softc->vnic_info,
 			    softc->vnic_info.rss_hash_type);
 	}
@@ -1365,7 +1365,7 @@ bnxt_rx_stall_sysctl(SYSCTL_HANDLER_ARGS) {
 	else
 		softc->vnic_info.flags &= ~BNXT_VNIC_FLAG_BD_STALL;
 
-	if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)
+	if (iflib_is_running(softc->ctx))
 		rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info);
 
 	return rc;
@@ -1390,7 +1390,7 @@ bnxt_vlan_strip_sysctl(SYSCTL_HANDLER_ARGS) {
 	else
 		softc->vnic_info.flags &= ~BNXT_VNIC_FLAG_VLAN_STRIP;
 
-	if (if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)
+	if (iflib_is_running(softc->ctx))
 		rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info);
 
 	return rc;
@@ -1675,7 +1675,7 @@ fn_name(SYSCTL_HANDLER_ARGS) {				                   \
 	if (rc || !req->newptr)				                   \
 		return rc;				                   \
 							                   \
-	if ((if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)) \
+	if (iflib_is_running(softc->ctx)) \
 		return EBUSY;				                   \
 							                   \
 	if (!(softc->flags & BNXT_FLAG_TPA))				   \
diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h
index 2e7f027380be..60ff33c1a802 100644
--- a/sys/dev/bnxt/bnxt_re/bnxt_re.h
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h
@@ -933,7 +933,7 @@ int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset);
 
 static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev)
 {
-	if (if_getdrvflags(rdev->netdev) & IFF_DRV_RUNNING &&
+	if (iflib_is_running(rdev->en_dev->softc->ctx) &&
 	    if_getlinkstate(rdev->netdev) == LINK_STATE_UP)
 		return IB_PORT_ACTIVE;
 	return IB_PORT_DOWN;
diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c
index e6ec286006fa..295125218d03 100644
--- a/sys/dev/bnxt/bnxt_re/main.c
+++ b/sys/dev/bnxt/bnxt_re/main.c
@@ -66,6 +66,7 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1);
 MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1);
 MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1);
+MODULE_DEPEND(bnxt_re, iflib, 1, 1, 1);
 MODULE_VERSION(bnxt_re, 1);
 
 
diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index 2e872b82680f..4961dd94072e 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -8631,7 +8631,6 @@ static void
 em_print_debug_info(struct e1000_softc *sc)
 {
 	device_t dev = iflib_get_dev(sc->ctx);
-	if_t ifp = iflib_get_ifp(sc->ctx);
 	struct tx_ring *txr;
 	struct rx_ring *rxr;
 
@@ -8639,15 +8638,8 @@ em_print_debug_info(struct e1000_softc *sc)
 		device_printf(dev, "queue state is unavailable\n");
 		return;
 	}
-	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
-		printf("Interface is RUNNING ");
-	else
-		printf("Interface is NOT RUNNING\n");
-
-	if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE)
-		printf("and INACTIVE\n");
-	else
-		printf("and ACTIVE\n");
+	device_printf(dev, "iflib software admission: %s\n",
+	    iflib_is_running(sc->ctx) ? "open" : "closed");
 
 	for (int i = 0; i < sc->tx_num_queues; i++) {
 		txr = &sc->tx_queues[i].txr;
diff --git a/sys/dev/e1000/if_igbv.c b/sys/dev/e1000/if_igbv.c
index 9947f4c19e2e..355b37fdcbee 100644
--- a/sys/dev/e1000/if_igbv.c
+++ b/sys/dev/e1000/if_igbv.c
@@ -446,7 +446,7 @@ igbv_if_update_admin_status(if_ctx_t ctx)
 	dev = iflib_get_dev(ctx);
 	KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
 
-	if ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) == 0 ||
+	if (!iflib_is_running(ctx) ||
 	    !sc->vf_queues_sanitized ||
 	    atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
 		if (sc->link_state != EM_LINK_STATE_DOWN) {
@@ -508,9 +508,8 @@ igbv_if_update_admin_status(if_ctx_t ctx)
 	    atomic_readandclear_32(&sc->stats_pending) != 0;
 	if (timer_tick) {
 		em_update_stats_counters(sc);
-		/* iflib clears RUNNING before stop; do not replay after reset. */
-		if ((if_getdrvflags(iflib_get_ifp(ctx)) &
-		    IFF_DRV_RUNNING) != 0)
+		/* iflib closes admission before stop; do not replay after reset. */
+		if (iflib_is_running(ctx))
 			igbv_vlan_retry_tick(sc);
 	}
 }
diff --git a/sys/dev/enic/if_enic.c b/sys/dev/enic/if_enic.c
index 8d8f75d4611d..82bed15e3f67 100644
--- a/sys/dev/enic/if_enic.c
+++ b/sys/dev/enic/if_enic.c
@@ -877,11 +877,9 @@ static int
 enic_rxq_intr(void *rxq)
 {
 	struct vnic_rq *rq;
-	if_t ifp;
 
 	rq = (struct vnic_rq *)rxq;
-	ifp = iflib_get_ifp(rq->vdev->softc->ctx);
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
+	if (!iflib_is_running(rq->vdev->softc->ctx))
 		return (FILTER_HANDLED);
 
 	return (FILTER_SCHEDULE_THREAD);
diff --git a/sys/dev/iavf/if_iavf_iflib.c b/sys/dev/iavf/if_iavf_iflib.c
index aba31d05ee1d..25aee839648a 100644
--- a/sys/dev/iavf/if_iavf_iflib.c
+++ b/sys/dev/iavf/if_iavf_iflib.c
@@ -1558,10 +1558,9 @@ iavf_if_update_admin_status(if_ctx_t ctx)
 	struct iavf_sc *sc = iavf_sc_from_ctx(ctx);
 	struct iavf_hw *hw = &sc->hw;
 	struct iavf_vsi *vsi = &sc->vsi;
-	if_t ifp = iflib_get_ifp(ctx);
 	u16 pending = 0;
 
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
+	if (!iflib_is_running(ctx) ||
 	    atomic_load_acq_32(&sc->mbx_ready) == 0) {
 		if (vsi->link_active) {
 			vsi->link_active = false;
@@ -2075,11 +2074,9 @@ void
 iavf_update_link_status(struct iavf_sc *sc)
 {
 	struct iavf_vsi *vsi = &sc->vsi;
-	if_t ifp;
 	u64 baudrate;
 
-	ifp = iflib_get_ifp(vsi->ctx);
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
+	if (!iflib_is_running(vsi->ctx)) {
 		if (vsi->link_active) {
 			vsi->link_active = false;
 			iflib_link_state_change(vsi->ctx, LINK_STATE_DOWN, 0);
@@ -2419,7 +2416,7 @@ iavf_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS)
 }
 
 #ifdef IAVF_DEBUG
-#define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
+#define CTX_ACTIVE(ctx) iflib_is_running(ctx)
 
 /**
  * iavf_sysctl_vf_reset - Request a VF reset
diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c
index 41c34bcfc1e4..f6e619b2ebe5 100644
--- a/sys/dev/ice/if_ice_iflib.c
+++ b/sys/dev/ice/if_ice_iflib.c
@@ -2610,7 +2610,7 @@ err_release_tx_queues:
 }
 
 /* determine if the iflib context is active */
-#define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
+#define CTX_ACTIVE(ctx) iflib_is_running(ctx)
 
 /**
  * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
diff --git a/sys/dev/igc/if_igc.c b/sys/dev/igc/if_igc.c
index b07548821172..ed90021573ab 100644
--- a/sys/dev/igc/if_igc.c
+++ b/sys/dev/igc/if_igc.c
@@ -3887,17 +3887,9 @@ static void
 igc_print_debug_info(struct igc_softc *sc)
 {
 	device_t dev = iflib_get_dev(sc->ctx);
-	if_t ifp = iflib_get_ifp(sc->ctx);
 
-	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
-		printf("Interface is RUNNING ");
-	else
-		printf("Interface is NOT RUNNING\n");
-
-	if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE)
-		printf("and INACTIVE\n");
-	else
-		printf("and ACTIVE\n");
+	device_printf(dev, "iflib software admission: %s\n",
+	    iflib_is_running(sc->ctx) ? "open" : "closed");
 
 	for (int i = 0; i < sc->tx_num_queues; i++) {
 		device_printf(dev, "TX Queue %d ------\n", i);
diff --git a/sys/dev/ixgbe/if_ix.c b/sys/dev/ixgbe/if_ix.c
index b2a133c80b55..f5ceb1f0b5eb 100644
--- a/sys/dev/ixgbe/if_ix.c
+++ b/sys/dev/ixgbe/if_ix.c
@@ -2973,10 +2973,9 @@ ixgbe_msix_que(void *arg)
 {
 	struct ix_rx_queue *que = arg;
 	struct ixgbe_softc *sc = que->sc;
-	if_t ifp = iflib_get_ifp(que->sc->ctx);
 
 	/* Protect against spurious interrupts */
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
+	if (!iflib_is_running(sc->ctx))
 		return (FILTER_HANDLED);
 
 	ixgbe_disable_queue(sc, que->msix);
@@ -5226,7 +5225,7 @@ ixgbe_if_update_admin_status(if_ctx_t ctx)
 	 * MOD and firmware events can produce dependent requests.  Fold those
 	 * into the claimed batch so link state is sampled after any link setup.
 	 */
-	if ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) != 0 &&
+	if (iflib_is_running(ctx) &&
 	    (sc->iov_mbx_cleanup_pending || ixgbe_mbx_pending(sc)))
 		atomic_set_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX);
 	for (pass = 0; pass < IXGBE_ADMIN_TASK_BUDGET; pass++) {
@@ -5245,7 +5244,7 @@ ixgbe_if_update_admin_status(if_ctx_t ctx)
 			ixgbe_handle_msf(ctx);
 		/* A reset request can re-enable VF traffic; skip it while stopped. */
 		if ((requests & IXGBE_REQUEST_TASK_MBX) != 0 &&
-		    (if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING) != 0)
+		    iflib_is_running(ctx))
 			ixgbe_handle_mbx(ctx);
 		if (requests & IXGBE_REQUEST_TASK_FDIR)
 			ixgbe_reinit_fdir(ctx);
@@ -5683,7 +5682,7 @@ ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc)
 	/* Updating SRRCTL on a live queue is itself an MDD violation. */
 	mdd_active = sc->num_rx_queues > 1 &&
 	    (sc->feat_en & IXGBE_FEATURE_SRIOV) != 0 &&
-	    (if_getdrvflags(iflib_get_ifp(sc->ctx)) & IFF_DRV_RUNNING) != 0;
+	    iflib_is_running(sc->ctx);
 	if (mdd_active)
 		ixgbe_disable_mdd(&sc->hw);
 	if (sc->num_rx_queues > 1) {
@@ -6004,7 +6003,7 @@ ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS)
 	}
 
 	/* Re-initialize hardware if it's already running */
-	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
+	if (iflib_is_running(sc->ctx))
 		if_init(ifp, ifp);
 
 	return (0);
diff --git a/sys/dev/ixgbe/if_ixv.c b/sys/dev/ixgbe/if_ixv.c
index 62e92be75197..b2c192caaae1 100644
--- a/sys/dev/ixgbe/if_ixv.c
+++ b/sys/dev/ixgbe/if_ixv.c
@@ -1187,11 +1187,9 @@ static int
 ixv_if_promisc_set(if_ctx_t ctx, int flags)
 {
 	struct ixgbe_softc *sc;
-	if_t ifp;
 
 	sc = iflib_get_softc(ctx);
-	ifp = iflib_get_ifp(ctx);
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
+	if (!iflib_is_running(ctx))
 		return (0);
 	if (ixv_update_xcast_mode(sc, flags) != IXGBE_SUCCESS)
 		return (EOPNOTSUPP);
@@ -1306,12 +1304,11 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 {
 	struct ixgbe_softc *sc = iflib_get_softc(ctx);
 	device_t dev = iflib_get_dev(ctx);
-	if_t ifp = iflib_get_ifp(ctx);
 	bool check_link, reset_seen;
 	s32 status;
 	uint64_t baudrate;
 
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 ||
+	if (!iflib_is_running(ctx) ||
 	    atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
 		if (sc->link_active) {
 			sc->link_active = false;
@@ -1391,8 +1388,8 @@ ixv_if_update_admin_status(if_ctx_t ctx)
 		}
 	}
 
-	/* iflib clears RUNNING before stop; do not replay after VF reset. */
-	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 &&
+	/* iflib closes admission before stop; do not replay after VF reset. */
+	if (iflib_is_running(ctx) &&
 	    atomic_readandclear_32(&sc->vf_vlan_retry_tick) != 0)
 		ixv_vlan_retry_tick(sc);
 
diff --git a/sys/dev/ixl/ixl_pf_main.c b/sys/dev/ixl/ixl_pf_main.c
index edc3581bd53f..4c7907a5a9be 100644
--- a/sys/dev/ixl/ixl_pf_main.c
+++ b/sys/dev/ixl/ixl_pf_main.c
@@ -2054,7 +2054,7 @@ void
 ixl_handle_empr_reset(struct ixl_pf *pf)
 {
 	struct ixl_vsi	*vsi = &pf->vsi;
-	bool is_up = !!(if_getdrvflags(vsi->ifp) & IFF_DRV_RUNNING);
+	bool is_up = iflib_is_running(vsi->ctx);
 
 	ixl_prepare_for_reset(pf, is_up);
 	/*