iflib: Add restart transactions for IOV reconfiguration

Some devices remap the PF queues when entering or leaving SR-IOV. Add
opt-in PCI IOV helpers that hold the iflib context lock across the
complete stop, driver callback, and restart transaction.

Existing drivers continue to use the non-restarting helpers.

Sponsored by:	BBOX.io

2 files changed, 55 insertions, 0 deletions

diff --git a/sys/net/iflib.c b/sys/net/iflib.c
index 87626ae59815..b6a8ec1eb6ce 100644
--- a/sys/net/iflib.c
+++ b/sys/net/iflib.c
@@ -5545,6 +5545,37 @@ iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
 	return (error);
 }
 
+int
+iflib_device_iov_init_restart(device_t dev, uint16_t num_vfs,
+    const nvlist_t *params)
+{
+	if_ctx_t ctx;
+	if_t ifp;
+	int error;
+
+	ctx = device_get_softc(dev);
+	ifp = ctx->ifc_ifp;
+
+	CTX_LOCK(ctx);
+	/*
+	 * Drivers which change the PF queue layout need the complete iflib
+	 * stop/init sequence around their IOV callback.  Keep that transition
+	 * within one context-lock critical section.
+	 */
+	if ((if_getflags(ifp) & IFF_UP) == 0 ||
+	    (if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
+		error = ENETDOWN;
+		goto out;
+	}
+
+	iflib_stop(ctx);
+	error = IFDI_IOV_INIT(ctx, num_vfs, params);
+	iflib_init_locked(ctx);
+out:
+	CTX_UNLOCK(ctx);
+	return (error);
+}
+
 void
 iflib_device_iov_uninit(device_t dev)
 {
@@ -5555,6 +5586,28 @@ iflib_device_iov_uninit(device_t dev)
 	CTX_UNLOCK(ctx);
 }
 
+void
+iflib_device_iov_uninit_restart(device_t dev)
+{
+	if_ctx_t ctx;
+	bool restart;
+
+	ctx = device_get_softc(dev);
+
+	CTX_LOCK(ctx);
+	/*
+	 * RUNNING can be clear while a watchdog reset is pending but the
+	 * hardware is still live.  Always stop before the driver changes its
+	 * queue layout, and use IFF_UP only to preserve administrative state.
+	 */
+	restart = (if_getflags(ctx->ifc_ifp) & IFF_UP) != 0;
+	iflib_stop(ctx);
+	IFDI_IOV_UNINIT(ctx);
+	if (restart)
+		iflib_init_locked(ctx);
+	CTX_UNLOCK(ctx);
+}
+
 int
 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
 {
diff --git a/sys/net/iflib.h b/sys/net/iflib.h
index c73c50e5a3b3..6e9c6e10cc5d 100644
--- a/sys/net/iflib.h
+++ b/sys/net/iflib.h
@@ -450,7 +450,9 @@ int iflib_device_shutdown(device_t);
 int iflib_device_probe_vendor(device_t);
 
 int iflib_device_iov_init(device_t, uint16_t, const nvlist_t *);
+int iflib_device_iov_init_restart(device_t, uint16_t, const nvlist_t *);
 void iflib_device_iov_uninit(device_t);
+void iflib_device_iov_uninit_restart(device_t);
 int iflib_device_iov_add_vf(device_t, uint16_t, const nvlist_t *);
 
 /*