cxgbe(4): Fix virtual interface reattach.

Replace the DOOMED flag with a transient DETACHING flag that is cleared
when VI is detached.  This fixes VI reattach when only the VI and not
the parent nexus is detached.  The old flag was never cleared and
prevented subsequent synch op's related to the VI.

PR:		275260
Reviewed by:	jhb
Differential Revision: https://reviews.freebsd.org/D43287
Sponsored by:	Chelsio Communications

(cherry picked from commit 0201eb29dde9dccbf796f81fbf18f3b21798183d)

2 files changed, 25 insertions, 18 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index ddb8621a2300..c3105ecacffd 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -170,7 +170,7 @@ enum {
 	FIXED_IFMEDIA	= (1 << 4),	/* ifmedia list doesn't change. */
 
 	/* VI flags */
-	DOOMED		= (1 << 0),
+	VI_DETACHING	= (1 << 0),
 	VI_INIT_DONE	= (1 << 1),
 	/* 1 << 2 is unused, was VI_SYSCTL_CTX */
 	TX_USES_VM_WR	= (1 << 3),
@@ -184,8 +184,9 @@ enum {
 	DF_VERBOSE_SLOWINTR	= (1 << 4),	/* Chatty slow intr handler */
 };
 
-#define IS_DOOMED(vi)	((vi)->flags & DOOMED)
-#define SET_DOOMED(vi)	do {(vi)->flags |= DOOMED;} while (0)
+#define IS_DETACHING(vi)	((vi)->flags & VI_DETACHING)
+#define SET_DETACHING(vi)	do {(vi)->flags |= VI_DETACHING;} while (0)
+#define CLR_DETACHING(vi)	do {(vi)->flags &= ~VI_DETACHING;} while (0)
 #define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
 #define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
 #define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
@@ -1355,8 +1356,9 @@ int t4_map_bar_2(struct adapter *);
 int t4_setup_intr_handlers(struct adapter *);
 void t4_sysctls(struct adapter *);
 int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *);
-void doom_vi(struct adapter *, struct vi_info *);
 void end_synchronized_op(struct adapter *, int);
+void begin_vi_detach(struct adapter *, struct vi_info *);
+void end_vi_detach(struct adapter *, struct vi_info *);
 int update_mac_settings(if_t, int);
 int adapter_init(struct adapter *);
 int vi_init(struct vi_info *);
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 570208e2c562..3b6cb7285e64 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -2755,17 +2755,14 @@ cxgbe_detach(device_t dev)
 	device_delete_children(dev);
 
 	sysctl_ctx_free(&pi->ctx);
-	doom_vi(sc, &pi->vi[0]);
-
+	begin_vi_detach(sc, &pi->vi[0]);
 	if (pi->flags & HAS_TRACEQ) {
 		sc->traceq = -1;	/* cloner should not create ifnet */
 		t4_tracer_port_detach(sc);
 	}
-
 	cxgbe_vi_detach(&pi->vi[0]);
 	ifmedia_removeall(&pi->media);
-
-	end_synchronized_op(sc, 0);
+	end_vi_detach(sc, &pi->vi[0]);
 
 	return (0);
 }
@@ -3595,12 +3592,10 @@ vcxgbe_detach(device_t dev)
 	vi = device_get_softc(dev);
 	sc = vi->adapter;
 
-	doom_vi(sc, vi);
-
+	begin_vi_detach(sc, vi);
 	cxgbe_vi_detach(vi);
 	t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
-
-	end_synchronized_op(sc, 0);
+	end_vi_detach(sc, vi);
 
 	return (0);
 }
@@ -6245,7 +6240,7 @@ begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
 	ADAPTER_LOCK(sc);
 	for (;;) {
 
-		if (vi && IS_DOOMED(vi)) {
+		if (vi && IS_DETACHING(vi)) {
 			rc = ENXIO;
 			goto done;
 		}
@@ -6284,14 +6279,13 @@ done:
 /*
  * Tell if_ioctl and if_init that the VI is going away.  This is
  * special variant of begin_synchronized_op and must be paired with a
- * call to end_synchronized_op.
+ * call to end_vi_detach.
  */
 void
-doom_vi(struct adapter *sc, struct vi_info *vi)
+begin_vi_detach(struct adapter *sc, struct vi_info *vi)
 {
-
 	ADAPTER_LOCK(sc);
-	SET_DOOMED(vi);
+	SET_DETACHING(vi);
 	wakeup(&sc->flags);
 	while (IS_BUSY(sc))
 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
@@ -6304,6 +6298,17 @@ doom_vi(struct adapter *sc, struct vi_info *vi)
 	ADAPTER_UNLOCK(sc);
 }
 
+void
+end_vi_detach(struct adapter *sc, struct vi_info *vi)
+{
+	ADAPTER_LOCK(sc);
+	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
+	CLR_BUSY(sc);
+	CLR_DETACHING(vi);
+	wakeup(&sc->flags);
+	ADAPTER_UNLOCK(sc);
+}
+
 /*
  * {begin|end}_synchronized_op must be called from the same thread.
  */