if_ntb: Transport link cleanup needs to be on a taskqueue

Because it can sleep drainking link work callout(s).  Linux (dual
BSD/GPL driver) does something very similar.

At the same time, switch the NTB CTX lock to a non-spin mutex, because
the taskqueue_swi lock can't be taken after a spin mutex.

Suggested by:	Witness
Sponsored by:	EMC / Isilon Storage Division

2 files changed, 14 insertions, 5 deletions

diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c
index 234795309599..34d8f26289e3 100644
--- a/sys/dev/ntb/if_ntb/if_ntb.c
+++ b/sys/dev/ntb/if_ntb/if_ntb.c
@@ -216,6 +216,7 @@ struct ntb_transport_ctx {
 	unsigned		qp_count;
 	enum ntb_link_event	link_is_up;
 	struct callout		link_work;
+	struct task		link_cleanup;
 	uint64_t		bufsize;
 	u_char			eaddr[ETHER_ADDR_LEN];
 	struct mtx		tx_lock;
@@ -304,6 +305,7 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
     unsigned int qp_num);
 static void ntb_qp_link_work(void *arg);
 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt);
+static void ntb_transport_link_cleanup_work(void *, int);
 static void ntb_qp_link_down(struct ntb_transport_qp *qp);
 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp);
 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
@@ -588,6 +590,7 @@ ntb_transport_probe(struct ntb_softc *ntb)
 	}
 
 	callout_init(&nt->link_work, 0);
+	TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt);
 
 	rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops);
 	if (rc != 0)
@@ -614,7 +617,7 @@ ntb_transport_free(struct ntb_transport_ctx *nt)
 	uint8_t i;
 
 	ntb_transport_link_cleanup(nt);
-
+	taskqueue_drain(taskqueue_swi, &nt->link_cleanup);
 	callout_drain(&nt->link_work);
 
 	BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc);
@@ -1178,7 +1181,7 @@ ntb_transport_event_callback(void *data)
 	} else {
 		if (bootverbose)
 			if_printf(nt->ifp, "HW link down\n");
-		ntb_transport_link_cleanup(nt);
+		taskqueue_enqueue(taskqueue_swi, &nt->link_cleanup);
 	}
 }
 
@@ -1448,6 +1451,12 @@ ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
 		ntb_spad_write(nt->ntb, i, 0);
 }
 
+static void
+ntb_transport_link_cleanup_work(void *arg, int pending __unused)
+{
+
+	ntb_transport_link_cleanup(arg);
+}
 
 static void
 ntb_qp_link_down(struct ntb_transport_qp *qp)
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c
index 395824be5cd1..e2bcd9cb4c5f 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw.c
@@ -182,8 +182,8 @@ struct ntb_softc {
 	void			*ntb_ctx;
 	const struct ntb_ctx_ops *ctx_ops;
 	struct ntb_vec		*msix_vec;
-#define CTX_LOCK(sc)		mtx_lock_spin(&(sc)->ctx_lock)
-#define CTX_UNLOCK(sc)		mtx_unlock_spin(&(sc)->ctx_lock)
+#define CTX_LOCK(sc)		mtx_lock(&(sc)->ctx_lock)
+#define CTX_UNLOCK(sc)		mtx_unlock(&(sc)->ctx_lock)
 #define CTX_ASSERT(sc,f)	mtx_assert(&(sc)->ctx_lock, (f))
 	struct mtx		ctx_lock;
 
@@ -512,7 +512,7 @@ ntb_attach(device_t device)
 	callout_init(&ntb->heartbeat_timer, 1);
 	callout_init(&ntb->lr_timer, 1);
 	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
-	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_SPIN);
+	mtx_init(&ntb->ctx_lock, "ntb ctx", NULL, MTX_DEF);
 
 	if (ntb->type == NTB_ATOM)
 		error = ntb_detect_atom(ntb);