IPv4 multicast: fix LOR in shutdown path

X_ip_mrouter_done() was calling the interface ioctl routines via
if_allmulti() while holding a write lock.  However, some interface
ioctl routines, including em/iflib and tap, use sxlocks, which are
not permitted while holding a non-sleepable lock, and this elicits
a warning from WITNESS.  Fix the locking issue by recording the
affected interface pointers in a malloc'ed array, and call
if_allmulti() on each after dropping the rwlock.

Reviewed by:	bz
Differential Revision: https://reviews.freebsd.org/D34845

1 files changed, 20 insertions, 6 deletions

diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c
index cc356e3678f3..f77898fb3aa4 100644
--- a/sys/netinet/ip_mroute.c
+++ b/sys/netinet/ip_mroute.c
@@ -747,7 +747,8 @@ ip_mrouter_init(struct socket *so, int version)
 static int
 X_ip_mrouter_done(void)
 {
-    struct ifnet *ifp;
+    struct ifnet **ifps;
+    int nifp;
     u_long i;
     vifi_t vifi;
     struct bw_upcall *bu;
@@ -774,6 +775,8 @@ X_ip_mrouter_done(void)
     	taskqueue_drain(V_task_queue, &V_task);
     }
 
+    ifps = malloc(MAXVIFS * sizeof(*ifps), M_TEMP, M_WAITOK);
+
     MRW_WLOCK();
     taskqueue_cancel(V_task_queue, &V_task, NULL);
 
@@ -785,14 +788,17 @@ X_ip_mrouter_done(void)
     mtx_destroy(&V_bw_upcalls_ring_mtx);
 
     /*
-     * For each phyint in use, disable promiscuous reception of all IP
-     * multicasts.
+     * For each phyint in use, prepare to disable promiscuous reception
+     * of all IP multicasts.  Defer the actual call until the lock is released;
+     * just record the list of interfaces while locked.  Some interfaces use
+     * sx locks in their ioctl routines, which is not allowed while holding
+     * a non-sleepable lock.
      */
-    for (vifi = 0; vifi < V_numvifs; vifi++) {
+    KASSERT(V_numvifs <= MAXVIFS, ("More vifs than possible"));
+    for (vifi = 0, nifp = 0; vifi < V_numvifs; vifi++) {
 	if (!in_nullhost(V_viftable[vifi].v_lcl_addr) &&
 		!(V_viftable[vifi].v_flags & (VIFF_TUNNEL | VIFF_REGISTER))) {
-	    ifp = V_viftable[vifi].v_ifp;
-	    if_allmulti(ifp, 0);
+	    ifps[nifp++] = V_viftable[vifi].v_ifp;
 	}
     }
     bzero((caddr_t)V_viftable, sizeof(*V_viftable) * MAXVIFS);
@@ -824,6 +830,14 @@ X_ip_mrouter_done(void)
 
     MRW_WUNLOCK();
 
+    /*
+     * Now drop our claim on promiscuous multicast on the interfaces recorded
+     * above.  This is safe to do now because ALLMULTI is reference counted.
+     */
+    for (vifi = 0; vifi < nifp; vifi++)
+	    if_allmulti(ifps[vifi], 0);
+    free(ifps, M_TEMP);
+
     CTR1(KTR_IPMF, "%s: done", __func__);
 
     return 0;