SMR protection for inpcbs

With introduction of epoch(9) synchronization to network stack the
inpcb database became protected by the network epoch together with
static network data (interfaces, addresses, etc).  However, inpcb
aren't static in nature, they are created and destroyed all the
time, which creates some traffic on the epoch(9) garbage collector.

Fairly new feature of uma(9) - Safe Memory Reclamation allows to
safely free memory in page-sized batches, with virtually zero
overhead compared to uma_zfree().  However, unlike epoch(9), it
puts stricter requirement on the access to the protected memory,
needing the critical(9) section to access it.  Details:

- The database is already build on CK lists, thanks to epoch(9).
- For write access nothing is changed.
- For a lookup in the database SMR section is now required.
  Once the desired inpcb is found we need to transition from SMR
  section to r/w lock on the inpcb itself, with a check that inpcb
  isn't yet freed.  This requires some compexity, since SMR section
  itself is a critical(9) section.  The complexity is hidden from
  KPI users in inp_smr_lock().
- For a inpcb list traversal (a pcblist sysctl, or broadcast
  notification) also a new KPI is provided, that hides internals of
  the database - inp_next(struct inp_iterator *).

Reviewed by:		rrs
Differential revision:	https://reviews.freebsd.org/D33022

1 files changed, 142 insertions, 161 deletions

diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index efd5c77ca8c5..cd21b8640441 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -147,9 +147,7 @@ u_long	udp_recvspace = 40 * (1024 +
 SYSCTL_ULONG(_net_inet_udp, UDPCTL_RECVSPACE, recvspace, CTLFLAG_RW,
     &udp_recvspace, 0, "Maximum space for incoming UDP datagrams");
 
-VNET_DEFINE(struct inpcbhead, udb);		/* from udp_var.h */
 VNET_DEFINE(struct inpcbinfo, udbinfo);
-VNET_DEFINE(struct inpcbhead, ulitecb);
 VNET_DEFINE(struct inpcbinfo, ulitecbinfo);
 VNET_DEFINE_STATIC(uma_zone_t, udpcb_zone);
 #define	V_udpcb_zone			VNET(udpcb_zone)
@@ -211,8 +209,8 @@ udp_init(void)
 	 * Once we can calculate the flowid that way and re-establish
 	 * a 4-tuple, flip this to 4-tuple.
 	 */
-	in_pcbinfo_init(&V_udbinfo, "udp", &V_udb, UDBHASHSIZE, UDBHASHSIZE,
-	    "udp_inpcb", udp_inpcb_init, IPI_HASHFIELDS_2TUPLE);
+	in_pcbinfo_init(&V_udbinfo, "udp", UDBHASHSIZE, UDBHASHSIZE,
+	    "udp_inpcb", udp_inpcb_init);
 	V_udpcb_zone = uma_zcreate("udpcb", sizeof(struct udpcb),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 	uma_zone_set_max(V_udpcb_zone, maxsockets);
@@ -225,9 +223,8 @@ void
 udplite_init(void)
 {
 
-	in_pcbinfo_init(&V_ulitecbinfo, "udplite", &V_ulitecb, UDBHASHSIZE,
-	    UDBHASHSIZE, "udplite_inpcb", udplite_inpcb_init,
-	    IPI_HASHFIELDS_2TUPLE);
+	in_pcbinfo_init(&V_ulitecbinfo, "udplite", UDBHASHSIZE,
+	    UDBHASHSIZE, "udplite_inpcb", udplite_inpcb_init);
 }
 
 /*
@@ -393,6 +390,123 @@ udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off,
 	return (0);
 }
 
+static bool
+udp_multi_match(const struct inpcb *inp, void *v)
+{
+	struct ip *ip = v;
+	struct udphdr *uh = (struct udphdr *)(ip + 1);
+
+	if (inp->inp_lport != uh->uh_dport)
+		return (false);
+#ifdef INET6
+	if ((inp->inp_vflag & INP_IPV4) == 0)
+		return (false);
+#endif
+	if (inp->inp_laddr.s_addr != INADDR_ANY &&
+	    inp->inp_laddr.s_addr != ip->ip_dst.s_addr)
+		return (false);
+	if (inp->inp_faddr.s_addr != INADDR_ANY &&
+	    inp->inp_faddr.s_addr != ip->ip_src.s_addr)
+		return (false);
+	if (inp->inp_fport != 0 &&
+	    inp->inp_fport != uh->uh_sport)
+		return (false);
+
+	return (true);
+}
+
+static int
+udp_multi_input(struct mbuf *m, int proto, struct sockaddr_in *udp_in)
+{
+	struct ip *ip = mtod(m, struct ip *);
+	struct inpcb_iterator inpi = INP_ITERATOR(udp_get_inpcbinfo(proto),
+	    INPLOOKUP_RLOCKPCB, udp_multi_match, ip);
+	struct udphdr *uh = (struct udphdr *)(ip + 1);
+	struct inpcb *inp;
+	struct mbuf *n;
+	int appends = 0;
+
+	MPASS(ip->ip_hl == sizeof(struct ip) >> 2);
+
+	while ((inp = inp_next(&inpi)) != NULL) {
+		/*
+		 * XXXRW: Because we weren't holding either the inpcb
+		 * or the hash lock when we checked for a match
+		 * before, we should probably recheck now that the
+		 * inpcb lock is held.
+		 */
+		/*
+		 * Handle socket delivery policy for any-source
+		 * and source-specific multicast. [RFC3678]
+		 */
+		if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
+			struct ip_moptions	*imo;
+			struct sockaddr_in	 group;
+			int			 blocked;
+
+			imo = inp->inp_moptions;
+			if (imo == NULL)
+				continue;
+			bzero(&group, sizeof(struct sockaddr_in));
+			group.sin_len = sizeof(struct sockaddr_in);
+			group.sin_family = AF_INET;
+			group.sin_addr = ip->ip_dst;
+
+			blocked = imo_multi_filter(imo, m->m_pkthdr.rcvif,
+				(struct sockaddr *)&group,
+				(struct sockaddr *)&udp_in[0]);
+			if (blocked != MCAST_PASS) {
+				if (blocked == MCAST_NOTGMEMBER)
+					IPSTAT_INC(ips_notmember);
+				if (blocked == MCAST_NOTSMEMBER ||
+				    blocked == MCAST_MUTED)
+					UDPSTAT_INC(udps_filtermcast);
+				continue;
+			}
+		}
+		if ((n = m_copym(m, 0, M_COPYALL, M_NOWAIT)) != NULL) {
+			if (proto == IPPROTO_UDPLITE)
+				UDPLITE_PROBE(receive, NULL, inp, ip, inp, uh);
+			else
+				UDP_PROBE(receive, NULL, inp, ip, inp, uh);
+			if (udp_append(inp, ip, n, sizeof(struct ip), udp_in)) {
+				INP_RUNLOCK(inp);
+				break;
+			} else
+				appends++;
+		}
+		/*
+		 * Don't look for additional matches if this one does
+		 * not have either the SO_REUSEPORT or SO_REUSEADDR
+		 * socket options set.  This heuristic avoids
+		 * searching through all pcbs in the common case of a
+		 * non-shared port.  It assumes that an application
+		 * will never clear these options after setting them.
+		 */
+		if ((inp->inp_socket->so_options &
+		    (SO_REUSEPORT|SO_REUSEPORT_LB|SO_REUSEADDR)) == 0) {
+			INP_RUNLOCK(inp);
+			break;
+		}
+	}
+	m_freem(m);
+
+	if (appends == 0) {
+		/*
+		 * No matching pcb found; discard datagram.  (No need
+		 * to send an ICMP Port Unreachable for a broadcast
+		 * or multicast datgram.)
+		 */
+		UDPSTAT_INC(udps_noport);
+		if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)))
+			UDPSTAT_INC(udps_noportmcast);
+		else
+			UDPSTAT_INC(udps_noportbcast);
+	}
+
+	return (IPPROTO_DONE);
+}
+
 int
 udp_input(struct mbuf **mp, int *offp, int proto)
 {
@@ -519,140 +633,15 @@ udp_input(struct mbuf **mp, int *offp, int proto)
 		}
 	}
 
-	pcbinfo = udp_get_inpcbinfo(proto);
 	if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
-	    in_broadcast(ip->ip_dst, ifp)) {
-		struct inpcb *last;
-		struct inpcbhead *pcblist;
-
-		NET_EPOCH_ASSERT();
-
-		pcblist = udp_get_pcblist(proto);
-		last = NULL;
-		CK_LIST_FOREACH(inp, pcblist, inp_list) {
-			if (inp->inp_lport != uh->uh_dport)
-				continue;
-#ifdef INET6
-			if ((inp->inp_vflag & INP_IPV4) == 0)
-				continue;
-#endif
-			if (inp->inp_laddr.s_addr != INADDR_ANY &&
-			    inp->inp_laddr.s_addr != ip->ip_dst.s_addr)
-				continue;
-			if (inp->inp_faddr.s_addr != INADDR_ANY &&
-			    inp->inp_faddr.s_addr != ip->ip_src.s_addr)
-				continue;
-			if (inp->inp_fport != 0 &&
-			    inp->inp_fport != uh->uh_sport)
-				continue;
+	    in_broadcast(ip->ip_dst, ifp))
+			return (udp_multi_input(m, proto, udp_in));
 
-			INP_RLOCK(inp);
-
-			if (__predict_false(inp->inp_flags2 & INP_FREED)) {
-				INP_RUNLOCK(inp);
-				continue;
-			}
-
-			/*
-			 * XXXRW: Because we weren't holding either the inpcb
-			 * or the hash lock when we checked for a match
-			 * before, we should probably recheck now that the
-			 * inpcb lock is held.
-			 */
-
-			/*
-			 * Handle socket delivery policy for any-source
-			 * and source-specific multicast. [RFC3678]
-			 */
-			if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
-				struct ip_moptions	*imo;
-				struct sockaddr_in	 group;
-				int			 blocked;
-
-				imo = inp->inp_moptions;
-				if (imo == NULL) {
-					INP_RUNLOCK(inp);
-					continue;
-				}
-				bzero(&group, sizeof(struct sockaddr_in));
-				group.sin_len = sizeof(struct sockaddr_in);
-				group.sin_family = AF_INET;
-				group.sin_addr = ip->ip_dst;
-
-				blocked = imo_multi_filter(imo, ifp,
-					(struct sockaddr *)&group,
-					(struct sockaddr *)&udp_in[0]);
-				if (blocked != MCAST_PASS) {
-					if (blocked == MCAST_NOTGMEMBER)
-						IPSTAT_INC(ips_notmember);
-					if (blocked == MCAST_NOTSMEMBER ||
-					    blocked == MCAST_MUTED)
-						UDPSTAT_INC(udps_filtermcast);
-					INP_RUNLOCK(inp);
-					continue;
-				}
-			}
-			if (last != NULL) {
-				struct mbuf *n;
-
-				if ((n = m_copym(m, 0, M_COPYALL, M_NOWAIT)) !=
-				    NULL) {
-					if (proto == IPPROTO_UDPLITE)
-						UDPLITE_PROBE(receive, NULL, last, ip,
-						    last, uh);
-					else
-						UDP_PROBE(receive, NULL, last, ip, last,
-						    uh);
-					if (udp_append(last, ip, n, iphlen,
-						udp_in)) {
-						INP_RUNLOCK(inp);
-						goto badunlocked;
-					}
-				}
-				/* Release PCB lock taken on previous pass. */
-				INP_RUNLOCK(last);
-			}
-			last = inp;
-			/*
-			 * Don't look for additional matches if this one does
-			 * not have either the SO_REUSEPORT or SO_REUSEADDR
-			 * socket options set.  This heuristic avoids
-			 * searching through all pcbs in the common case of a
-			 * non-shared port.  It assumes that an application
-			 * will never clear these options after setting them.
-			 */
-			if ((last->inp_socket->so_options &
-			    (SO_REUSEPORT|SO_REUSEPORT_LB|SO_REUSEADDR)) == 0)
-				break;
-		}
-
-		if (last == NULL) {
-			/*
-			 * No matching pcb found; discard datagram.  (No need
-			 * to send an ICMP Port Unreachable for a broadcast
-			 * or multicast datgram.)
-			 */
-			UDPSTAT_INC(udps_noport);
-			if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)))
-				UDPSTAT_INC(udps_noportmcast);
-			else
-				UDPSTAT_INC(udps_noportbcast);
-			goto badunlocked;
-		}
-		if (proto == IPPROTO_UDPLITE)
-			UDPLITE_PROBE(receive, NULL, last, ip, last, uh);
-		else
-			UDP_PROBE(receive, NULL, last, ip, last, uh);
-		if (udp_append(last, ip, m, iphlen, udp_in) == 0)
-			INP_RUNLOCK(last);
-		return (IPPROTO_DONE);
-	}
+	pcbinfo = udp_get_inpcbinfo(proto);
 
 	/*
 	 * Locate pcb for datagram.
-	 */
-
-	/*
+	 *
 	 * Grab info from PACKET_TAG_IPFORWARD tag prepended to the chain.
 	 */
 	if ((m->m_flags & M_IP_NEXTHOP) &&
@@ -852,8 +841,9 @@ udplite_ctlinput(int cmd, struct sockaddr *sa, void *vip)
 static int
 udp_pcblist(SYSCTL_HANDLER_ARGS)
 {
+	struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_udbinfo,
+	    INPLOOKUP_RLOCKPCB);
 	struct xinpgen xig;
-	struct epoch_tracker et;
 	struct inpcb *inp;
 	int error;
 
@@ -881,24 +871,19 @@ udp_pcblist(SYSCTL_HANDLER_ARGS)
 	if (error)
 		return (error);
 
-	NET_EPOCH_ENTER(et);
-	for (inp = CK_LIST_FIRST(V_udbinfo.ipi_listhead);
-	    inp != NULL;
-	    inp = CK_LIST_NEXT(inp, inp_list)) {
-		INP_RLOCK(inp);
+	while ((inp = inp_next(&inpi)) != NULL) {
 		if (inp->inp_gencnt <= xig.xig_gen &&
 		    cr_canseeinpcb(req->td->td_ucred, inp) == 0) {
 			struct xinpcb xi;
 
 			in_pcbtoxinpcb(inp, &xi);
-			INP_RUNLOCK(inp);
 			error = SYSCTL_OUT(req, &xi, sizeof xi);
-			if (error)
+			if (error) {
+				INP_RUNLOCK(inp);
 				break;
-		} else
-			INP_RUNLOCK(inp);
+			}
+		}
 	}
-	NET_EPOCH_EXIT(et);
 
 	if (!error) {
 		/*
@@ -1284,15 +1269,16 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 	laddr = inp->inp_laddr;
 	lport = inp->inp_lport;
 	if (src.sin_family == AF_INET) {
-		INP_HASH_LOCK_ASSERT(pcbinfo);
 		if ((lport == 0) ||
 		    (laddr.s_addr == INADDR_ANY &&
 		     src.sin_addr.s_addr == INADDR_ANY)) {
 			error = EINVAL;
 			goto release;
 		}
+		INP_HASH_WLOCK(pcbinfo);
 		error = in_pcbbind_setup(inp, (struct sockaddr *)&src,
 		    &laddr.s_addr, &lport, td->td_ucred);
+		INP_HASH_WUNLOCK(pcbinfo);
 		if (error)
 			goto release;
 	}
@@ -1335,12 +1321,14 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 		    inp->inp_lport == 0 ||
 		    sin->sin_addr.s_addr == INADDR_ANY ||
 		    sin->sin_addr.s_addr == INADDR_BROADCAST) {
-			INP_HASH_LOCK_ASSERT(pcbinfo);
+			INP_HASH_WLOCK(pcbinfo);
 			error = in_pcbconnect_setup(inp, addr, &laddr.s_addr,
 			    &lport, &faddr.s_addr, &fport, NULL,
 			    td->td_ucred);
-			if (error)
+			if (error) {
+				INP_HASH_WUNLOCK(pcbinfo);
 				goto release;
+			}
 
 			/*
 			 * XXXRW: Why not commit the port if the address is
@@ -1357,7 +1345,6 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 				if (prison_flag(td->td_ucred, PR_IP4))
 					inp->inp_laddr = laddr;
 				inp->inp_lport = lport;
-				INP_HASH_WLOCK(pcbinfo);
 				error = in_pcbinshash(inp);
 				INP_HASH_WUNLOCK(pcbinfo);
 				if (error != 0) {
@@ -1366,7 +1353,8 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 					goto release;
 				}
 				inp->inp_flags |= INP_ANONPORT;
-			}
+			} else
+				INP_HASH_WUNLOCK(pcbinfo);
 		} else {
 			faddr = sin->sin_addr;
 			fport = sin->sin_port;
@@ -1560,12 +1548,9 @@ udp_attach(struct socket *so, int proto, struct thread *td)
 	error = soreserve(so, udp_sendspace, udp_recvspace);
 	if (error)
 		return (error);
-	INP_INFO_WLOCK(pcbinfo);
 	error = in_pcballoc(so, pcbinfo);
-	if (error) {
-		INP_INFO_WUNLOCK(pcbinfo);
+	if (error)
 		return (error);
-	}
 
 	inp = sotoinpcb(so);
 	inp->inp_vflag |= INP_IPV4;
@@ -1577,12 +1562,10 @@ udp_attach(struct socket *so, int proto, struct thread *td)
 	if (error) {
 		in_pcbdetach(inp);
 		in_pcbfree(inp);
-		INP_INFO_WUNLOCK(pcbinfo);
 		return (error);
 	}
-
 	INP_WUNLOCK(inp);
-	INP_INFO_WUNLOCK(pcbinfo);
+
 	return (0);
 }
 #endif /* INET */
@@ -1718,14 +1701,12 @@ udp_detach(struct socket *so)
 	KASSERT(inp != NULL, ("udp_detach: inp == NULL"));
 	KASSERT(inp->inp_faddr.s_addr == INADDR_ANY,
 	    ("udp_detach: not disconnected"));
-	INP_INFO_WLOCK(pcbinfo);
 	INP_WLOCK(inp);
 	up = intoudpcb(inp);
 	KASSERT(up != NULL, ("%s: up == NULL", __func__));
 	inp->inp_ppcb = NULL;
 	in_pcbdetach(inp);
 	in_pcbfree(inp);
-	INP_INFO_WUNLOCK(pcbinfo);
 	udp_discardcb(up);
 }