Embed dst sockaddr into rtentry and remove rte packet counter

Currently each rtentry has dst&gateway allocated separately from another zone,
 bloating cache accesses.

Current 'struct rtentry' has 12 "mandatory" radix pointers in the beginning,
 leaving 4 usable pointers/32 bytes in the first 2 cache lines (amd64).
Fields needed for the datapath are destination sockaddr and rt_nhop.

So far it doesn't look like there is other routable addressing protocol other
 than IPv4/IPv6/MPLS, which uses keys longer than 20 bytes.
With that in mind, embed dst into struct rtentry, making the first 24 bytes
 of rtentry within 128 bytes. That is enough to make IPv6 address within first
 128 bytes.

It is still pretty easy to add code for supporting separately-allocated dst,
 however it doesn't make a lot of sense in having such code without a use case.

As rS359823 moved the gateway to the nexthop structure, the dst embedding change
 removes the need for any additional allocations done by rt_setgate().

Lastly, as a part of cleanup, remove counter(9) allocation code, as this field
 is not used in packet processing anymore.

Reviewed by:	ae
Differential Revision:	https://reviews.freebsd.org/D24669

5 files changed, 30 insertions, 64 deletions

diff --git a/sys/net/route.c b/sys/net/route.c
index 5de7b39e2cd2..666cba5dd28e 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -241,10 +241,6 @@ rtentry_zinit(void *mem, int size, int how)
 {
 	struct rtentry *rt = mem;
 
-	rt->rt_pksent = counter_u64_alloc(how);
-	if (rt->rt_pksent == NULL)
-		return (ENOMEM);
-
 	RT_LOCK_INIT(rt);
 
 	return (0);
@@ -256,7 +252,6 @@ rtentry_zfini(void *mem, int size)
 	struct rtentry *rt = mem;
 
 	RT_LOCK_DESTROY(rt);
-	counter_u64_free(rt->rt_pksent);
 }
 
 static int
@@ -265,7 +260,6 @@ rtentry_ctor(void *mem, int size, void *arg, int how)
 	struct rtentry *rt = mem;
 
 	bzero(rt, offsetof(struct rtentry, rt_endzero));
-	counter_u64_zero(rt->rt_pksent);
 	rt->rt_chain = NULL;
 
 	return (0);
@@ -551,12 +545,6 @@ rtfree(struct rtentry *rt)
 			goto done;
 		}
 #endif
-		/*
-		 * The key is separatly alloc'd so free it (see rt_setgate()).
-		 * This also frees the gateway, as they are always malloc'd
-		 * together.
-		 */
-		R_Free(rt_key(rt));
 
 		/* Unreference nexthop */
 		nhop_free(rt->rt_nhop);
@@ -1557,6 +1545,9 @@ add_route(struct rib_head *rnh, struct rt_addrinfo *info,
 	    (gateway->sa_family != AF_UNSPEC) && (gateway->sa_family != AF_LINK))
 		return (EINVAL);
 
+	if (dst->sa_len > sizeof(((struct rtentry *)NULL)->rt_dstb))
+		return (EINVAL);
+
 	if (info->rti_ifa == NULL) {
 		error = rt_getifa_fib(info, rnh->rib_fibnum);
 		if (error)
@@ -1582,15 +1573,10 @@ add_route(struct rib_head *rnh, struct rt_addrinfo *info,
 	rt->rt_flags = RTF_UP | flags;
 	rt->rt_fibnum = rnh->rib_fibnum;
 	rt->rt_nhop = nh;
-	/*
-	 * Add the gateway. Possibly re-malloc-ing the storage for it.
-	 */
-	if ((error = rt_setgate(rt, dst, gateway)) != 0) {
-		ifa_free(info->rti_ifa);
-		nhop_free(nh);
-		uma_zfree(V_rtzone, rt);
-		return (error);
-	}
+
+	/* Fill in dst */
+	memcpy(&rt->rt_dst, dst, dst->sa_len);
+	rt_key(rt) = &rt->rt_dst;
 
 	/*
 	 * point to the (possibly newly malloc'd) dest address.
@@ -1623,7 +1609,6 @@ add_route(struct rib_head *rnh, struct rt_addrinfo *info,
 		rt_mpath_conflict(rnh, rt, netmask)) {
 		RIB_WUNLOCK(rnh);
 
-		R_Free(rt_key(rt));
 		nhop_free(nh);
 		uma_zfree(V_rtzone, rt);
 		return (EEXIST);
@@ -1663,7 +1648,6 @@ add_route(struct rib_head *rnh, struct rt_addrinfo *info,
 	 * then un-make it (this should be a function)
 	 */
 	if (rn == NULL) {
-		R_Free(rt_key(rt));
 		nhop_free(nh);
 		uma_zfree(V_rtzone, rt);
 		return (EEXIST);
@@ -1899,40 +1883,6 @@ rt_setmetrics(const struct rt_addrinfo *info, struct rtentry *rt)
 		    info->rti_rmx->rmx_expire - time_second + time_uptime : 0;
 }
 
-int
-rt_setgate(struct rtentry *rt, struct sockaddr *dst, struct sockaddr *gate)
-{
-	/* XXX dst may be overwritten, can we move this to below */
-	int dlen = SA_SIZE(dst), glen = SA_SIZE(gate);
-
-	/*
-	 * Prepare to store the gateway in rt->rt_gateway.
-	 * Both dst and gateway are stored one after the other in the same
-	 * malloc'd chunk. If we have room, we can reuse the old buffer,
-	 * rt_gateway already points to the right place.
-	 * Otherwise, malloc a new block and update the 'dst' address.
-	 */
-	if (rt_key(rt) == NULL) {
-		caddr_t new;
-
-		R_Malloc(new, caddr_t, dlen + glen);
-		if (new == NULL)
-			return ENOBUFS;
-		/*
-		 * XXX note, we copy from *dst and not *rt_key(rt) because
-		 * rt_setgate() can be called to initialize a newly
-		 * allocated route entry, in which case rt_key(rt) == NULL
-		 * (and also rt->rt_gateway == NULL).
-		 * Free()/free() handle a NULL argument just fine.
-		 */
-		bcopy(dst, new, dlen);
-		R_Free(rt_key(rt));	/* free old block, if any */
-		rt_key(rt) = (struct sockaddr *)new;
-	}
-
-	return (0);
-}
-
 void
 rt_maskedcopy(struct sockaddr *src, struct sockaddr *dst, struct sockaddr *netmask)
 {
diff --git a/sys/net/route.h b/sys/net/route.h
index e834105f55a8..c13e627c54bd 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -378,7 +378,6 @@ int	 rt_addrmsg(int, struct ifaddr *, int);
 int	 rt_routemsg(int, struct rtentry *, struct ifnet *ifp, int, int);
 int	 rt_routemsg_info(int, struct rt_addrinfo *, int);
 void	 rt_newmaddrmsg(int, struct ifmultiaddr *);
-int	 rt_setgate(struct rtentry *, struct sockaddr *, struct sockaddr *);
 void 	 rt_maskedcopy(struct sockaddr *, struct sockaddr *, struct sockaddr *);
 struct rib_head *rt_table_init(int, int, u_int);
 void	rt_table_destroy(struct rib_head *);
diff --git a/sys/net/route/route_ddb.c b/sys/net/route/route_ddb.c
index 216f94f0c682..3a129cb634ac 100644
--- a/sys/net/route/route_ddb.c
+++ b/sys/net/route/route_ddb.c
@@ -48,11 +48,9 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_dl.h>
-#include <net/vnet.h>
 #include <net/route.h>
-#include <net/route/route_var.h>
 #include <net/route/nhop.h>
-#include <netinet/in.h>
+#include <net/route/route_var.h>
 
 /*
  * Unfortunately, RTF_ values are expressed as raw masks rather than powers of
diff --git a/sys/net/route/route_var.h b/sys/net/route/route_var.h
index f6d1d4874863..02bf15204fe6 100644
--- a/sys/net/route/route_var.h
+++ b/sys/net/route/route_var.h
@@ -35,6 +35,7 @@
 #ifndef RNF_NORMAL
 #include <net/radix.h>
 #endif
+#include <netinet/in.h>		/* struct sockaddr_in */
 #include <sys/counter.h>
 
 struct nh_control;
@@ -120,14 +121,33 @@ struct rtentry {
 #define	rt_mask(r)	(*((struct sockaddr **)(&(r)->rt_nodes->rn_mask)))
 #define	rt_key_const(r)		(*((const struct sockaddr * const *)(&(r)->rt_nodes->rn_key)))
 #define	rt_mask_const(r)	(*((const struct sockaddr * const *)(&(r)->rt_nodes->rn_mask)))
+
+	/*
+	 * 2 radix_node structurs above consists of 2x6 pointers, leaving
+	 * 4 pointers (32 bytes) of the second cache line on amd64.
+	 *
+	 */
 	struct nhop_object	*rt_nhop;	/* nexthop data */
+	union {
+		/*
+		 * Destination address storage.
+		 * sizeof(struct sockaddr_in6) == 28, however
+		 * the dataplane-relevant part (e.g. address) lies
+		 * at offset 8..24, making the address not crossing
+		 * cacheline boundary.
+		 */
+		struct sockaddr_in	rt_dst4;
+		struct sockaddr_in6	rt_dst6;
+		struct sockaddr		rt_dst;
+		char			rt_dstb[28];
+	};
+
 	int		rt_flags;	/* up/down?, host/net */
 	int		rt_refcnt;	/* # held references */
 	u_int		rt_fibnum;	/* which FIB */
 	u_long		rt_weight;	/* absolute weight */ 
 	u_long		rt_expire;	/* lifetime for route, e.g. redirect */
-#define	rt_endzero	rt_pksent
-	counter_u64_t	rt_pksent;	/* packets sent using this route */
+#define	rt_endzero	rt_mtx
 	struct mtx	rt_mtx;		/* mutex for routing entry */
 	struct rtentry	*rt_chain;	/* pointer to next rtentry to delete */
 };
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index 2e1dc85927c0..e474da739cbb 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -1078,7 +1078,6 @@ rt_getmetrics(const struct rtentry *rt, struct rt_metrics *out)
 	bzero(out, sizeof(*out));
 	out->rmx_mtu = rt->rt_nhop->nh_mtu;
 	out->rmx_weight = rt->rt_weight;
-	out->rmx_pksent = counter_u64_fetch(rt->rt_pksent);
 	out->rmx_nhidx = nhop_get_idx(rt->rt_nhop);
 	/* Kernel -> userland timebase conversion. */
 	out->rmx_expire = rt->rt_expire ?