aboutsummaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
authorAlexander V. Chernikov <melifaro@FreeBSD.org>2020-10-18 17:15:47 +0000
committerAlexander V. Chernikov <melifaro@FreeBSD.org>2020-10-18 17:15:47 +0000
commit0c325f53f16731f608919a4489f96fbbe28d2344 (patch)
tree47edfa6af78113b91e7dad7f002817c6e110b01f /sys
parent186bcdaac7c70e59eb04412ad402a6fb57b97d11 (diff)
downloadsrc-0c325f53f16731f608919a4489f96fbbe28d2344.tar.gz
src-0c325f53f16731f608919a4489f96fbbe28d2344.zip
Implement flowid calculation for outbound connections to balance
connections over multiple paths. Multipath routing relies on mbuf flowid data for both transit and outbound traffic. Current code fills mbuf flowid from inp_flowid for connection-oriented sockets. However, inp_flowid is currently not calculated for outbound connections. This change creates simple hashing functions and starts calculating hashes for TCP,UDP/UDP-Lite and raw IP if multipath routes are present in the system. Reviewed by: glebius (previous version),ae Differential Revision: https://reviews.freebsd.org/D26523
Notes
Notes: svn path=/head/; revision=366813
Diffstat (limited to 'sys')
-rw-r--r--sys/conf/files2
-rw-r--r--sys/net/radix.c17
-rw-r--r--sys/net/route.h34
-rw-r--r--sys/net/route/mpath_ctl.c25
-rw-r--r--sys/net/route/route_var.h4
-rw-r--r--sys/netinet/in_fib.c35
-rw-r--r--sys/netinet/in_fib.h3
-rw-r--r--sys/netinet/in_pcb.c11
-rw-r--r--sys/netinet/in_rss.c42
-rw-r--r--sys/netinet/in_rss.h2
-rw-r--r--sys/netinet/raw_ip.c25
-rw-r--r--sys/netinet/udp_usrreq.c32
-rw-r--r--sys/netinet6/in6_fib.c34
-rw-r--r--sys/netinet6/in6_fib.h3
-rw-r--r--sys/netinet6/in6_pcb.c12
-rw-r--r--sys/netinet6/in6_rss.c44
-rw-r--r--sys/netinet6/in6_rss.h3
-rw-r--r--sys/netinet6/ip6_output.c6
-rw-r--r--sys/netinet6/raw_ip6.c13
-rw-r--r--sys/netinet6/udp6_usrreq.c42
20 files changed, 312 insertions, 77 deletions
diff --git a/sys/conf/files b/sys/conf/files
index 9b1a52f9ba59..f4a296546f1e 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4180,7 +4180,7 @@ net/rss_config.c optional inet rss | inet6 rss
net/rtsock.c standard
net/slcompress.c optional netgraph_vjc | sppp | \
netgraph_sppp
-net/toeplitz.c optional inet rss | inet6 rss
+net/toeplitz.c optional inet rss | inet6 rss | route_mpath
net/vnet.c optional vimage
net80211/ieee80211.c optional wlan
net80211/ieee80211_acl.c optional wlan wlan_acl
diff --git a/sys/net/radix.c b/sys/net/radix.c
index f65153393d74..410278e9a2e9 100644
--- a/sys/net/radix.c
+++ b/sys/net/radix.c
@@ -624,21 +624,6 @@ rn_addroute(void *v_arg, void *n_arg, struct radix_head *head,
saved_tt = tt = rn_insert(v, head, &keyduplicated, treenodes);
if (keyduplicated) {
for (t = tt; tt; t = tt, tt = tt->rn_dupedkey) {
-#ifdef RADIX_MPATH
- /* permit multipath, if enabled for the family */
- if (rn_mpath_capable(head) && netmask == tt->rn_mask) {
- /*
- * go down to the end of multipaths, so that
- * new entry goes into the end of rn_dupedkey
- * chain.
- */
- do {
- t = tt;
- tt = tt->rn_dupedkey;
- } while (tt && t->rn_mask == tt->rn_mask);
- break;
- }
-#endif
if (tt->rn_mask == netmask)
return (0);
if (netmask == 0 ||
@@ -744,10 +729,8 @@ on2:
if (m->rm_flags & RNF_NORMAL) {
mmask = m->rm_leaf->rn_mask;
if (tt->rn_flags & RNF_NORMAL) {
-#if !defined(RADIX_MPATH)
log(LOG_ERR,
"Non-unique normal route, mask not entered\n");
-#endif
return (tt);
}
} else
diff --git a/sys/net/route.h b/sys/net/route.h
index 021b4621692b..1ce808b6c1e1 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -125,8 +125,42 @@ VNET_DECLARE(uint32_t, _rt_numfibs); /* number of existing route tables */
#define rt_numfibs V_rt_numfibs
VNET_DECLARE(u_int, rt_add_addr_allfibs); /* Announce interfaces to all fibs */
#define V_rt_add_addr_allfibs VNET(rt_add_addr_allfibs)
+
+/* Calculate flowid for locally-originated packets */
+#define V_fib_hash_outbound VNET(fib_hash_outbound)
+VNET_DECLARE(u_int, fib_hash_outbound);
+
+/* Outbound flowid generation rules */
+#ifdef RSS
+
+#define fib4_calc_packet_hash xps_proto_software_hash_v4
+#define fib6_calc_packet_hash xps_proto_software_hash_v6
+#define CALC_FLOWID_OUTBOUND_SENDTO true
+
+#ifdef ROUTE_MPATH
+#define CALC_FLOWID_OUTBOUND V_fib_hash_outbound
+#else
+#define CALC_FLOWID_OUTBOUND false
#endif
+#else /* !RSS */
+
+#define fib4_calc_packet_hash fib4_calc_software_hash
+#define fib6_calc_packet_hash fib6_calc_software_hash
+
+#ifdef ROUTE_MPATH
+#define CALC_FLOWID_OUTBOUND_SENDTO V_fib_hash_outbound
+#define CALC_FLOWID_OUTBOUND V_fib_hash_outbound
+#else
+#define CALC_FLOWID_OUTBOUND_SENDTO false
+#define CALC_FLOWID_OUTBOUND false
+#endif
+
+#endif /* RSS */
+
+
+#endif /* _KERNEL */
+
/*
* We distinguish between routes to hosts and routes to networks,
* preferring the former if available. For each route we infer
diff --git a/sys/net/route/mpath_ctl.c b/sys/net/route/mpath_ctl.c
index 1ac7c191ed05..5632750f466d 100644
--- a/sys/net/route/mpath_ctl.c
+++ b/sys/net/route/mpath_ctl.c
@@ -64,6 +64,20 @@
*/
SYSCTL_DECL(_net_route);
+VNET_DEFINE(u_int, fib_hash_outbound) = 0;
+SYSCTL_UINT(_net_route, OID_AUTO, hash_outbound, CTLFLAG_RD | CTLFLAG_VNET,
+ &VNET_NAME(fib_hash_outbound), 0,
+ "Compute flowid for locally-originated packets");
+
+/* Default entropy to add to the hash calculation for the outbound connections*/
+uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN] = {
+ 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
+ 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
+ 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
+ 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
+ 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
+};
+
/*
* Tries to add @rnd_add nhop to the existing set of nhops (@nh_orig) for the
@@ -115,6 +129,17 @@ add_route_mpath(struct rib_head *rnh, struct rt_addrinfo *info,
RTSTAT_INC(rts_add_retry);
}
+ if (V_fib_hash_outbound == 0 && error == 0 &&
+ NH_IS_NHGRP(rc->rc_nh_new)) {
+ /*
+ * First multipath route got installed. Enable local
+ * outbound connections hashing.
+ */
+ if (bootverbose)
+ printf("FIB: enabled flowid calculation for locally-originated packets\n");
+ V_fib_hash_outbound = 1;
+ }
+
return (error);
}
diff --git a/sys/net/route/route_var.h b/sys/net/route/route_var.h
index 12d081d410a2..00e6c3a59b04 100644
--- a/sys/net/route/route_var.h
+++ b/sys/net/route/route_var.h
@@ -307,4 +307,8 @@ int nhgrp_get_addition_group(struct rib_head *rnh,
void nhgrp_free(struct nhgrp_object *nhg);
+/* Entropy data used for outbound hashing */
+#define MPATH_ENTROPY_KEY_LEN 40
+extern uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN];
+
#endif
diff --git a/sys/netinet/in_fib.c b/sys/netinet/in_fib.c
index 4c84de2c7281..031277add777 100644
--- a/sys/netinet/in_fib.c
+++ b/sys/netinet/in_fib.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
#include <net/route/route_ctl.h>
#include <net/route/route_var.h>
#include <net/route/nhop.h>
+#include <net/toeplitz.h>
#include <net/vnet.h>
#include <netinet/in.h>
@@ -62,6 +63,40 @@ __FBSDID("$FreeBSD$");
/* Assert 'struct route_in' is compatible with 'struct route' */
CHK_STRUCT_ROUTE_COMPAT(struct route_in, ro_dst4);
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv4 {
+ struct in_addr src;
+ struct in_addr dst;
+ unsigned short src_port;
+ unsigned short dst_port;
+ char proto;
+ char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv4) == 16,
+ "_hash_5tuple_ipv4 size is wrong");
+
+
+uint32_t
+fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype)
+{
+ struct _hash_5tuple_ipv4 data;
+
+ data.src = src;
+ data.dst = dst;
+ data.src_port = src_port;
+ data.dst_port = dst_port;
+ data.proto = proto;
+ data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+ *phashtype = M_HASHTYPE_OPAQUE;
+
+ return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+ sizeof(data), (uint8_t *)&data));
+}
+#endif
+
/*
* Looks up path in fib @fibnum specified by @dst.
* Returns path nexthop on success. Nexthop is safe to use
diff --git a/sys/netinet/in_fib.h b/sys/netinet/in_fib.h
index bc2a2ad6a30e..0d93086bc116 100644
--- a/sys/netinet/in_fib.h
+++ b/sys/netinet/in_fib.h
@@ -51,4 +51,7 @@ int fib4_check_urpf(uint32_t fibnum, struct in_addr dst, uint32_t scopeid,
uint32_t flags, const struct ifnet *src_if);
struct nhop_object *fib4_lookup_debugnet(uint32_t fibnum, struct in_addr dst,
uint32_t scopeid, uint32_t flags);
+uint32_t fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype);
#endif
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 30bf72f83e33..37438dd68c71 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_ratelimit.h"
#include "opt_pcbgroup.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -1327,7 +1328,17 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
lport = *lportp;
faddr = sin->sin_addr;
fport = sin->sin_port;
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_val, hash_type;
+ hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
+ inp->inp_socket->so_proto->pr_protocol, &hash_type);
+
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
/*
* If the destination address is INADDR_ANY,
diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c
index f3184175a7cf..05659b97fe7c 100644
--- a/sys/netinet/in_rss.c
+++ b/sys/netinet/in_rss.c
@@ -152,6 +152,48 @@ rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
}
/*
+ * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
+ * IPv4 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple. This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoing" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+ uint32_t hash;
+
+ /*
+ * Next, choose the hash type depending upon the protocol
+ * identifier.
+ */
+ if ((proto == IPPROTO_TCP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+ hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_TCP_IPV4;
+ return (hash);
+ } else if ((proto == IPPROTO_UDP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+ hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_UDP_IPV4;
+ return (hash);
+ } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
+ /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+ hash = rss_hash_ip4_2tuple(d, s);
+ *hashtype = M_HASHTYPE_RSS_IPV4;
+ return (hash);
+ }
+
+ *hashtype = M_HASHTYPE_NONE;
+ return (0);
+}
+
+/*
* Do a software calculation of the RSS for the given mbuf.
*
* This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet/in_rss.h b/sys/netinet/in_rss.h
index fd300ac5fdff..cdc69bc64709 100644
--- a/sys/netinet/in_rss.h
+++ b/sys/netinet/in_rss.h
@@ -53,5 +53,7 @@ int rss_proto_software_hash_v4(struct in_addr src,
uint32_t *hashtype);
struct mbuf * rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source,
u_int *cpuid);
+uint32_t xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype);
#endif /* !_NETINET_IN_RSS_H_ */
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index 6240a77bdb39..a63fc19587f9 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include <sys/param.h>
#include <sys/jail.h>
@@ -67,6 +68,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/in.h>
#include <netinet/in_systm.h>
+#include <netinet/in_fib.h>
#include <netinet/in_pcb.h>
#include <netinet/in_var.h>
#include <netinet/if_ether.h>
@@ -484,6 +486,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
ip->ip_len = htons(m->m_pkthdr.len);
ip->ip_src = inp->inp_laddr;
ip->ip_dst.s_addr = dst;
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib4_calc_software_hash(ip->ip_src,
+ ip->ip_dst, 0, 0, ip->ip_p, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
+ flags |= IP_NODEFAULTFLOWID;
+ }
+#endif
if (jailed(inp->inp_cred)) {
/*
* prison_local_ip4() would be good enough but would
@@ -519,7 +532,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
return (EINVAL);
ip = mtod(m, struct ip *);
}
-
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib4_calc_software_hash(ip->ip_dst,
+ ip->ip_src, 0, 0, ip->ip_p, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
+ flags |= IP_NODEFAULTFLOWID;
+ }
+#endif
INP_RLOCK(inp);
/*
* Don't allow both user specified and setsockopt options,
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index e1bb5f07c0d0..52304ddd6584 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -76,6 +77,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
+#include <netinet/in_fib.h>
#include <netinet/in_pcb.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
@@ -1483,30 +1485,14 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
m->m_pkthdr.flowid = flowid;
M_HASHTYPE_SET(m, flowtype);
}
-#ifdef RSS
- else {
+#if defined(ROUTE_MPATH) || defined(RSS)
+ else if (CALC_FLOWID_OUTBOUND_SENDTO) {
uint32_t hash_val, hash_type;
- /*
- * Calculate an appropriate RSS hash for UDP and
- * UDP Lite.
- *
- * The called function will take care of figuring out
- * whether a 2-tuple or 4-tuple hash is required based
- * on the currently configured scheme.
- *
- * Later later on connected socket values should be
- * cached in the inpcb and reused, rather than constantly
- * re-calculating it.
- *
- * UDP Lite is a different protocol number and will
- * likely end up being hashed as a 2-tuple until
- * RSS / NICs grow UDP Lite protocol awareness.
- */
- if (rss_proto_software_hash_v4(faddr, laddr, fport, lport,
- pr, &hash_val, &hash_type) == 0) {
- m->m_pkthdr.flowid = hash_val;
- M_HASHTYPE_SET(m, hash_type);
- }
+
+ hash_val = fib4_calc_packet_hash(laddr, faddr,
+ lport, fport, pr, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
}
/*
diff --git a/sys/netinet6/in6_fib.c b/sys/netinet6/in6_fib.c
index 9fd869b2730b..a3fd20f8ba25 100644
--- a/sys/netinet6/in6_fib.c
+++ b/sys/netinet6/in6_fib.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
#include <net/route/route_ctl.h>
#include <net/route/route_var.h>
#include <net/route/nhop.h>
+#include <net/toeplitz.h>
#include <net/vnet.h>
#include <netinet/in.h>
@@ -68,6 +69,39 @@ __FBSDID("$FreeBSD$");
CHK_STRUCT_ROUTE_COMPAT(struct route_in6, ro_dst);
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv6 {
+ struct in6_addr src;
+ struct in6_addr dst;
+ unsigned short src_port;
+ unsigned short dst_port;
+ char proto;
+ char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv6) == 40,
+ "_hash_5tuple_ipv6 size is wrong");
+
+uint32_t
+fib6_calc_software_hash(const struct in6_addr *src, const struct in6_addr *dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype)
+{
+ struct _hash_5tuple_ipv6 data;
+
+ data.src = *src;
+ data.dst = *dst;
+ data.src_port = src_port;
+ data.dst_port = dst_port;
+ data.proto = proto;
+ data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+ *phashtype = M_HASHTYPE_OPAQUE_HASH;
+
+ return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+ sizeof(data), (uint8_t *)&data));
+}
+#endif
+
/*
* Looks up path in fib @fibnum specified by @dst.
* Assumes scope is deembedded and provided in @scopeid.
diff --git a/sys/netinet6/in6_fib.h b/sys/netinet6/in6_fib.h
index a716674bce23..78edb5e5b776 100644
--- a/sys/netinet6/in6_fib.h
+++ b/sys/netinet6/in6_fib.h
@@ -39,4 +39,7 @@ int fib6_check_urpf(uint32_t fibnum, const struct in6_addr *dst6,
uint32_t scopeid, uint32_t flags, const struct ifnet *src_if);
struct nhop_object *fib6_lookup_debugnet(uint32_t fibnum,
const struct in6_addr *dst6, uint32_t scopeid, uint32_t flags);
+uint32_t fib6_calc_software_hash(const struct in6_addr *src,
+ const struct in6_addr *dst, unsigned short src_port, unsigned short dst_port,
+ char proto, uint32_t *phashtype);
#endif
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 3444ca1b2fe7..ab498bf91734 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include "opt_pcbgroup.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -423,6 +424,17 @@ in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(pcbinfo);
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+ &sin6->sin6_addr, 0, sin6->sin6_port,
+ inp->inp_socket->so_proto->pr_protocol, &hash_type);
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
/*
* Call inner routine, to assign local interface address.
* in6_pcbladdr() may automatically fill in sin6_scope_id.
diff --git a/sys/netinet6/in6_rss.c b/sys/netinet6/in6_rss.c
index 72dd0a8831af..2b86d961515b 100644
--- a/sys/netinet6/in6_rss.c
+++ b/sys/netinet6/in6_rss.c
@@ -153,6 +153,50 @@ rss_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
}
/*
+ * Calculate an appropriate ipv6 2-tuple or 4-tuple given the given
+ * IPv6 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple. This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoin" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+
+ uint32_t hash;
+
+ /*
+ * Next, choose the hash type depending upon the protocol
+ * identifier.
+ */
+ if ((proto == IPPROTO_TCP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
+ hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_TCP_IPV6;
+ return (hash);
+ } else if ((proto == IPPROTO_UDP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
+ hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_UDP_IPV6;
+ return (hash);
+ } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) {
+ /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+ hash = rss_hash_ip6_2tuple(d, s);
+ *hashtype = M_HASHTYPE_RSS_IPV6;
+ return (hash);
+ }
+
+ *hashtype = M_HASHTYPE_NONE;
+ return (0);
+}
+
+
+/*
* Do a software calculation of the RSS for the given mbuf.
*
* This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet6/in6_rss.h b/sys/netinet6/in6_rss.h
index f5b48c71408a..5846e515d185 100644
--- a/sys/netinet6/in6_rss.h
+++ b/sys/netinet6/in6_rss.h
@@ -54,5 +54,8 @@ int rss_proto_software_hash_v6(const struct in6_addr *src,
uint32_t *hashtype);
struct mbuf * rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source,
u_int *cpuid);
+uint32_t xps_proto_software_hash_v6(const struct in6_addr *s,
+ const struct in6_addr *d, u_short sp, u_short dp,
+ int proto, uint32_t *hashtype);
#endif /* !_NETINET6_IN6_RSS_H_ */
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 59aabaa258f5..4e8f22a01b0a 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -419,9 +419,6 @@ done:
*
* ifpp - XXX: just for statistics
*/
-/*
- * XXX TODO: no flowid is assigned for outbound flows?
- */
int
ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
struct route_in6 *ro, int flags, struct ip6_moptions *im6o,
@@ -775,7 +772,8 @@ again:
}
}
- nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
+ nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE,
+ m->m_pkthdr.flowid);
if (nh == NULL) {
IP6STAT_INC(ip6s_noroute);
/* No ifp in6_ifstat_inc(ifp, ifs6_out_discard); */
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index 636828eb4ef4..aea99add4391 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
#include "opt_ipsec.h"
#include "opt_inet6.h"
+#include "opt_route.h"
#include <sys/param.h>
#include <sys/errno.h>
@@ -103,6 +104,7 @@ __FBSDID("$FreeBSD$");
#include <netinet6/ip6_var.h>
#include <netinet6/nd6.h>
#include <netinet6/raw_ip6.h>
+#include <netinet6/in6_fib.h>
#include <netinet6/scope6_var.h>
#include <netinet6/send.h>
@@ -462,6 +464,17 @@ rip6_output(struct mbuf *m, struct socket *so, ...)
}
ip6 = mtod(m, struct ip6_hdr *);
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+ &dstsock->sin6_addr, 0, 0, so->so_proto->pr_protocol,
+ &hash_type);
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
/*
* Source address selection.
*/
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 32aa21cb24a1..1535be90e1b0 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -115,6 +116,7 @@ __FBSDID("$FreeBSD$");
#include <netinet6/ip6protosw.h>
#include <netinet6/ip6_var.h>
+#include <netinet6/in6_fib.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/in6_rss.h>
#include <netinet6/udp6_var.h>
@@ -954,42 +956,20 @@ udp6_output(struct socket *so, int flags_arg, struct mbuf *m,
}
flags = 0;
-#ifdef RSS
- {
- uint32_t hash_val, hash_type;
+#if defined(ROUTE_MPATH) || defined(RSS)
+ if (CALC_FLOWID_OUTBOUND_SENDTO) {
+ uint32_t hash_type, hash_val;
uint8_t pr;
pr = inp->inp_socket->so_proto->pr_protocol;
- /*
- * Calculate an appropriate RSS hash for UDP and
- * UDP Lite.
- *
- * The called function will take care of figuring out
- * whether a 2-tuple or 4-tuple hash is required based
- * on the currently configured scheme.
- *
- * Later later on connected socket values should be
- * cached in the inpcb and reused, rather than constantly
- * re-calculating it.
- *
- * UDP Lite is a different protocol number and will
- * likely end up being hashed as a 2-tuple until
- * RSS / NICs grow UDP Lite protocol awareness.
- */
- if (rss_proto_software_hash_v6(faddr, laddr, fport,
- inp->inp_lport, pr, &hash_val, &hash_type) == 0) {
- m->m_pkthdr.flowid = hash_val;
- M_HASHTYPE_SET(m, hash_type);
- }
- /*
- * Don't override with the inp cached flowid.
- *
- * Until the whole UDP path is vetted, it may actually
- * be incorrect.
- */
- flags |= IP_NODEFAULTFLOWID;
+ hash_val = fib6_calc_packet_hash(laddr, faddr,
+ inp->inp_lport, fport, pr, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
}
+ /* do not use inp flowid */
+ flags |= IP_NODEFAULTFLOWID;
#endif
UDPSTAT_INC(udps_opackets);