Add full support for Receive Side Scaling, RSS, to the mlx5en

driver. This includes binding all interrupt and worker threads
according to the RSS configuration, setting up correct Toeplitz
hashing keys as given by RSS and setting the correct mbuf
hashtype for all received traffic.

MFC after:	1 week
Sponsored by:	Mellanox Technologies
Differential Revision:	https://reviews.freebsd.org/D4410

5 files changed, 136 insertions, 11 deletions

diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
index 11b560a295c4..59efb00cbf7e 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
@@ -31,6 +31,13 @@
 #include <dev/mlx5/mlx5_ifc.h>
 #include "mlx5_core.h"
 
+#include "opt_rss.h"
+
+#ifdef  RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
 enum {
 	MLX5_EQE_SIZE		= sizeof(struct mlx5_eqe),
 	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
@@ -389,6 +396,18 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 			  priv->irq_info[vecidx].name, eq);
 	if (err)
 		goto err_eq;
+#ifdef RSS
+	if (vecidx >= MLX5_EQ_VEC_COMP_BASE) {
+		u8 bucket = vecidx - MLX5_EQ_VEC_COMP_BASE;
+		err = bind_irq_to_cpu(priv->msix_arr[vecidx].vector,
+				      rss_getcpu(bucket % rss_getnumbuckets()));
+		if (err)
+			goto err_irq;
+	}
+#else
+	if (0)
+		goto err_irq;
+#endif
 
 
 	/* EQs are created in ARMED state
@@ -398,6 +417,8 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 	kvfree(in);
 	return 0;
 
+err_irq:
+	free_irq(priv->msix_arr[vecidx].vector, eq);
 
 err_eq:
 	mlx5_cmd_destroy_eq(dev, eq->eqn);
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 64976030c4ad..23e4fd0facb7 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -50,6 +50,13 @@
 #include <net/ethernet.h>
 #include <sys/buf_ring.h>
 
+#include "opt_rss.h"
+
+#ifdef	RSS
+#include <net/rss_config.h>
+#include <netinet/in_rss.h>
+#endif
+
 #include <machine/bus.h>
 
 #ifdef HAVE_TURBO_LRO
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 71a708ea22eb..77aaf7ba2636 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -931,6 +931,10 @@ mlx5e_create_sq(struct mlx5e_channel *c,
 
 	void *sqc = param->sqc;
 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
+#ifdef RSS
+	cpuset_t cpu_mask;
+	int cpu_id;
+#endif
 	int err;
 
 	/* Create DMA descriptor TAG */
@@ -991,9 +995,15 @@ mlx5e_create_sq(struct mlx5e_channel *c,
 	}
 
 	TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq);
-	taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s tx sq",
-	    c->ifp->if_xname);
-
+#ifdef RSS
+	cpu_id = rss_getcpu(c->ix % rss_getnumbuckets());
+	CPU_SETOF(cpu_id, &cpu_mask);
+	taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask,
+	    "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id);
+#else
+	taskqueue_start_threads(&sq->sq_tq, 1, PI_NET,
+	    "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc);
+#endif
 	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
 	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
 	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
@@ -1768,8 +1778,14 @@ mlx5e_open_rqt(struct mlx5e_priv *priv)
 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
 
 	for (i = 0; i < sz; i++) {
-		int ix = i % priv->params.num_channels;
-
+		int ix;
+#ifdef RSS
+		ix = rss_get_indirection_to_bucket(i);
+#else
+		ix = i;
+#endif
+		/* ensure we don't overflow */
+		ix %= priv->params.num_channels;
 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn);
 	}
 
@@ -1834,6 +1850,8 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    MLX5_CAP_ETH(priv->mdev,
 		    lro_timer_supported_periods[2]));
 	}
+
+	/* setup parameters for hashing TIR type, if any */
 	switch (tt) {
 	case MLX5E_TT_ANY:
 		MLX5_SET(tirc, tirc, disp_type,
@@ -1848,8 +1866,16 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    priv->rqtn);
 		MLX5_SET(tirc, tirc, rx_hash_fn,
 		    MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
-		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
 		hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
+#ifdef RSS
+		/*
+		 * The FreeBSD RSS implementation does currently not
+		 * support symmetric Toeplitz hashes:
+		 */
+		MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
+		rss_getkey((uint8_t *)hkey);
+#else
+		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
 		hkey[0] = cpu_to_be32(0xD181C62C);
 		hkey[1] = cpu_to_be32(0xF7F4DB5B);
 		hkey[2] = cpu_to_be32(0x1983A2FC);
@@ -1860,6 +1886,7 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		hkey[7] = cpu_to_be32(0x593D56D9);
 		hkey[8] = cpu_to_be32(0xF3253C06);
 		hkey[9] = cpu_to_be32(0x2ADC1FFC);
+#endif
 		break;
 	}
 
@@ -1869,6 +1896,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    MLX5_L3_PROT_TYPE_IPV4);
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 		    MLX5_L4_PROT_TYPE_TCP);
+#ifdef RSS
+		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+			    MLX5_HASH_IP);
+		} else
+#endif
 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
 		    MLX5_HASH_ALL);
 		break;
@@ -1878,6 +1911,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    MLX5_L3_PROT_TYPE_IPV6);
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 		    MLX5_L4_PROT_TYPE_TCP);
+#ifdef RSS
+		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
+			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+			    MLX5_HASH_IP);
+		} else
+#endif
 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
 		    MLX5_HASH_ALL);
 		break;
@@ -1887,6 +1926,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    MLX5_L3_PROT_TYPE_IPV4);
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 		    MLX5_L4_PROT_TYPE_UDP);
+#ifdef RSS
+		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+			    MLX5_HASH_IP);
+		} else
+#endif
 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
 		    MLX5_HASH_ALL);
 		break;
@@ -1896,6 +1941,12 @@ mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
 		    MLX5_L3_PROT_TYPE_IPV6);
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 		    MLX5_L4_PROT_TYPE_UDP);
+#ifdef RSS
+		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
+			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+			    MLX5_HASH_IP);
+		} else
+#endif
 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
 		    MLX5_HASH_ALL);
 		break;
@@ -2052,6 +2103,13 @@ mlx5e_open_locked(struct ifnet *ifp)
 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
 		return (0);
 
+#ifdef RSS
+	if (rss_getnumbuckets() > priv->params.num_channels) {
+		if_printf(ifp, "NOTE: There are more RSS buckets(%u) than "
+		    "channels(%u) available\n", rss_getnumbuckets(),
+		    priv->params.num_channels);
+	}
+#endif
 	err = mlx5e_open_tises(priv);
 	if (err) {
 		if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index 3e827bd31709..87fc5247bf9e 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -192,12 +192,43 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
 
 	mb->m_pkthdr.len = mb->m_len = cqe_bcnt;
 	/* check if a Toeplitz hash was computed */
-	if (cqe->rss_hash_type != 0)
+	if (cqe->rss_hash_type != 0) {
 		mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
-	else
+#ifdef RSS
+		/* decode the RSS hash type */
+		switch (cqe->rss_hash_type &
+		    (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
+		/* IPv4 */
+		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
+			break;
+		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
+			break;
+		case CQE_RSS_DST_HTYPE_IPV4:
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
+			break;
+		/* IPv6 */
+		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
+			break;
+		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
+			break;
+		case CQE_RSS_DST_HTYPE_IPV6:
+			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
+			break;
+		default:	/* Other */
+			M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+			break;
+		}
+#else
+		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+#endif
+	} else {
 		mb->m_pkthdr.flowid = rq->ix;
-
-	M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+	}
 	mb->m_pkthdr.rcvif = ifp;
 
 	if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) &&
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index 0256fc70c295..cb087270f72c 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -85,7 +85,15 @@ mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb)
 
 	/* check if flowid is set */
 	if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
-		ch = (mb->m_pkthdr.flowid % 128) % ch;
+#ifdef RSS
+		u32 temp;
+
+		if (rss_hash2bucket(mb->m_pkthdr.flowid,
+		    M_HASHTYPE_GET(mb), &temp) == 0)
+			ch = temp % ch;
+		else
+#endif
+			ch = (mb->m_pkthdr.flowid % 128) % ch;
 	} else {
 #if (__FreeBSD_version >= 1100000)
 		ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |