RoCE/infiniband upgrade to Linux v4.9 for kernel and userspace.

This commit merges projects/bsd_rdma_4_9 to head.

List of kernel sources used:
============================

1) kernel sources were cloned from git://github.com/torvalds/linux.git
Top commit 69973b830859bc6529a7a0468ba0d80ee5117826 - tag: v4.9, linux-4.9

2) krping was cloned from https://github.com/larrystevenwise/krping
Top commit 292a2f1abf0348285e678a82264740d52e4dcfe4

List of userspace sources used:
===============================

1) rdma-core was cloned from https://github.com/linux-rdma/rdma-core.git
Top commit d65138ef93af30b3ea249f3a84aa6a24ba7f8a75

2) OpenSM was cloned from git://git.openfabrics.org/~halr/opensm.git
Top commit 85f841cf209f791c89a075048a907020e924528d

3) libibmad was cloned from git://git.openfabrics.org/~iraweiny/libibmad.git
Tag 1.3.13 with some additional patches from Mellanox.

4) infiniband-diags was cloned from git://git.openfabrics.org/~iraweiny/infiniband-diags.git
Tag 1.6.7 with some additional patches from Mellanox.

NOTES:
======

1) The mthca driver has been removed in kernel and in userspace.
2) All GPLv2 only sources have been removed and where applicable
   rewritten from scratch under a BSD license.
3) List of fully supported drivers in userspace and kernel:
   a) iw_cxgbe (Chelsio)
   b) mlx4ib (Mellanox)
   c) mlx5ib (Mellanox)
4) WITH_OFED=YES is still required by make in order to build
   OFED userspace and kernel code.
5) Full support has been added for routable RoCE, RoCE v2.

Sponsored by:	Mellanox Technologies

2 files changed, 63 insertions, 95 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 6054e1e5a2a0..a6f0403417b6 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4453,58 +4453,62 @@ compat/linuxkpi/common/src/linux_work.c		optional compat_linuxkpi \
 	compile-with "${LINUXKPI_C}"
 
 # OpenFabrics Enterprise Distribution (Infiniband)
-ofed/drivers/infiniband/core/addr.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/agent.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/cache.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-# XXX Mad.c must be ordered before cm.c for sysinit sets to occur in
-# the correct order.
-ofed/drivers/infiniband/core/mad.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/cm.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/ -Wno-unused-function"
-ofed/drivers/infiniband/core/cma.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/device.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/fmr_pool.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/iwcm.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/mad_rmpp.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/multicast.c	optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/packer.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/peer_mem.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/sa_query.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/smi.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/sysfs.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/ucm.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/ucma.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/ud_header.c	optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/umem.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/user_mad.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/uverbs_cmd.c	optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/uverbs_main.c	optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/uverbs_marshall.c	optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
-ofed/drivers/infiniband/core/verbs.c		optional ofed		\
-	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/core/"
+ofed/drivers/infiniband/core/ib_addr.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_agent.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_cache.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_cm.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_cma.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_cq.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_device.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_fmr_pool.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_iwcm.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_iwpm_msg.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_iwpm_util.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_mad.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_mad_rmpp.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_multicast.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_packer.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_sa_query.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_smi.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_sysfs.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_ucm.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_ucma.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_ud_header.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_umem.c			optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_user_mad.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_uverbs_cmd.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_uverbs_main.c		optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_uverbs_marshall.c	optional ofed	\
+	compile-with "${OFED_C}"
+ofed/drivers/infiniband/core/ib_verbs.c			optional ofed	\
+	compile-with "${OFED_C}"
 
 ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c	optional ipoib		\
 	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
@@ -4550,8 +4554,6 @@ dev/mlx4/mlx4_ib/mlx4_ib_mad.c			optional mlx4ib pci ofed \
 	compile-with "${OFED_C}"
 dev/mlx4/mlx4_ib/mlx4_ib_main.c			optional mlx4ib pci ofed \
 	compile-with "${OFED_C}"
-dev/mlx4/mlx4_ib/mlx4_ib_exp.c			optional mlx4ib pci ofed \
-	compile-with "${OFED_C}"
 dev/mlx4/mlx4_ib/mlx4_ib_mr.c			optional mlx4ib pci ofed \
 	compile-with "${OFED_C}"
 dev/mlx4/mlx4_ib/mlx4_ib_qp.c			optional mlx4ib pci ofed \
@@ -4623,6 +4625,8 @@ dev/mlx5/mlx5_ib/mlx5_ib_cq.c			optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_ib/mlx5_ib_doorbell.c		optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_ib/mlx5_ib_gsi.c			optional mlx5ib pci ofed \
+	compile-with "${OFED_C}"
 dev/mlx5/mlx5_ib/mlx5_ib_mad.c			optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_ib/mlx5_ib_main.c			optional mlx5ib pci ofed \
@@ -4633,10 +4637,10 @@ dev/mlx5/mlx5_ib/mlx5_ib_mr.c			optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_ib/mlx5_ib_qp.c			optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
-dev/mlx5/mlx5_ib/mlx5_ib_roce.c			optional mlx5ib pci ofed \
-	compile-with "${OFED_C}"
 dev/mlx5/mlx5_ib/mlx5_ib_srq.c			optional mlx5ib pci ofed \
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_ib/mlx5_ib_virt.c			optional mlx5ib pci ofed \
+	compile-with "${OFED_C}"
 
 dev/mlx5/mlx5_core/mlx5_alloc.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
@@ -4696,43 +4700,6 @@ dev/mlx5/mlx5_en/mlx5_en_rx.c			optional mlx5en pci inet inet6	\
 dev/mlx5/mlx5_en/mlx5_en_txrx.c			optional mlx5en pci inet inet6	\
 	compile-with "${OFED_C}"
 
-ofed/drivers/infiniband/hw/mthca/mthca_allocator.c	optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_av.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_catas.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_cmd.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_cq.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_eq.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_mad.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_main.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_mcg.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_memfree.c	optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_mr.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_pd.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_profile.c	optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_provider.c	optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_qp.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_reset.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_srq.c		optional mthca	\
-	compile-with "${OFED_C}"
-ofed/drivers/infiniband/hw/mthca/mthca_uar.c		optional mthca	\
-	compile-with "${OFED_C}"
-
 # crypto support
 opencrypto/cast.c		optional crypto | ipsec | ipsec_support
 opencrypto/criov.c		optional crypto | ipsec | ipsec_support
diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk
index 28528b095300..5c5a847ddc0e 100644
--- a/sys/conf/kern.pre.mk
+++ b/sys/conf/kern.pre.mk
@@ -178,9 +178,10 @@ LINUXKPI_C=		${NORMAL_C} ${LINUXKPI_INCLUDES}
 
 # Infiniband C flags.  Correct include paths and omit errors that linux
 # does not honor.
-OFEDINCLUDES=	-I$S/ofed/include ${LINUXKPI_INCLUDES}
+OFEDINCLUDES=	-I$S/ofed/include -I$S/ofed/include/uapi ${LINUXKPI_INCLUDES}
 OFEDNOERR=	-Wno-cast-qual -Wno-pointer-arith
-OFEDCFLAGS=	${CFLAGS:N-I*} ${OFEDINCLUDES} ${CFLAGS:M-I*} ${OFEDNOERR}
+OFEDCFLAGS=	${CFLAGS:N-I*} -DCONFIG_INFINIBAND_USER_MEM \
+		${OFEDINCLUDES} ${CFLAGS:M-I*} ${OFEDNOERR}
 OFED_C_NOIMP=	${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF}
 OFED_C=		${OFED_C_NOIMP} ${.IMPSRC}