Add RDMA (iWARP and RoCEv1) support

David Somayajulu (davidcs): Overall RDMA Driver infrastructure and iWARP
Anand Khoje (akhoje@marvell.com): RoCEv1 verbs implementation

MFC after:5 days

15 files changed, 23125 insertions, 0 deletions

diff --git a/sys/dev/qlnx/qlnxe/ecore_iwarp.c b/sys/dev/qlnx/qlnxe/ecore_iwarp.c
new file mode 100644
index 000000000000..eec3613499a9
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/ecore_iwarp.c
@@ -0,0 +1,3970 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : ecore_iwarp.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "bcm_osal.h"
+#include "ecore.h"
+#include "ecore_status.h"
+#include "ecore_sp_commands.h"
+#include "ecore_cxt.h"
+#include "ecore_rdma.h"
+#include "reg_addr.h"
+#include "ecore_hw.h"
+#include "ecore_hsi_iwarp.h"
+#include "ecore_ll2.h"
+#include "ecore_ooo.h"
+#ifndef LINUX_REMOVE
+#include "ecore_tcp_ip.h"
+#endif
+
+#ifdef _NTDDK_
+#pragma warning(push)
+#pragma warning(disable : 28123)
+#pragma warning(disable : 28167)
+#endif
+
+/* Default values used for MPA Rev 1 */
+#define ECORE_IWARP_ORD_DEFAULT 32
+#define ECORE_IWARP_IRD_DEFAULT 32
+
+#define ECORE_IWARP_MAX_FW_MSS  4120
+
+struct mpa_v2_hdr {
+	__be16 ird;
+	__be16 ord;
+};
+
+#define MPA_V2_PEER2PEER_MODEL	0x8000
+#define MPA_V2_SEND_RTR		0x4000 /* on ird */
+#define MPA_V2_READ_RTR		0x4000 /* on ord */
+#define MPA_V2_WRITE_RTR	0x8000
+#define MPA_V2_IRD_ORD_MASK	0x3FFF
+
+#define MPA_REV2(_mpa_rev) (_mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED)
+
+#define ECORE_IWARP_INVALID_TCP_CID 0xffffffff
+/* How many times fin will be sent before FW aborts and send RST */
+#define ECORE_IWARP_MAX_FIN_RT_DEFAULT 2
+#define ECORE_IWARP_RCV_WND_SIZE_MIN (0xffff)
+/* INTERNAL: These numbers are derived from BRB buffer sizes to obtain optimal performance */
+#define ECORE_IWARP_RCV_WND_SIZE_BB_DEF_2_PORTS (200*1024)
+#define ECORE_IWARP_RCV_WND_SIZE_BB_DEF_4_PORTS (100*1024)
+#define ECORE_IWARP_RCV_WND_SIZE_AH_DEF_2_PORTS (150*1024)
+#define ECORE_IWARP_RCV_WND_SIZE_AH_DEF_4_PORTS (90*1024)
+#define ECORE_IWARP_MAX_WND_SCALE    (14)
+/* Timestamp header is the length of the timestamp option (10):
+ * kind:8 bit, length:8 bit, timestamp:32 bit, ack: 32bit
+ * rounded up to a multiple of 4
+ */
+#define TIMESTAMP_HEADER_SIZE (12)
+
+static enum _ecore_status_t
+ecore_iwarp_async_event(struct ecore_hwfn *p_hwfn,
+			u8 fw_event_code,
+			u16 OSAL_UNUSED echo,
+			union event_ring_data *data,
+			u8 fw_return_code);
+
+static enum _ecore_status_t
+ecore_iwarp_empty_ramrod(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_listener *listener);
+
+static OSAL_INLINE struct ecore_iwarp_fpdu *
+ecore_iwarp_get_curr_fpdu(struct ecore_hwfn *p_hwfn, u16 cid);
+
+/* Override devinfo with iWARP specific values */
+void
+ecore_iwarp_init_devinfo(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+
+	dev->max_inline = IWARP_REQ_MAX_INLINE_DATA_SIZE;
+	dev->max_qp = OSAL_MIN_T(u64,
+				 IWARP_MAX_QPS,
+				 p_hwfn->p_rdma_info->num_qps) -
+		ECORE_IWARP_PREALLOC_CNT;
+
+	dev->max_cq = dev->max_qp;
+
+	dev->max_qp_resp_rd_atomic_resc = ECORE_IWARP_IRD_DEFAULT;
+	dev->max_qp_req_rd_atomic_resc = ECORE_IWARP_ORD_DEFAULT;
+}
+
+enum _ecore_status_t
+ecore_iwarp_init_hw(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
+{
+	p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_TCP;
+	ecore_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1);
+	p_hwfn->b_rdma_enabled_in_prs = true;
+
+	return 0;
+}
+
+void
+ecore_iwarp_init_fw_ramrod(struct ecore_hwfn *p_hwfn,
+			   struct iwarp_init_func_ramrod_data *p_ramrod)
+{
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "ooo handle = %d\n",
+		   p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle);
+
+	p_ramrod->iwarp.ll2_ooo_q_index =
+		p_hwfn->hw_info.resc_start[ECORE_LL2_QUEUE] +
+		p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle;
+
+	p_ramrod->tcp.max_fin_rt = ECORE_IWARP_MAX_FIN_RT_DEFAULT;
+	return;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_alloc_cid(struct ecore_hwfn *p_hwfn, u32 *cid)
+{
+	enum _ecore_status_t rc;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->cid_map,
+				      cid);
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	*cid += ecore_cxt_get_proto_cid_start(p_hwfn,
+					      p_hwfn->p_rdma_info->proto);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "Failed in allocating iwarp cid\n");
+		return rc;
+	}
+
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, *cid);
+
+	if (rc != ECORE_SUCCESS) {
+		OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+		*cid -= ecore_cxt_get_proto_cid_start(p_hwfn,
+					     p_hwfn->p_rdma_info->proto);
+
+		ecore_bmap_release_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->cid_map,
+				      *cid);
+
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	}
+
+	return rc;
+}
+
+static void
+ecore_iwarp_set_tcp_cid(struct ecore_hwfn *p_hwfn, u32 cid)
+{
+	cid -= ecore_cxt_get_proto_cid_start(p_hwfn,
+					     p_hwfn->p_rdma_info->proto);
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_set_id(p_hwfn,
+			  &p_hwfn->p_rdma_info->tcp_cid_map,
+			  cid);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+/* This function allocates a cid for passive tcp ( called from syn receive)
+ * the reason it's separate from the regular cid allocation is because it
+ * is assured that these cids already have ilt alloacted. They are preallocated
+ * to ensure that we won't need to allocate memory during syn processing
+ */
+static enum _ecore_status_t
+ecore_iwarp_alloc_tcp_cid(struct ecore_hwfn *p_hwfn, u32 *cid)
+{
+	enum _ecore_status_t rc;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->tcp_cid_map,
+				      cid);
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	*cid += ecore_cxt_get_proto_cid_start(p_hwfn,
+					      p_hwfn->p_rdma_info->proto);
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "can't allocate iwarp tcp cid max-count=%d\n",
+			   p_hwfn->p_rdma_info->tcp_cid_map.max_count);
+
+		*cid = ECORE_IWARP_INVALID_TCP_CID;
+	}
+
+	return rc;
+}
+
+/* We have two cid maps, one for tcp which should be used only from passive
+ * syn processing and replacing a pre-allocated ep in the list. the second
+ * for active tcp and for QPs.
+ */
+static void ecore_iwarp_cid_cleaned(struct ecore_hwfn *p_hwfn, u32 cid)
+{
+	cid -= ecore_cxt_get_proto_cid_start(p_hwfn,
+					     p_hwfn->p_rdma_info->proto);
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+
+	if (cid < ECORE_IWARP_PREALLOC_CNT) {
+		ecore_bmap_release_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->tcp_cid_map,
+				      cid);
+	} else {
+		ecore_bmap_release_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->cid_map,
+				      cid);
+	}
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+enum _ecore_status_t
+ecore_iwarp_create_qp(struct ecore_hwfn *p_hwfn,
+		      struct ecore_rdma_qp *qp,
+		      struct ecore_rdma_create_qp_out_params *out_params)
+{
+	struct iwarp_create_qp_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+	u16 physical_queue;
+	u32 cid;
+
+	qp->shared_queue =
+		OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+					&qp->shared_queue_phys_addr,
+					IWARP_SHARED_QUEUE_PAGE_SIZE);
+	if (!qp->shared_queue) {
+		DP_NOTICE(p_hwfn, false,
+			  "ecore iwarp create qp failed: cannot allocate memory (shared queue).\n");
+		return ECORE_NOMEM;
+	} else {
+		out_params->sq_pbl_virt = (u8 *)qp->shared_queue +
+			IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET;
+		out_params->sq_pbl_phys = qp->shared_queue_phys_addr +
+			IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET;
+		out_params->rq_pbl_virt = (u8 *)qp->shared_queue +
+			IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET;
+		out_params->rq_pbl_phys = qp->shared_queue_phys_addr +
+			IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET;
+	}
+
+	rc = ecore_iwarp_alloc_cid(p_hwfn, &cid);
+	if (rc != ECORE_SUCCESS)
+		goto err1;
+
+	qp->icid = (u16)cid;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.cid = qp->icid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_CREATE_QP,
+				   PROTOCOLID_IWARP, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.iwarp_create_qp;
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN,
+		  qp->fmr_and_reserved_lkey);
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP,
+		  qp->signal_all);
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN,
+		  qp->incoming_rdma_read_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN,
+		  qp->incoming_rdma_write_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN,
+		  qp->incoming_atomic_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG,
+		  qp->use_srq);
+
+	p_ramrod->pd = qp->pd;
+	p_ramrod->sq_num_pages = qp->sq_num_pages;
+	p_ramrod->rq_num_pages = qp->rq_num_pages;
+
+	p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi);
+	p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo);
+
+	p_ramrod->cq_cid_for_sq =
+		OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) |
+				 qp->sq_cq_id);
+	p_ramrod->cq_cid_for_rq =
+		OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) |
+				 qp->rq_cq_id);
+
+	p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi);
+
+	physical_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
+	p_ramrod->physical_q0 = OSAL_CPU_TO_LE16(physical_queue);
+	physical_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK);
+	p_ramrod->physical_q1 = OSAL_CPU_TO_LE16(physical_queue);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	if (rc != ECORE_SUCCESS)
+		goto err1;
+
+	return rc;
+
+err1:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       qp->shared_queue,
+			       qp->shared_queue_phys_addr,
+			       IWARP_SHARED_QUEUE_PAGE_SIZE);
+
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_modify_fw(struct ecore_hwfn *p_hwfn,
+		      struct ecore_rdma_qp *qp)
+{
+	struct iwarp_modify_qp_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_MODIFY_QP,
+				   p_hwfn->p_rdma_info->proto,
+				   &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.iwarp_modify_qp;
+	SET_FIELD(p_ramrod->flags, IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN,
+		  0x1);
+	if (qp->iwarp_state == ECORE_IWARP_QP_STATE_CLOSING)
+		p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_CLOSING;
+	else
+		p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_ERROR;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x)rc=%d\n",
+		   qp->icid, rc);
+
+	return rc;
+}
+
+enum ecore_iwarp_qp_state
+ecore_roce2iwarp_state(enum ecore_roce_qp_state state)
+{
+	switch (state) {
+	case ECORE_ROCE_QP_STATE_RESET:
+	case ECORE_ROCE_QP_STATE_INIT:
+	case ECORE_ROCE_QP_STATE_RTR:
+		return ECORE_IWARP_QP_STATE_IDLE;
+	case ECORE_ROCE_QP_STATE_RTS:
+		return ECORE_IWARP_QP_STATE_RTS;
+	case ECORE_ROCE_QP_STATE_SQD:
+		return ECORE_IWARP_QP_STATE_CLOSING;
+	case ECORE_ROCE_QP_STATE_ERR:
+		return ECORE_IWARP_QP_STATE_ERROR;
+	case ECORE_ROCE_QP_STATE_SQE:
+		return ECORE_IWARP_QP_STATE_TERMINATE;
+	}
+	return ECORE_IWARP_QP_STATE_ERROR;
+}
+
+static enum ecore_roce_qp_state
+ecore_iwarp2roce_state(enum ecore_iwarp_qp_state state)
+{
+	switch (state) {
+	case ECORE_IWARP_QP_STATE_IDLE:
+		return ECORE_ROCE_QP_STATE_INIT;
+	case ECORE_IWARP_QP_STATE_RTS:
+		return ECORE_ROCE_QP_STATE_RTS;
+	case ECORE_IWARP_QP_STATE_TERMINATE:
+		return ECORE_ROCE_QP_STATE_SQE;
+	case ECORE_IWARP_QP_STATE_CLOSING:
+		return ECORE_ROCE_QP_STATE_SQD;
+	case ECORE_IWARP_QP_STATE_ERROR:
+		return ECORE_ROCE_QP_STATE_ERR;
+	}
+	return ECORE_ROCE_QP_STATE_ERR;
+}
+
+const char *iwarp_state_names[] = {
+	"IDLE",
+	"RTS",
+	"TERMINATE",
+	"CLOSING",
+	"ERROR",
+};
+
+enum _ecore_status_t
+ecore_iwarp_modify_qp(struct ecore_hwfn *p_hwfn,
+		      struct ecore_rdma_qp *qp,
+		      enum ecore_iwarp_qp_state new_state,
+		      bool internal)
+{
+	enum ecore_iwarp_qp_state prev_iw_state;
+	enum _ecore_status_t rc = 0;
+	bool modify_fw = false;
+
+	/* modify QP can be called from upper-layer or as a result of async
+	 * RST/FIN... therefore need to protect
+	 */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock);
+	prev_iw_state = qp->iwarp_state;
+
+	if (prev_iw_state == new_state) {
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock);
+		return ECORE_SUCCESS;
+	}
+
+	switch (prev_iw_state) {
+	case ECORE_IWARP_QP_STATE_IDLE:
+		switch (new_state) {
+		case ECORE_IWARP_QP_STATE_RTS:
+			qp->iwarp_state = ECORE_IWARP_QP_STATE_RTS;
+			break;
+		case ECORE_IWARP_QP_STATE_ERROR:
+			qp->iwarp_state = ECORE_IWARP_QP_STATE_ERROR;
+			if (!internal)
+				modify_fw = true;
+			break;
+		default:
+			break;
+		}
+		break;
+	case ECORE_IWARP_QP_STATE_RTS:
+		switch (new_state) {
+		case ECORE_IWARP_QP_STATE_CLOSING:
+			if (!internal)
+				modify_fw = true;
+
+			qp->iwarp_state = ECORE_IWARP_QP_STATE_CLOSING;
+			break;
+		case ECORE_IWARP_QP_STATE_ERROR:
+			if (!internal)
+				modify_fw = true;
+			qp->iwarp_state = ECORE_IWARP_QP_STATE_ERROR;
+			break;
+		default:
+			break;
+		}
+		break;
+	case ECORE_IWARP_QP_STATE_ERROR:
+		switch (new_state) {
+		case ECORE_IWARP_QP_STATE_IDLE:
+			/* TODO: destroy flow -> need to destroy EP&QP */
+			qp->iwarp_state = new_state;
+			break;
+		case ECORE_IWARP_QP_STATE_CLOSING:
+			/* could happen due to race... do nothing.... */
+			break;
+		default:
+			rc = ECORE_INVAL;
+		}
+		break;
+	case ECORE_IWARP_QP_STATE_TERMINATE:
+	case ECORE_IWARP_QP_STATE_CLOSING:
+		qp->iwarp_state = new_state;
+		break;
+	default:
+		break;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) %s --> %s %s\n",
+		   qp->icid,
+		   iwarp_state_names[prev_iw_state],
+		   iwarp_state_names[qp->iwarp_state],
+		   internal ? "internal" : " ");
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock);
+
+	if (modify_fw)
+		ecore_iwarp_modify_fw(p_hwfn, qp);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_fw_destroy(struct ecore_hwfn *p_hwfn,
+		       struct ecore_rdma_qp *qp)
+{
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_DESTROY_QP,
+				   p_hwfn->p_rdma_info->proto,
+				   &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) rc = %d\n",  qp->icid, rc);
+
+	return rc;
+}
+
+static void ecore_iwarp_destroy_ep(struct ecore_hwfn *p_hwfn,
+				   struct ecore_iwarp_ep *ep,
+				   bool remove_from_active_list)
+{
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       ep->ep_buffer_virt,
+			       ep->ep_buffer_phys,
+			       sizeof(*ep->ep_buffer_virt));
+
+	if (remove_from_active_list) {
+		OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+		OSAL_LIST_REMOVE_ENTRY(&ep->list_entry,
+				       &p_hwfn->p_rdma_info->iwarp.ep_list);
+
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	}
+
+	if (ep->qp)
+		ep->qp->ep = OSAL_NULL;
+
+	OSAL_FREE(p_hwfn->p_dev, ep);
+}
+
+enum _ecore_status_t
+ecore_iwarp_destroy_qp(struct ecore_hwfn *p_hwfn,
+		       struct ecore_rdma_qp *qp)
+{
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	struct ecore_iwarp_ep *ep = qp->ep;
+	struct ecore_iwarp_fpdu *fpdu;
+	int wait_count = 0;
+
+	fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, qp->icid);
+	if (fpdu && fpdu->incomplete_bytes)
+		DP_NOTICE(p_hwfn, false,
+			  "Pending Partial fpdu with incomplete bytes=%d\n",
+			  fpdu->incomplete_bytes);
+
+	if (qp->iwarp_state != ECORE_IWARP_QP_STATE_ERROR) {
+
+		rc = ecore_iwarp_modify_qp(p_hwfn, qp,
+					   ECORE_IWARP_QP_STATE_ERROR,
+					   false);
+
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
+	/* Make sure ep is closed before returning and freeing memory. */
+	if (ep) {
+		while (ep->state != ECORE_IWARP_EP_CLOSED) {
+			DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+				   "Waiting for ep->state to be closed...state=%x\n",
+				   ep->state);
+
+			OSAL_MSLEEP(100);
+			if (wait_count++ > 200) {
+				DP_NOTICE(p_hwfn, false, "ep state close timeout state=%x\n",
+					  ep->state);
+				break;
+			}
+		}
+
+		ecore_iwarp_destroy_ep(p_hwfn, ep, false);
+	}
+
+	rc = ecore_iwarp_fw_destroy(p_hwfn, qp);
+
+	if (qp->shared_queue)
+		OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+				       qp->shared_queue,
+				       qp->shared_queue_phys_addr,
+				       IWARP_SHARED_QUEUE_PAGE_SIZE);
+
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_create_ep(struct ecore_hwfn *p_hwfn,
+		      struct ecore_iwarp_ep **ep_out)
+{
+	struct ecore_iwarp_ep *ep;
+	enum _ecore_status_t rc;
+
+	ep = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*ep));
+	if (!ep) {
+		DP_NOTICE(p_hwfn, false,
+			  "ecore create ep failed: cannot allocate memory (ep). rc = %d\n",
+			  ECORE_NOMEM);
+		return ECORE_NOMEM;
+	}
+
+	ep->state = ECORE_IWARP_EP_INIT;
+
+	/* ep_buffer is allocated once and is structured as follows:
+	 * [MAX_PRIV_DATA_LEN][MAX_PRIV_DATA_LEN][union async_output]
+	 * We could have allocated this in three calls but since all together
+	 * it is less than a page, we do one allocation and initialize pointers
+	 * accordingly
+	 */
+	ep->ep_buffer_virt = OSAL_DMA_ALLOC_COHERENT(
+		p_hwfn->p_dev,
+		&ep->ep_buffer_phys,
+		sizeof(*ep->ep_buffer_virt));
+
+	if (!ep->ep_buffer_virt) {
+		DP_NOTICE(p_hwfn, false,
+			  "ecore create ep failed: cannot allocate memory (ulp buffer). rc = %d\n",
+			  ECORE_NOMEM);
+		rc = ECORE_NOMEM;
+		goto err;
+	}
+
+	ep->sig = 0xdeadbeef;
+
+	*ep_out = ep;
+
+	return ECORE_SUCCESS;
+
+err:
+	OSAL_FREE(p_hwfn->p_dev, ep);
+	return rc;
+}
+
+static void
+ecore_iwarp_print_tcp_ramrod(struct ecore_hwfn *p_hwfn,
+			     struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod)
+{
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, ">>> PRINT TCP RAMROD\n");
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_mac=%x %x %x\n",
+		   p_tcp_ramrod->tcp.local_mac_addr_lo,
+		   p_tcp_ramrod->tcp.local_mac_addr_mid,
+		   p_tcp_ramrod->tcp.local_mac_addr_hi);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_mac=%x %x %x\n",
+		   p_tcp_ramrod->tcp.remote_mac_addr_lo,
+		   p_tcp_ramrod->tcp.remote_mac_addr_mid,
+		   p_tcp_ramrod->tcp.remote_mac_addr_hi);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "vlan_id=%x\n",
+		   p_tcp_ramrod->tcp.vlan_id);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "flags=%x\n",
+		   p_tcp_ramrod->tcp.flags);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ip_version=%x\n",
+		   p_tcp_ramrod->tcp.ip_version);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_ip=%x.%x.%x.%x\n",
+		   p_tcp_ramrod->tcp.local_ip[0],
+		   p_tcp_ramrod->tcp.local_ip[1],
+		   p_tcp_ramrod->tcp.local_ip[2],
+		   p_tcp_ramrod->tcp.local_ip[3]);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_ip=%x.%x.%x.%x\n",
+		   p_tcp_ramrod->tcp.remote_ip[0],
+		   p_tcp_ramrod->tcp.remote_ip[1],
+		   p_tcp_ramrod->tcp.remote_ip[2],
+		   p_tcp_ramrod->tcp.remote_ip[3]);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "flow_label=%x\n",
+		   p_tcp_ramrod->tcp.flow_label);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ttl=%x\n",
+		   p_tcp_ramrod->tcp.ttl);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "tos_or_tc=%x\n",
+		   p_tcp_ramrod->tcp.tos_or_tc);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_port=%x\n",
+		   p_tcp_ramrod->tcp.local_port);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_port=%x\n",
+		   p_tcp_ramrod->tcp.remote_port);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "mss=%x\n",
+		   p_tcp_ramrod->tcp.mss);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rcv_wnd_scale=%x\n",
+		   p_tcp_ramrod->tcp.rcv_wnd_scale);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "connect_mode=%x\n",
+		   p_tcp_ramrod->tcp.connect_mode);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_ip_payload_length=%x\n",
+		   p_tcp_ramrod->tcp.syn_ip_payload_length);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_phy_addr_lo=%x\n",
+		   p_tcp_ramrod->tcp.syn_phy_addr_lo);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_phy_addr_hi=%x\n",
+		   p_tcp_ramrod->tcp.syn_phy_addr_hi);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "<<<f  PRINT TCP RAMROD\n");
+}
+
+/* Default values for tcp option2 */
+#define ECORE_IWARP_DEF_MAX_RT_TIME (0)
+#define ECORE_IWARP_DEF_CWND_FACTOR (4)
+#define ECORE_IWARP_DEF_KA_MAX_PROBE_CNT (5)
+#define ECORE_IWARP_DEF_KA_TIMEOUT (1200000) /* 20 min */
+#define ECORE_IWARP_DEF_KA_INTERVAL (1000) /* 1 sec */
+
+static enum _ecore_status_t
+ecore_iwarp_tcp_offload(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_ep *ep)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	dma_addr_t async_output_phys;
+	dma_addr_t in_pdata_phys;
+	enum _ecore_status_t rc;
+	u16 physical_q;
+	u8 tcp_flags;
+	int i;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = ep->tcp_cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+
+	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
+		init_data.comp_mode = ECORE_SPQ_MODE_CB;
+	} else {
+		init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+	}
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_TCP_OFFLOAD,
+				   PROTOCOLID_IWARP, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_tcp_ramrod = &p_ent->ramrod.iwarp_tcp_offload;
+
+	/* Point to the "second half" of the ulp buffer */
+	in_pdata_phys = ep->ep_buffer_phys +
+		OFFSETOF(struct ecore_iwarp_ep_memory, in_pdata);
+	p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr.hi =
+		DMA_HI_LE(in_pdata_phys);
+	p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr.lo =
+		DMA_LO_LE(in_pdata_phys);
+	p_tcp_ramrod->iwarp.incoming_ulp_buffer.len =
+		OSAL_CPU_TO_LE16(sizeof(ep->ep_buffer_virt->in_pdata));
+
+	async_output_phys = ep->ep_buffer_phys +
+		OFFSETOF(struct ecore_iwarp_ep_memory, async_output);
+
+	p_tcp_ramrod->iwarp.async_eqe_output_buf.hi =
+		DMA_HI_LE(async_output_phys);
+	p_tcp_ramrod->iwarp.async_eqe_output_buf.lo =
+		DMA_LO_LE(async_output_phys);
+	p_tcp_ramrod->iwarp.handle_for_async.hi = OSAL_CPU_TO_LE32(PTR_HI(ep));
+	p_tcp_ramrod->iwarp.handle_for_async.lo = OSAL_CPU_TO_LE32(PTR_LO(ep));
+
+	physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
+	p_tcp_ramrod->iwarp.physical_q0 = OSAL_CPU_TO_LE16(physical_q);
+	physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK);
+	p_tcp_ramrod->iwarp.physical_q1 = OSAL_CPU_TO_LE16(physical_q);
+	p_tcp_ramrod->iwarp.mpa_mode = iwarp_info->mpa_rev;
+
+	ecore_set_fw_mac_addr(&p_tcp_ramrod->tcp.remote_mac_addr_hi,
+			      &p_tcp_ramrod->tcp.remote_mac_addr_mid,
+			      &p_tcp_ramrod->tcp.remote_mac_addr_lo,
+			      ep->remote_mac_addr);
+	ecore_set_fw_mac_addr(&p_tcp_ramrod->tcp.local_mac_addr_hi,
+			      &p_tcp_ramrod->tcp.local_mac_addr_mid,
+			      &p_tcp_ramrod->tcp.local_mac_addr_lo,
+			      ep->local_mac_addr);
+
+	p_tcp_ramrod->tcp.vlan_id = OSAL_CPU_TO_LE16(ep->cm_info.vlan);
+
+	tcp_flags = p_hwfn->p_rdma_info->iwarp.tcp_flags;
+	p_tcp_ramrod->tcp.flags = 0;
+	SET_FIELD(p_tcp_ramrod->tcp.flags,
+		  TCP_OFFLOAD_PARAMS_OPT2_TS_EN,
+		  !!(tcp_flags & ECORE_IWARP_TS_EN));
+
+	SET_FIELD(p_tcp_ramrod->tcp.flags,
+		  TCP_OFFLOAD_PARAMS_OPT2_DA_EN,
+		  !!(tcp_flags & ECORE_IWARP_DA_EN));
+
+	p_tcp_ramrod->tcp.ip_version = ep->cm_info.ip_version;
+
+	for (i = 0; i < 4; i++) {
+		p_tcp_ramrod->tcp.remote_ip[i] =
+			OSAL_CPU_TO_LE32(ep->cm_info.remote_ip[i]);
+		p_tcp_ramrod->tcp.local_ip[i] =
+			OSAL_CPU_TO_LE32(ep->cm_info.local_ip[i]);
+	}
+
+	p_tcp_ramrod->tcp.remote_port =
+		OSAL_CPU_TO_LE16(ep->cm_info.remote_port);
+	p_tcp_ramrod->tcp.local_port = OSAL_CPU_TO_LE16(ep->cm_info.local_port);
+	p_tcp_ramrod->tcp.mss = OSAL_CPU_TO_LE16(ep->mss);
+	p_tcp_ramrod->tcp.flow_label = 0;
+	p_tcp_ramrod->tcp.ttl = 0x40;
+	p_tcp_ramrod->tcp.tos_or_tc = 0;
+
+	p_tcp_ramrod->tcp.max_rt_time = ECORE_IWARP_DEF_MAX_RT_TIME;
+	p_tcp_ramrod->tcp.cwnd = ECORE_IWARP_DEF_CWND_FACTOR * p_tcp_ramrod->tcp.mss;
+	p_tcp_ramrod->tcp.ka_max_probe_cnt = ECORE_IWARP_DEF_KA_MAX_PROBE_CNT;
+	p_tcp_ramrod->tcp.ka_timeout = ECORE_IWARP_DEF_KA_TIMEOUT;
+	p_tcp_ramrod->tcp.ka_interval = ECORE_IWARP_DEF_KA_INTERVAL;
+
+	p_tcp_ramrod->tcp.rcv_wnd_scale =
+		(u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale;
+	p_tcp_ramrod->tcp.connect_mode = ep->connect_mode;
+
+	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
+		p_tcp_ramrod->tcp.syn_ip_payload_length =
+			OSAL_CPU_TO_LE16(ep->syn_ip_payload_length);
+		p_tcp_ramrod->tcp.syn_phy_addr_hi =
+			DMA_HI_LE(ep->syn_phy_addr);
+		p_tcp_ramrod->tcp.syn_phy_addr_lo =
+			DMA_LO_LE(ep->syn_phy_addr);
+	}
+
+	ecore_iwarp_print_tcp_ramrod(p_hwfn, p_tcp_ramrod);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "EP(0x%x) Offload completed rc=%d\n" , ep->tcp_cid, rc);
+
+	return rc;
+}
+
+/* This function should be called after IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE
+ * is received. it will be called from the dpc context.
+ */
+static enum _ecore_status_t
+ecore_iwarp_mpa_offload(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_ep *ep)
+{
+	struct iwarp_mpa_offload_ramrod_data *p_mpa_ramrod;
+	struct ecore_iwarp_info *iwarp_info;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	dma_addr_t async_output_phys;
+	dma_addr_t out_pdata_phys;
+	dma_addr_t in_pdata_phys;
+	struct ecore_rdma_qp *qp;
+	bool reject;
+	enum _ecore_status_t rc;
+
+	if (!ep)
+		return ECORE_INVAL;
+
+	qp = ep->qp;
+	reject = (qp == OSAL_NULL);
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = reject ? ep->tcp_cid : qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+
+	if (ep->connect_mode == TCP_CONNECT_ACTIVE || !ep->event_cb)
+		init_data.comp_mode = ECORE_SPQ_MODE_CB;
+	else
+		init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_MPA_OFFLOAD,
+				   PROTOCOLID_IWARP, &init_data);
+
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_mpa_ramrod = &p_ent->ramrod.iwarp_mpa_offload;
+	out_pdata_phys = ep->ep_buffer_phys +
+		OFFSETOF(struct ecore_iwarp_ep_memory, out_pdata);
+	p_mpa_ramrod->common.outgoing_ulp_buffer.addr.hi =
+		DMA_HI_LE(out_pdata_phys);
+	p_mpa_ramrod->common.outgoing_ulp_buffer.addr.lo =
+		DMA_LO_LE(out_pdata_phys);
+	p_mpa_ramrod->common.outgoing_ulp_buffer.len =
+		ep->cm_info.private_data_len;
+	p_mpa_ramrod->common.crc_needed = p_hwfn->p_rdma_info->iwarp.crc_needed;
+
+	p_mpa_ramrod->common.out_rq.ord = ep->cm_info.ord;
+	p_mpa_ramrod->common.out_rq.ird = ep->cm_info.ird;
+
+	p_mpa_ramrod->tcp_cid = p_hwfn->hw_info.opaque_fid << 16 | ep->tcp_cid;
+
+	in_pdata_phys = ep->ep_buffer_phys +
+		OFFSETOF(struct ecore_iwarp_ep_memory, in_pdata);
+	p_mpa_ramrod->tcp_connect_side = ep->connect_mode;
+	p_mpa_ramrod->incoming_ulp_buffer.addr.hi =
+		DMA_HI_LE(in_pdata_phys);
+	p_mpa_ramrod->incoming_ulp_buffer.addr.lo =
+		DMA_LO_LE(in_pdata_phys);
+	p_mpa_ramrod->incoming_ulp_buffer.len =
+		OSAL_CPU_TO_LE16(sizeof(ep->ep_buffer_virt->in_pdata));
+	async_output_phys = ep->ep_buffer_phys +
+		OFFSETOF(struct ecore_iwarp_ep_memory, async_output);
+	p_mpa_ramrod->async_eqe_output_buf.hi =
+		DMA_HI_LE(async_output_phys);
+	p_mpa_ramrod->async_eqe_output_buf.lo =
+		DMA_LO_LE(async_output_phys);
+	p_mpa_ramrod->handle_for_async.hi = OSAL_CPU_TO_LE32(PTR_HI(ep));
+	p_mpa_ramrod->handle_for_async.lo = OSAL_CPU_TO_LE32(PTR_LO(ep));
+
+	if (!reject) {
+		p_mpa_ramrod->shared_queue_addr.hi =
+			DMA_HI_LE(qp->shared_queue_phys_addr);
+		p_mpa_ramrod->shared_queue_addr.lo =
+			DMA_LO_LE(qp->shared_queue_phys_addr);
+
+		p_mpa_ramrod->stats_counter_id =
+			RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) +
+			qp->stats_queue;
+	} else {
+		p_mpa_ramrod->common.reject = 1;
+	}
+
+	iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	p_mpa_ramrod->rcv_wnd = iwarp_info->rcv_wnd_size;
+	p_mpa_ramrod->mode = ep->mpa_rev;
+	SET_FIELD(p_mpa_ramrod->rtr_pref,
+		  IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED,
+		  ep->rtr_type);
+
+	ep->state = ECORE_IWARP_EP_MPA_OFFLOADED;
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (!reject)
+		ep->cid = qp->icid; /* Now they're migrated. */
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "QP(0x%x) EP(0x%x) MPA Offload rc = %d IRD=0x%x ORD=0x%x rtr_type=%d mpa_rev=%d reject=%d\n",
+		   reject ? 0xffff : qp->icid, ep->tcp_cid, rc, ep->cm_info.ird,
+		   ep->cm_info.ord, ep->rtr_type, ep->mpa_rev, reject);
+	return rc;
+}
+
+static void
+ecore_iwarp_mpa_received(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_ep *ep)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	struct ecore_iwarp_cm_event_params params;
+	struct mpa_v2_hdr *mpa_v2_params;
+	union async_output *async_data;
+	u16 mpa_ord, mpa_ird;
+	u8 mpa_hdr_size = 0;
+	u8 mpa_rev;
+
+	async_data = &ep->ep_buffer_virt->async_output;
+
+	mpa_rev = async_data->mpa_request.mpa_handshake_mode;
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "private_data_len=%x handshake_mode=%x private_data=(%x)\n",
+		   async_data->mpa_request.ulp_data_len,
+		   mpa_rev,
+		   *((u32 *)((u8 *)ep->ep_buffer_virt->in_pdata)));
+
+	if (ep->listener->state > ECORE_IWARP_LISTENER_STATE_UNPAUSE) {
+		/* MPA reject initiated by ecore */
+		OSAL_MEMSET(&ep->cm_info, 0, sizeof(ep->cm_info));
+		ep->event_cb = OSAL_NULL;
+		ecore_iwarp_mpa_offload(p_hwfn, ep);
+		return;
+	}
+
+	if (mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) {
+		if (iwarp_info->mpa_rev == MPA_NEGOTIATION_TYPE_BASIC) {
+			DP_ERR(p_hwfn, "MPA_NEGOTIATE Received MPA rev 2 on driver supporting only MPA rev 1\n");
+			/* MPA_REV2 ToDo: close the tcp connection. */
+			return;
+		}
+
+		/* Read ord/ird values from private data buffer */
+		mpa_v2_params =
+			(struct mpa_v2_hdr *)(ep->ep_buffer_virt->in_pdata);
+		mpa_hdr_size = sizeof(*mpa_v2_params);
+
+		mpa_ord = ntohs(mpa_v2_params->ord);
+		mpa_ird = ntohs(mpa_v2_params->ird);
+
+		/* Temprary store in cm_info incoming ord/ird requested, later
+		 * replace with negotiated value during accept
+		 */
+		ep->cm_info.ord = (u8)OSAL_MIN_T(u16,
+						(mpa_ord & MPA_V2_IRD_ORD_MASK),
+						ECORE_IWARP_ORD_DEFAULT);
+
+		ep->cm_info.ird = (u8)OSAL_MIN_T(u16,
+						(mpa_ird & MPA_V2_IRD_ORD_MASK),
+						ECORE_IWARP_IRD_DEFAULT);
+
+		/* Peer2Peer negotiation */
+		ep->rtr_type = MPA_RTR_TYPE_NONE;
+		if (mpa_ird & MPA_V2_PEER2PEER_MODEL) {
+			if (mpa_ord & MPA_V2_WRITE_RTR)
+				ep->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE;
+
+			if (mpa_ord & MPA_V2_READ_RTR)
+				ep->rtr_type |= MPA_RTR_TYPE_ZERO_READ;
+
+			if (mpa_ird & MPA_V2_SEND_RTR)
+				ep->rtr_type |= MPA_RTR_TYPE_ZERO_SEND;
+
+			ep->rtr_type &= iwarp_info->rtr_type;
+			/* if we're left with no match send our capabilities */
+			if (ep->rtr_type == MPA_RTR_TYPE_NONE)
+				ep->rtr_type = iwarp_info->rtr_type;
+
+			/* prioritize write over send and read */
+			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE)
+					ep->rtr_type = MPA_RTR_TYPE_ZERO_WRITE;
+		}
+
+		ep->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;
+	} else {
+		ep->cm_info.ord = ECORE_IWARP_ORD_DEFAULT;
+		ep->cm_info.ird = ECORE_IWARP_IRD_DEFAULT;
+		ep->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x rtr:0x%x ulp_data_len = %x mpa_hdr_size = %x\n",
+		   mpa_rev, ep->cm_info.ord, ep->cm_info.ird, ep->rtr_type,
+		   async_data->mpa_request.ulp_data_len,
+		   mpa_hdr_size);
+
+	/* Strip mpa v2 hdr from private data before sending to upper layer */
+	ep->cm_info.private_data =
+		ep->ep_buffer_virt->in_pdata + mpa_hdr_size;
+
+	ep->cm_info.private_data_len =
+		async_data->mpa_request.ulp_data_len - mpa_hdr_size;
+
+	params.event = ECORE_IWARP_EVENT_MPA_REQUEST;
+	params.cm_info = &ep->cm_info;
+	params.ep_context = ep;
+	params.status = ECORE_SUCCESS;
+
+	ep->state = ECORE_IWARP_EP_MPA_REQ_RCVD;
+	ep->event_cb(ep->cb_context, &params);
+}
+
+static void
+ecore_iwarp_move_to_ep_list(struct ecore_hwfn *p_hwfn,
+			    osal_list_t *list, struct ecore_iwarp_ep *ep)
+{
+	OSAL_SPIN_LOCK(&ep->listener->lock);
+	OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, &ep->listener->ep_list);
+	OSAL_SPIN_UNLOCK(&ep->listener->lock);
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_LIST_PUSH_TAIL(&ep->list_entry, list);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+}
+
+static void
+ecore_iwarp_return_ep(struct ecore_hwfn *p_hwfn,
+		      struct ecore_iwarp_ep *ep)
+{
+	ep->state = ECORE_IWARP_EP_INIT;
+	if (ep->qp)
+		ep->qp->ep = OSAL_NULL;
+	ep->qp = OSAL_NULL;
+	OSAL_MEMSET(&ep->cm_info, 0, sizeof(ep->cm_info));
+
+	if (ep->tcp_cid == ECORE_IWARP_INVALID_TCP_CID) {
+		/* We don't care about the return code, it's ok if tcp_cid
+		 * remains invalid...in this case we'll defer allocation
+		 */
+		ecore_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);
+	}
+
+	ecore_iwarp_move_to_ep_list(p_hwfn,
+				    &p_hwfn->p_rdma_info->iwarp.ep_free_list,
+				    ep);
+}
+
+static void
+ecore_iwarp_parse_private_data(struct ecore_hwfn *p_hwfn,
+			       struct ecore_iwarp_ep *ep)
+{
+	struct mpa_v2_hdr *mpa_v2_params;
+	union async_output *async_data;
+	u16 mpa_ird, mpa_ord;
+	u8 mpa_data_size = 0;
+
+	if (MPA_REV2(p_hwfn->p_rdma_info->iwarp.mpa_rev)) {
+		mpa_v2_params = (struct mpa_v2_hdr *)
+		((u8 *)ep->ep_buffer_virt->in_pdata);
+		mpa_data_size = sizeof(*mpa_v2_params);
+		mpa_ird = ntohs(mpa_v2_params->ird);
+		mpa_ord = ntohs(mpa_v2_params->ord);
+
+		ep->cm_info.ird = (u8)(mpa_ord & MPA_V2_IRD_ORD_MASK);
+		ep->cm_info.ord = (u8)(mpa_ird & MPA_V2_IRD_ORD_MASK);
+	} /* else: Ord / Ird already configured */
+
+	async_data = &ep->ep_buffer_virt->async_output;
+
+	ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_data_size;
+	ep->cm_info.private_data_len =
+		async_data->mpa_response.ulp_data_len - mpa_data_size;
+}
+
+static void
+ecore_iwarp_mpa_reply_arrived(struct ecore_hwfn *p_hwfn,
+			      struct ecore_iwarp_ep *ep)
+{
+	struct ecore_iwarp_cm_event_params params;
+
+	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
+		DP_NOTICE(p_hwfn, true, "MPA reply event not expected on passive side!\n");
+		return;
+	}
+
+	params.event = ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY;
+
+	ecore_iwarp_parse_private_data(p_hwfn, ep);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",
+		   ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);
+
+	params.cm_info = &ep->cm_info;
+	params.ep_context = ep;
+	params.status = ECORE_SUCCESS;
+
+	ep->mpa_reply_processed = true;
+
+	ep->event_cb(ep->cb_context, &params);
+}
+
+#define ECORE_IWARP_CONNECT_MODE_STRING(ep) \
+	(ep->connect_mode == TCP_CONNECT_PASSIVE) ? "Passive" : "Active"
+
+/* Called as a result of the event:
+ * IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE
+ */
+static void
+ecore_iwarp_mpa_complete(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_ep *ep,
+			 u8 fw_return_code)
+{
+	struct ecore_iwarp_cm_event_params params;
+
+	if (ep->connect_mode == TCP_CONNECT_ACTIVE)
+		params.event = ECORE_IWARP_EVENT_ACTIVE_COMPLETE;
+	else
+		params.event = ECORE_IWARP_EVENT_PASSIVE_COMPLETE;
+
+	if (ep->connect_mode == TCP_CONNECT_ACTIVE &&
+	    !ep->mpa_reply_processed) {
+		ecore_iwarp_parse_private_data(p_hwfn, ep);
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",
+		   ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);
+
+	params.cm_info = &ep->cm_info;
+
+	params.ep_context = ep;
+
+	if ((ep->connect_mode == TCP_CONNECT_PASSIVE) &&
+	    (ep->state != ECORE_IWARP_EP_MPA_OFFLOADED)) {
+		/* This is a FW bug. Shouldn't get complete without offload */
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) ERROR: Got MPA complete without MPA offload fw_return_code=%d ep->state=%d\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid,
+			  fw_return_code, ep->state);
+		ep->state = ECORE_IWARP_EP_CLOSED;
+		return;
+	}
+
+	if ((ep->connect_mode == TCP_CONNECT_PASSIVE) &&
+	    (ep->state == ECORE_IWARP_EP_ABORTING))
+		return;
+
+	ep->state = ECORE_IWARP_EP_CLOSED;
+
+	switch (fw_return_code) {
+	case RDMA_RETURN_OK:
+		ep->qp->max_rd_atomic_req = ep->cm_info.ord;
+		ep->qp->max_rd_atomic_resp = ep->cm_info.ird;
+		ecore_iwarp_modify_qp(p_hwfn, ep->qp,
+				      ECORE_IWARP_QP_STATE_RTS,
+				      1);
+		ep->state = ECORE_IWARP_EP_ESTABLISHED;
+		params.status = ECORE_SUCCESS;
+		break;
+	case IWARP_CONN_ERROR_MPA_TIMEOUT:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA timeout\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_TIMEOUT;
+		break;
+	case IWARP_CONN_ERROR_MPA_ERROR_REJECT:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Reject\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_RST:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA reset(tcp cid: 0x%x)\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid,
+			  ep->tcp_cid);
+		params.status = ECORE_CONN_RESET;
+		break;
+	case IWARP_CONN_ERROR_MPA_FIN:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA received FIN\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_INSUF_IRD:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA insufficient ird\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_RTR_MISMATCH:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA RTR MISMATCH\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_INVALID_PACKET:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Invalid Packet\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_LOCAL_ERROR:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Local Error\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_TERMINATE:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA TERMINATE\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	default:
+		params.status = ECORE_CONN_RESET;
+		break;
+	}
+
+	if (ep->event_cb)
+		ep->event_cb(ep->cb_context, &params);
+
+	/* on passive side, if there is no associated QP (REJECT) we need to
+	 * return the ep to the pool, otherwise we wait for QP to release it.
+	 * Since we add an element in accept instead of this one. in anycase
+	 * we need to remove it from the ep_list (active connections)...
+	 */
+	if (fw_return_code != RDMA_RETURN_OK) {
+		ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID;
+		if ((ep->connect_mode == TCP_CONNECT_PASSIVE) &&
+		    (ep->qp == OSAL_NULL)) { /* Rejected */
+			ecore_iwarp_return_ep(p_hwfn, ep);
+		} else {
+			OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+			OSAL_LIST_REMOVE_ENTRY(
+				&ep->list_entry,
+				&p_hwfn->p_rdma_info->iwarp.ep_list);
+			OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+		}
+	}
+}
+
+static void
+ecore_iwarp_mpa_v2_set_private(struct ecore_hwfn *p_hwfn,
+			       struct ecore_iwarp_ep *ep,
+			       u8 *mpa_data_size)
+{
+	struct mpa_v2_hdr *mpa_v2_params;
+	u16 mpa_ird, mpa_ord;
+
+	*mpa_data_size = 0;
+	if (MPA_REV2(ep->mpa_rev)) {
+		mpa_v2_params =
+			(struct mpa_v2_hdr *)ep->ep_buffer_virt->out_pdata;
+		*mpa_data_size = sizeof(*mpa_v2_params);
+
+		mpa_ird = (u16)ep->cm_info.ird;
+		mpa_ord = (u16)ep->cm_info.ord;
+
+		if (ep->rtr_type != MPA_RTR_TYPE_NONE) {
+			mpa_ird |= MPA_V2_PEER2PEER_MODEL;
+
+			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_SEND)
+				mpa_ird |= MPA_V2_SEND_RTR;
+
+			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE)
+				mpa_ord |= MPA_V2_WRITE_RTR;
+
+			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_READ)
+				mpa_ord |= MPA_V2_READ_RTR;
+		}
+
+		mpa_v2_params->ird = htons(mpa_ird);
+		mpa_v2_params->ord = htons(mpa_ord);
+
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "MPA_NEGOTIATE Header: [%x ord:%x ird] %x ord:%x ird:%x peer2peer:%x rtr_send:%x rtr_write:%x rtr_read:%x\n",
+			   mpa_v2_params->ird,
+			   mpa_v2_params->ord,
+			   *((u32 *)mpa_v2_params),
+			   mpa_ord & MPA_V2_IRD_ORD_MASK,
+			   mpa_ird & MPA_V2_IRD_ORD_MASK,
+			   !!(mpa_ird & MPA_V2_PEER2PEER_MODEL),
+			   !!(mpa_ird & MPA_V2_SEND_RTR),
+			   !!(mpa_ord & MPA_V2_WRITE_RTR),
+			   !!(mpa_ord & MPA_V2_READ_RTR));
+	}
+}
+
+enum _ecore_status_t
+ecore_iwarp_connect(void *rdma_cxt,
+		    struct ecore_iwarp_connect_in *iparams,
+		    struct ecore_iwarp_connect_out *oparams)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_info *iwarp_info;
+	struct ecore_iwarp_ep *ep;
+	enum _ecore_status_t rc;
+	u8 mpa_data_size = 0;
+	u8 ts_hdr_size = 0;
+	u32 cid;
+
+	if ((iparams->cm_info.ord > ECORE_IWARP_ORD_DEFAULT) ||
+	    (iparams->cm_info.ird > ECORE_IWARP_IRD_DEFAULT)) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",
+			   iparams->qp->icid, iparams->cm_info.ord,
+			   iparams->cm_info.ird);
+
+		return ECORE_INVAL;
+	}
+
+	iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+
+	/* Allocate ep object */
+	rc = ecore_iwarp_alloc_cid(p_hwfn, &cid);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (iparams->qp->ep == OSAL_NULL) {
+		rc = ecore_iwarp_create_ep(p_hwfn, &ep);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	} else {
+		ep = iparams->qp->ep;
+		DP_ERR(p_hwfn, "Note re-use of QP for different connect\n");
+		ep->state = ECORE_IWARP_EP_INIT;
+	}
+
+	ep->tcp_cid = cid;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_LIST_PUSH_TAIL(&ep->list_entry,
+			    &p_hwfn->p_rdma_info->iwarp.ep_list);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+	ep->qp = iparams->qp;
+	ep->qp->ep = ep;
+	OSAL_MEMCPY(ep->remote_mac_addr,
+		    iparams->remote_mac_addr,
+		    ETH_ALEN);
+	OSAL_MEMCPY(ep->local_mac_addr,
+		    iparams->local_mac_addr,
+		    ETH_ALEN);
+	OSAL_MEMCPY(&ep->cm_info, &iparams->cm_info, sizeof(ep->cm_info));
+
+	ep->cm_info.ord = iparams->cm_info.ord;
+	ep->cm_info.ird = iparams->cm_info.ird;
+
+	ep->rtr_type = iwarp_info->rtr_type;
+	if (iwarp_info->peer2peer == 0)
+		ep->rtr_type = MPA_RTR_TYPE_NONE;
+
+	if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) &&
+	    (ep->cm_info.ord == 0))
+		ep->cm_info.ord = 1;
+
+	ep->mpa_rev = iwarp_info->mpa_rev;
+
+	ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
+
+	ep->cm_info.private_data = (u8 *)ep->ep_buffer_virt->out_pdata;
+	ep->cm_info.private_data_len =
+		iparams->cm_info.private_data_len + mpa_data_size;
+
+	OSAL_MEMCPY((u8 *)(u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
+		    iparams->cm_info.private_data,
+		    iparams->cm_info.private_data_len);
+
+	if (p_hwfn->p_rdma_info->iwarp.tcp_flags & ECORE_IWARP_TS_EN)
+		ts_hdr_size = TIMESTAMP_HEADER_SIZE;
+
+	ep->mss = iparams->mss - ts_hdr_size;
+	ep->mss = OSAL_MIN_T(u16, ECORE_IWARP_MAX_FW_MSS, ep->mss);
+
+	ep->event_cb = iparams->event_cb;
+	ep->cb_context = iparams->cb_context;
+	ep->connect_mode = TCP_CONNECT_ACTIVE;
+
+	oparams->ep_context = ep;
+
+	rc = ecore_iwarp_tcp_offload(p_hwfn, ep);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x) rc = %d\n",
+		   iparams->qp->icid, ep->tcp_cid, rc);
+
+	if (rc != ECORE_SUCCESS)
+		ecore_iwarp_destroy_ep(p_hwfn, ep, true);
+
+	return rc;
+}
+
+static struct ecore_iwarp_ep *
+ecore_iwarp_get_free_ep(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_iwarp_ep *ep = OSAL_NULL;
+	enum _ecore_status_t rc;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+	if (OSAL_LIST_IS_EMPTY(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {
+		DP_ERR(p_hwfn, "Ep list is empty\n");
+		goto out;
+	}
+
+	ep = OSAL_LIST_FIRST_ENTRY(&p_hwfn->p_rdma_info->iwarp.ep_free_list,
+				   struct ecore_iwarp_ep,
+				   list_entry);
+
+	/* in some cases we could have failed allocating a tcp cid when added
+	 * from accept / failure... retry now..this is not the common case.
+	 */
+	if (ep->tcp_cid == ECORE_IWARP_INVALID_TCP_CID) {
+		rc = ecore_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);
+		/* if we fail we could look for another entry with a valid
+		 * tcp_cid, but since we don't expect to reach this anyway
+		 * it's not worth the handling
+		 */
+		if (rc) {
+			ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID;
+			ep = OSAL_NULL;
+			goto out;
+		}
+	}
+
+	OSAL_LIST_REMOVE_ENTRY(&ep->list_entry,
+			       &p_hwfn->p_rdma_info->iwarp.ep_free_list);
+
+out:
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	return ep;
+}
+
+/* takes into account timer scan ~20 ms and interrupt/dpc overhead */
+#define ECORE_IWARP_MAX_CID_CLEAN_TIME  100
+/* Technically we shouldn't reach this count with 100 ms iteration sleep */
+#define ECORE_IWARP_MAX_NO_PROGRESS_CNT 5
+
+/* This function waits for all the bits of a bmap to be cleared, as long as
+ * there is progress ( i.e. the number of bits left to be cleared decreases )
+ * the function continues.
+ */
+static enum _ecore_status_t
+ecore_iwarp_wait_cid_map_cleared(struct ecore_hwfn *p_hwfn,
+				 struct ecore_bmap *bmap)
+{
+	int prev_weight = 0;
+	int wait_count = 0;
+	int weight = 0;
+
+	weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count);
+	prev_weight = weight;
+
+	while (weight) {
+		OSAL_MSLEEP(ECORE_IWARP_MAX_CID_CLEAN_TIME);
+
+		weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count);
+
+		if (prev_weight == weight) {
+			wait_count++;
+		} else {
+			prev_weight = weight;
+			wait_count = 0;
+		}
+
+		if (wait_count > ECORE_IWARP_MAX_NO_PROGRESS_CNT) {
+			DP_NOTICE(p_hwfn, false,
+				  "%s bitmap wait timed out (%d cids pending)\n",
+				  bmap->name, weight);
+			return ECORE_TIMEOUT;
+		}
+	}
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_wait_for_all_cids(struct ecore_hwfn *p_hwfn)
+{
+	enum _ecore_status_t rc;
+	int i;
+
+	rc = ecore_iwarp_wait_cid_map_cleared(
+		p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map);
+	if (rc)
+		return rc;
+
+	/* Now free the tcp cids from the main cid map */
+	for (i = 0; i < ECORE_IWARP_PREALLOC_CNT; i++) {
+		ecore_bmap_release_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->cid_map,
+				      i);
+	}
+
+	/* Now wait for all cids to be completed */
+	rc = ecore_iwarp_wait_cid_map_cleared(
+		p_hwfn, &p_hwfn->p_rdma_info->cid_map);
+
+	return rc;
+}
+
+static void
+ecore_iwarp_free_prealloc_ep(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_iwarp_ep *ep;
+	u32 cid;
+
+	while (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {
+		OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+		ep = OSAL_LIST_FIRST_ENTRY(
+			&p_hwfn->p_rdma_info->iwarp.ep_free_list,
+			struct ecore_iwarp_ep, list_entry);
+
+		if (ep == OSAL_NULL) {
+			OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+			break;
+		}
+
+#ifdef _NTDDK_
+#pragma warning(suppress : 6011)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(
+			&ep->list_entry,
+			&p_hwfn->p_rdma_info->iwarp.ep_free_list);
+
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+		if (ep->tcp_cid != ECORE_IWARP_INVALID_TCP_CID) {
+			cid = ep->tcp_cid - ecore_cxt_get_proto_cid_start(
+				p_hwfn, p_hwfn->p_rdma_info->proto);
+
+			OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+
+			ecore_bmap_release_id(p_hwfn,
+					      &p_hwfn->p_rdma_info->tcp_cid_map,
+					      cid);
+
+			OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+		}
+
+		ecore_iwarp_destroy_ep(p_hwfn, ep, false);
+	}
+}
+
+static enum _ecore_status_t
+ecore_iwarp_prealloc_ep(struct ecore_hwfn *p_hwfn, bool init)
+{
+	struct ecore_iwarp_ep *ep;
+	int rc = ECORE_SUCCESS;
+	u32 cid;
+	int count;
+	int i;
+
+	if (init)
+		count = ECORE_IWARP_PREALLOC_CNT;
+	else
+		count = 1;
+
+	for (i = 0; i < count; i++) {
+		rc = ecore_iwarp_create_ep(p_hwfn, &ep);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		/* During initialization we allocate from the main pool,
+		 * afterwards we allocate only from the tcp_cid.
+		 */
+		if (init) {
+			rc = ecore_iwarp_alloc_cid(p_hwfn, &cid);
+			if (rc != ECORE_SUCCESS)
+				goto err;
+			ecore_iwarp_set_tcp_cid(p_hwfn, cid);
+		} else {
+			/* We don't care about the return code, it's ok if
+			 * tcp_cid remains invalid...in this case we'll
+			 * defer allocation
+			 */
+			ecore_iwarp_alloc_tcp_cid(p_hwfn, &cid);
+		}
+
+		ep->tcp_cid = cid;
+
+		OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+		OSAL_LIST_PUSH_TAIL(&ep->list_entry,
+				    &p_hwfn->p_rdma_info->iwarp.ep_free_list);
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	}
+
+	return rc;
+
+err:
+	ecore_iwarp_destroy_ep(p_hwfn, ep, false);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_alloc(struct ecore_hwfn *p_hwfn)
+{
+	enum _ecore_status_t rc;
+
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_hwfn->p_rdma_info->iwarp.qp_lock);
+#endif
+	OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.qp_lock);
+
+	/* Allocate bitmap for tcp cid. These are used by passive side
+	 * to ensure it can allocate a tcp cid during dpc that was
+	 * pre-acquired and doesn't require dynamic allocation of ilt
+	 */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map,
+				   ECORE_IWARP_PREALLOC_CNT,
+				   "TCP_CID");
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate tcp cid, rc = %d\n",
+			   rc);
+		return rc;
+	}
+
+	OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.ep_free_list);
+//DAVIDS	OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	rc = ecore_iwarp_prealloc_ep(p_hwfn, true);
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "ecore_iwarp_prealloc_ep failed, rc = %d\n",
+			   rc);
+		return rc;
+	}
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "ecore_iwarp_prealloc_ep success, rc = %d\n",
+			   rc);
+
+	return ecore_ooo_alloc(p_hwfn);
+}
+
+void
+ecore_iwarp_resc_free(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	OSAL_SPIN_LOCK_DEALLOC(iwarp_info->iw_lock);
+	OSAL_SPIN_LOCK_DEALLOC(iwarp_info->qp_lock);
+#endif
+	ecore_ooo_free(p_hwfn);
+	if (iwarp_info->partial_fpdus)
+		OSAL_FREE(p_hwfn->p_dev, iwarp_info->partial_fpdus);
+	if (iwarp_info->mpa_bufs)
+		OSAL_FREE(p_hwfn->p_dev, iwarp_info->mpa_bufs);
+	if (iwarp_info->mpa_intermediate_buf)
+		OSAL_FREE(p_hwfn->p_dev, iwarp_info->mpa_intermediate_buf);
+
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1);
+}
+
+
+enum _ecore_status_t
+ecore_iwarp_accept(void *rdma_cxt,
+		   struct ecore_iwarp_accept_in *iparams)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_ep *ep;
+	u8 mpa_data_size = 0;
+	enum _ecore_status_t rc;
+
+	ep = (struct ecore_iwarp_ep *)iparams->ep_context;
+	if (!ep) {
+		DP_ERR(p_hwfn, "Ep Context receive in accept is NULL\n");
+		return ECORE_INVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",
+		   iparams->qp->icid, ep->tcp_cid);
+
+	if ((iparams->ord > ECORE_IWARP_ORD_DEFAULT) ||
+	    (iparams->ird > ECORE_IWARP_IRD_DEFAULT)) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) EP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",
+			   iparams->qp->icid, ep->tcp_cid,
+			   iparams->ord, iparams->ord);
+		return ECORE_INVAL;
+	}
+
+	/* We could reach qp->ep != OSAL NULL if we do accept on the same qp */
+	if (iparams->qp->ep == OSAL_NULL) {
+		/* We need to add a replacement for the ep to the free list */
+		ecore_iwarp_prealloc_ep(p_hwfn, false);
+	} else {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Note re-use of QP for different connect\n");
+		/* Return the old ep to the free_pool */
+		ecore_iwarp_return_ep(p_hwfn, iparams->qp->ep);
+	}
+
+	ecore_iwarp_move_to_ep_list(p_hwfn,
+				    &p_hwfn->p_rdma_info->iwarp.ep_list,
+				    ep);
+	ep->listener = OSAL_NULL;
+	ep->cb_context = iparams->cb_context;
+	ep->qp = iparams->qp;
+	ep->qp->ep = ep;
+
+	if (ep->mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) {
+		/* Negotiate ord/ird: if upperlayer requested ord larger than
+		 * ird advertised by remote, we need to decrease our ord
+		 * to match remote ord
+		 */
+		if (iparams->ord > ep->cm_info.ird) {
+			iparams->ord = ep->cm_info.ird;
+		}
+
+		/* For chelsio compatability, if rtr_zero read is requested
+		 * we can't set ird to zero
+		 */
+		if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) &&
+		    (iparams->ird == 0))
+			iparams->ird = 1;
+	}
+
+	/* Update cm_info ord/ird to be negotiated values */
+	ep->cm_info.ord = iparams->ord;
+	ep->cm_info.ird = iparams->ird;
+
+	ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
+
+	ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;
+	ep->cm_info.private_data_len =
+		iparams->private_data_len + mpa_data_size;
+
+	OSAL_MEMCPY((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
+		    iparams->private_data,
+		    iparams->private_data_len);
+
+	if (ep->state == ECORE_IWARP_EP_CLOSED) {
+		DP_NOTICE(p_hwfn, false,
+			  "(0x%x) Accept called on EP in CLOSED state\n",
+			  ep->tcp_cid);
+		ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID;
+		ecore_iwarp_return_ep(p_hwfn, ep);
+		return ECORE_CONN_RESET;
+	}
+
+	rc = ecore_iwarp_mpa_offload(p_hwfn, ep);
+	if (rc) {
+		ecore_iwarp_modify_qp(p_hwfn,
+				      iparams->qp,
+				      ECORE_IWARP_QP_STATE_ERROR,
+				      1);
+	}
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_reject(void *rdma_cxt,
+		   struct ecore_iwarp_reject_in *iparams)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_ep *ep;
+	u8 mpa_data_size = 0;
+	enum _ecore_status_t rc;
+
+	ep = (struct ecore_iwarp_ep *)iparams->ep_context;
+	if (!ep) {
+		DP_ERR(p_hwfn, "Ep Context receive in reject is NULL\n");
+		return ECORE_INVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "EP(0x%x)\n", ep->tcp_cid);
+
+	ep->cb_context = iparams->cb_context;
+	ep->qp = OSAL_NULL;
+
+	ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
+
+	ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;
+	ep->cm_info.private_data_len =
+		iparams->private_data_len + mpa_data_size;
+
+	OSAL_MEMCPY((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
+		    iparams->private_data,
+		    iparams->private_data_len);
+
+	if (ep->state == ECORE_IWARP_EP_CLOSED) {
+		DP_NOTICE(p_hwfn, false,
+			  "(0x%x) Reject called on EP in CLOSED state\n",
+			  ep->tcp_cid);
+		ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID;
+		ecore_iwarp_return_ep(p_hwfn, ep);
+		return ECORE_CONN_RESET;
+	}
+
+	rc = ecore_iwarp_mpa_offload(p_hwfn, ep);
+	return rc;
+}
+
+static void
+ecore_iwarp_print_cm_info(struct ecore_hwfn *p_hwfn,
+			  struct ecore_iwarp_cm_info *cm_info)
+{
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ip_version = %d\n",
+		   cm_info->ip_version);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_ip %x.%x.%x.%x\n",
+		   cm_info->remote_ip[0],
+		   cm_info->remote_ip[1],
+		   cm_info->remote_ip[2],
+		   cm_info->remote_ip[3]);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_ip %x.%x.%x.%x\n",
+		   cm_info->local_ip[0],
+		   cm_info->local_ip[1],
+		   cm_info->local_ip[2],
+		   cm_info->local_ip[3]);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_port = %x\n",
+		   cm_info->remote_port);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_port = %x\n",
+		   cm_info->local_port);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "vlan = %x\n",
+		   cm_info->vlan);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "private_data_len = %x\n",
+		   cm_info->private_data_len);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ord = %d\n",
+		   cm_info->ord);
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ird = %d\n",
+		   cm_info->ird);
+}
+
+static int
+ecore_iwarp_ll2_post_rx(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_ll2_buff *buf,
+			u8 handle)
+{
+	enum _ecore_status_t rc;
+
+	rc = ecore_ll2_post_rx_buffer(
+		p_hwfn,
+		handle,
+		buf->data_phys_addr,
+		(u16)buf->buff_size,
+		buf, 1);
+
+	if (rc) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to repost rx buffer to ll2 rc = %d, handle=%d\n",
+			  rc, handle);
+		OSAL_DMA_FREE_COHERENT(
+			p_hwfn->p_dev,
+			buf->data,
+			buf->data_phys_addr,
+			buf->buff_size);
+		OSAL_FREE(p_hwfn->p_dev, buf);
+	}
+
+	return rc;
+}
+
+static bool
+ecore_iwarp_ep_exists(struct ecore_hwfn *p_hwfn,
+		      struct ecore_iwarp_listener *listener,
+		      struct ecore_iwarp_cm_info *cm_info)
+{
+	struct ecore_iwarp_ep *ep = OSAL_NULL;
+	bool found = false;
+
+	OSAL_SPIN_LOCK(&listener->lock);
+	OSAL_LIST_FOR_EACH_ENTRY(ep, &listener->ep_list,
+				 list_entry, struct ecore_iwarp_ep) {
+		if ((ep->cm_info.local_port == cm_info->local_port) &&
+		    (ep->cm_info.remote_port == cm_info->remote_port) &&
+		    (ep->cm_info.vlan == cm_info->vlan) &&
+		    !OSAL_MEMCMP(&(ep->cm_info.local_ip), cm_info->local_ip,
+				 sizeof(cm_info->local_ip)) &&
+		    !OSAL_MEMCMP(&(ep->cm_info.remote_ip), cm_info->remote_ip,
+				 sizeof(cm_info->remote_ip))) {
+				found = true;
+				break;
+		}
+	}
+
+	OSAL_SPIN_UNLOCK(&listener->lock);
+
+	if (found) {
+		DP_NOTICE(p_hwfn, false, "SYN received on active connection - dropping\n");
+		ecore_iwarp_print_cm_info(p_hwfn, cm_info);
+
+		return true;
+	}
+
+	return false;
+}
+
+static struct ecore_iwarp_listener *
+ecore_iwarp_get_listener(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_cm_info *cm_info)
+{
+	struct ecore_iwarp_listener *listener = OSAL_NULL;
+	static const u32 ip_zero[4] = {0, 0, 0, 0};
+	bool found = false;
+
+	ecore_iwarp_print_cm_info(p_hwfn, cm_info);
+
+	OSAL_LIST_FOR_EACH_ENTRY(listener,
+				 &p_hwfn->p_rdma_info->iwarp.listen_list,
+				 list_entry, struct ecore_iwarp_listener) {
+
+		if (listener->port == cm_info->local_port) {
+			/* Any IP (i.e. 0.0.0.0 ) will be treated as any vlan */
+			if (!OSAL_MEMCMP(listener->ip_addr,
+					 ip_zero,
+					 sizeof(ip_zero))) {
+				found = true;
+				break;
+			}
+
+			/* If not any IP -> check vlan as well */
+			if (!OSAL_MEMCMP(listener->ip_addr,
+					 cm_info->local_ip,
+					 sizeof(cm_info->local_ip)) &&
+
+			     (listener->vlan == cm_info->vlan)) {
+				found = true;
+				break;
+			}
+		}
+	}
+
+	if (found && listener->state == ECORE_IWARP_LISTENER_STATE_ACTIVE) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener found = %p\n",
+			   listener);
+		return listener;
+	}
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener not found\n");
+	return OSAL_NULL;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_parse_rx_pkt(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_cm_info *cm_info,
+			 void *buf,
+			 u8 *remote_mac_addr,
+			 u8 *local_mac_addr,
+			 int *payload_len,
+			 int *tcp_start_offset)
+{
+	struct ecore_vlan_ethhdr *vethh;
+	struct ecore_ethhdr *ethh;
+	struct ecore_iphdr *iph;
+	struct ecore_ipv6hdr *ip6h;
+	struct ecore_tcphdr *tcph;
+	bool vlan_valid = false;
+	int eth_hlen, ip_hlen;
+	u16 eth_type;
+	int i;
+
+	ethh = (struct ecore_ethhdr *)buf;
+	eth_type = ntohs(ethh->h_proto);
+	if (eth_type == ETH_P_8021Q) {
+		vlan_valid = true;
+		vethh = (struct ecore_vlan_ethhdr *)ethh;
+		cm_info->vlan = ntohs(vethh->h_vlan_TCI) & VLAN_VID_MASK;
+		eth_type = ntohs(vethh->h_vlan_encapsulated_proto);
+	}
+
+	eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0);
+
+	OSAL_MEMCPY(remote_mac_addr,
+		    ethh->h_source,
+		    ETH_ALEN);
+
+	OSAL_MEMCPY(local_mac_addr,
+		    ethh->h_dest,
+		    ETH_ALEN);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_type =%d Source mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n",
+		   eth_type, ethh->h_source[0], ethh->h_source[1],
+		   ethh->h_source[2], ethh->h_source[3],
+		   ethh->h_source[4], ethh->h_source[5]);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_hlen=%d destination mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n",
+		   eth_hlen, ethh->h_dest[0], ethh->h_dest[1],
+		   ethh->h_dest[2], ethh->h_dest[3],
+		   ethh->h_dest[4], ethh->h_dest[5]);
+
+	iph = (struct ecore_iphdr *)((u8 *)(ethh) + eth_hlen);
+
+	if (eth_type == ETH_P_IP) {
+		if (iph->protocol != IPPROTO_TCP) {
+			DP_NOTICE(p_hwfn, false,
+				  "Unexpected ip protocol on ll2 %x\n",
+				  iph->protocol);
+			return ECORE_INVAL;
+		}
+
+		cm_info->local_ip[0] = ntohl(iph->daddr);
+		cm_info->remote_ip[0] = ntohl(iph->saddr);
+		cm_info->ip_version = (enum ecore_tcp_ip_version)TCP_IPV4;
+
+		ip_hlen = (iph->ihl)*sizeof(u32);
+		*payload_len = ntohs(iph->tot_len) - ip_hlen;
+
+	} else if (eth_type == ETH_P_IPV6) {
+		ip6h = (struct ecore_ipv6hdr *)iph;
+
+		if (ip6h->nexthdr != IPPROTO_TCP) {
+			DP_NOTICE(p_hwfn, false,
+				  "Unexpected ip protocol on ll2 %x\n",
+				  iph->protocol);
+			return ECORE_INVAL;
+		}
+
+		for (i = 0; i < 4; i++) {
+			cm_info->local_ip[i] =
+				ntohl(ip6h->daddr.in6_u.u6_addr32[i]);
+			cm_info->remote_ip[i] =
+				ntohl(ip6h->saddr.in6_u.u6_addr32[i]);
+		}
+		cm_info->ip_version = (enum ecore_tcp_ip_version)TCP_IPV6;
+
+		ip_hlen = sizeof(*ip6h);
+		*payload_len = ntohs(ip6h->payload_len);
+	} else {
+		DP_NOTICE(p_hwfn, false,
+			  "Unexpected ethertype on ll2 %x\n", eth_type);
+		return ECORE_INVAL;
+	}
+
+	tcph = (struct ecore_tcphdr *)((u8 *)iph + ip_hlen);
+
+	if (!tcph->syn) {
+		DP_NOTICE(p_hwfn, false,
+			  "Only SYN type packet expected on this ll2 conn, iph->ihl=%d source=%d dest=%d\n",
+			  iph->ihl, tcph->source, tcph->dest);
+		return ECORE_INVAL;
+	}
+
+	cm_info->local_port = ntohs(tcph->dest);
+	cm_info->remote_port = ntohs(tcph->source);
+
+	ecore_iwarp_print_cm_info(p_hwfn, cm_info);
+
+	*tcp_start_offset = eth_hlen + ip_hlen;
+
+	return ECORE_SUCCESS;
+}
+
+static struct ecore_iwarp_fpdu *
+ecore_iwarp_get_curr_fpdu(struct ecore_hwfn *p_hwfn, u16 cid)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	struct ecore_iwarp_fpdu *partial_fpdu;
+	u32 idx = cid - ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_IWARP);
+
+	if (idx >= iwarp_info->max_num_partial_fpdus) {
+		DP_ERR(p_hwfn, "Invalid cid %x max_num_partial_fpdus=%x\n", cid,
+		       iwarp_info->max_num_partial_fpdus);
+		return OSAL_NULL;
+	}
+
+	partial_fpdu = &iwarp_info->partial_fpdus[idx];
+
+	return partial_fpdu;
+}
+
+enum ecore_iwarp_mpa_pkt_type {
+	ECORE_IWARP_MPA_PKT_PACKED,
+	ECORE_IWARP_MPA_PKT_PARTIAL,
+	ECORE_IWARP_MPA_PKT_UNALIGNED
+};
+
+#define ECORE_IWARP_INVALID_FPDU_LENGTH 0xffff
+#define ECORE_IWARP_MPA_FPDU_LENGTH_SIZE (2)
+#define ECORE_IWARP_MPA_CRC32_DIGEST_SIZE (4)
+
+/* Pad to multiple of 4 */
+#define ECORE_IWARP_PDU_DATA_LEN_WITH_PAD(data_len) (((data_len) + 3) & ~3)
+
+#define ECORE_IWARP_FPDU_LEN_WITH_PAD(_mpa_len) \
+	(ECORE_IWARP_PDU_DATA_LEN_WITH_PAD(_mpa_len + \
+					   ECORE_IWARP_MPA_FPDU_LENGTH_SIZE) + \
+					   ECORE_IWARP_MPA_CRC32_DIGEST_SIZE)
+
+/* fpdu can be fragmented over maximum 3 bds: header, partial mpa, unaligned */
+#define ECORE_IWARP_MAX_BDS_PER_FPDU 3
+
+char *pkt_type_str[] = {
+	"ECORE_IWARP_MPA_PKT_PACKED",
+	"ECORE_IWARP_MPA_PKT_PARTIAL",
+	"ECORE_IWARP_MPA_PKT_UNALIGNED"
+};
+
+static enum _ecore_status_t
+ecore_iwarp_recycle_pkt(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_fpdu *fpdu,
+			struct ecore_iwarp_ll2_buff *buf);
+
+static enum ecore_iwarp_mpa_pkt_type
+ecore_iwarp_mpa_classify(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_fpdu *fpdu,
+			 u16 tcp_payload_len,
+			 u8 *mpa_data)
+
+{
+	enum ecore_iwarp_mpa_pkt_type pkt_type;
+	u16 mpa_len;
+
+	if (fpdu->incomplete_bytes) {
+		pkt_type = ECORE_IWARP_MPA_PKT_UNALIGNED;
+		goto out;
+	}
+
+	/* special case of one byte remaining... */
+	if (tcp_payload_len == 1) {
+		/* lower byte will be read next packet */
+		fpdu->fpdu_length = *mpa_data << 8;
+		pkt_type = ECORE_IWARP_MPA_PKT_PARTIAL;
+		goto out;
+	}
+
+	mpa_len = ntohs(*((u16 *)(mpa_data)));
+	fpdu->fpdu_length = ECORE_IWARP_FPDU_LEN_WITH_PAD(mpa_len);
+
+	if (fpdu->fpdu_length <= tcp_payload_len)
+		pkt_type = ECORE_IWARP_MPA_PKT_PACKED;
+	else
+		pkt_type = ECORE_IWARP_MPA_PKT_PARTIAL;
+
+out:
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_ALIGN: %s: fpdu_length=0x%x tcp_payload_len:0x%x\n",
+		   pkt_type_str[pkt_type], fpdu->fpdu_length, tcp_payload_len);
+
+	return pkt_type;
+}
+
+static void
+ecore_iwarp_init_fpdu(struct ecore_iwarp_ll2_buff *buf,
+		      struct ecore_iwarp_fpdu *fpdu,
+		      struct unaligned_opaque_data *pkt_data,
+		      u16 tcp_payload_size, u8 placement_offset)
+{
+	fpdu->mpa_buf = buf;
+	fpdu->pkt_hdr = buf->data_phys_addr + placement_offset;
+	fpdu->pkt_hdr_size = pkt_data->tcp_payload_offset;
+
+	fpdu->mpa_frag = buf->data_phys_addr + pkt_data->first_mpa_offset;
+	fpdu->mpa_frag_virt = (u8 *)(buf->data) + pkt_data->first_mpa_offset;
+
+	if (tcp_payload_size == 1)
+		fpdu->incomplete_bytes = ECORE_IWARP_INVALID_FPDU_LENGTH;
+	else if (tcp_payload_size < fpdu->fpdu_length)
+		fpdu->incomplete_bytes = fpdu->fpdu_length - tcp_payload_size;
+	else
+		fpdu->incomplete_bytes = 0; /* complete fpdu */
+
+	fpdu->mpa_frag_len = fpdu->fpdu_length - fpdu->incomplete_bytes;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_copy_fpdu(struct ecore_hwfn *p_hwfn,
+		      struct ecore_iwarp_fpdu *fpdu,
+		      struct unaligned_opaque_data *pkt_data,
+		      struct ecore_iwarp_ll2_buff *buf,
+		      u16 tcp_payload_size)
+
+{
+	u8 *tmp_buf = p_hwfn->p_rdma_info->iwarp.mpa_intermediate_buf;
+	enum _ecore_status_t rc;
+
+	/* need to copy the data from the partial packet stored in fpdu
+	 * to the new buf, for this we also need to move the data currently
+	 * placed on the buf. The assumption is that the buffer is big enough
+	 * since fpdu_length <= mss, we use an intermediate buffer since
+	 * we may need to copy the new data to an overlapping location
+	 */
+	if ((fpdu->mpa_frag_len + tcp_payload_size) > (u16)buf->buff_size) {
+		DP_ERR(p_hwfn,
+		       "MPA ALIGN: Unexpected: buffer is not large enough for split fpdu buff_size = %d mpa_frag_len = %d, tcp_payload_size = %d, incomplete_bytes = %d\n",
+		       buf->buff_size, fpdu->mpa_frag_len, tcp_payload_size,
+		       fpdu->incomplete_bytes);
+		return ECORE_INVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA ALIGN Copying fpdu: [%p, %d] [%p, %d]\n",
+		   fpdu->mpa_frag_virt, fpdu->mpa_frag_len,
+		   (u8 *)(buf->data) + pkt_data->first_mpa_offset,
+		   tcp_payload_size);
+
+	OSAL_MEMCPY(tmp_buf, fpdu->mpa_frag_virt, fpdu->mpa_frag_len);
+	OSAL_MEMCPY(tmp_buf + fpdu->mpa_frag_len,
+		    (u8 *)(buf->data) + pkt_data->first_mpa_offset,
+		    tcp_payload_size);
+
+	rc = ecore_iwarp_recycle_pkt(p_hwfn, fpdu, fpdu->mpa_buf);
+	if (rc)
+		return rc;
+
+	/* If we managed to post the buffer copy the data to the new buffer
+	 * o/w this will occur in the next round...
+	 */
+	OSAL_MEMCPY((u8 *)(buf->data), tmp_buf,
+		    fpdu->mpa_frag_len + tcp_payload_size);
+
+	fpdu->mpa_buf = buf;
+	/* fpdu->pkt_hdr remains as is */
+	/* fpdu->mpa_frag is overriden with new buf */
+	fpdu->mpa_frag = buf->data_phys_addr;
+	fpdu->mpa_frag_virt = buf->data;
+	fpdu->mpa_frag_len += tcp_payload_size;
+
+	fpdu->incomplete_bytes -= tcp_payload_size;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA ALIGN: split fpdu buff_size = %d mpa_frag_len = %d, tcp_payload_size = %d, incomplete_bytes = %d\n",
+		   buf->buff_size, fpdu->mpa_frag_len, tcp_payload_size,
+		   fpdu->incomplete_bytes);
+
+	return 0;
+}
+
+static void
+ecore_iwarp_update_fpdu_length(struct ecore_hwfn *p_hwfn,
+			       struct ecore_iwarp_fpdu *fpdu,
+			       u8 *mpa_data)
+{
+	u16 mpa_len;
+
+	/* Update incomplete packets if needed */
+	if (fpdu->incomplete_bytes == ECORE_IWARP_INVALID_FPDU_LENGTH) {
+		mpa_len = fpdu->fpdu_length | *mpa_data;
+		fpdu->fpdu_length = ECORE_IWARP_FPDU_LEN_WITH_PAD(mpa_len);
+		fpdu->mpa_frag_len = fpdu->fpdu_length;
+		/* one byte of hdr */
+		fpdu->incomplete_bytes = fpdu->fpdu_length - 1;
+		DP_VERBOSE(p_hwfn,
+			   ECORE_MSG_RDMA,
+			   "MPA_ALIGN: Partial header mpa_len=%x fpdu_length=%x incomplete_bytes=%x\n",
+			   mpa_len, fpdu->fpdu_length, fpdu->incomplete_bytes);
+	}
+}
+
+#define ECORE_IWARP_IS_RIGHT_EDGE(_curr_pkt) \
+	(GET_FIELD(_curr_pkt->flags, \
+		   UNALIGNED_OPAQUE_DATA_PKT_REACHED_WIN_RIGHT_EDGE))
+
+/* This function is used to recycle a buffer using the ll2 drop option. It
+ * uses the mechanism to ensure that all buffers posted to tx before this one
+ * were completed. The buffer sent here will be sent as a cookie in the tx
+ * completion function and can then be reposted to rx chain when done. The flow
+ * that requires this is the flow where a FPDU splits over more than 3 tcp
+ * segments. In this case the driver needs to re-post a rx buffer instead of
+ * the one received, but driver can't simply repost a buffer it copied from
+ * as there is a case where the buffer was originally a packed FPDU, and is
+ * partially posted to FW. Driver needs to ensure FW is done with it.
+ */
+static enum _ecore_status_t
+ecore_iwarp_recycle_pkt(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_fpdu *fpdu,
+			struct ecore_iwarp_ll2_buff *buf)
+{
+	struct ecore_ll2_tx_pkt_info tx_pkt;
+	enum _ecore_status_t rc;
+	u8 ll2_handle;
+
+	OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt));
+	tx_pkt.num_of_bds = 1;
+	tx_pkt.tx_dest = ECORE_LL2_TX_DEST_DROP;
+	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2;
+	tx_pkt.first_frag = fpdu->pkt_hdr;
+	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;
+	buf->piggy_buf = OSAL_NULL;
+	tx_pkt.cookie = buf;
+
+	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;
+
+	rc = ecore_ll2_prepare_tx_packet(p_hwfn,
+					 ll2_handle,
+					 &tx_pkt, true);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_ALIGN: send drop tx packet [%lx, 0x%x], buf=%p, rc=%d\n",
+		   (long unsigned int)tx_pkt.first_frag,
+		   tx_pkt.first_frag_len, buf, rc);
+
+	if (rc)
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Can't drop packet rc=%d\n", rc);
+
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_win_right_edge(struct ecore_hwfn *p_hwfn,
+			   struct ecore_iwarp_fpdu *fpdu)
+{
+	struct ecore_ll2_tx_pkt_info tx_pkt;
+	enum _ecore_status_t rc;
+	u8 ll2_handle;
+
+	OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt));
+	tx_pkt.num_of_bds = 1;
+	tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB;
+	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2;
+
+	tx_pkt.first_frag = fpdu->pkt_hdr;
+	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;
+	tx_pkt.enable_ip_cksum = true;
+	tx_pkt.enable_l4_cksum = true;
+	tx_pkt.calc_ip_len = true;
+	/* vlan overload with enum iwarp_ll2_tx_queues */
+	tx_pkt.vlan = IWARP_LL2_ALIGNED_RIGHT_TRIMMED_TX_QUEUE;
+
+	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;
+
+	rc = ecore_ll2_prepare_tx_packet(p_hwfn,
+					 ll2_handle,
+					 &tx_pkt, true);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_ALIGN: Sent right edge FPDU num_bds=%d [%lx, 0x%x], rc=%d\n",
+		   tx_pkt.num_of_bds, (long unsigned int)tx_pkt.first_frag,
+		   tx_pkt.first_frag_len, rc);
+
+	if (rc)
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Can't send right edge rc=%d\n", rc);
+
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_send_fpdu(struct ecore_hwfn *p_hwfn,
+		      struct ecore_iwarp_fpdu *fpdu,
+		      struct unaligned_opaque_data *curr_pkt,
+		      struct ecore_iwarp_ll2_buff *buf,
+		      u16 tcp_payload_size,
+		      enum ecore_iwarp_mpa_pkt_type pkt_type)
+{
+	struct ecore_ll2_tx_pkt_info tx_pkt;
+	enum _ecore_status_t rc;
+	u8 ll2_handle;
+
+	OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt));
+
+	tx_pkt.num_of_bds = (pkt_type == ECORE_IWARP_MPA_PKT_UNALIGNED) ? 3 : 2;
+	tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB;
+	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2;
+
+	/* Send the mpa_buf only with the last fpdu (in case of packed) */
+	if ((pkt_type == ECORE_IWARP_MPA_PKT_UNALIGNED) ||
+	    (tcp_payload_size <= fpdu->fpdu_length))
+		tx_pkt.cookie = fpdu->mpa_buf;
+
+	tx_pkt.first_frag = fpdu->pkt_hdr;
+	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;
+	tx_pkt.enable_ip_cksum = true;
+	tx_pkt.enable_l4_cksum = true;
+	tx_pkt.calc_ip_len = true;
+	/* vlan overload with enum iwarp_ll2_tx_queues */
+	tx_pkt.vlan = IWARP_LL2_ALIGNED_TX_QUEUE;
+
+	/* special case of unaligned packet and not packed, need to send
+	 * both buffers as cookie to release.
+	 */
+	if (tcp_payload_size == fpdu->incomplete_bytes) {
+		fpdu->mpa_buf->piggy_buf = buf;
+	}
+
+	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;
+
+	rc = ecore_ll2_prepare_tx_packet(p_hwfn,
+					 ll2_handle,
+					 &tx_pkt, true);
+	if (rc)
+		goto err;
+
+	rc = ecore_ll2_set_fragment_of_tx_packet(p_hwfn, ll2_handle,
+						 fpdu->mpa_frag,
+						 fpdu->mpa_frag_len);
+	if (rc)
+		goto err;
+
+	if (fpdu->incomplete_bytes) {
+		rc = ecore_ll2_set_fragment_of_tx_packet(
+			p_hwfn, ll2_handle,
+			buf->data_phys_addr + curr_pkt->first_mpa_offset,
+			fpdu->incomplete_bytes);
+
+		if (rc)
+			goto err;
+	}
+
+err:
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_ALIGN: Sent FPDU num_bds=%d [%lx, 0x%x], [0x%lx, 0x%x], [0x%lx, 0x%x] (cookie %p) rc=%d\n",
+		   tx_pkt.num_of_bds, (long unsigned int)tx_pkt.first_frag,
+		   tx_pkt.first_frag_len, (long unsigned int)fpdu->mpa_frag,
+		   fpdu->mpa_frag_len, (long unsigned int)buf->data_phys_addr +
+		   curr_pkt->first_mpa_offset, fpdu->incomplete_bytes,
+		   tx_pkt.cookie, rc);
+
+	return rc;
+}
+
+static void
+ecore_iwarp_mpa_get_data(struct ecore_hwfn *p_hwfn,
+			 struct unaligned_opaque_data *curr_pkt,
+			 u32 opaque_data0, u32 opaque_data1)
+{
+	u64 opaque_data;
+
+	opaque_data = HILO_64(opaque_data1, opaque_data0);
+	*curr_pkt = *((struct unaligned_opaque_data *)&opaque_data);
+
+	/* fix endianity */
+	curr_pkt->first_mpa_offset = curr_pkt->tcp_payload_offset +
+		OSAL_LE16_TO_CPU(curr_pkt->first_mpa_offset);
+	curr_pkt->cid = OSAL_LE32_TO_CPU(curr_pkt->cid);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "OPAQUE0=0x%x OPAQUE1=0x%x first_mpa_offset:0x%x\ttcp_payload_offset:0x%x\tflags:0x%x\tcid:0x%x\n",
+		   opaque_data0, opaque_data1, curr_pkt->first_mpa_offset,
+		   curr_pkt->tcp_payload_offset, curr_pkt->flags,
+		   curr_pkt->cid);
+}
+
+static void
+ecore_iwarp_mpa_print_tcp_seq(struct ecore_hwfn *p_hwfn,
+			      void *buf)
+{
+	struct ecore_vlan_ethhdr *vethh;
+	struct ecore_ethhdr *ethh;
+	struct ecore_iphdr *iph;
+	struct ecore_ipv6hdr *ip6h;
+	struct ecore_tcphdr *tcph;
+	bool vlan_valid = false;
+	int eth_hlen, ip_hlen;
+	u16 eth_type;
+
+	if ((p_hwfn->dp_level > ECORE_LEVEL_VERBOSE) ||
+	    !(p_hwfn->dp_module & ECORE_MSG_RDMA))
+		return;
+
+	ethh = (struct ecore_ethhdr *)buf;
+	eth_type = ntohs(ethh->h_proto);
+	if (eth_type == ETH_P_8021Q) {
+		vlan_valid = true;
+		vethh = (struct ecore_vlan_ethhdr *)ethh;
+		eth_type = ntohs(vethh->h_vlan_encapsulated_proto);
+	}
+
+	eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0);
+
+	iph = (struct ecore_iphdr *)((u8 *)(ethh) + eth_hlen);
+
+	if (eth_type == ETH_P_IP) {
+		ip_hlen = (iph->ihl)*sizeof(u32);
+	} else if (eth_type == ETH_P_IPV6) {
+		ip6h = (struct ecore_ipv6hdr *)iph;
+		ip_hlen = sizeof(*ip6h);
+	} else {
+		DP_ERR(p_hwfn, "Unexpected ethertype on ll2 %x\n", eth_type);
+		return;
+	}
+
+	tcph = (struct ecore_tcphdr *)((u8 *)iph + ip_hlen);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Processing MPA PKT: tcp_seq=0x%x tcp_ack_seq=0x%x\n",
+		   ntohl(tcph->seq), ntohl(tcph->ack_seq));
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_type =%d Source mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n",
+		   eth_type, ethh->h_source[0], ethh->h_source[1],
+		   ethh->h_source[2], ethh->h_source[3],
+		   ethh->h_source[4], ethh->h_source[5]);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_hlen=%d destination mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n",
+		   eth_hlen, ethh->h_dest[0], ethh->h_dest[1],
+		   ethh->h_dest[2], ethh->h_dest[3],
+		   ethh->h_dest[4], ethh->h_dest[5]);
+
+	return;
+}
+
+/* This function is called when an unaligned or incomplete MPA packet arrives
+ * driver needs to align the packet, perhaps using previous data and send
+ * it down to FW once it is aligned.
+ */
+static enum _ecore_status_t
+ecore_iwarp_process_mpa_pkt(struct ecore_hwfn *p_hwfn,
+			    struct ecore_iwarp_ll2_mpa_buf *mpa_buf)
+{
+	struct ecore_iwarp_ll2_buff *buf = mpa_buf->ll2_buf;
+	enum ecore_iwarp_mpa_pkt_type pkt_type;
+	struct unaligned_opaque_data *curr_pkt = &mpa_buf->data;
+	struct ecore_iwarp_fpdu *fpdu;
+	u8 *mpa_data;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	ecore_iwarp_mpa_print_tcp_seq(
+		p_hwfn, (u8 *)(buf->data) + mpa_buf->placement_offset);
+
+	fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, curr_pkt->cid & 0xffff);
+	if (!fpdu) {/* something corrupt with cid, post rx back */
+		DP_ERR(p_hwfn, "Invalid cid, drop and post back to rx cid=%x\n",
+		       curr_pkt->cid);
+		rc = ecore_iwarp_ll2_post_rx(
+			p_hwfn, buf, p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle);
+
+		if (rc) { /* not much we can do here except log and free */
+			DP_ERR(p_hwfn, "Post rx buffer failed\n");
+
+			/* we don't expect any failures from rx, not even
+			 * busy since we allocate #bufs=#descs
+			 */
+			rc = ECORE_UNKNOWN_ERROR;
+		}
+		return rc;
+	}
+
+	do {
+		mpa_data = ((u8 *)(buf->data) + curr_pkt->first_mpa_offset);
+
+		pkt_type = ecore_iwarp_mpa_classify(p_hwfn, fpdu,
+						    mpa_buf->tcp_payload_len,
+						    mpa_data);
+
+		switch (pkt_type) {
+		case ECORE_IWARP_MPA_PKT_PARTIAL:
+			ecore_iwarp_init_fpdu(buf, fpdu,
+					      curr_pkt,
+					      mpa_buf->tcp_payload_len,
+					      mpa_buf->placement_offset);
+
+			if (!ECORE_IWARP_IS_RIGHT_EDGE(curr_pkt)) {
+				mpa_buf->tcp_payload_len = 0;
+				break;
+			}
+
+			rc = ecore_iwarp_win_right_edge(p_hwfn, fpdu);
+
+			if (rc) {
+				DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+					   "Can't send FPDU:reset rc=%d\n", rc);
+				OSAL_MEM_ZERO(fpdu, sizeof(*fpdu));
+				break;
+			}
+
+			mpa_buf->tcp_payload_len = 0;
+			break;
+		case ECORE_IWARP_MPA_PKT_PACKED:
+			if (fpdu->fpdu_length == 8) {
+				DP_ERR(p_hwfn, "SUSPICIOUS fpdu_length = 0x%x: assuming bug...aborting this packet...\n",
+				       fpdu->fpdu_length);
+				mpa_buf->tcp_payload_len = 0;
+				break;
+			}
+
+			ecore_iwarp_init_fpdu(buf, fpdu,
+					      curr_pkt,
+					      mpa_buf->tcp_payload_len,
+					      mpa_buf->placement_offset);
+
+			rc = ecore_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf,
+						   mpa_buf->tcp_payload_len,
+						   pkt_type);
+			if (rc) {
+				DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+					   "Can't send FPDU:reset rc=%d\n", rc);
+				OSAL_MEM_ZERO(fpdu, sizeof(*fpdu));
+				break;
+			}
+			mpa_buf->tcp_payload_len -= fpdu->fpdu_length;
+			curr_pkt->first_mpa_offset += fpdu->fpdu_length;
+			break;
+		case ECORE_IWARP_MPA_PKT_UNALIGNED:
+			ecore_iwarp_update_fpdu_length(p_hwfn, fpdu, mpa_data);
+			if (mpa_buf->tcp_payload_len < fpdu->incomplete_bytes) {
+				/* special handling of fpdu split over more
+				 * than 2 segments
+				 */
+				if (ECORE_IWARP_IS_RIGHT_EDGE(curr_pkt)) {
+					rc = ecore_iwarp_win_right_edge(p_hwfn,
+									fpdu);
+					/* packet will be re-processed later */
+					if (rc)
+						return rc;
+				}
+
+				rc = ecore_iwarp_copy_fpdu(
+					p_hwfn, fpdu, curr_pkt,
+					buf, mpa_buf->tcp_payload_len);
+
+				/* packet will be re-processed later */
+				if (rc)
+					return rc;
+
+				mpa_buf->tcp_payload_len = 0;
+
+				break;
+			}
+
+			rc = ecore_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf,
+						   mpa_buf->tcp_payload_len,
+						   pkt_type);
+			if (rc) {
+				DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+					   "Can't send FPDU:delay rc=%d\n", rc);
+				/* don't reset fpdu -> we need it for next
+				 * classify
+				 */
+				break;
+			}
+			mpa_buf->tcp_payload_len -= fpdu->incomplete_bytes;
+			curr_pkt->first_mpa_offset += fpdu->incomplete_bytes;
+			/* The framed PDU was sent - no more incomplete bytes */
+			fpdu->incomplete_bytes = 0;
+			break;
+		}
+
+	} while (mpa_buf->tcp_payload_len && !rc);
+
+	return rc;
+}
+
+static void
+ecore_iwarp_process_pending_pkts(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	struct ecore_iwarp_ll2_mpa_buf *mpa_buf = OSAL_NULL;
+	enum _ecore_status_t rc;
+
+	while (!OSAL_LIST_IS_EMPTY(&iwarp_info->mpa_buf_pending_list)) {
+		mpa_buf = OSAL_LIST_FIRST_ENTRY(
+			&iwarp_info->mpa_buf_pending_list,
+			struct ecore_iwarp_ll2_mpa_buf,
+			list_entry);
+
+		rc = ecore_iwarp_process_mpa_pkt(p_hwfn, mpa_buf);
+
+		 /* busy means break and continue processing later, don't
+		  * remove the buf from the pending list.
+		  */
+		if (rc == ECORE_BUSY)
+			break;
+
+#ifdef _NTDDK_
+#pragma warning(suppress : 6011)
+#pragma warning(suppress : 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(
+			&mpa_buf->list_entry,
+			&iwarp_info->mpa_buf_pending_list);
+
+		OSAL_LIST_PUSH_TAIL(&mpa_buf->list_entry,
+				    &iwarp_info->mpa_buf_list);
+
+		if (rc) { /* different error, don't continue */
+			DP_NOTICE(p_hwfn, false, "process pkts failed rc=%d\n",
+				  rc);
+			break;
+		}
+	}
+}
+
+static void
+ecore_iwarp_ll2_comp_mpa_pkt(void *cxt,
+			     struct ecore_ll2_comp_rx_data *data)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	struct ecore_iwarp_ll2_mpa_buf *mpa_buf;
+
+	iwarp_info->unalign_rx_comp++;
+
+	mpa_buf = OSAL_LIST_FIRST_ENTRY(&iwarp_info->mpa_buf_list,
+					struct ecore_iwarp_ll2_mpa_buf,
+					list_entry);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "LL2 MPA CompRx buf=%p placement_offset=%d, payload_len=0x%x mpa_buf=%p\n",
+		   data->cookie, data->u.placement_offset,
+		   data->length.packet_length, mpa_buf);
+
+	if (!mpa_buf) {
+		DP_ERR(p_hwfn, "no free mpa buf. this is a driver bug.\n");
+		return;
+	}
+	OSAL_LIST_REMOVE_ENTRY(&mpa_buf->list_entry, &iwarp_info->mpa_buf_list);
+
+	ecore_iwarp_mpa_get_data(p_hwfn, &mpa_buf->data,
+				 data->opaque_data_0, data->opaque_data_1);
+
+	mpa_buf->tcp_payload_len = data->length.packet_length -
+		 mpa_buf->data.first_mpa_offset;
+	mpa_buf->ll2_buf = (struct ecore_iwarp_ll2_buff *)data->cookie;
+	mpa_buf->data.first_mpa_offset += data->u.placement_offset;
+	mpa_buf->placement_offset = data->u.placement_offset;
+
+	OSAL_LIST_PUSH_TAIL(&mpa_buf->list_entry,
+			    &iwarp_info->mpa_buf_pending_list);
+
+	ecore_iwarp_process_pending_pkts(p_hwfn);
+}
+
+static void
+ecore_iwarp_ll2_comp_syn_pkt(void *cxt, struct ecore_ll2_comp_rx_data *data)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_iwarp_ll2_buff *buf =
+		(struct ecore_iwarp_ll2_buff *)data->cookie;
+	struct ecore_iwarp_listener *listener;
+	struct ecore_iwarp_cm_info cm_info;
+	struct ecore_ll2_tx_pkt_info tx_pkt;
+	u8 remote_mac_addr[ETH_ALEN];
+	u8 local_mac_addr[ETH_ALEN];
+	struct ecore_iwarp_ep *ep;
+	enum _ecore_status_t rc;
+	int tcp_start_offset;
+	u8 ts_hdr_size = 0;
+	int payload_len;
+	u32 hdr_size;
+
+	OSAL_MEM_ZERO(&cm_info, sizeof(cm_info));
+
+	/* Check if packet was received with errors... */
+	if (data->err_flags != 0) {
+		DP_NOTICE(p_hwfn, false, "Error received on SYN packet: 0x%x\n",
+			  data->err_flags);
+		goto err;
+	}
+
+	if (GET_FIELD(data->parse_flags,
+		      PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED) &&
+	    GET_FIELD(data->parse_flags,
+		      PARSING_AND_ERR_FLAGS_L4CHKSMERROR)) {
+		DP_NOTICE(p_hwfn, false, "Syn packet received with checksum error\n");
+		goto err;
+	}
+
+	rc = ecore_iwarp_parse_rx_pkt(
+		p_hwfn, &cm_info, (u8 *)(buf->data) + data->u.placement_offset,
+		remote_mac_addr, local_mac_addr, &payload_len,
+		&tcp_start_offset);
+	if (rc)
+		goto err;
+
+	/* Check if there is a listener for this 4-tuple */
+	listener = ecore_iwarp_get_listener(p_hwfn, &cm_info);
+	if (!listener) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "SYN received on tuple not listened on parse_flags=%d packet len=%d\n",
+			   data->parse_flags, data->length.packet_length);
+
+		OSAL_MEMSET(&tx_pkt, 0, sizeof(tx_pkt));
+		tx_pkt.num_of_bds = 1;
+		tx_pkt.bd_flags = 0;
+		tx_pkt.l4_hdr_offset_w = (data->length.packet_length) >> 2;
+		tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB;
+		tx_pkt.first_frag = buf->data_phys_addr +
+			data->u.placement_offset;
+		tx_pkt.first_frag_len = data->length.packet_length;
+		tx_pkt.cookie = buf;
+
+		rc = ecore_ll2_prepare_tx_packet(
+			p_hwfn,
+			p_hwfn->p_rdma_info->iwarp.ll2_syn_handle,
+			&tx_pkt, true);
+
+		if (rc) {
+			DP_NOTICE(p_hwfn, false,
+				  "Can't post SYN back to chip rc=%d\n", rc);
+			goto err;
+		}
+		return;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Received syn on listening port\n");
+
+	/* For debugging purpose... */
+	if (listener->drop)
+		goto err;
+
+	/* There may be an open ep on this connection if this is a syn
+	 * retrasnmit... need to make sure there isn't...
+	 */
+	if (ecore_iwarp_ep_exists(p_hwfn, listener, &cm_info))
+		goto err;
+
+	ep = ecore_iwarp_get_free_ep(p_hwfn);
+	if (ep == OSAL_NULL)
+		goto err;
+
+	OSAL_SPIN_LOCK(&listener->lock);
+	OSAL_LIST_PUSH_TAIL(&ep->list_entry, &listener->ep_list);
+	OSAL_SPIN_UNLOCK(&listener->lock);
+
+	OSAL_MEMCPY(ep->remote_mac_addr,
+		    remote_mac_addr,
+		    ETH_ALEN);
+	OSAL_MEMCPY(ep->local_mac_addr,
+		    local_mac_addr,
+		    ETH_ALEN);
+
+	OSAL_MEMCPY(&ep->cm_info, &cm_info, sizeof(ep->cm_info));
+
+	if (p_hwfn->p_rdma_info->iwarp.tcp_flags & ECORE_IWARP_TS_EN)
+		ts_hdr_size = TIMESTAMP_HEADER_SIZE;
+
+	hdr_size = ((cm_info.ip_version == ECORE_TCP_IPV4) ? 40 : 60) +
+		ts_hdr_size;
+	ep->mss = p_hwfn->p_rdma_info->iwarp.max_mtu - hdr_size;
+	ep->mss = OSAL_MIN_T(u16, ECORE_IWARP_MAX_FW_MSS, ep->mss);
+
+	ep->listener = listener;
+	ep->event_cb = listener->event_cb;
+	ep->cb_context = listener->cb_context;
+	ep->connect_mode = TCP_CONNECT_PASSIVE;
+
+	ep->syn = buf;
+	ep->syn_ip_payload_length = (u16)payload_len;
+	ep->syn_phy_addr = buf->data_phys_addr + data->u.placement_offset +
+		tcp_start_offset;
+
+	rc = ecore_iwarp_tcp_offload(p_hwfn, ep);
+	if (rc != ECORE_SUCCESS) {
+		ecore_iwarp_return_ep(p_hwfn, ep);
+		goto err;
+	}
+	return;
+
+err:
+	ecore_iwarp_ll2_post_rx(
+		p_hwfn, buf, p_hwfn->p_rdma_info->iwarp.ll2_syn_handle);
+}
+
+static void
+ecore_iwarp_ll2_rel_rx_pkt(void *cxt,
+			   u8 OSAL_UNUSED connection_handle,
+			   void *cookie,
+			   dma_addr_t OSAL_UNUSED rx_buf_addr,
+			   bool OSAL_UNUSED b_last_packet)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_iwarp_ll2_buff *buffer =
+		(struct ecore_iwarp_ll2_buff *)cookie;
+
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       buffer->data,
+			       buffer->data_phys_addr,
+			       buffer->buff_size);
+
+	OSAL_FREE(p_hwfn->p_dev, buffer);
+}
+
+static void
+ecore_iwarp_ll2_comp_tx_pkt(void *cxt,
+			    u8 connection_handle,
+			    void *cookie,
+			    dma_addr_t OSAL_UNUSED first_frag_addr,
+			    bool OSAL_UNUSED b_last_fragment,
+			    bool OSAL_UNUSED b_last_packet)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_iwarp_ll2_buff *buffer =
+		(struct ecore_iwarp_ll2_buff *)cookie;
+	struct ecore_iwarp_ll2_buff *piggy;
+
+	if (!buffer) /* can happen in packed mpa unaligned... */
+		return;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "LL2 CompTX buf=%p piggy_buf=%p handle=%d\n",
+		   buffer, buffer->piggy_buf, connection_handle);
+
+	/* we got a tx packet -> this was originally a rx packet... now we
+	 * can post it back...
+	 */
+	piggy = buffer->piggy_buf;
+	if (piggy) {
+		buffer->piggy_buf = OSAL_NULL;
+		ecore_iwarp_ll2_post_rx(p_hwfn, piggy,
+					connection_handle);
+	}
+
+	ecore_iwarp_ll2_post_rx(p_hwfn, buffer,
+				connection_handle);
+
+	if (connection_handle == p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle)
+		ecore_iwarp_process_pending_pkts(p_hwfn);
+
+	return;
+}
+
+static void
+ecore_iwarp_ll2_rel_tx_pkt(void *cxt,
+			   u8 OSAL_UNUSED connection_handle,
+			   void *cookie,
+			   dma_addr_t OSAL_UNUSED first_frag_addr,
+			   bool OSAL_UNUSED b_last_fragment,
+			   bool OSAL_UNUSED b_last_packet)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_iwarp_ll2_buff *buffer =
+		(struct ecore_iwarp_ll2_buff *)cookie;
+
+	if (!buffer)
+		return;
+
+	if (buffer->piggy_buf) {
+		OSAL_DMA_FREE_COHERENT(
+			p_hwfn->p_dev,
+			buffer->piggy_buf->data,
+			buffer->piggy_buf->data_phys_addr,
+			buffer->piggy_buf->buff_size);
+
+		OSAL_FREE(p_hwfn->p_dev, buffer->piggy_buf);
+	}
+
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       buffer->data,
+			       buffer->data_phys_addr,
+			       buffer->buff_size);
+
+	OSAL_FREE(p_hwfn->p_dev, buffer);
+	return;
+}
+
+/* Current known slowpath for iwarp ll2 is unalign flush. When this completion
+ * is received, need to reset the FPDU.
+ */
+static void
+ecore_iwarp_ll2_slowpath(void *cxt,
+			 u8 OSAL_UNUSED connection_handle,
+			 u32 opaque_data_0,
+			 u32 opaque_data_1)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct unaligned_opaque_data unalign_data;
+	struct ecore_iwarp_fpdu *fpdu;
+
+	ecore_iwarp_mpa_get_data(p_hwfn, &unalign_data,
+				 opaque_data_0, opaque_data_1);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "(0x%x) Flush fpdu\n",
+		   unalign_data.cid);
+
+	fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, (u16)unalign_data.cid);
+	if (fpdu)
+		OSAL_MEM_ZERO(fpdu, sizeof(*fpdu));
+}
+
+static int
+ecore_iwarp_ll2_stop(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	int rc = 0;
+
+	if (iwarp_info->ll2_syn_handle != ECORE_IWARP_HANDLE_INVAL) {
+
+		rc = ecore_ll2_terminate_connection(p_hwfn,
+						    iwarp_info->ll2_syn_handle);
+		if (rc)
+			DP_INFO(p_hwfn, "Failed to terminate syn connection\n");
+
+		ecore_ll2_release_connection(p_hwfn,
+					     iwarp_info->ll2_syn_handle);
+		iwarp_info->ll2_syn_handle = ECORE_IWARP_HANDLE_INVAL;
+	}
+
+	if (iwarp_info->ll2_ooo_handle != ECORE_IWARP_HANDLE_INVAL) {
+		rc = ecore_ll2_terminate_connection(p_hwfn,
+						    iwarp_info->ll2_ooo_handle);
+		if (rc)
+			DP_INFO(p_hwfn, "Failed to terminate ooo connection\n");
+
+		ecore_ll2_release_connection(p_hwfn,
+					     iwarp_info->ll2_ooo_handle);
+		iwarp_info->ll2_ooo_handle = ECORE_IWARP_HANDLE_INVAL;
+	}
+
+	if (iwarp_info->ll2_mpa_handle != ECORE_IWARP_HANDLE_INVAL) {
+		rc = ecore_ll2_terminate_connection(p_hwfn,
+						    iwarp_info->ll2_mpa_handle);
+		if (rc)
+			DP_INFO(p_hwfn, "Failed to terminate mpa connection\n");
+
+		ecore_ll2_release_connection(p_hwfn,
+					     iwarp_info->ll2_mpa_handle);
+		iwarp_info->ll2_mpa_handle = ECORE_IWARP_HANDLE_INVAL;
+	}
+
+	ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0,
+				    p_hwfn->p_rdma_info->iwarp.mac_addr);
+
+	return rc;
+}
+
+static int
+ecore_iwarp_ll2_alloc_buffers(struct ecore_hwfn *p_hwfn,
+			      int num_rx_bufs,
+			      int buff_size,
+			      u8 ll2_handle)
+{
+	struct ecore_iwarp_ll2_buff *buffer;
+	int rc = 0;
+	int i;
+
+	for (i = 0; i < num_rx_bufs; i++) {
+		buffer = OSAL_ZALLOC(p_hwfn->p_dev,
+				     GFP_KERNEL, sizeof(*buffer));
+		if (!buffer) {
+			DP_INFO(p_hwfn, "Failed to allocate LL2 buffer desc\n");
+			break;
+		}
+
+		buffer->data =
+			OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+						&buffer->data_phys_addr,
+						buff_size);
+
+		if (!buffer->data) {
+			DP_INFO(p_hwfn, "Failed to allocate LL2 buffers\n");
+			OSAL_FREE(p_hwfn->p_dev, buffer);
+			rc = ECORE_NOMEM;
+			break;
+		}
+
+		buffer->buff_size = buff_size;
+		rc = ecore_iwarp_ll2_post_rx(p_hwfn, buffer, ll2_handle);
+
+		if (rc)
+			break; /* buffers will be deallocated by ecore_ll2 */
+	}
+	return rc;
+}
+
+#define ECORE_IWARP_CACHE_PADDING(size) \
+	(((size) + ETH_CACHE_LINE_SIZE - 1) & ~(ETH_CACHE_LINE_SIZE - 1))
+
+#define ECORE_IWARP_MAX_BUF_SIZE(mtu) \
+	ECORE_IWARP_CACHE_PADDING(mtu + ETH_HLEN + 2*VLAN_HLEN + 2 +\
+				  ETH_CACHE_LINE_SIZE)
+
+static int
+ecore_iwarp_ll2_start(struct ecore_hwfn *p_hwfn,
+		      struct ecore_rdma_start_in_params *params)
+{
+	struct ecore_iwarp_info *iwarp_info;
+	struct ecore_ll2_acquire_data data;
+	struct ecore_ll2_cbs cbs;
+	u32 mpa_buff_size;
+	int rc = ECORE_SUCCESS;
+	u16 n_ooo_bufs;
+	int i;
+
+	iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	iwarp_info->ll2_syn_handle = ECORE_IWARP_HANDLE_INVAL;
+	iwarp_info->ll2_ooo_handle = ECORE_IWARP_HANDLE_INVAL;
+	iwarp_info->ll2_mpa_handle = ECORE_IWARP_HANDLE_INVAL;
+
+	iwarp_info->max_mtu = params->max_mtu;
+
+	OSAL_MEMCPY(p_hwfn->p_rdma_info->iwarp.mac_addr, params->mac_addr,
+		    ETH_ALEN);
+
+	rc = ecore_llh_add_mac_filter(p_hwfn->p_dev, 0, params->mac_addr);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	/* Start SYN connection */
+	cbs.rx_comp_cb = ecore_iwarp_ll2_comp_syn_pkt;
+	cbs.rx_release_cb = ecore_iwarp_ll2_rel_rx_pkt;
+	cbs.tx_comp_cb = ecore_iwarp_ll2_comp_tx_pkt;
+	cbs.tx_release_cb = ecore_iwarp_ll2_rel_tx_pkt;
+	cbs.cookie = p_hwfn;
+
+	OSAL_MEMSET(&data, 0, sizeof(data));
+	data.input.conn_type = ECORE_LL2_TYPE_IWARP;
+	data.input.mtu = ECORE_IWARP_MAX_SYN_PKT_SIZE;
+	data.input.rx_num_desc = ECORE_IWARP_LL2_SYN_RX_SIZE;
+	data.input.tx_num_desc = ECORE_IWARP_LL2_SYN_TX_SIZE;
+	data.input.tx_max_bds_per_packet = 1; /* will never be fragmented */
+	data.input.tx_tc = PKT_LB_TC;
+	data.input.tx_dest = ECORE_LL2_TX_DEST_LB;
+	data.p_connection_handle = &iwarp_info->ll2_syn_handle;
+	data.cbs = &cbs;
+
+	rc = ecore_ll2_acquire_connection(p_hwfn, &data);
+	if (rc) {
+		DP_NOTICE(p_hwfn, false, "Failed to acquire LL2 connection\n");
+		ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, params->mac_addr);
+		return rc;
+	}
+
+	rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_syn_handle);
+	if (rc) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to establish LL2 connection\n");
+		goto err;
+	}
+
+	rc = ecore_iwarp_ll2_alloc_buffers(p_hwfn,
+					   ECORE_IWARP_LL2_SYN_RX_SIZE,
+					   ECORE_IWARP_MAX_SYN_PKT_SIZE,
+					   iwarp_info->ll2_syn_handle);
+	if (rc)
+		goto err;
+
+	/* Start OOO connection */
+	data.input.conn_type = ECORE_LL2_TYPE_OOO;
+	data.input.mtu = params->max_mtu;
+
+	n_ooo_bufs = params->iwarp.ooo_num_rx_bufs;
+
+	if (n_ooo_bufs > ECORE_IWARP_LL2_OOO_MAX_RX_SIZE)
+		n_ooo_bufs = ECORE_IWARP_LL2_OOO_MAX_RX_SIZE;
+
+	data.input.rx_num_desc = n_ooo_bufs;
+	data.input.rx_num_ooo_buffers = n_ooo_bufs;
+
+	p_hwfn->p_rdma_info->iwarp.num_ooo_rx_bufs = data.input.rx_num_desc;
+	data.input.tx_max_bds_per_packet = 1; /* will never be fragmented */
+	data.input.tx_num_desc = ECORE_IWARP_LL2_OOO_DEF_TX_SIZE;
+	data.p_connection_handle = &iwarp_info->ll2_ooo_handle;
+	data.input.secondary_queue = true;
+
+	rc = ecore_ll2_acquire_connection(p_hwfn, &data);
+	if (rc)
+		goto err;
+
+	rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_ooo_handle);
+	if (rc)
+		goto err;
+
+	/* Start MPA connection */
+	cbs.rx_comp_cb = ecore_iwarp_ll2_comp_mpa_pkt;
+	cbs.slowpath_cb = ecore_iwarp_ll2_slowpath;
+
+	OSAL_MEMSET(&data, 0, sizeof(data));
+	data.input.conn_type = ECORE_LL2_TYPE_IWARP;
+	data.input.mtu = params->max_mtu;
+	data.input.rx_num_desc = n_ooo_bufs * 2;
+	/* we allocate the same amount for TX to reduce the chance we
+	 * run out of tx descriptors
+	 */
+	data.input.tx_num_desc = data.input.rx_num_desc;
+	data.input.tx_max_bds_per_packet = ECORE_IWARP_MAX_BDS_PER_FPDU;
+	data.p_connection_handle = &iwarp_info->ll2_mpa_handle;
+	data.input.secondary_queue = true;
+	data.cbs = &cbs;
+
+	rc = ecore_ll2_acquire_connection(p_hwfn, &data);
+	if (rc)
+		goto err;
+
+	rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_mpa_handle);
+	if (rc)
+		goto err;
+
+	mpa_buff_size = ECORE_IWARP_MAX_BUF_SIZE(params->max_mtu);
+	rc = ecore_iwarp_ll2_alloc_buffers(p_hwfn,
+					   data.input.rx_num_desc,
+					   mpa_buff_size,
+					   iwarp_info->ll2_mpa_handle);
+	if (rc)
+		goto err;
+
+	iwarp_info->partial_fpdus =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			    sizeof(*iwarp_info->partial_fpdus) *
+			    (u16)p_hwfn->p_rdma_info->num_qps);
+
+	if (!iwarp_info->partial_fpdus) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate ecore_iwarp_info(partial_fpdus)\n");
+		goto err;
+	}
+
+	iwarp_info->max_num_partial_fpdus = (u16)p_hwfn->p_rdma_info->num_qps;
+
+	/* The mpa_bufs array serves for pending RX packets received on the
+	 * mpa ll2 that don't have place on the tx ring and require later
+	 * processing. We can't fail on allocation of such a struct therefore
+	 * we allocate enough to take care of all rx packets
+	 */
+	iwarp_info->mpa_bufs =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			    sizeof(*iwarp_info->mpa_bufs) *
+				   data.input.rx_num_desc);
+
+	if (!iwarp_info->mpa_bufs) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate mpa_bufs array mem_size=%d\n",
+			  (u32)(sizeof(*iwarp_info->mpa_bufs) *
+				data.input.rx_num_desc));
+		goto err;
+	}
+
+	iwarp_info->mpa_intermediate_buf =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, mpa_buff_size);
+	if (!iwarp_info->mpa_intermediate_buf) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate mpa_intermediate_buf mem_size=%d\n",
+			  mpa_buff_size);
+		goto err;
+	}
+
+	OSAL_LIST_INIT(&iwarp_info->mpa_buf_pending_list);
+	OSAL_LIST_INIT(&iwarp_info->mpa_buf_list);
+	for (i = 0; i < data.input.rx_num_desc; i++) {
+		OSAL_LIST_PUSH_TAIL(&iwarp_info->mpa_bufs[i].list_entry,
+				    &iwarp_info->mpa_buf_list);
+	}
+
+	return rc;
+
+err:
+	ecore_iwarp_ll2_stop(p_hwfn);
+
+	return rc;
+}
+
+static void
+ecore_iwarp_set_defaults(struct ecore_hwfn *p_hwfn,
+			 struct ecore_rdma_start_in_params *params)
+{
+	u32 rcv_wnd_size;
+	u32 n_ooo_bufs;
+
+	/* rcv_wnd_size = 0: use defaults */
+	rcv_wnd_size = params->iwarp.rcv_wnd_size;
+	if (!rcv_wnd_size) {
+		if (ecore_device_num_ports(p_hwfn->p_dev) == 4) {
+			rcv_wnd_size = ECORE_IS_AH(p_hwfn->p_dev) ?
+				       ECORE_IWARP_RCV_WND_SIZE_AH_DEF_4_PORTS :
+				       ECORE_IWARP_RCV_WND_SIZE_BB_DEF_4_PORTS;
+		} else {
+			rcv_wnd_size = ECORE_IS_AH(p_hwfn->p_dev) ?
+				       ECORE_IWARP_RCV_WND_SIZE_AH_DEF_2_PORTS :
+				       ECORE_IWARP_RCV_WND_SIZE_BB_DEF_2_PORTS;
+		}
+		params->iwarp.rcv_wnd_size = rcv_wnd_size;
+	}
+
+	n_ooo_bufs = params->iwarp.ooo_num_rx_bufs;
+	if (!n_ooo_bufs) {
+		n_ooo_bufs = (u32)(((u64)ECORE_MAX_OOO *
+			      params->iwarp.rcv_wnd_size) /
+			      params->max_mtu);
+		n_ooo_bufs = OSAL_MIN_T(u32, n_ooo_bufs, USHRT_MAX);
+		params->iwarp.ooo_num_rx_bufs = (u16)n_ooo_bufs;
+	}
+}
+
+enum _ecore_status_t
+ecore_iwarp_setup(struct ecore_hwfn		    *p_hwfn,
+		  struct ecore_rdma_start_in_params *params)
+{
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	struct ecore_iwarp_info *iwarp_info;
+	u32 rcv_wnd_size;
+
+	iwarp_info = &(p_hwfn->p_rdma_info->iwarp);
+
+	if (!params->iwarp.rcv_wnd_size || !params->iwarp.ooo_num_rx_bufs)
+		ecore_iwarp_set_defaults(p_hwfn, params);
+
+	/* Scale 0 will set window of 0xFFFC (64K -4).
+	 * Scale x will set window of 0xFFFC << (x)
+	 * Therefore we subtract log2(64K) so that result is 0
+	 */
+	rcv_wnd_size = params->iwarp.rcv_wnd_size;
+	if (rcv_wnd_size < ECORE_IWARP_RCV_WND_SIZE_MIN)
+		rcv_wnd_size = ECORE_IWARP_RCV_WND_SIZE_MIN;
+
+	iwarp_info->rcv_wnd_scale = OSAL_MIN_T(u32, OSAL_LOG2(rcv_wnd_size) -
+		OSAL_LOG2(ECORE_IWARP_RCV_WND_SIZE_MIN), ECORE_IWARP_MAX_WND_SCALE);
+	iwarp_info->rcv_wnd_size = rcv_wnd_size >> iwarp_info->rcv_wnd_scale;
+
+	iwarp_info->tcp_flags = params->iwarp.flags;
+	iwarp_info->crc_needed = params->iwarp.crc_needed;
+	switch (params->iwarp.mpa_rev) {
+	case ECORE_MPA_REV1:
+		iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC;
+		break;
+	case ECORE_MPA_REV2:
+		iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;
+		break;
+	}
+
+	iwarp_info->peer2peer = params->iwarp.mpa_peer2peer;
+	iwarp_info->rtr_type = MPA_RTR_TYPE_NONE;
+
+	if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_SEND)
+		iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_SEND;
+
+	if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_WRITE)
+		iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE;
+
+	if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_READ)
+		iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_READ;
+
+	//DAVIDS OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.qp_lock);
+	OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.ep_list);
+	OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.listen_list);
+
+	ecore_spq_register_async_cb(p_hwfn, PROTOCOLID_IWARP,
+				    ecore_iwarp_async_event);
+	ecore_ooo_setup(p_hwfn);
+
+	rc = ecore_iwarp_ll2_start(p_hwfn, params);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "MPA_REV = %d. peer2peer=%d rtr=%x\n",
+		   iwarp_info->mpa_rev,
+		   iwarp_info->peer2peer,
+		   iwarp_info->rtr_type);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_stop(struct ecore_hwfn *p_hwfn)
+{
+	enum _ecore_status_t rc;
+
+	ecore_iwarp_free_prealloc_ep(p_hwfn);
+	rc = ecore_iwarp_wait_for_all_cids(p_hwfn);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	ecore_spq_unregister_async_cb(p_hwfn, PROTOCOLID_IWARP);
+
+	return ecore_iwarp_ll2_stop(p_hwfn);
+}
+
+static void
+ecore_iwarp_qp_in_error(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_ep *ep,
+			u8 fw_return_code)
+{
+	struct ecore_iwarp_cm_event_params params;
+
+	ecore_iwarp_modify_qp(p_hwfn, ep->qp, ECORE_IWARP_QP_STATE_ERROR, true);
+
+	params.event = ECORE_IWARP_EVENT_CLOSE;
+	params.ep_context = ep;
+	params.cm_info = &ep->cm_info;
+	params.status = (fw_return_code == IWARP_QP_IN_ERROR_GOOD_CLOSE) ?
+		ECORE_SUCCESS : ECORE_CONN_RESET;
+
+	ep->state = ECORE_IWARP_EP_CLOSED;
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_LIST_REMOVE_ENTRY(&ep->list_entry,
+			       &p_hwfn->p_rdma_info->iwarp.ep_list);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+	ep->event_cb(ep->cb_context, &params);
+}
+
+static void
+ecore_iwarp_exception_received(struct ecore_hwfn *p_hwfn,
+			       struct ecore_iwarp_ep *ep,
+			       int fw_ret_code)
+{
+	struct ecore_iwarp_cm_event_params params;
+	bool event_cb = false;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "EP(0x%x) fw_ret_code=%d\n",
+		   ep->cid, fw_ret_code);
+
+	switch (fw_ret_code) {
+	case IWARP_EXCEPTION_DETECTED_LLP_CLOSED:
+		params.status = ECORE_SUCCESS;
+		params.event = ECORE_IWARP_EVENT_DISCONNECT;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_LLP_RESET:
+		params.status = ECORE_CONN_RESET;
+		params.event = ECORE_IWARP_EVENT_DISCONNECT;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_RQ_EMPTY:
+		params.event = ECORE_IWARP_EVENT_RQ_EMPTY;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_IRQ_FULL:
+		params.event = ECORE_IWARP_EVENT_IRQ_FULL;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_LLP_TIMEOUT:
+		params.event = ECORE_IWARP_EVENT_LLP_TIMEOUT;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_REMOTE_PROTECTION_ERROR:
+		params.event = ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_CQ_OVERFLOW:
+		params.event = ECORE_IWARP_EVENT_CQ_OVERFLOW;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_LOCAL_CATASTROPHIC:
+		params.event = ECORE_IWARP_EVENT_QP_CATASTROPHIC;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_LOCAL_ACCESS_ERROR:
+		params.event = ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_REMOTE_OPERATION_ERROR:
+		params.event = ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR;
+		event_cb = true;
+		break;
+	case IWARP_EXCEPTION_DETECTED_TERMINATE_RECEIVED:
+		params.event = ECORE_IWARP_EVENT_TERMINATE_RECEIVED;
+		event_cb = true;
+		break;
+	default:
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Unhandled exception received...\n");
+		break;
+	}
+
+	if (event_cb) {
+		params.ep_context = ep;
+		params.cm_info = &ep->cm_info;
+		ep->event_cb(ep->cb_context, &params);
+	}
+}
+
+static void
+ecore_iwarp_tcp_connect_unsuccessful(struct ecore_hwfn *p_hwfn,
+				     struct ecore_iwarp_ep *ep,
+				     u8 fw_return_code)
+{
+	struct ecore_iwarp_cm_event_params params;
+
+	OSAL_MEM_ZERO(&params, sizeof(params));
+	params.event = ECORE_IWARP_EVENT_ACTIVE_COMPLETE;
+	params.ep_context = ep;
+	params.cm_info = &ep->cm_info;
+	ep->state = ECORE_IWARP_EP_CLOSED;
+
+	switch (fw_return_code) {
+	case IWARP_CONN_ERROR_TCP_CONNECT_INVALID_PACKET:
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "%s(0x%x) TCP connect got invalid packet\n",
+			   ECORE_IWARP_CONNECT_MODE_STRING(ep),
+			   ep->tcp_cid);
+		params.status = ECORE_CONN_RESET;
+		break;
+	case IWARP_CONN_ERROR_TCP_CONNECTION_RST:
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "%s(0x%x) TCP Connection Reset\n",
+			   ECORE_IWARP_CONNECT_MODE_STRING(ep),
+			   ep->tcp_cid);
+		params.status = ECORE_CONN_RESET;
+		break;
+	case IWARP_CONN_ERROR_TCP_CONNECT_TIMEOUT:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) TCP timeout\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep),
+			  ep->tcp_cid);
+		params.status = ECORE_TIMEOUT;
+		break;
+	case IWARP_CONN_ERROR_MPA_NOT_SUPPORTED_VER:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA not supported VER\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep),
+			  ep->tcp_cid);
+		params.status = ECORE_CONN_REFUSED;
+		break;
+	case IWARP_CONN_ERROR_MPA_INVALID_PACKET:
+		DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Invalid Packet\n",
+			  ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
+		params.status = ECORE_CONN_RESET;
+		break;
+	default:
+		DP_ERR(p_hwfn, "%s(0x%x) Unexpected return code tcp connect: %d\n",
+		       ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid,
+		       fw_return_code);
+		params.status = ECORE_CONN_RESET;
+		break;
+	}
+
+	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
+		ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID;
+		ecore_iwarp_return_ep(p_hwfn, ep);
+	} else {
+		ep->event_cb(ep->cb_context, &params);
+		OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+		OSAL_LIST_REMOVE_ENTRY(&ep->list_entry,
+				       &p_hwfn->p_rdma_info->iwarp.ep_list);
+		OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	}
+}
+
+static void
+ecore_iwarp_connect_complete(struct ecore_hwfn *p_hwfn,
+			     struct ecore_iwarp_ep *ep,
+			     u8 fw_return_code)
+{
+	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
+		/* Done with the SYN packet, post back to ll2 rx */
+		ecore_iwarp_ll2_post_rx(
+			p_hwfn, ep->syn,
+			p_hwfn->p_rdma_info->iwarp.ll2_syn_handle);
+
+		ep->syn = OSAL_NULL;
+
+		if (ep->state == ECORE_IWARP_EP_ABORTING)
+			return;
+
+		/* If connect failed - upper layer doesn't know about it */
+		if (fw_return_code == RDMA_RETURN_OK)
+			ecore_iwarp_mpa_received(p_hwfn, ep);
+		else
+			ecore_iwarp_tcp_connect_unsuccessful(p_hwfn, ep,
+							     fw_return_code);
+
+	} else {
+		if (fw_return_code == RDMA_RETURN_OK)
+			ecore_iwarp_mpa_offload(p_hwfn, ep);
+		else
+			ecore_iwarp_tcp_connect_unsuccessful(p_hwfn, ep,
+							     fw_return_code);
+	}
+}
+
+static OSAL_INLINE bool
+ecore_iwarp_check_ep_ok(struct ecore_hwfn *p_hwfn,
+			struct ecore_iwarp_ep *ep)
+{
+	if (ep == OSAL_NULL) {
+		DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep);
+		return false;
+	}
+
+	if (ep->sig != 0xdeadbeef) {
+		DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep);
+		return false;
+	}
+
+	return true;
+}
+
+static enum _ecore_status_t
+ecore_iwarp_async_event(struct ecore_hwfn *p_hwfn,
+			u8 fw_event_code,
+			u16 OSAL_UNUSED echo,
+			union event_ring_data *data,
+			u8 fw_return_code)
+{
+	struct regpair *fw_handle = &data->rdma_data.async_handle;
+	struct ecore_iwarp_ep *ep = OSAL_NULL;
+	u16 cid;
+
+	ep = (struct ecore_iwarp_ep *)(osal_uintptr_t)HILO_64(fw_handle->hi,
+							      fw_handle->lo);
+
+	switch (fw_event_code) {
+	/* Async completion after TCP 3-way handshake */
+	case IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE:
+		if (!ecore_iwarp_check_ep_ok(p_hwfn, ep))
+			return ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "EP(0x%x) IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE fw_ret_code=%d\n",
+			   ep->tcp_cid, fw_return_code);
+		ecore_iwarp_connect_complete(p_hwfn, ep, fw_return_code);
+		break;
+	case IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED:
+		if (!ecore_iwarp_check_ep_ok(p_hwfn, ep))
+			return ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED fw_ret_code=%d\n",
+			   ep->cid, fw_return_code);
+		ecore_iwarp_exception_received(p_hwfn, ep, fw_return_code);
+		break;
+	/* Async completion for Close Connection ramrod */
+	case IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE:
+		if (!ecore_iwarp_check_ep_ok(p_hwfn, ep))
+			return ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE fw_ret_code=%d\n",
+			   ep->cid, fw_return_code);
+		ecore_iwarp_qp_in_error(p_hwfn, ep, fw_return_code);
+		break;
+	/* Async event for active side only */
+	case IWARP_EVENT_TYPE_ASYNC_ENHANCED_MPA_REPLY_ARRIVED:
+		if (!ecore_iwarp_check_ep_ok(p_hwfn, ep))
+			return ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_MPA_REPLY_ARRIVED fw_ret_code=%d\n",
+			   ep->cid, fw_return_code);
+		ecore_iwarp_mpa_reply_arrived(p_hwfn, ep);
+		break;
+	/* MPA Negotiations completed */
+	case IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE:
+		if (!ecore_iwarp_check_ep_ok(p_hwfn, ep))
+			return ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE fw_ret_code=%d\n",
+			   ep->cid, fw_return_code);
+		ecore_iwarp_mpa_complete(p_hwfn, ep, fw_return_code);
+		break;
+	case IWARP_EVENT_TYPE_ASYNC_CID_CLEANED:
+		cid = (u16)OSAL_LE32_TO_CPU(fw_handle->lo);
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "(0x%x)IWARP_EVENT_TYPE_ASYNC_CID_CLEANED\n",
+			   cid);
+		ecore_iwarp_cid_cleaned(p_hwfn, cid);
+
+		break;
+	case IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW:
+		DP_NOTICE(p_hwfn, false,
+			  "IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW\n");
+
+		p_hwfn->p_rdma_info->events.affiliated_event(
+			p_hwfn->p_rdma_info->events.context,
+			ECORE_IWARP_EVENT_CQ_OVERFLOW,
+			(void *)fw_handle);
+		break;
+	default:
+		DP_ERR(p_hwfn, "Received unexpected async iwarp event %d\n",
+		       fw_event_code);
+		return ECORE_INVAL;
+	}
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t
+ecore_iwarp_create_listen(void *rdma_cxt,
+			  struct ecore_iwarp_listen_in *iparams,
+			  struct ecore_iwarp_listen_out *oparams)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_listener *listener;
+
+	listener = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*listener));
+
+	if (!listener) {
+		DP_NOTICE(p_hwfn,
+			  false,
+			  "ecore iwarp create listener failed: cannot allocate memory (listener). rc = %d\n",
+			  ECORE_NOMEM);
+		return ECORE_NOMEM;
+	}
+	listener->ip_version = iparams->ip_version;
+	OSAL_MEMCPY(listener->ip_addr,
+		    iparams->ip_addr,
+		    sizeof(listener->ip_addr));
+	listener->port = iparams->port;
+	listener->vlan = iparams->vlan;
+
+	listener->event_cb = iparams->event_cb;
+	listener->cb_context = iparams->cb_context;
+	listener->max_backlog = iparams->max_backlog;
+	listener->state = ECORE_IWARP_LISTENER_STATE_ACTIVE;
+	oparams->handle = listener;
+
+	OSAL_SPIN_LOCK_INIT(&listener->lock);
+	OSAL_LIST_INIT(&listener->ep_list);
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+	OSAL_LIST_PUSH_TAIL(&listener->list_entry,
+			    &p_hwfn->p_rdma_info->iwarp.listen_list);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "callback=%p handle=%p ip=%x:%x:%x:%x port=0x%x vlan=0x%x\n",
+		   listener->event_cb,
+		   listener,
+		   listener->ip_addr[0],
+		   listener->ip_addr[1],
+		   listener->ip_addr[2],
+		   listener->ip_addr[3],
+		   listener->port,
+		   listener->vlan);
+
+	return ECORE_SUCCESS;
+}
+
+static void
+ecore_iwarp_pause_complete(struct ecore_iwarp_listener *listener)
+{
+	struct ecore_iwarp_cm_event_params params;
+
+	if (listener->state == ECORE_IWARP_LISTENER_STATE_UNPAUSE)
+		listener->state = ECORE_IWARP_LISTENER_STATE_ACTIVE;
+
+	params.event = ECORE_IWARP_EVENT_LISTEN_PAUSE_COMP;
+	listener->event_cb(listener->cb_context, &params);
+}
+
+static void
+ecore_iwarp_tcp_abort_comp(struct ecore_hwfn *p_hwfn, void *cookie,
+			   union event_ring_data OSAL_UNUSED *data,
+			   u8 OSAL_UNUSED fw_return_code)
+{
+	struct ecore_iwarp_ep *ep = (struct ecore_iwarp_ep *)cookie;
+	struct ecore_iwarp_listener *listener = ep->listener;
+
+	ecore_iwarp_return_ep(p_hwfn, ep);
+
+	if (OSAL_LIST_IS_EMPTY(&listener->ep_list))
+		listener->done = true;
+}
+
+static void
+ecore_iwarp_abort_inflight_connections(struct ecore_hwfn *p_hwfn,
+				       struct ecore_iwarp_listener *listener)
+{
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_iwarp_ep *ep = OSAL_NULL;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_comp_cb comp_data;
+	enum _ecore_status_t rc;
+
+	/* remove listener from list before destroying listener */
+	OSAL_LIST_REMOVE_ENTRY(&listener->list_entry,
+			       &p_hwfn->p_rdma_info->iwarp.listen_list);
+	if (OSAL_LIST_IS_EMPTY(&listener->ep_list)) {
+		listener->done = true;
+		return;
+	}
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.p_comp_data = &comp_data;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_CB;
+	init_data.p_comp_data->function = ecore_iwarp_tcp_abort_comp;
+
+	OSAL_LIST_FOR_EACH_ENTRY(ep, &listener->ep_list,
+				 list_entry, struct ecore_iwarp_ep) {
+		ep->state = ECORE_IWARP_EP_ABORTING;
+		init_data.p_comp_data->cookie = ep;
+		init_data.cid = ep->tcp_cid;
+		rc = ecore_sp_init_request(p_hwfn, &p_ent,
+					   IWARP_RAMROD_CMD_ID_ABORT_TCP_OFFLOAD,
+					   PROTOCOLID_IWARP,
+					   &init_data);
+		if (rc == ECORE_SUCCESS)
+			ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	}
+}
+
+static void
+ecore_iwarp_listener_state_transition(struct ecore_hwfn *p_hwfn, void *cookie,
+				      union event_ring_data OSAL_UNUSED *data,
+				      u8 OSAL_UNUSED fw_return_code)
+{
+	struct ecore_iwarp_listener *listener = (struct ecore_iwarp_listener *)cookie;
+
+	switch (listener->state) {
+	case ECORE_IWARP_LISTENER_STATE_PAUSE:
+	case ECORE_IWARP_LISTENER_STATE_UNPAUSE:
+		ecore_iwarp_pause_complete(listener);
+		break;
+	case ECORE_IWARP_LISTENER_STATE_DESTROYING:
+		ecore_iwarp_abort_inflight_connections(p_hwfn, listener);
+		break;
+	default:
+		break;
+	}
+}
+
+static enum _ecore_status_t
+ecore_iwarp_empty_ramrod(struct ecore_hwfn *p_hwfn,
+			 struct ecore_iwarp_listener *listener)
+{
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_spq_comp_cb comp_data;
+	struct ecore_sp_init_data init_data;
+	enum _ecore_status_t rc;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.p_comp_data = &comp_data;
+	init_data.cid = ecore_spq_get_cid(p_hwfn);
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_CB;
+	init_data.p_comp_data->function = ecore_iwarp_listener_state_transition;
+	init_data.p_comp_data->cookie = listener;
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   COMMON_RAMROD_EMPTY,
+				   PROTOCOLID_COMMON,
+				   &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_pause_listen(void *rdma_cxt, void *handle,
+			 bool pause, bool comp)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_listener *listener =
+		(struct ecore_iwarp_listener *)handle;
+	enum _ecore_status_t rc;
+
+	listener->state = pause ?
+		ECORE_IWARP_LISTENER_STATE_PAUSE :
+		ECORE_IWARP_LISTENER_STATE_UNPAUSE;
+	if (!comp)
+		return ECORE_SUCCESS;
+
+	rc = ecore_iwarp_empty_ramrod(p_hwfn, listener);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener=%p, state=%d\n",
+		   listener, listener->state);
+
+	return ECORE_PENDING;
+}
+
+enum _ecore_status_t
+ecore_iwarp_destroy_listen(void *rdma_cxt, void *handle)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_iwarp_listener *listener =
+		(struct ecore_iwarp_listener *)handle;
+	enum _ecore_status_t rc;
+	int wait_count = 0;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "handle=%p\n", handle);
+
+	listener->state = ECORE_IWARP_LISTENER_STATE_DESTROYING;
+	rc = ecore_iwarp_empty_ramrod(p_hwfn, listener);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	while (!listener->done) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Waiting for ep list to be empty...\n");
+		OSAL_MSLEEP(100);
+		if (wait_count++ > 200) {
+			DP_NOTICE(p_hwfn, false, "ep list close timeout\n");
+			break;
+		}
+	}
+
+	OSAL_FREE(p_hwfn->p_dev, listener);
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t
+ecore_iwarp_send_rtr(void *rdma_cxt, struct ecore_iwarp_send_rtr_in *iparams)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	struct ecore_rdma_qp *qp;
+	struct ecore_iwarp_ep *ep;
+	enum _ecore_status_t rc;
+
+	ep = (struct ecore_iwarp_ep *)iparams->ep_context;
+	if (!ep) {
+		DP_ERR(p_hwfn, "Ep Context receive in send_rtr is NULL\n");
+		return ECORE_INVAL;
+	}
+
+	qp = ep->qp;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",
+		   qp->icid, ep->tcp_cid);
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_CB;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR,
+				   PROTOCOLID_IWARP, &init_data);
+
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ecore_iwarp_send_rtr, rc = 0x%x\n",
+		   rc);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_iwarp_query_qp(struct ecore_rdma_qp *qp,
+		     struct ecore_rdma_query_qp_out_params *out_params)
+{
+	out_params->state = ecore_iwarp2roce_state(qp->iwarp_state);
+	return ECORE_SUCCESS;
+}
+
+#ifdef _NTDDK_
+#pragma warning(pop)
+#endif
diff --git a/sys/dev/qlnx/qlnxe/ecore_ll2.c b/sys/dev/qlnx/qlnxe/ecore_ll2.c
new file mode 100644
index 000000000000..95b31d3bebc9
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/ecore_ll2.c
@@ -0,0 +1,2211 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : ecore_ll2.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "bcm_osal.h"
+
+#include "ecore.h"
+#include "ecore_status.h"
+#include "ecore_ll2.h"
+#include "reg_addr.h"
+#include "ecore_int.h"
+#include "ecore_cxt.h"
+#include "ecore_sp_commands.h"
+#include "ecore_hw.h"
+#include "reg_addr.h"
+#include "ecore_dev_api.h"
+#include "ecore_iro.h"
+#include "ecore_gtt_reg_addr.h"
+#include "ecore_ooo.h"
+#include "ecore_hw.h"
+#include "ecore_mcp.h"
+
+#define ECORE_LL2_RX_REGISTERED(ll2)	((ll2)->rx_queue.b_cb_registred)
+#define ECORE_LL2_TX_REGISTERED(ll2)	((ll2)->tx_queue.b_cb_registred)
+
+#ifdef _NTDDK_
+#pragma warning(push)
+#pragma warning(disable : 28167)
+#pragma warning(disable : 28123)
+#pragma warning(disable : 28121)
+#endif
+
+static struct ecore_ll2_info *
+__ecore_ll2_handle_sanity(struct ecore_hwfn *p_hwfn,
+			  u8 connection_handle,
+			  bool b_lock, bool b_only_active)
+{
+	struct ecore_ll2_info *p_ll2_conn, *p_ret = OSAL_NULL;
+
+	if (connection_handle >= ECORE_MAX_NUM_OF_LL2_CONNECTIONS)
+		return OSAL_NULL;
+
+	if (!p_hwfn->p_ll2_info)
+		return OSAL_NULL;
+
+	/* TODO - is there really need for the locked vs. unlocked
+	 * variant? I simply used what was already there.
+	 */
+	p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
+
+	if (b_only_active) {
+		if (b_lock)
+			OSAL_MUTEX_ACQUIRE(&p_ll2_conn->mutex);
+		if (p_ll2_conn->b_active)
+			p_ret = p_ll2_conn;
+		if (b_lock)
+			OSAL_MUTEX_RELEASE(&p_ll2_conn->mutex);
+	} else {
+		p_ret = p_ll2_conn;
+	}
+
+	return p_ret;
+}
+
+static struct ecore_ll2_info *
+ecore_ll2_handle_sanity(struct ecore_hwfn *p_hwfn,
+			u8 connection_handle)
+{
+	return __ecore_ll2_handle_sanity(p_hwfn, connection_handle,
+					 false, true);
+}
+
+static struct ecore_ll2_info *
+ecore_ll2_handle_sanity_lock(struct ecore_hwfn *p_hwfn,
+			     u8 connection_handle)
+{
+	return __ecore_ll2_handle_sanity(p_hwfn, connection_handle,
+					 true, true);
+}
+
+static struct ecore_ll2_info *
+ecore_ll2_handle_sanity_inactive(struct ecore_hwfn *p_hwfn,
+				 u8 connection_handle)
+{
+	return __ecore_ll2_handle_sanity(p_hwfn, connection_handle,
+					 false, false);
+}
+
+#ifndef LINUX_REMOVE
+/* TODO - is this really been used by anyone? Is it a on future todo list? */
+enum _ecore_status_t
+ecore_ll2_get_fragment_of_tx_packet(struct ecore_hwfn *p_hwfn,
+				    u8 connection_handle,
+				    dma_addr_t *p_addr,
+				    bool *b_last_fragment)
+{
+	struct ecore_ll2_tx_packet *p_pkt;
+	struct ecore_ll2_info *p_ll2_conn;
+	u16 cur_frag_idx = 0;
+
+	p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return ECORE_INVAL;
+	p_pkt = &p_ll2_conn->tx_queue.cur_completing_packet;
+
+	if (!p_ll2_conn->tx_queue.b_completing_packet || !p_addr)
+		return ECORE_INVAL;
+
+	if (p_ll2_conn->tx_queue.cur_completing_bd_idx == p_pkt->bd_used)
+		return ECORE_INVAL;
+
+	/* Packet is available and has at least one more frag - provide it */
+	cur_frag_idx = p_ll2_conn->tx_queue.cur_completing_bd_idx++;
+	*p_addr = p_pkt->bds_set[cur_frag_idx].tx_frag;
+	if (b_last_fragment)
+		*b_last_fragment = p_pkt->bd_used ==
+				   p_ll2_conn->tx_queue.cur_completing_bd_idx;
+
+	return ECORE_SUCCESS;
+}
+#endif
+
+static void ecore_ll2_txq_flush(struct ecore_hwfn *p_hwfn,
+				u8 connection_handle)
+{
+	bool b_last_packet = false, b_last_frag = false;
+	struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL;
+	struct ecore_ll2_info *p_ll2_conn;
+	struct ecore_ll2_tx_queue *p_tx;
+	unsigned long flags = 0;
+	dma_addr_t tx_frag;
+
+	p_ll2_conn = ecore_ll2_handle_sanity_inactive(p_hwfn,
+						      connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return;
+	p_tx = &p_ll2_conn->tx_queue;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags);
+	while (!OSAL_LIST_IS_EMPTY(&p_tx->active_descq)) {
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq,
+					      struct ecore_ll2_tx_packet,
+					      list_entry);
+
+		if (p_pkt == OSAL_NULL)
+			break;
+
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+				       &p_tx->active_descq);
+		b_last_packet = OSAL_LIST_IS_EMPTY(&p_tx->active_descq);
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+				    &p_tx->free_descq);
+		OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags);
+		if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) {
+			struct ecore_ooo_buffer	*p_buffer;
+
+			p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie;
+			ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer);
+		} else {
+			p_tx->cur_completing_packet = *p_pkt;
+			p_tx->cur_completing_bd_idx = 1;
+			b_last_frag = p_tx->cur_completing_bd_idx ==
+				      p_pkt->bd_used;
+
+			tx_frag = p_pkt->bds_set[0].tx_frag;
+			p_ll2_conn->cbs.tx_release_cb(p_ll2_conn->cbs.cookie,
+						      p_ll2_conn->my_id,
+						      p_pkt->cookie,
+						      tx_frag,
+						      b_last_frag,
+						      b_last_packet);
+		}
+		OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags);
+	}
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags);
+}
+
+static enum _ecore_status_t
+ecore_ll2_txq_completion(struct ecore_hwfn *p_hwfn,
+			 void *p_cookie)
+{
+	struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info*)p_cookie;
+	struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	u16 new_idx = 0, num_bds = 0, num_bds_in_packet = 0;
+	struct ecore_ll2_tx_packet *p_pkt;
+	bool b_last_frag = false;
+	unsigned long flags;
+	enum _ecore_status_t rc = ECORE_INVAL;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags);
+	if (p_tx->b_completing_packet) {
+		/* TODO - this looks completely unnecessary to me - the only
+		 * way we can re-enter is by the DPC calling us again, but this
+		 * would only happen AFTER we return, and we unset this at end
+		 * of the function.
+		 */
+		rc = ECORE_BUSY;
+		goto out;
+	}
+
+	new_idx = OSAL_LE16_TO_CPU(*p_tx->p_fw_cons);
+	num_bds = ((s16)new_idx - (s16)p_tx->bds_idx);
+	while (num_bds) {
+		if (OSAL_LIST_IS_EMPTY(&p_tx->active_descq))
+			goto out;
+
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq,
+					      struct ecore_ll2_tx_packet,
+					      list_entry);
+		if (!p_pkt)
+			goto out;
+
+		p_tx->b_completing_packet = true;
+		p_tx->cur_completing_packet = *p_pkt;
+		num_bds_in_packet = p_pkt->bd_used;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+				       &p_tx->active_descq);
+
+		if (num_bds < num_bds_in_packet) {
+			DP_NOTICE(p_hwfn, true,
+				  "Rest of BDs does not cover whole packet\n");
+			goto out;
+		}
+
+		num_bds -= num_bds_in_packet;
+		p_tx->bds_idx += num_bds_in_packet;
+		while (num_bds_in_packet--)
+			ecore_chain_consume(&p_tx->txq_chain);
+
+		p_tx->cur_completing_bd_idx = 1;
+		b_last_frag = p_tx->cur_completing_bd_idx ==
+			      p_pkt->bd_used;
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+				    &p_tx->free_descq);
+
+		OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags);
+
+		p_ll2_conn->cbs.tx_comp_cb(p_ll2_conn->cbs.cookie,
+					   p_ll2_conn->my_id,
+					   p_pkt->cookie,
+					   p_pkt->bds_set[0].tx_frag,
+					   b_last_frag,
+					   !num_bds);
+
+		OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags);
+	}
+
+	p_tx->b_completing_packet = false;
+	rc = ECORE_SUCCESS;
+out:
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags);
+	return rc;
+}
+
+static void ecore_ll2_rxq_parse_gsi(union core_rx_cqe_union *p_cqe,
+				    struct ecore_ll2_comp_rx_data *data)
+{
+	data->parse_flags =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.parse_flags.flags);
+	data->length.data_length =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.data_length);
+	data->vlan =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.vlan);
+	data->opaque_data_0 =
+		OSAL_LE32_TO_CPU(p_cqe->rx_cqe_gsi.src_mac_addrhi);
+	data->opaque_data_1 =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.src_mac_addrlo);
+	data->u.data_length_error =
+		p_cqe->rx_cqe_gsi.data_length_error;
+	data->qp_id = OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.qp_id);
+
+	data->src_qp = OSAL_LE32_TO_CPU(p_cqe->rx_cqe_gsi.src_qp);
+}
+
+static void ecore_ll2_rxq_parse_reg(union core_rx_cqe_union *p_cqe,
+				    struct ecore_ll2_comp_rx_data *data)
+{
+	data->parse_flags =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.parse_flags.flags);
+	data->err_flags =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.err_flags.flags);
+	data->length.packet_length =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.packet_length);
+	data->vlan =
+		OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.vlan);
+	data->opaque_data_0 =
+		OSAL_LE32_TO_CPU(p_cqe->rx_cqe_fp.opaque_data.data[0]);
+	data->opaque_data_1 =
+		OSAL_LE32_TO_CPU(p_cqe->rx_cqe_fp.opaque_data.data[1]);
+	data->u.placement_offset =
+		p_cqe->rx_cqe_fp.placement_offset;
+}
+
+#if defined(_NTDDK_)
+#pragma warning(suppress : 28167 26110)
+#endif
+static enum _ecore_status_t
+ecore_ll2_handle_slowpath(struct ecore_hwfn *p_hwfn,
+			  struct ecore_ll2_info *p_ll2_conn,
+			  union core_rx_cqe_union *p_cqe,
+			  unsigned long *p_lock_flags)
+{
+	struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	struct core_rx_slow_path_cqe *sp_cqe;
+
+	sp_cqe = &p_cqe->rx_cqe_sp;
+	if (sp_cqe->ramrod_cmd_id != CORE_RAMROD_RX_QUEUE_FLUSH) {
+		DP_NOTICE(p_hwfn, true,
+			  "LL2 - unexpected Rx CQE slowpath ramrod_cmd_id:%d\n",
+			  sp_cqe->ramrod_cmd_id);
+		return ECORE_INVAL;
+	}
+
+	if (p_ll2_conn->cbs.slowpath_cb == OSAL_NULL) {
+		DP_NOTICE(p_hwfn, true,
+			  "LL2 - received RX_QUEUE_FLUSH but no callback was provided\n");
+		return ECORE_INVAL;
+	}
+
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, *p_lock_flags);
+
+	p_ll2_conn->cbs.slowpath_cb(p_ll2_conn->cbs.cookie,
+				    p_ll2_conn->my_id,
+				    OSAL_LE32_TO_CPU(sp_cqe->opaque_data.data[0]),
+				    OSAL_LE32_TO_CPU(sp_cqe->opaque_data.data[1]));
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, *p_lock_flags);
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_ll2_rxq_handle_completion(struct ecore_hwfn *p_hwfn,
+				struct ecore_ll2_info *p_ll2_conn,
+				union core_rx_cqe_union *p_cqe,
+				unsigned long *p_lock_flags,
+				bool b_last_cqe)
+{
+	struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL;
+	struct ecore_ll2_comp_rx_data data;
+
+	if (!OSAL_LIST_IS_EMPTY(&p_rx->active_descq))
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq,
+					      struct ecore_ll2_rx_packet,
+					      list_entry);
+	if (!p_pkt) {
+		DP_NOTICE(p_hwfn, false,
+			  "[%d] LL2 Rx completion but active_descq is empty\n",
+			  p_ll2_conn->input.conn_type);
+
+		return ECORE_IO;
+	}
+
+	OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, &p_rx->active_descq);
+
+	if (p_cqe->rx_cqe_sp.type == CORE_RX_CQE_TYPE_REGULAR)
+		ecore_ll2_rxq_parse_reg(p_cqe, &data);
+	else
+		ecore_ll2_rxq_parse_gsi(p_cqe, &data);
+
+	if (ecore_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd) {
+		DP_NOTICE(p_hwfn, false,
+			  "Mismatch between active_descq and the LL2 Rx chain\n");
+		/* TODO - didn't return error value since this wasn't handled
+		 * before, but this is obviously lacking.
+		 */
+	}
+
+	OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, &p_rx->free_descq);
+
+	data.connection_handle = p_ll2_conn->my_id;
+	data.cookie = p_pkt->cookie;
+	data.rx_buf_addr = p_pkt->rx_buf_addr;
+	data.b_last_packet = b_last_cqe;
+
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, *p_lock_flags);
+	p_ll2_conn->cbs.rx_comp_cb(p_ll2_conn->cbs.cookie,
+				   &data);
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, *p_lock_flags);
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_ll2_rxq_completion(struct ecore_hwfn *p_hwfn,
+						     void *cookie)
+{
+	struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info*)cookie;
+	struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	union core_rx_cqe_union *cqe = OSAL_NULL;
+	u16 cq_new_idx = 0, cq_old_idx = 0;
+	unsigned long flags = 0;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags);
+	cq_new_idx = OSAL_LE16_TO_CPU(*p_rx->p_fw_cons);
+	cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain);
+
+	while (cq_new_idx != cq_old_idx) {
+		bool b_last_cqe = (cq_new_idx == cq_old_idx);
+
+		cqe = (union core_rx_cqe_union *)ecore_chain_consume(&p_rx->rcq_chain);
+		cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain);
+
+		DP_VERBOSE(p_hwfn, ECORE_MSG_LL2,
+			   "LL2 [sw. cons %04x, fw. at %04x] - Got Packet of type %02x\n",
+			   cq_old_idx, cq_new_idx, cqe->rx_cqe_sp.type);
+
+		switch (cqe->rx_cqe_sp.type) {
+		case CORE_RX_CQE_TYPE_SLOW_PATH:
+			rc = ecore_ll2_handle_slowpath(p_hwfn, p_ll2_conn,
+						       cqe, &flags);
+			break;
+		case CORE_RX_CQE_TYPE_GSI_OFFLOAD:
+		case CORE_RX_CQE_TYPE_REGULAR:
+			rc = ecore_ll2_rxq_handle_completion(p_hwfn, p_ll2_conn,
+							     cqe, &flags,
+							     b_last_cqe);
+			break;
+		default:
+			rc = ECORE_IO;
+		}
+	}
+
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags);
+	return rc;
+}
+
+static void ecore_ll2_rxq_flush(struct ecore_hwfn *p_hwfn,
+			 u8 connection_handle)
+{
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+	struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL;
+	struct ecore_ll2_rx_queue *p_rx;
+	unsigned long flags = 0;
+
+	p_ll2_conn = ecore_ll2_handle_sanity_inactive(p_hwfn,
+						      connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return;
+	p_rx = &p_ll2_conn->rx_queue;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags);
+	while (!OSAL_LIST_IS_EMPTY(&p_rx->active_descq)) {
+		bool b_last;
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq,
+					      struct ecore_ll2_rx_packet,
+					      list_entry);
+		if (p_pkt == OSAL_NULL)
+			break;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+				       &p_rx->active_descq);
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+				    &p_rx->free_descq);
+		b_last = OSAL_LIST_IS_EMPTY(&p_rx->active_descq);
+		OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags);
+
+		if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) {
+			struct ecore_ooo_buffer	*p_buffer;
+
+			p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie;
+			ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer);
+		} else {
+			dma_addr_t rx_buf_addr = p_pkt->rx_buf_addr;
+			void *cookie  = p_pkt->cookie;
+
+			p_ll2_conn->cbs.rx_release_cb(p_ll2_conn->cbs.cookie,
+						      p_ll2_conn->my_id,
+						      cookie,
+						      rx_buf_addr,
+						      b_last);
+		}
+		OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags);
+	}
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags);
+}
+
+static bool
+ecore_ll2_lb_rxq_handler_slowpath(struct ecore_hwfn *p_hwfn,
+				  struct core_rx_slow_path_cqe *p_cqe)
+{
+	struct ooo_opaque *iscsi_ooo;
+	u32 cid;
+
+	if (p_cqe->ramrod_cmd_id != CORE_RAMROD_RX_QUEUE_FLUSH)
+		return false;
+
+	iscsi_ooo = (struct ooo_opaque *)&p_cqe->opaque_data;
+	if (iscsi_ooo->ooo_opcode != TCP_EVENT_DELETE_ISLES)
+		return false;
+
+	/* Need to make a flush */
+	cid = OSAL_LE32_TO_CPU(iscsi_ooo->cid);
+	ecore_ooo_release_connection_isles(p_hwfn->p_ooo_info, cid);
+
+	return true;
+}
+
+static enum _ecore_status_t
+ecore_ll2_lb_rxq_handler(struct ecore_hwfn *p_hwfn,
+			 struct ecore_ll2_info *p_ll2_conn)
+{
+	struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	u16 packet_length = 0, parse_flags = 0, vlan = 0;
+	struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL;
+	u32 num_ooo_add_to_peninsula = 0, cid;
+	union core_rx_cqe_union *cqe = OSAL_NULL;
+	u16 cq_new_idx = 0, cq_old_idx = 0;
+	struct ecore_ooo_buffer	*p_buffer;
+	struct ooo_opaque *iscsi_ooo;
+	u8 placement_offset = 0;
+	u8 cqe_type;
+
+	cq_new_idx = OSAL_LE16_TO_CPU(*p_rx->p_fw_cons);
+	cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain);
+	if (cq_new_idx == cq_old_idx)
+		return ECORE_SUCCESS;
+
+	while (cq_new_idx != cq_old_idx) {
+		struct core_rx_fast_path_cqe *p_cqe_fp;
+
+		cqe = (union core_rx_cqe_union *)ecore_chain_consume(&p_rx->rcq_chain);
+		cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain);
+		cqe_type = cqe->rx_cqe_sp.type;
+
+		if (cqe_type == CORE_RX_CQE_TYPE_SLOW_PATH)
+			if (ecore_ll2_lb_rxq_handler_slowpath(p_hwfn,
+							      &cqe->rx_cqe_sp))
+				continue;
+
+		if (cqe_type != CORE_RX_CQE_TYPE_REGULAR) {
+			DP_NOTICE(p_hwfn, true,
+				  "Got a non-regular LB LL2 completion [type 0x%02x]\n",
+				  cqe_type);
+			return ECORE_INVAL;
+		}
+		p_cqe_fp = &cqe->rx_cqe_fp;
+
+		placement_offset = p_cqe_fp->placement_offset;
+		parse_flags = OSAL_LE16_TO_CPU(p_cqe_fp->parse_flags.flags);
+		packet_length = OSAL_LE16_TO_CPU(p_cqe_fp->packet_length);
+		vlan = OSAL_LE16_TO_CPU(p_cqe_fp->vlan);
+		iscsi_ooo = (struct ooo_opaque *)&p_cqe_fp->opaque_data;
+		ecore_ooo_save_history_entry(p_hwfn->p_ooo_info, iscsi_ooo);
+		cid = OSAL_LE32_TO_CPU(iscsi_ooo->cid);
+
+		/* Process delete isle first*/
+		if (iscsi_ooo->drop_size)
+			ecore_ooo_delete_isles(p_hwfn, p_hwfn->p_ooo_info, cid,
+					       iscsi_ooo->drop_isle,
+					       iscsi_ooo->drop_size);
+
+		if (iscsi_ooo->ooo_opcode == TCP_EVENT_NOP)
+			continue;
+
+		/* Now process create/add/join isles */
+		if (OSAL_LIST_IS_EMPTY(&p_rx->active_descq)) {
+			DP_NOTICE(p_hwfn, true,
+				  "LL2 OOO RX chain has no submitted buffers\n");
+			return ECORE_IO;
+		}
+
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq,
+					      struct ecore_ll2_rx_packet,
+					      list_entry);
+
+		if ((iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_NEW_ISLE) ||
+		    (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_RIGHT) ||
+		    (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_LEFT) ||
+		    (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_PEN) ||
+		    (iscsi_ooo->ooo_opcode == TCP_EVENT_JOIN)) {
+			if (!p_pkt) {
+				DP_NOTICE(p_hwfn, true,
+					  "LL2 OOO RX packet is not valid\n");
+				return ECORE_IO;
+			}
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+			OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+					       &p_rx->active_descq);
+			p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie;
+			p_buffer->packet_length = packet_length;
+			p_buffer->parse_flags = parse_flags;
+			p_buffer->vlan = vlan;
+			p_buffer->placement_offset = placement_offset;
+			if (ecore_chain_consume(&p_rx->rxq_chain) !=
+			    p_pkt->rxq_bd) {
+				/**/
+			}
+			ecore_ooo_dump_rx_event(p_hwfn, iscsi_ooo, p_buffer);
+			OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+					    &p_rx->free_descq);
+
+			switch (iscsi_ooo->ooo_opcode) {
+			case TCP_EVENT_ADD_NEW_ISLE:
+				ecore_ooo_add_new_isle(p_hwfn,
+						       p_hwfn->p_ooo_info,
+						       cid,
+						       iscsi_ooo->ooo_isle,
+						       p_buffer);
+				break;
+			case TCP_EVENT_ADD_ISLE_RIGHT:
+				ecore_ooo_add_new_buffer(p_hwfn,
+							 p_hwfn->p_ooo_info,
+							 cid,
+							 iscsi_ooo->ooo_isle,
+							 p_buffer,
+							 ECORE_OOO_RIGHT_BUF);
+				break;
+			case TCP_EVENT_ADD_ISLE_LEFT:
+				ecore_ooo_add_new_buffer(p_hwfn,
+							 p_hwfn->p_ooo_info,
+							 cid,
+							 iscsi_ooo->ooo_isle,
+							 p_buffer,
+							 ECORE_OOO_LEFT_BUF);
+				break;
+			case TCP_EVENT_JOIN:
+				ecore_ooo_add_new_buffer(p_hwfn,
+							 p_hwfn->p_ooo_info,
+							 cid,
+							 iscsi_ooo->ooo_isle +
+							 1,
+							 p_buffer,
+							 ECORE_OOO_LEFT_BUF);
+				ecore_ooo_join_isles(p_hwfn,
+						     p_hwfn->p_ooo_info,
+						     cid,
+						     iscsi_ooo->ooo_isle);
+				break;
+			case TCP_EVENT_ADD_PEN:
+				num_ooo_add_to_peninsula++;
+				ecore_ooo_put_ready_buffer(p_hwfn->p_ooo_info,
+							   p_buffer, true);
+				break;
+			}
+		} else {
+			DP_NOTICE(p_hwfn, true,
+				  "Unexpected event (%d) TX OOO completion\n",
+				  iscsi_ooo->ooo_opcode);
+		}
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static void
+ecore_ooo_submit_tx_buffers(struct ecore_hwfn *p_hwfn,
+			    struct ecore_ll2_info *p_ll2_conn)
+{
+	struct ecore_ll2_tx_pkt_info tx_pkt;
+	struct ecore_ooo_buffer *p_buffer;
+	dma_addr_t first_frag;
+	u16 l4_hdr_offset_w;
+	u8 bd_flags;
+	enum _ecore_status_t rc;
+
+	/* Submit Tx buffers here */
+	while ((p_buffer = ecore_ooo_get_ready_buffer(p_hwfn->p_ooo_info))) {
+		l4_hdr_offset_w = 0;
+		bd_flags = 0;
+
+		first_frag = p_buffer->rx_buffer_phys_addr +
+			     p_buffer->placement_offset;
+		SET_FIELD(bd_flags, CORE_TX_BD_DATA_FORCE_VLAN_MODE, 1);
+		SET_FIELD(bd_flags, CORE_TX_BD_DATA_L4_PROTOCOL, 1);
+
+		OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt));
+		tx_pkt.num_of_bds = 1;
+		tx_pkt.vlan = p_buffer->vlan;
+		tx_pkt.bd_flags = bd_flags;
+		tx_pkt.l4_hdr_offset_w = l4_hdr_offset_w;
+		tx_pkt.tx_dest = (enum ecore_ll2_tx_dest)p_ll2_conn->tx_dest;
+		tx_pkt.first_frag = first_frag;
+		tx_pkt.first_frag_len = p_buffer->packet_length;
+		tx_pkt.cookie = p_buffer;
+
+		rc = ecore_ll2_prepare_tx_packet(p_hwfn, p_ll2_conn->my_id,
+						 &tx_pkt, true);
+		if (rc != ECORE_SUCCESS) {
+			ecore_ooo_put_ready_buffer(p_hwfn->p_ooo_info,
+						   p_buffer, false);
+			break;
+		}
+	}
+}
+
+static void
+ecore_ooo_submit_rx_buffers(struct ecore_hwfn *p_hwfn,
+			    struct ecore_ll2_info *p_ll2_conn)
+{
+	struct ecore_ooo_buffer *p_buffer;
+	enum _ecore_status_t rc;
+
+	while ((p_buffer = ecore_ooo_get_free_buffer(p_hwfn->p_ooo_info))) {
+		rc = ecore_ll2_post_rx_buffer(p_hwfn,
+					    p_ll2_conn->my_id,
+					    p_buffer->rx_buffer_phys_addr,
+					    0, p_buffer, true);
+		if (rc != ECORE_SUCCESS) {
+			ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer);
+			break;
+		}
+	}
+}
+
+static enum _ecore_status_t
+ecore_ll2_lb_rxq_completion(struct ecore_hwfn *p_hwfn,
+			    void *p_cookie)
+{
+	struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info *)p_cookie;
+	enum _ecore_status_t rc;
+
+	rc = ecore_ll2_lb_rxq_handler(p_hwfn, p_ll2_conn);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	ecore_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn);
+	ecore_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn);
+
+	return 0;
+}
+
+static enum _ecore_status_t
+ecore_ll2_lb_txq_completion(struct ecore_hwfn *p_hwfn,
+			    void *p_cookie)
+{
+	struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info *)p_cookie;
+	struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL;
+	struct ecore_ooo_buffer	*p_buffer;
+	bool b_dont_submit_rx = false;
+	u16 new_idx = 0, num_bds = 0;
+	enum _ecore_status_t rc;
+
+	new_idx = OSAL_LE16_TO_CPU(*p_tx->p_fw_cons);
+	num_bds = ((s16)new_idx - (s16)p_tx->bds_idx);
+
+	if (!num_bds)
+		return ECORE_SUCCESS;
+
+	while (num_bds) {
+
+		if (OSAL_LIST_IS_EMPTY(&p_tx->active_descq))
+			return ECORE_INVAL;
+
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq,
+					      struct ecore_ll2_tx_packet,
+					      list_entry);
+		if (!p_pkt)
+			return ECORE_INVAL;
+
+		if (p_pkt->bd_used != 1) {
+			DP_NOTICE(p_hwfn, true,
+				  "Unexpectedly many BDs(%d) in TX OOO completion\n",
+				  p_pkt->bd_used);
+			return ECORE_INVAL;
+		}
+
+		OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+				       &p_tx->active_descq);
+
+		num_bds--;
+		p_tx->bds_idx++;
+		ecore_chain_consume(&p_tx->txq_chain);
+
+		p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie;
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+				    &p_tx->free_descq);
+
+		if (b_dont_submit_rx) {
+			ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer);
+			continue;
+		}
+
+		rc = ecore_ll2_post_rx_buffer(p_hwfn, p_ll2_conn->my_id,
+					      p_buffer->rx_buffer_phys_addr, 0,
+					      p_buffer, true);
+		if (rc != ECORE_SUCCESS) {
+			ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer);
+			b_dont_submit_rx = true;
+		}
+	}
+
+	ecore_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn);
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_sp_ll2_rx_queue_start(struct ecore_hwfn *p_hwfn,
+							struct ecore_ll2_info *p_ll2_conn,
+							u8 action_on_error)
+{
+	enum ecore_ll2_conn_type conn_type = p_ll2_conn->input.conn_type;
+	struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	struct core_rx_start_ramrod_data *p_ramrod = OSAL_NULL;
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_sp_init_data init_data;
+	u16 cqe_pbl_size;
+	enum _ecore_status_t rc	= ECORE_SUCCESS;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   CORE_RAMROD_RX_QUEUE_START,
+				   PROTOCOLID_CORE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_rx_queue_start;
+
+	p_ramrod->sb_id = OSAL_CPU_TO_LE16(ecore_int_get_sp_sb_id(p_hwfn));
+	p_ramrod->sb_index = p_rx->rx_sb_index;
+	p_ramrod->complete_event_flg = 1;
+
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(p_ll2_conn->input.mtu);
+	DMA_REGPAIR_LE(p_ramrod->bd_base,
+		       p_rx->rxq_chain.p_phys_addr);
+	cqe_pbl_size = (u16)ecore_chain_get_page_cnt(&p_rx->rcq_chain);
+	p_ramrod->num_of_pbl_pages = OSAL_CPU_TO_LE16(cqe_pbl_size);
+	DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr,
+		       ecore_chain_get_pbl_phys(&p_rx->rcq_chain));
+
+	p_ramrod->drop_ttl0_flg = p_ll2_conn->input.rx_drop_ttl0_flg;
+	p_ramrod->inner_vlan_stripping_en =
+		p_ll2_conn->input.rx_vlan_removal_en;
+
+	if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits) &&
+	    (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE))
+		p_ramrod->report_outer_vlan = 1;
+	p_ramrod->queue_id = p_ll2_conn->queue_id;
+	p_ramrod->main_func_queue = p_ll2_conn->main_func_queue;
+
+	if (OSAL_TEST_BIT(ECORE_MF_LL2_NON_UNICAST,
+			  &p_hwfn->p_dev->mf_bits) &&
+	    p_ramrod->main_func_queue &&
+	    ((conn_type != ECORE_LL2_TYPE_ROCE) &&
+	     (conn_type != ECORE_LL2_TYPE_IWARP))) {
+		p_ramrod->mf_si_bcast_accept_all = 1;
+		p_ramrod->mf_si_mcast_accept_all = 1;
+	} else {
+		p_ramrod->mf_si_bcast_accept_all = 0;
+		p_ramrod->mf_si_mcast_accept_all = 0;
+	}
+
+	p_ramrod->action_on_error.error_type = action_on_error;
+	p_ramrod->gsi_offload_flag = p_ll2_conn->input.gsi_enable;
+	return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+}
+
+static enum _ecore_status_t ecore_sp_ll2_tx_queue_start(struct ecore_hwfn *p_hwfn,
+							struct ecore_ll2_info *p_ll2_conn)
+{
+	enum ecore_ll2_conn_type conn_type = p_ll2_conn->input.conn_type;
+	struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct core_tx_start_ramrod_data *p_ramrod = OSAL_NULL;
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_sp_init_data init_data;
+	u16 pq_id = 0, pbl_size;
+	enum _ecore_status_t rc = ECORE_NOTIMPL;
+
+	if (!ECORE_LL2_TX_REGISTERED(p_ll2_conn))
+		return ECORE_SUCCESS;
+
+	if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO)
+		p_ll2_conn->tx_stats_en = 0;
+	else
+		p_ll2_conn->tx_stats_en = 1;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   CORE_RAMROD_TX_QUEUE_START,
+				   PROTOCOLID_CORE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_tx_queue_start;
+
+	p_ramrod->sb_id = OSAL_CPU_TO_LE16(ecore_int_get_sp_sb_id(p_hwfn));
+	p_ramrod->sb_index = p_tx->tx_sb_index;
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(p_ll2_conn->input.mtu);
+	p_ramrod->stats_en = p_ll2_conn->tx_stats_en;
+	p_ramrod->stats_id = p_ll2_conn->tx_stats_id;
+
+	DMA_REGPAIR_LE(p_ramrod->pbl_base_addr,
+		       ecore_chain_get_pbl_phys(&p_tx->txq_chain));
+	pbl_size = (u16)ecore_chain_get_page_cnt(&p_tx->txq_chain);
+	p_ramrod->pbl_size = OSAL_CPU_TO_LE16(pbl_size);
+
+	/* TODO RESC_ALLOC pq for ll2 */
+	switch (p_ll2_conn->input.tx_tc) {
+	case PURE_LB_TC:
+		pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);
+		break;
+	case PKT_LB_TC:
+		pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OOO);
+		break;
+	default:
+		pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
+	}
+
+	p_ramrod->qm_pq_id = OSAL_CPU_TO_LE16(pq_id);
+
+	switch (conn_type) {
+	case ECORE_LL2_TYPE_FCOE:
+		p_ramrod->conn_type = PROTOCOLID_FCOE;
+		break;
+	case ECORE_LL2_TYPE_ISCSI:
+		p_ramrod->conn_type = PROTOCOLID_ISCSI;
+		break;
+	case ECORE_LL2_TYPE_ROCE:
+		p_ramrod->conn_type = PROTOCOLID_ROCE;
+		break;
+	case ECORE_LL2_TYPE_IWARP:
+		p_ramrod->conn_type = PROTOCOLID_IWARP;
+		break;
+	case ECORE_LL2_TYPE_OOO:
+		if (p_hwfn->hw_info.personality == ECORE_PCI_ISCSI) {
+			p_ramrod->conn_type = PROTOCOLID_ISCSI;
+		} else {
+			p_ramrod->conn_type = PROTOCOLID_IWARP;
+		}
+		break;
+	default:
+		p_ramrod->conn_type = PROTOCOLID_ETH;
+		DP_NOTICE(p_hwfn, false, "Unknown connection type: %d\n",
+			  conn_type);
+	}
+
+	p_ramrod->gsi_offload_flag = p_ll2_conn->input.gsi_enable;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	rc = ecore_db_recovery_add(p_hwfn->p_dev, p_tx->doorbell_addr,
+				   &p_tx->db_msg, DB_REC_WIDTH_32B,
+				   DB_REC_KERNEL);
+	return rc;
+}
+
+static enum _ecore_status_t ecore_sp_ll2_rx_queue_stop(struct ecore_hwfn *p_hwfn,
+						       struct ecore_ll2_info *p_ll2_conn)
+{
+	struct core_rx_stop_ramrod_data	*p_ramrod = OSAL_NULL;
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_sp_init_data init_data;
+	enum _ecore_status_t rc = ECORE_NOTIMPL;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   CORE_RAMROD_RX_QUEUE_STOP,
+				   PROTOCOLID_CORE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_rx_queue_stop;
+
+	p_ramrod->complete_event_flg = 1;
+	p_ramrod->queue_id = p_ll2_conn->queue_id;
+
+	return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+}
+
+static enum _ecore_status_t ecore_sp_ll2_tx_queue_stop(struct ecore_hwfn *p_hwfn,
+						       struct ecore_ll2_info *p_ll2_conn)
+{
+	struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct ecore_spq_entry *p_ent = OSAL_NULL;
+	struct ecore_sp_init_data init_data;
+	enum _ecore_status_t rc = ECORE_NOTIMPL;
+
+	ecore_db_recovery_del(p_hwfn->p_dev, p_tx->doorbell_addr,
+			      &p_tx->db_msg);
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   CORE_RAMROD_TX_QUEUE_STOP,
+				   PROTOCOLID_CORE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+}
+
+static enum _ecore_status_t
+ecore_ll2_acquire_connection_rx(struct ecore_hwfn *p_hwfn,
+				struct ecore_ll2_info *p_ll2_info)
+{
+	struct ecore_ll2_rx_packet *p_descq;
+	u32 capacity;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (!p_ll2_info->input.rx_num_desc)
+		goto out;
+
+	rc = ecore_chain_alloc(p_hwfn->p_dev,
+			       ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+			       ECORE_CHAIN_MODE_NEXT_PTR,
+			       ECORE_CHAIN_CNT_TYPE_U16,
+			       p_ll2_info->input.rx_num_desc,
+			       sizeof(struct core_rx_bd),
+			       &p_ll2_info->rx_queue.rxq_chain, OSAL_NULL);
+	if (rc) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate ll2 rxq chain\n");
+		goto out;
+	}
+
+	capacity = ecore_chain_get_capacity(&p_ll2_info->rx_queue.rxq_chain);
+	p_descq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			      capacity * sizeof(struct ecore_ll2_rx_packet));
+	if (!p_descq) {
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate ll2 Rx desc\n");
+		goto out;
+	}
+	p_ll2_info->rx_queue.descq_array = p_descq;
+
+	rc = ecore_chain_alloc(p_hwfn->p_dev,
+			       ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+			       ECORE_CHAIN_MODE_PBL,
+			       ECORE_CHAIN_CNT_TYPE_U16,
+			       p_ll2_info->input.rx_num_desc,
+			       sizeof(struct core_rx_fast_path_cqe),
+			       &p_ll2_info->rx_queue.rcq_chain, OSAL_NULL);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate ll2 rcq chain\n");
+		goto out;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_LL2,
+		   "Allocated LL2 Rxq [Type %08x] with 0x%08x buffers\n",
+		   p_ll2_info->input.conn_type,
+		   p_ll2_info->input.rx_num_desc);
+
+out:
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_ll2_acquire_connection_tx(struct ecore_hwfn *p_hwfn,
+				struct ecore_ll2_info *p_ll2_info)
+{
+	struct ecore_ll2_tx_packet *p_descq;
+	u32 capacity;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u32 desc_size;
+
+	if (!p_ll2_info->input.tx_num_desc)
+		goto out;
+
+	rc = ecore_chain_alloc(p_hwfn->p_dev,
+			       ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+			       ECORE_CHAIN_MODE_PBL,
+			       ECORE_CHAIN_CNT_TYPE_U16,
+			       p_ll2_info->input.tx_num_desc,
+			       sizeof(struct core_tx_bd),
+			       &p_ll2_info->tx_queue.txq_chain, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto out;
+
+	capacity = ecore_chain_get_capacity(&p_ll2_info->tx_queue.txq_chain);
+	desc_size = (sizeof(*p_descq) +
+		     (p_ll2_info->input.tx_max_bds_per_packet - 1) *
+		     sizeof(p_descq->bds_set));
+
+	p_descq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			      capacity * desc_size);
+	if (!p_descq) {
+		rc = ECORE_NOMEM;
+		goto out;
+	}
+	p_ll2_info->tx_queue.descq_array = p_descq;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_LL2,
+		   "Allocated LL2 Txq [Type %08x] with 0x%08x buffers\n",
+		   p_ll2_info->input.conn_type,
+		   p_ll2_info->input.tx_num_desc);
+
+out:
+	if (rc != ECORE_SUCCESS)
+		DP_NOTICE(p_hwfn, false,
+			  "Can't allocate memory for Tx LL2 with 0x%08x buffers\n",
+			  p_ll2_info->input.tx_num_desc);
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_ll2_acquire_connection_ooo(struct ecore_hwfn *p_hwfn,
+				 struct ecore_ll2_info *p_ll2_info, u16 mtu)
+{
+	struct ecore_ooo_buffer *p_buf = OSAL_NULL;
+	u32 rx_buffer_size = 0;
+	void *p_virt;
+	u16 buf_idx;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (p_ll2_info->input.conn_type != ECORE_LL2_TYPE_OOO)
+		return rc;
+
+	/* Correct number of requested OOO buffers if needed */
+	if (!p_ll2_info->input.rx_num_ooo_buffers) {
+		u16 num_desc = p_ll2_info->input.rx_num_desc;
+
+		if (!num_desc)
+			return ECORE_INVAL;
+		p_ll2_info->input.rx_num_ooo_buffers = num_desc * 2;
+	}
+
+	/* TODO - use some defines for buffer size */
+	rx_buffer_size = mtu + 14 + 4 + 8 + ETH_CACHE_LINE_SIZE;
+	rx_buffer_size = (rx_buffer_size + ETH_CACHE_LINE_SIZE - 1) &
+			 ~(ETH_CACHE_LINE_SIZE - 1);
+
+	for (buf_idx = 0; buf_idx < p_ll2_info->input.rx_num_ooo_buffers;
+	     buf_idx++) {
+		p_buf = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_buf));
+		if (!p_buf) {
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to allocate ooo descriptor\n");
+			rc = ECORE_NOMEM;
+			goto out;
+		}
+
+		p_buf->rx_buffer_size = rx_buffer_size;
+		p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+						 &p_buf->rx_buffer_phys_addr,
+						 p_buf->rx_buffer_size);
+		if (!p_virt) {
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to allocate ooo buffer\n");
+			OSAL_FREE(p_hwfn->p_dev, p_buf);
+			rc = ECORE_NOMEM;
+			goto out;
+		}
+		p_buf->rx_buffer_virt_addr = p_virt;
+		ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buf);
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_LL2,
+		   "Allocated [%04x] LL2 OOO buffers [each of size 0x%08x]\n",
+		   p_ll2_info->input.rx_num_ooo_buffers, rx_buffer_size);
+
+out:
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_ll2_set_cbs(struct ecore_ll2_info *p_ll2_info,
+		    const struct ecore_ll2_cbs *cbs)
+{
+	if (!cbs || (!cbs->rx_comp_cb ||
+		     !cbs->rx_release_cb ||
+		     !cbs->tx_comp_cb ||
+		     !cbs->tx_release_cb ||
+		     !cbs->cookie))
+		return ECORE_INVAL;
+
+	p_ll2_info->cbs.rx_comp_cb = cbs->rx_comp_cb;
+	p_ll2_info->cbs.rx_release_cb = cbs->rx_release_cb;
+	p_ll2_info->cbs.tx_comp_cb = cbs->tx_comp_cb;
+	p_ll2_info->cbs.tx_release_cb = cbs->tx_release_cb;
+	p_ll2_info->cbs.slowpath_cb = cbs->slowpath_cb;
+	p_ll2_info->cbs.cookie = cbs->cookie;
+
+	return ECORE_SUCCESS;
+}
+
+static enum core_error_handle
+ecore_ll2_get_error_choice(enum ecore_ll2_error_handle err)
+{
+	switch (err) {
+	case ECORE_LL2_DROP_PACKET:
+		return LL2_DROP_PACKET;
+	case ECORE_LL2_DO_NOTHING:
+		return LL2_DO_NOTHING;
+	case ECORE_LL2_ASSERT:
+		return LL2_ASSERT;
+	default:
+		return LL2_DO_NOTHING;
+	}
+}
+
+enum _ecore_status_t
+ecore_ll2_acquire_connection(void *cxt,
+			     struct ecore_ll2_acquire_data *data)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	ecore_int_comp_cb_t comp_rx_cb, comp_tx_cb;
+	struct ecore_ll2_info *p_ll2_info = OSAL_NULL;
+	enum _ecore_status_t rc;
+	u8 i, *p_tx_max;
+
+	if (!data->p_connection_handle || !p_hwfn->p_ll2_info) {
+		DP_NOTICE(p_hwfn, false, "Invalid connection handle, ll2_info not allocated\n");
+		return ECORE_INVAL;
+	}
+
+	/* Find a free connection to be used */
+	for (i = 0; (i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS); i++) {
+		OSAL_MUTEX_ACQUIRE(&p_hwfn->p_ll2_info[i].mutex);
+		if (p_hwfn->p_ll2_info[i].b_active) {
+			OSAL_MUTEX_RELEASE(&p_hwfn->p_ll2_info[i].mutex);
+			continue;
+		}
+
+		p_hwfn->p_ll2_info[i].b_active = true;
+		p_ll2_info = &p_hwfn->p_ll2_info[i];
+		OSAL_MUTEX_RELEASE(&p_hwfn->p_ll2_info[i].mutex);
+		break;
+	}
+	if (p_ll2_info == OSAL_NULL) {
+		DP_NOTICE(p_hwfn, false, "No available ll2 connection\n");
+		return ECORE_BUSY;
+	}
+
+	OSAL_MEMCPY(&p_ll2_info->input, &data->input,
+		    sizeof(p_ll2_info->input));
+
+	switch (data->input.tx_dest) {
+	case ECORE_LL2_TX_DEST_NW:
+		p_ll2_info->tx_dest = CORE_TX_DEST_NW;
+		break;
+	case ECORE_LL2_TX_DEST_LB:
+		p_ll2_info->tx_dest = CORE_TX_DEST_LB;
+		break;
+	case ECORE_LL2_TX_DEST_DROP:
+		p_ll2_info->tx_dest = CORE_TX_DEST_DROP;
+		break;
+	default:
+		return ECORE_INVAL;
+	}
+
+	if ((data->input.conn_type == ECORE_LL2_TYPE_OOO) ||
+	    data->input.secondary_queue)
+		p_ll2_info->main_func_queue = false;
+	else
+		p_ll2_info->main_func_queue = true;
+
+	/* Correct maximum number of Tx BDs */
+	p_tx_max = &p_ll2_info->input.tx_max_bds_per_packet;
+	if (*p_tx_max == 0)
+		*p_tx_max = CORE_LL2_TX_MAX_BDS_PER_PACKET;
+	else
+		*p_tx_max = OSAL_MIN_T(u8, *p_tx_max,
+				       CORE_LL2_TX_MAX_BDS_PER_PACKET);
+
+	rc = ecore_ll2_set_cbs(p_ll2_info, data->cbs);
+	if (rc) {
+		DP_NOTICE(p_hwfn, false, "Invalid callback functions\n");
+		goto q_allocate_fail;
+	}
+
+	rc = ecore_ll2_acquire_connection_rx(p_hwfn, p_ll2_info);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "ll2 acquire rx connection failed\n");
+		goto q_allocate_fail;
+	}
+
+	rc = ecore_ll2_acquire_connection_tx(p_hwfn, p_ll2_info);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "ll2 acquire tx connection failed\n");
+		goto q_allocate_fail;
+	}
+
+	rc = ecore_ll2_acquire_connection_ooo(p_hwfn, p_ll2_info,
+					      data->input.mtu);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "ll2 acquire ooo connection failed\n");
+		goto q_allocate_fail;
+	}
+
+	/* Register callbacks for the Rx/Tx queues */
+	if (data->input.conn_type == ECORE_LL2_TYPE_OOO) {
+		comp_rx_cb = ecore_ll2_lb_rxq_completion;
+		comp_tx_cb = ecore_ll2_lb_txq_completion;
+
+	} else {
+		comp_rx_cb = ecore_ll2_rxq_completion;
+		comp_tx_cb = ecore_ll2_txq_completion;
+	}
+
+	if (data->input.rx_num_desc) {
+		ecore_int_register_cb(p_hwfn, comp_rx_cb,
+				      &p_hwfn->p_ll2_info[i],
+				      &p_ll2_info->rx_queue.rx_sb_index,
+				      &p_ll2_info->rx_queue.p_fw_cons);
+		p_ll2_info->rx_queue.b_cb_registred   = true;
+	}
+
+	if (data->input.tx_num_desc) {
+		ecore_int_register_cb(p_hwfn,
+				      comp_tx_cb,
+				      &p_hwfn->p_ll2_info[i],
+				      &p_ll2_info->tx_queue.tx_sb_index,
+				      &p_ll2_info->tx_queue.p_fw_cons);
+		p_ll2_info->tx_queue.b_cb_registred   = true;
+	}
+
+	*(data->p_connection_handle) = i;
+	return rc;
+
+q_allocate_fail:
+	ecore_ll2_release_connection(p_hwfn, i);
+	return ECORE_NOMEM;
+}
+
+static enum _ecore_status_t ecore_ll2_establish_connection_rx(struct ecore_hwfn *p_hwfn,
+							      struct ecore_ll2_info *p_ll2_conn)
+{
+	enum ecore_ll2_error_handle error_input;
+	enum core_error_handle error_mode;
+	u8 action_on_error = 0;
+
+	if (!ECORE_LL2_RX_REGISTERED(p_ll2_conn))
+		return ECORE_SUCCESS;
+
+	DIRECT_REG_WR(p_hwfn, p_ll2_conn->rx_queue.set_prod_addr, 0x0);
+	error_input = p_ll2_conn->input.ai_err_packet_too_big;
+	error_mode = ecore_ll2_get_error_choice(error_input);
+	SET_FIELD(action_on_error,
+		  CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG, error_mode);
+	error_input = p_ll2_conn->input.ai_err_no_buf;
+	error_mode = ecore_ll2_get_error_choice(error_input);
+	SET_FIELD(action_on_error,
+		  CORE_RX_ACTION_ON_ERROR_NO_BUFF, error_mode);
+
+	return ecore_sp_ll2_rx_queue_start(p_hwfn, p_ll2_conn, action_on_error);
+}
+
+static void
+ecore_ll2_establish_connection_ooo(struct ecore_hwfn *p_hwfn,
+				   struct ecore_ll2_info *p_ll2_conn)
+{
+	if (p_ll2_conn->input.conn_type != ECORE_LL2_TYPE_OOO)
+		return;
+
+	ecore_ooo_release_all_isles(p_hwfn->p_ooo_info);
+	ecore_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn);
+}
+
+enum _ecore_status_t ecore_ll2_establish_connection(void *cxt,
+						    u8 connection_handle)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct e4_core_conn_context *p_cxt;
+	struct ecore_ll2_info *p_ll2_conn;
+	struct ecore_cxt_info cxt_info;
+	struct ecore_ll2_rx_queue *p_rx;
+	struct ecore_ll2_tx_queue *p_tx;
+	struct ecore_ll2_tx_packet *p_pkt;
+	struct ecore_ptt *p_ptt;
+	enum _ecore_status_t rc = ECORE_NOTIMPL;
+	u32 i, capacity;
+	u32 desc_size;
+	u8 qid;
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return ECORE_AGAIN;
+
+	p_ll2_conn = ecore_ll2_handle_sanity_lock(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL) {
+		rc = ECORE_INVAL;
+		goto out;
+	}
+
+	p_rx = &p_ll2_conn->rx_queue;
+	p_tx = &p_ll2_conn->tx_queue;
+
+	ecore_chain_reset(&p_rx->rxq_chain);
+	ecore_chain_reset(&p_rx->rcq_chain);
+	OSAL_LIST_INIT(&p_rx->active_descq);
+	OSAL_LIST_INIT(&p_rx->free_descq);
+	OSAL_LIST_INIT(&p_rx->posting_descq);
+	OSAL_SPIN_LOCK_INIT(&p_rx->lock);
+	capacity = ecore_chain_get_capacity(&p_rx->rxq_chain);
+	for (i = 0; i < capacity; i++)
+		OSAL_LIST_PUSH_TAIL(&p_rx->descq_array[i].list_entry,
+				    &p_rx->free_descq);
+	*p_rx->p_fw_cons = 0;
+
+	ecore_chain_reset(&p_tx->txq_chain);
+	OSAL_LIST_INIT(&p_tx->active_descq);
+	OSAL_LIST_INIT(&p_tx->free_descq);
+	OSAL_LIST_INIT(&p_tx->sending_descq);
+	OSAL_SPIN_LOCK_INIT(&p_tx->lock);
+	capacity = ecore_chain_get_capacity(&p_tx->txq_chain);
+	/* The size of the element in descq_array is flexible */
+	desc_size = (sizeof(*p_pkt) +
+		     (p_ll2_conn->input.tx_max_bds_per_packet - 1) *
+		     sizeof(p_pkt->bds_set));
+
+	for (i = 0; i < capacity; i++) {
+		p_pkt = (struct ecore_ll2_tx_packet *)((u8 *)p_tx->descq_array +
+						       desc_size*i);
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry,
+				    &p_tx->free_descq);
+	}
+	p_tx->cur_completing_bd_idx = 0;
+	p_tx->bds_idx = 0;
+	p_tx->b_completing_packet = false;
+	p_tx->cur_send_packet = OSAL_NULL;
+	p_tx->cur_send_frag_num = 0;
+	p_tx->cur_completing_frag_num = 0;
+	*p_tx->p_fw_cons = 0;
+
+	rc = ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_ll2_conn->cid);
+	if (rc)
+		goto out;
+	cxt_info.iid = p_ll2_conn->cid;
+	rc = ecore_cxt_get_cid_info(p_hwfn, &cxt_info);
+	if (rc) {
+		DP_NOTICE(p_hwfn, true, "Cannot find context info for cid=%d\n",
+			  p_ll2_conn->cid);
+		goto out;
+	}
+
+	p_cxt = cxt_info.p_cxt;
+
+	/* @@@TBD we zero the context until we have ilt_reset implemented. */
+	OSAL_MEM_ZERO(p_cxt, sizeof(*p_cxt));
+
+	qid = ecore_ll2_handle_to_queue_id(p_hwfn, connection_handle);
+	p_ll2_conn->queue_id = qid;
+	p_ll2_conn->tx_stats_id = qid;
+	p_rx->set_prod_addr = (u8 OSAL_IOMEM*)p_hwfn->regview +
+					      GTT_BAR0_MAP_REG_TSDM_RAM +
+					      TSTORM_LL2_RX_PRODS_OFFSET(qid);
+	p_tx->doorbell_addr = (u8 OSAL_IOMEM*)p_hwfn->doorbells +
+					      DB_ADDR(p_ll2_conn->cid,
+						      DQ_DEMS_LEGACY);
+
+	/* prepare db data */
+	SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_DEST, DB_DEST_XCM);
+	SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_AGG_CMD,
+		  DB_AGG_CMD_SET);
+	SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_AGG_VAL_SEL,
+		  DQ_XCM_CORE_TX_BD_PROD_CMD);
+	p_tx->db_msg.agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
+
+	rc = ecore_ll2_establish_connection_rx(p_hwfn, p_ll2_conn);
+	if (rc)
+		goto out;
+
+	rc = ecore_sp_ll2_tx_queue_start(p_hwfn, p_ll2_conn);
+	if (rc)
+		goto out;
+
+	if (!ECORE_IS_RDMA_PERSONALITY(p_hwfn))
+		ecore_wr(p_hwfn, p_ptt, PRS_REG_USE_LIGHT_L2, 1);
+
+	ecore_ll2_establish_connection_ooo(p_hwfn, p_ll2_conn);
+
+	if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE) {
+		if (!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC,
+				   &p_hwfn->p_dev->mf_bits))
+			ecore_llh_add_protocol_filter(p_hwfn->p_dev, 0,
+						      ECORE_LLH_FILTER_ETHERTYPE,
+						      0x8906, 0);
+		ecore_llh_add_protocol_filter(p_hwfn->p_dev, 0,
+					      ECORE_LLH_FILTER_ETHERTYPE,
+					      0x8914, 0);
+	}
+
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+static void ecore_ll2_post_rx_buffer_notify_fw(struct ecore_hwfn *p_hwfn,
+					       struct ecore_ll2_rx_queue *p_rx,
+					       struct ecore_ll2_rx_packet *p_curp)
+{
+	struct ecore_ll2_rx_packet *p_posting_packet = OSAL_NULL;
+	struct core_ll2_rx_prod rx_prod = {0, 0, 0};
+	bool b_notify_fw = false;
+	u16 bd_prod, cq_prod;
+
+	/* This handles the flushing of already posted buffers */
+	while (!OSAL_LIST_IS_EMPTY(&p_rx->posting_descq)) {
+		p_posting_packet = OSAL_LIST_FIRST_ENTRY(&p_rx->posting_descq,
+							 struct ecore_ll2_rx_packet,
+							 list_entry);
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_posting_packet->list_entry, &p_rx->posting_descq);
+		OSAL_LIST_PUSH_TAIL(&p_posting_packet->list_entry, &p_rx->active_descq);
+		b_notify_fw = true;
+	}
+
+	/* This handles the supplied packet [if there is one] */
+	if (p_curp) {
+		OSAL_LIST_PUSH_TAIL(&p_curp->list_entry,
+				    &p_rx->active_descq);
+		b_notify_fw = true;
+	}
+
+	if (!b_notify_fw)
+		return;
+
+	bd_prod = ecore_chain_get_prod_idx(&p_rx->rxq_chain);
+	cq_prod = ecore_chain_get_prod_idx(&p_rx->rcq_chain);
+	rx_prod.bd_prod = OSAL_CPU_TO_LE16(bd_prod);
+	rx_prod.cqe_prod = OSAL_CPU_TO_LE16(cq_prod);
+	DIRECT_REG_WR(p_hwfn, p_rx->set_prod_addr, *((u32 *)&rx_prod));
+}
+
+enum _ecore_status_t ecore_ll2_post_rx_buffer(void *cxt,
+					      u8 connection_handle,
+					      dma_addr_t addr,
+					      u16 buf_len,
+					      void *cookie,
+					      u8 notify_fw)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct core_rx_bd_with_buff_len *p_curb = OSAL_NULL;
+	struct ecore_ll2_rx_packet *p_curp = OSAL_NULL;
+	struct ecore_ll2_info *p_ll2_conn;
+	struct ecore_ll2_rx_queue *p_rx;
+	unsigned long flags;
+	void *p_data;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return ECORE_INVAL;
+	p_rx = &p_ll2_conn->rx_queue;
+	if (p_rx->set_prod_addr == OSAL_NULL)
+		return ECORE_IO;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags);
+	if (!OSAL_LIST_IS_EMPTY(&p_rx->free_descq))
+		p_curp = OSAL_LIST_FIRST_ENTRY(&p_rx->free_descq,
+					       struct ecore_ll2_rx_packet,
+					       list_entry);
+	if (p_curp) {
+		if (ecore_chain_get_elem_left(&p_rx->rxq_chain) &&
+		    ecore_chain_get_elem_left(&p_rx->rcq_chain)) {
+			p_data = ecore_chain_produce(&p_rx->rxq_chain);
+			p_curb = (struct core_rx_bd_with_buff_len *)p_data;
+			ecore_chain_produce(&p_rx->rcq_chain);
+		}
+	}
+
+	/* If we're lacking entires, let's try to flush buffers to FW */
+	if (!p_curp || !p_curb) {
+		rc =  ECORE_BUSY;
+		p_curp = OSAL_NULL;
+		goto out_notify;
+	}
+
+	/* We have an Rx packet we can fill */
+	DMA_REGPAIR_LE(p_curb->addr, addr);
+	p_curb->buff_length = OSAL_CPU_TO_LE16(buf_len);
+	p_curp->rx_buf_addr = addr;
+	p_curp->cookie = cookie;
+	p_curp->rxq_bd = p_curb;
+	p_curp->buf_length = buf_len;
+	OSAL_LIST_REMOVE_ENTRY(&p_curp->list_entry,
+			       &p_rx->free_descq);
+
+	/* Check if we only want to enqueue this packet without informing FW */
+	if (!notify_fw) {
+		OSAL_LIST_PUSH_TAIL(&p_curp->list_entry,
+				    &p_rx->posting_descq);
+		goto out;
+	}
+
+out_notify:
+	ecore_ll2_post_rx_buffer_notify_fw(p_hwfn, p_rx, p_curp);
+out:
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags);
+	return rc;
+}
+
+static void ecore_ll2_prepare_tx_packet_set(struct ecore_ll2_tx_queue *p_tx,
+					    struct ecore_ll2_tx_packet *p_curp,
+					    struct ecore_ll2_tx_pkt_info *pkt,
+					    u8 notify_fw)
+{
+	OSAL_LIST_REMOVE_ENTRY(&p_curp->list_entry,
+			       &p_tx->free_descq);
+	p_curp->cookie = pkt->cookie;
+	p_curp->bd_used = pkt->num_of_bds;
+	p_curp->notify_fw = notify_fw;
+	p_tx->cur_send_packet = p_curp;
+	p_tx->cur_send_frag_num = 0;
+
+	p_curp->bds_set[p_tx->cur_send_frag_num].tx_frag = pkt->first_frag;
+	p_curp->bds_set[p_tx->cur_send_frag_num].frag_len = pkt->first_frag_len;
+	p_tx->cur_send_frag_num++;
+}
+
+static void ecore_ll2_prepare_tx_packet_set_bd(
+		struct ecore_hwfn *p_hwfn,
+		struct ecore_ll2_info *p_ll2,
+		struct ecore_ll2_tx_packet *p_curp,
+		struct ecore_ll2_tx_pkt_info *pkt)
+{
+	struct ecore_chain *p_tx_chain = &p_ll2->tx_queue.txq_chain;
+	u16 prod_idx = ecore_chain_get_prod_idx(p_tx_chain);
+	struct core_tx_bd *start_bd = OSAL_NULL;
+	enum core_roce_flavor_type roce_flavor;
+	enum core_tx_dest tx_dest;
+	u16 bd_data = 0, frag_idx;
+
+	roce_flavor = (pkt->ecore_roce_flavor == ECORE_LL2_ROCE) ?
+		CORE_ROCE : CORE_RROCE;
+
+	switch (pkt->tx_dest) {
+	case ECORE_LL2_TX_DEST_NW:
+		tx_dest = CORE_TX_DEST_NW;
+		break;
+	case ECORE_LL2_TX_DEST_LB:
+		tx_dest = CORE_TX_DEST_LB;
+		break;
+	case ECORE_LL2_TX_DEST_DROP:
+		tx_dest = CORE_TX_DEST_DROP;
+		break;
+	default:
+		tx_dest = CORE_TX_DEST_LB;
+		break;
+	}
+
+	start_bd = (struct core_tx_bd*)ecore_chain_produce(p_tx_chain);
+
+	if (ECORE_IS_IWARP_PERSONALITY(p_hwfn) &&
+	    (p_ll2->input.conn_type == ECORE_LL2_TYPE_OOO)) {
+		start_bd->nw_vlan_or_lb_echo =
+			OSAL_CPU_TO_LE16(IWARP_LL2_IN_ORDER_TX_QUEUE);
+	} else {
+		start_bd->nw_vlan_or_lb_echo = OSAL_CPU_TO_LE16(pkt->vlan);
+		if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits) &&
+		    (p_ll2->input.conn_type == ECORE_LL2_TYPE_FCOE))
+			pkt->remove_stag = true;
+	}
+
+	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_L4_HDR_OFFSET_W,
+		  OSAL_CPU_TO_LE16(pkt->l4_hdr_offset_w));
+	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_TX_DST, tx_dest);
+	bd_data |= pkt->bd_flags;
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_START_BD, 0x1);
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_NBDS, pkt->num_of_bds);
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_ROCE_FLAV, roce_flavor);
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_CSUM, !!(pkt->enable_ip_cksum));
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_L4_CSUM, !!(pkt->enable_l4_cksum));
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_LEN, !!(pkt->calc_ip_len));
+	SET_FIELD(bd_data, CORE_TX_BD_DATA_DISABLE_STAG_INSERTION,
+		  !!(pkt->remove_stag));
+
+	start_bd->bd_data.as_bitfield = OSAL_CPU_TO_LE16(bd_data);
+	DMA_REGPAIR_LE(start_bd->addr, pkt->first_frag);
+	start_bd->nbytes = OSAL_CPU_TO_LE16(pkt->first_frag_len);
+
+	DP_VERBOSE(p_hwfn, (ECORE_MSG_TX_QUEUED | ECORE_MSG_LL2),
+		   "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n",
+		   p_ll2->queue_id, p_ll2->cid, p_ll2->input.conn_type,
+		   prod_idx, pkt->first_frag_len, pkt->num_of_bds,
+		   OSAL_LE32_TO_CPU(start_bd->addr.hi),
+		   OSAL_LE32_TO_CPU(start_bd->addr.lo));
+
+	if (p_ll2->tx_queue.cur_send_frag_num == pkt->num_of_bds)
+		return;
+
+	/* Need to provide the packet with additional BDs for frags */
+	for (frag_idx = p_ll2->tx_queue.cur_send_frag_num;
+	     frag_idx < pkt->num_of_bds; frag_idx++) {
+		struct core_tx_bd **p_bd = &p_curp->bds_set[frag_idx].txq_bd;
+
+		*p_bd = (struct core_tx_bd *)ecore_chain_produce(p_tx_chain);
+		(*p_bd)->bd_data.as_bitfield = 0;
+		(*p_bd)->bitfield1 = 0;
+		p_curp->bds_set[frag_idx].tx_frag = 0;
+		p_curp->bds_set[frag_idx].frag_len = 0;
+	}
+}
+
+/* This should be called while the Txq spinlock is being held */
+static void ecore_ll2_tx_packet_notify(struct ecore_hwfn *p_hwfn,
+				       struct ecore_ll2_info *p_ll2_conn)
+{
+	bool b_notify = p_ll2_conn->tx_queue.cur_send_packet->notify_fw;
+	struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL;
+	u16 bd_prod;
+
+	/* If there are missing BDs, don't do anything now */
+	if (p_ll2_conn->tx_queue.cur_send_frag_num !=
+	    p_ll2_conn->tx_queue.cur_send_packet->bd_used)
+		return;
+
+
+	/* Push the current packet to the list and clean after it */
+	OSAL_LIST_PUSH_TAIL(&p_ll2_conn->tx_queue.cur_send_packet->list_entry,
+			    &p_ll2_conn->tx_queue.sending_descq);
+	p_ll2_conn->tx_queue.cur_send_packet = OSAL_NULL;
+	p_ll2_conn->tx_queue.cur_send_frag_num = 0;
+
+	/* Notify FW of packet only if requested to */
+	if (!b_notify)
+		return;
+
+	bd_prod = ecore_chain_get_prod_idx(&p_ll2_conn->tx_queue.txq_chain);
+
+	while (!OSAL_LIST_IS_EMPTY(&p_tx->sending_descq)) {
+		p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->sending_descq,
+					      struct ecore_ll2_tx_packet,
+					      list_entry);
+		if (p_pkt == OSAL_NULL)
+			break;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry,
+				       &p_tx->sending_descq);
+		OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, &p_tx->active_descq);
+	}
+
+	p_tx->db_msg.spq_prod = OSAL_CPU_TO_LE16(bd_prod);
+
+	/* Make sure the BDs data is updated before ringing the doorbell */
+	OSAL_WMB(p_hwfn->p_dev);
+
+	//DIRECT_REG_WR(p_hwfn, p_tx->doorbell_addr, *((u32 *)&p_tx->db_msg));
+	DIRECT_REG_WR_DB(p_hwfn, p_tx->doorbell_addr, *((u32 *)&p_tx->db_msg));
+
+	DP_VERBOSE(p_hwfn, (ECORE_MSG_TX_QUEUED | ECORE_MSG_LL2),
+		   "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Doorbelled [producer 0x%04x]\n",
+		   p_ll2_conn->queue_id, p_ll2_conn->cid,
+		   p_ll2_conn->input.conn_type,
+		   p_tx->db_msg.spq_prod);
+}
+
+enum _ecore_status_t ecore_ll2_prepare_tx_packet(
+		void *cxt,
+		u8 connection_handle,
+		struct ecore_ll2_tx_pkt_info *pkt,
+		bool notify_fw)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_ll2_tx_packet *p_curp = OSAL_NULL;
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+	struct ecore_ll2_tx_queue *p_tx;
+	struct ecore_chain *p_tx_chain;
+	unsigned long flags;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return ECORE_INVAL;
+	p_tx = &p_ll2_conn->tx_queue;
+	p_tx_chain = &p_tx->txq_chain;
+
+	if (pkt->num_of_bds > p_ll2_conn->input.tx_max_bds_per_packet)
+		return ECORE_IO; /* coalescing is requireed */
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags);
+	if (p_tx->cur_send_packet) {
+		rc = ECORE_EXISTS;
+		goto out;
+	}
+
+	/* Get entry, but only if we have tx elements for it */
+	if (!OSAL_LIST_IS_EMPTY(&p_tx->free_descq))
+		p_curp = OSAL_LIST_FIRST_ENTRY(&p_tx->free_descq,
+					       struct ecore_ll2_tx_packet,
+					       list_entry);
+	if (p_curp && ecore_chain_get_elem_left(p_tx_chain) < pkt->num_of_bds)
+		p_curp = OSAL_NULL;
+
+	if (!p_curp) {
+		rc = ECORE_BUSY;
+		goto out;
+	}
+
+	/* Prepare packet and BD, and perhaps send a doorbell to FW */
+	ecore_ll2_prepare_tx_packet_set(p_tx, p_curp, pkt, notify_fw);
+
+	ecore_ll2_prepare_tx_packet_set_bd(p_hwfn, p_ll2_conn, p_curp,
+					   pkt);
+
+	ecore_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
+
+out:
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags);
+	return rc;
+}
+
+enum _ecore_status_t ecore_ll2_set_fragment_of_tx_packet(void *cxt,
+							 u8 connection_handle,
+							 dma_addr_t addr,
+							 u16 nbytes)
+{
+	struct ecore_ll2_tx_packet *p_cur_send_packet = OSAL_NULL;
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+	u16 cur_send_frag_num = 0;
+	struct core_tx_bd *p_bd;
+	unsigned long flags;
+
+	p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return ECORE_INVAL;
+
+	if (!p_ll2_conn->tx_queue.cur_send_packet)
+		return ECORE_INVAL;
+
+	p_cur_send_packet = p_ll2_conn->tx_queue.cur_send_packet;
+	cur_send_frag_num = p_ll2_conn->tx_queue.cur_send_frag_num;
+
+	if (cur_send_frag_num >= p_cur_send_packet->bd_used)
+		return ECORE_INVAL;
+
+	/* Fill the BD information, and possibly notify FW */
+	p_bd = p_cur_send_packet->bds_set[cur_send_frag_num].txq_bd;
+	DMA_REGPAIR_LE(p_bd->addr, addr);
+	p_bd->nbytes = OSAL_CPU_TO_LE16(nbytes);
+	p_cur_send_packet->bds_set[cur_send_frag_num].tx_frag = addr;
+	p_cur_send_packet->bds_set[cur_send_frag_num].frag_len = nbytes;
+
+	p_ll2_conn->tx_queue.cur_send_frag_num++;
+
+	OSAL_SPIN_LOCK_IRQSAVE(&p_ll2_conn->tx_queue.lock, flags);
+	ecore_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
+	OSAL_SPIN_UNLOCK_IRQSAVE(&p_ll2_conn->tx_queue.lock, flags);
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_ll2_terminate_connection(void *cxt,
+						    u8 connection_handle)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+	enum _ecore_status_t rc = ECORE_NOTIMPL;
+	struct ecore_ptt *p_ptt;
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return ECORE_AGAIN;
+
+	p_ll2_conn = ecore_ll2_handle_sanity_lock(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL) {
+		rc = ECORE_INVAL;
+		goto out;
+	}
+
+	/* Stop Tx & Rx of connection, if needed */
+	if (ECORE_LL2_TX_REGISTERED(p_ll2_conn)) {
+		rc = ecore_sp_ll2_tx_queue_stop(p_hwfn, p_ll2_conn);
+		if (rc != ECORE_SUCCESS)
+			goto out;
+		ecore_ll2_txq_flush(p_hwfn, connection_handle);
+	}
+
+	if (ECORE_LL2_RX_REGISTERED(p_ll2_conn)) {
+		rc = ecore_sp_ll2_rx_queue_stop(p_hwfn, p_ll2_conn);
+		if (rc)
+			goto out;
+		ecore_ll2_rxq_flush(p_hwfn, connection_handle);
+	}
+
+	if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO)
+		ecore_ooo_release_all_isles(p_hwfn->p_ooo_info);
+
+	if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE) {
+		if (!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC,
+				   &p_hwfn->p_dev->mf_bits))
+			ecore_llh_remove_protocol_filter(p_hwfn->p_dev, 0,
+							 ECORE_LLH_FILTER_ETHERTYPE,
+							 0x8906, 0);
+		ecore_llh_remove_protocol_filter(p_hwfn->p_dev, 0,
+						 ECORE_LLH_FILTER_ETHERTYPE,
+						 0x8914, 0);
+	}
+
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+static void ecore_ll2_release_connection_ooo(struct ecore_hwfn *p_hwfn,
+					     struct ecore_ll2_info *p_ll2_conn)
+{
+	struct ecore_ooo_buffer *p_buffer;
+
+	if (p_ll2_conn->input.conn_type != ECORE_LL2_TYPE_OOO)
+		return;
+
+	ecore_ooo_release_all_isles(p_hwfn->p_ooo_info);
+	while ((p_buffer = ecore_ooo_get_free_buffer(p_hwfn->p_ooo_info))) {
+		OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+				       p_buffer->rx_buffer_virt_addr,
+				       p_buffer->rx_buffer_phys_addr,
+				       p_buffer->rx_buffer_size);
+		OSAL_FREE(p_hwfn->p_dev, p_buffer);
+	}
+}
+
+void ecore_ll2_release_connection(void *cxt,
+				  u8 connection_handle)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+
+	p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (p_ll2_conn == OSAL_NULL)
+		return;
+
+	if (ECORE_LL2_RX_REGISTERED(p_ll2_conn)) {
+		p_ll2_conn->rx_queue.b_cb_registred = false;
+		ecore_int_unregister_cb(p_hwfn,
+					p_ll2_conn->rx_queue.rx_sb_index);
+	}
+
+	if (ECORE_LL2_TX_REGISTERED(p_ll2_conn)) {
+		p_ll2_conn->tx_queue.b_cb_registred = false;
+		ecore_int_unregister_cb(p_hwfn,
+					p_ll2_conn->tx_queue.tx_sb_index);
+	}
+
+	OSAL_FREE(p_hwfn->p_dev, p_ll2_conn->tx_queue.descq_array);
+	ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->tx_queue.txq_chain);
+
+	OSAL_FREE(p_hwfn->p_dev, p_ll2_conn->rx_queue.descq_array);
+	ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->rx_queue.rxq_chain);
+	ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->rx_queue.rcq_chain);
+
+	ecore_cxt_release_cid(p_hwfn, p_ll2_conn->cid);
+
+	ecore_ll2_release_connection_ooo(p_hwfn, p_ll2_conn);
+
+	OSAL_MUTEX_ACQUIRE(&p_ll2_conn->mutex);
+	p_ll2_conn->b_active = false;
+	OSAL_MUTEX_RELEASE(&p_ll2_conn->mutex);
+}
+
+/* ECORE LL2: internal functions */
+
+enum _ecore_status_t ecore_ll2_alloc(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_ll2_info *p_ll2_info;
+	u8 i;
+
+	/* Allocate LL2's set struct */
+	p_ll2_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+				 sizeof(struct ecore_ll2_info) *
+				 ECORE_MAX_NUM_OF_LL2_CONNECTIONS);
+	if (!p_ll2_info) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to allocate `struct ecore_ll2'\n");
+		return ECORE_NOMEM;
+	}
+
+	p_hwfn->p_ll2_info = p_ll2_info;
+
+	for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++) {
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+		if (OSAL_MUTEX_ALLOC(p_hwfn, &p_ll2_info[i].mutex))
+			goto handle_err;
+		if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_ll2_info[i].rx_queue.lock))
+			goto handle_err;
+		if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_ll2_info[i].tx_queue.lock))
+			goto handle_err;
+#endif
+		p_ll2_info[i].my_id = i;
+	}
+
+	return ECORE_SUCCESS;
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+handle_err:
+	ecore_ll2_free(p_hwfn);
+	return ECORE_NOMEM;
+#endif
+}
+
+void ecore_ll2_setup(struct ecore_hwfn *p_hwfn)
+{
+	int  i;
+
+	for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++)
+		OSAL_MUTEX_INIT(&p_hwfn->p_ll2_info[i].mutex);
+}
+
+void ecore_ll2_free(struct ecore_hwfn *p_hwfn)
+{
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	int i;
+#endif
+	if (!p_hwfn->p_ll2_info)
+		return;
+
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++) {
+		OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ll2_info[i].rx_queue.lock);
+		OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ll2_info[i].tx_queue.lock);
+		OSAL_MUTEX_DEALLOC(&p_hwfn->p_ll2_info[i].mutex);
+	}
+#endif
+	OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_ll2_info);
+	p_hwfn->p_ll2_info = OSAL_NULL;
+}
+
+static void _ecore_ll2_get_port_stats(struct ecore_hwfn *p_hwfn,
+				      struct ecore_ptt *p_ptt,
+				      struct ecore_ll2_stats *p_stats)
+{
+	struct core_ll2_port_stats port_stats;
+
+	OSAL_MEMSET(&port_stats, 0, sizeof(port_stats));
+	ecore_memcpy_from(p_hwfn, p_ptt, &port_stats,
+			  BAR0_MAP_REG_TSDM_RAM +
+			  TSTORM_LL2_PORT_STAT_OFFSET(MFW_PORT(p_hwfn)),
+			  sizeof(port_stats));
+
+	p_stats->gsi_invalid_hdr +=
+		HILO_64_REGPAIR(port_stats.gsi_invalid_hdr);
+	p_stats->gsi_invalid_pkt_length +=
+		HILO_64_REGPAIR(port_stats.gsi_invalid_pkt_length);
+	p_stats->gsi_unsupported_pkt_typ +=
+		HILO_64_REGPAIR(port_stats.gsi_unsupported_pkt_typ);
+	p_stats->gsi_crcchksm_error +=
+		HILO_64_REGPAIR(port_stats.gsi_crcchksm_error);
+}
+
+static void _ecore_ll2_get_tstats(struct ecore_hwfn *p_hwfn,
+				  struct ecore_ptt *p_ptt,
+				  struct ecore_ll2_info *p_ll2_conn,
+				  struct ecore_ll2_stats *p_stats)
+{
+	struct core_ll2_tstorm_per_queue_stat tstats;
+	u8 qid = p_ll2_conn->queue_id;
+	u32 tstats_addr;
+
+	OSAL_MEMSET(&tstats, 0, sizeof(tstats));
+	tstats_addr = BAR0_MAP_REG_TSDM_RAM +
+		      CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(qid);
+	ecore_memcpy_from(p_hwfn, p_ptt, &tstats,
+			  tstats_addr,
+			  sizeof(tstats));
+
+	p_stats->packet_too_big_discard +=
+		HILO_64_REGPAIR(tstats.packet_too_big_discard);
+	p_stats->no_buff_discard +=
+		HILO_64_REGPAIR(tstats.no_buff_discard);
+}
+
+static void _ecore_ll2_get_ustats(struct ecore_hwfn *p_hwfn,
+				  struct ecore_ptt *p_ptt,
+				  struct ecore_ll2_info *p_ll2_conn,
+				  struct ecore_ll2_stats *p_stats)
+{
+	struct core_ll2_ustorm_per_queue_stat ustats;
+	u8 qid = p_ll2_conn->queue_id;
+	u32 ustats_addr;
+
+	OSAL_MEMSET(&ustats, 0, sizeof(ustats));
+	ustats_addr = BAR0_MAP_REG_USDM_RAM +
+		      CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(qid);
+	ecore_memcpy_from(p_hwfn, p_ptt, &ustats,
+			  ustats_addr,
+			  sizeof(ustats));
+
+	p_stats->rcv_ucast_bytes += HILO_64_REGPAIR(ustats.rcv_ucast_bytes);
+	p_stats->rcv_mcast_bytes += HILO_64_REGPAIR(ustats.rcv_mcast_bytes);
+	p_stats->rcv_bcast_bytes += HILO_64_REGPAIR(ustats.rcv_bcast_bytes);
+	p_stats->rcv_ucast_pkts += HILO_64_REGPAIR(ustats.rcv_ucast_pkts);
+	p_stats->rcv_mcast_pkts += HILO_64_REGPAIR(ustats.rcv_mcast_pkts);
+	p_stats->rcv_bcast_pkts += HILO_64_REGPAIR(ustats.rcv_bcast_pkts);
+}
+
+static void _ecore_ll2_get_pstats(struct ecore_hwfn *p_hwfn,
+				  struct ecore_ptt *p_ptt,
+				  struct ecore_ll2_info *p_ll2_conn,
+				  struct ecore_ll2_stats *p_stats)
+{
+	struct core_ll2_pstorm_per_queue_stat pstats;
+	u8 stats_id = p_ll2_conn->tx_stats_id;
+	u32 pstats_addr;
+
+	OSAL_MEMSET(&pstats, 0, sizeof(pstats));
+	pstats_addr = BAR0_MAP_REG_PSDM_RAM +
+		      CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(stats_id);
+	ecore_memcpy_from(p_hwfn, p_ptt, &pstats,
+			  pstats_addr,
+			  sizeof(pstats));
+
+	p_stats->sent_ucast_bytes += HILO_64_REGPAIR(pstats.sent_ucast_bytes);
+	p_stats->sent_mcast_bytes += HILO_64_REGPAIR(pstats.sent_mcast_bytes);
+	p_stats->sent_bcast_bytes += HILO_64_REGPAIR(pstats.sent_bcast_bytes);
+	p_stats->sent_ucast_pkts += HILO_64_REGPAIR(pstats.sent_ucast_pkts);
+	p_stats->sent_mcast_pkts += HILO_64_REGPAIR(pstats.sent_mcast_pkts);
+	p_stats->sent_bcast_pkts += HILO_64_REGPAIR(pstats.sent_bcast_pkts);
+}
+
+enum _ecore_status_t __ecore_ll2_get_stats(void *cxt,
+					   u8 connection_handle,
+					   struct ecore_ll2_stats *p_stats)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt;
+	struct ecore_ll2_info *p_ll2_conn = OSAL_NULL;
+	struct ecore_ptt *p_ptt;
+
+	if ((connection_handle >= ECORE_MAX_NUM_OF_LL2_CONNECTIONS) ||
+	    !p_hwfn->p_ll2_info) {
+		return ECORE_INVAL;
+	}
+
+	p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt) {
+		DP_ERR(p_hwfn, "Failed to acquire ptt\n");
+		return ECORE_INVAL;
+	}
+
+	if (p_ll2_conn->input.gsi_enable)
+		_ecore_ll2_get_port_stats(p_hwfn, p_ptt, p_stats);
+
+	_ecore_ll2_get_tstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+
+	_ecore_ll2_get_ustats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+
+	if (p_ll2_conn->tx_stats_en)
+		_ecore_ll2_get_pstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_ll2_get_stats(void *cxt,
+					 u8 connection_handle,
+					 struct ecore_ll2_stats	*p_stats)
+{
+	OSAL_MEMSET(p_stats, 0, sizeof(*p_stats));
+
+	return __ecore_ll2_get_stats(cxt, connection_handle, p_stats);
+}
+
+/**/
+
+#ifdef _NTDDK_
+#pragma warning(pop)
+#endif
diff --git a/sys/dev/qlnx/qlnxe/ecore_ooo.c b/sys/dev/qlnx/qlnxe/ecore_ooo.c
new file mode 100644
index 000000000000..50359b6e20c6
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/ecore_ooo.c
@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : ecore_ooo.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "bcm_osal.h"
+
+#include "ecore.h"
+#include "ecore_status.h"
+#include "ecore_ll2.h"
+#include "ecore_ooo.h"
+#include "ecore_iscsi.h"
+#include "ecore_cxt.h"
+/*
+ * Static OOO functions
+ */
+
+static struct ecore_ooo_archipelago *
+ecore_ooo_seek_archipelago(struct ecore_ooo_info *p_ooo_info, u32 cid)
+{
+	u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
+	struct ecore_ooo_archipelago *p_archipelago;
+
+	if (idx >= p_ooo_info->max_num_archipelagos)
+		return OSAL_NULL;
+
+	p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
+
+	if (OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list))
+		return OSAL_NULL;
+
+	return p_archipelago;
+}
+
+static struct ecore_ooo_isle *ecore_ooo_seek_isle(struct ecore_hwfn *p_hwfn,
+						  struct ecore_ooo_info *p_ooo_info,
+						  u32 cid, u8 isle)
+{
+	struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
+	struct ecore_ooo_isle *p_isle = OSAL_NULL;
+	u8 the_num_of_isle = 1;
+
+	p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
+	if (!p_archipelago) {
+		DP_NOTICE(p_hwfn, true,
+			 "Connection %d is not found in OOO list\n", cid);
+		return OSAL_NULL;
+	}
+
+	OSAL_LIST_FOR_EACH_ENTRY(p_isle,
+				 &p_archipelago->isles_list,
+				 list_entry, struct ecore_ooo_isle) {
+		if (the_num_of_isle == isle)
+			return p_isle;
+		the_num_of_isle++;
+	}
+
+	return OSAL_NULL;
+}
+
+void ecore_ooo_save_history_entry(struct ecore_ooo_info *p_ooo_info,
+				  struct ooo_opaque *p_cqe)
+{
+	struct ecore_ooo_history *p_history = &p_ooo_info->ooo_history;
+
+	if (p_history->head_idx == p_history->num_of_cqes)
+			p_history->head_idx = 0;
+	p_history->p_cqes[p_history->head_idx] = *p_cqe;
+	p_history->head_idx++;
+}
+
+//#ifdef CONFIG_ECORE_ISCSI
+#if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP)
+enum _ecore_status_t ecore_ooo_alloc(struct ecore_hwfn *p_hwfn)
+{
+	u16 max_num_archipelagos = 0, cid_base;
+	struct ecore_ooo_info *p_ooo_info;
+	u16 max_num_isles = 0;
+	u32 i;
+
+	switch (p_hwfn->hw_info.personality) {
+	case ECORE_PCI_ISCSI:
+		max_num_archipelagos =
+			p_hwfn->pf_params.iscsi_pf_params.num_cons;
+		cid_base =(u16)ecore_cxt_get_proto_cid_start(p_hwfn,
+							     PROTOCOLID_ISCSI);
+		break;
+	case ECORE_PCI_ETH_RDMA:
+	case ECORE_PCI_ETH_IWARP:
+		max_num_archipelagos =
+			(u16)ecore_cxt_get_proto_cid_count(p_hwfn,
+							   PROTOCOLID_IWARP,
+							   OSAL_NULL);
+		cid_base = (u16)ecore_cxt_get_proto_cid_start(p_hwfn,
+							      PROTOCOLID_IWARP);
+		break;
+	default:
+		DP_NOTICE(p_hwfn, true,
+			  "Failed to allocate ecore_ooo_info: unknown personalization\n");
+		return ECORE_INVAL;
+	}
+
+	max_num_isles = ECORE_MAX_NUM_ISLES + max_num_archipelagos;
+
+	if (!max_num_archipelagos) {
+		DP_NOTICE(p_hwfn, true,
+			  "Failed to allocate ecore_ooo_info: unknown amount of connections\n");
+		return ECORE_INVAL;
+	}
+
+	p_ooo_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+				 sizeof(*p_ooo_info));
+	if (!p_ooo_info) {
+		DP_NOTICE(p_hwfn, true, "Failed to allocate ecore_ooo_info\n");
+		return ECORE_NOMEM;
+	}
+	p_ooo_info->cid_base = cid_base; /* We look only at the icid */
+	p_ooo_info->max_num_archipelagos = max_num_archipelagos;
+
+	OSAL_LIST_INIT(&p_ooo_info->free_buffers_list);
+	OSAL_LIST_INIT(&p_ooo_info->ready_buffers_list);
+	OSAL_LIST_INIT(&p_ooo_info->free_isles_list);
+
+	p_ooo_info->p_isles_mem =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			    sizeof(struct ecore_ooo_isle) *
+			    max_num_isles);
+	if (!p_ooo_info->p_isles_mem) {
+		DP_NOTICE(p_hwfn,true,
+			  "Failed to allocate ecore_ooo_info (isles)\n");
+		goto no_isles_mem;
+	}
+
+	for (i = 0; i < max_num_isles; i++) {
+		OSAL_LIST_INIT(&p_ooo_info->p_isles_mem[i].buffers_list);
+		OSAL_LIST_PUSH_TAIL(&p_ooo_info->p_isles_mem[i].list_entry,
+				    &p_ooo_info->free_isles_list);
+	}
+
+	p_ooo_info->p_archipelagos_mem =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			    sizeof(struct ecore_ooo_archipelago) *
+			    max_num_archipelagos);
+	if (!p_ooo_info->p_archipelagos_mem) {
+		DP_NOTICE(p_hwfn,true,
+			 "Failed to allocate ecore_ooo_info(archpelagos)\n");
+		goto no_archipelagos_mem;
+	}
+
+	for (i = 0; i < max_num_archipelagos; i++) {
+		OSAL_LIST_INIT(&p_ooo_info->p_archipelagos_mem[i].isles_list);
+	}
+
+	p_ooo_info->ooo_history.p_cqes =
+		OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
+			    sizeof(struct ooo_opaque) *
+			    ECORE_MAX_NUM_OOO_HISTORY_ENTRIES);
+	if (!p_ooo_info->ooo_history.p_cqes) {
+		DP_NOTICE(p_hwfn,true,
+			  "Failed to allocate ecore_ooo_info(history)\n");
+		goto no_history_mem;
+	}
+	p_ooo_info->ooo_history.num_of_cqes =
+		ECORE_MAX_NUM_OOO_HISTORY_ENTRIES;
+
+	p_hwfn->p_ooo_info = p_ooo_info;
+	return ECORE_SUCCESS;
+
+no_history_mem:
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem);
+no_archipelagos_mem:
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem);
+no_isles_mem:
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info);
+	return ECORE_NOMEM;
+}
+#endif
+
+void ecore_ooo_release_connection_isles(struct ecore_ooo_info *p_ooo_info,
+					u32 cid)
+{
+	struct ecore_ooo_archipelago *p_archipelago;
+	struct ecore_ooo_buffer *p_buffer;
+	struct ecore_ooo_isle *p_isle;
+
+	p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
+	if (!p_archipelago)
+		return;
+
+	while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) {
+		p_isle = OSAL_LIST_FIRST_ENTRY(
+				&p_archipelago->isles_list,
+				struct ecore_ooo_isle, list_entry);
+
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
+				       &p_archipelago->isles_list);
+
+		while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
+			p_buffer =
+				OSAL_LIST_FIRST_ENTRY(
+				&p_isle->buffers_list ,
+				struct ecore_ooo_buffer, list_entry);
+
+			if (p_buffer == OSAL_NULL)
+				break;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+			OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
+				      	       &p_isle->buffers_list);
+				OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
+						&p_ooo_info->free_buffers_list);
+			}
+			OSAL_LIST_PUSH_TAIL(&p_isle->list_entry,
+					&p_ooo_info->free_isles_list);
+		}
+
+}
+
+void ecore_ooo_release_all_isles(struct ecore_ooo_info *p_ooo_info)
+{
+	struct ecore_ooo_archipelago *p_archipelago;
+	struct ecore_ooo_buffer *p_buffer;
+	struct ecore_ooo_isle *p_isle;
+	u32 i;
+
+	for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
+		p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
+
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) {
+			p_isle = OSAL_LIST_FIRST_ENTRY(
+					&p_archipelago->isles_list,
+					struct ecore_ooo_isle, list_entry);
+
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+			OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
+					       &p_archipelago->isles_list);
+
+			while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
+				p_buffer =
+					OSAL_LIST_FIRST_ENTRY(
+					&p_isle->buffers_list ,
+					struct ecore_ooo_buffer, list_entry);
+
+				if (p_buffer == OSAL_NULL)
+					break;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+				OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
+						      &p_isle->buffers_list);
+				OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
+					&p_ooo_info->free_buffers_list);
+			}
+			OSAL_LIST_PUSH_TAIL(&p_isle->list_entry,
+				&p_ooo_info->free_isles_list);
+		}
+	}
+	if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) {
+		OSAL_LIST_SPLICE_TAIL_INIT(&p_ooo_info->ready_buffers_list,
+					  &p_ooo_info->free_buffers_list);
+	}
+}
+
+//#ifdef CONFIG_ECORE_ISCSI
+#if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP)
+void ecore_ooo_setup(struct ecore_hwfn *p_hwfn)
+{
+	ecore_ooo_release_all_isles(p_hwfn->p_ooo_info);
+	OSAL_MEM_ZERO(p_hwfn->p_ooo_info->ooo_history.p_cqes,
+		      p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
+		      sizeof(struct ooo_opaque));
+	p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
+}
+
+void ecore_ooo_free(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_ooo_info *p_ooo_info = p_hwfn->p_ooo_info;
+	struct ecore_ooo_buffer *p_buffer;
+
+	if (!p_ooo_info)
+		return;
+
+	ecore_ooo_release_all_isles(p_ooo_info);
+	while (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) {
+		p_buffer = OSAL_LIST_FIRST_ENTRY(&p_ooo_info->
+						 free_buffers_list,
+						 struct ecore_ooo_buffer,
+						 list_entry);
+		if (p_buffer == OSAL_NULL)
+			break;
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011 28182)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
+				       &p_ooo_info->free_buffers_list);
+		OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+				       p_buffer->rx_buffer_virt_addr,
+				       p_buffer->rx_buffer_phys_addr,
+				       p_buffer->rx_buffer_size);
+		OSAL_FREE(p_hwfn->p_dev, p_buffer);
+	}
+
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem);
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem);
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info->ooo_history.p_cqes);
+	OSAL_FREE(p_hwfn->p_dev, p_ooo_info);
+	p_hwfn->p_ooo_info = OSAL_NULL;
+}
+#endif
+
+void ecore_ooo_put_free_buffer(struct ecore_ooo_info *p_ooo_info,
+			       struct ecore_ooo_buffer *p_buffer)
+{
+	OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
+			    &p_ooo_info->free_buffers_list);
+}
+
+struct ecore_ooo_buffer *
+ecore_ooo_get_free_buffer(struct ecore_ooo_info *p_ooo_info)
+{
+	struct ecore_ooo_buffer	*p_buffer = OSAL_NULL;
+
+	if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) {
+		p_buffer =
+			OSAL_LIST_FIRST_ENTRY(
+			&p_ooo_info->free_buffers_list,
+			struct ecore_ooo_buffer, list_entry);
+
+		OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
+				      &p_ooo_info->free_buffers_list);
+	}
+
+	return p_buffer;
+}
+
+void ecore_ooo_put_ready_buffer(struct ecore_ooo_info *p_ooo_info,
+				struct ecore_ooo_buffer *p_buffer, u8 on_tail)
+{
+	if (on_tail) {
+		OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
+				   &p_ooo_info->ready_buffers_list);
+	} else {
+		OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry,
+				   &p_ooo_info->ready_buffers_list);
+	}
+}
+
+struct ecore_ooo_buffer *
+ecore_ooo_get_ready_buffer(struct ecore_ooo_info *p_ooo_info)
+{
+	struct ecore_ooo_buffer	*p_buffer = OSAL_NULL;
+
+	if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) {
+		p_buffer =
+			OSAL_LIST_FIRST_ENTRY(
+			&p_ooo_info->ready_buffers_list,
+			struct ecore_ooo_buffer, list_entry);
+
+		OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
+				      &p_ooo_info->ready_buffers_list);
+	}
+
+	return p_buffer;
+}
+
+void ecore_ooo_delete_isles(struct ecore_hwfn *p_hwfn,
+			   struct ecore_ooo_info *p_ooo_info,
+			   u32 cid,
+			   u8 drop_isle,
+			   u8 drop_size)
+{
+	struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
+	struct ecore_ooo_isle *p_isle = OSAL_NULL;
+	u8 isle_idx;
+
+	p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
+	for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
+		p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info,
+					    cid, drop_isle);
+		if (!p_isle) {
+			DP_NOTICE(p_hwfn, true,
+				 "Isle %d is not found(cid %d)\n",
+				 drop_isle, cid);
+			return;
+		}
+		if (OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
+			DP_NOTICE(p_hwfn, true,
+				 "Isle %d is empty(cid %d)\n",
+				 drop_isle, cid);
+		} else {
+			OSAL_LIST_SPLICE_TAIL_INIT(&p_isle->buffers_list,
+					&p_ooo_info->free_buffers_list);
+		}
+#if defined(_NTDDK_)
+#pragma warning(suppress : 6011)
+#endif
+		OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
+				      &p_archipelago->isles_list);
+		p_ooo_info->cur_isles_number--;
+		OSAL_LIST_PUSH_HEAD(&p_isle->list_entry,
+				   &p_ooo_info->free_isles_list);
+	}
+}
+
+void ecore_ooo_add_new_isle(struct ecore_hwfn *p_hwfn,
+			   struct ecore_ooo_info *p_ooo_info,
+			   u32 cid, u8 ooo_isle,
+			   struct ecore_ooo_buffer *p_buffer)
+{
+	struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
+	struct ecore_ooo_isle *p_prev_isle = OSAL_NULL;
+	struct ecore_ooo_isle *p_isle = OSAL_NULL;
+
+	if (ooo_isle > 1) {
+		p_prev_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle - 1);
+		if (!p_prev_isle) {
+			DP_NOTICE(p_hwfn, true,
+				 "Isle %d is not found(cid %d)\n",
+				 ooo_isle - 1, cid);
+			return;
+		}
+	}
+	p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
+	if (!p_archipelago && (ooo_isle != 1)) {
+		DP_NOTICE(p_hwfn, true,
+			 "Connection %d is not found in OOO list\n", cid);
+		return;
+	}
+
+	if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_isles_list)) {
+		p_isle =
+			OSAL_LIST_FIRST_ENTRY(
+			&p_ooo_info->free_isles_list,
+			struct ecore_ooo_isle, list_entry);
+
+		OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
+				      &p_ooo_info->free_isles_list);
+		if (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
+			DP_NOTICE(p_hwfn, true, "Free isle is not empty\n");
+			OSAL_LIST_INIT(&p_isle->buffers_list);
+		}
+	} else {
+		DP_NOTICE(p_hwfn, true, "No more free isles\n");
+		return;
+	}
+
+	if (!p_archipelago) {
+		u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
+
+		p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
+	}
+	OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry, &p_isle->buffers_list);
+	p_ooo_info->cur_isles_number++;
+	p_ooo_info->gen_isles_number++;
+	if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
+		p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
+	if (!p_prev_isle) {
+		OSAL_LIST_PUSH_HEAD(&p_isle->list_entry, &p_archipelago->isles_list);
+	} else {
+		OSAL_LIST_INSERT_ENTRY_AFTER(&p_isle->list_entry,
+			                    &p_prev_isle->list_entry,
+					    &p_archipelago->isles_list);
+	}
+}
+
+void ecore_ooo_add_new_buffer(struct ecore_hwfn	*p_hwfn,
+			     struct ecore_ooo_info *p_ooo_info,
+			     u32 cid,
+			     u8 ooo_isle,
+			     struct ecore_ooo_buffer *p_buffer,
+		             u8 buffer_side)
+{
+	struct ecore_ooo_isle	* p_isle = OSAL_NULL;
+	p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
+	if (!p_isle) {
+		DP_NOTICE(p_hwfn, true,
+			 "Isle %d is not found(cid %d)\n",
+			 ooo_isle, cid);
+		return;
+	}
+	if (buffer_side == ECORE_OOO_LEFT_BUF) {
+		OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry,
+				   &p_isle->buffers_list);
+	} else {
+		OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
+				   &p_isle->buffers_list);
+	}
+}
+
+void ecore_ooo_join_isles(struct ecore_hwfn *p_hwfn,
+			  struct ecore_ooo_info *p_ooo_info,
+			  u32 cid, u8 left_isle)
+{
+	struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
+	struct ecore_ooo_isle *p_right_isle = OSAL_NULL;
+	struct ecore_ooo_isle *p_left_isle = OSAL_NULL;
+
+	p_right_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
+					  left_isle + 1);
+	if (!p_right_isle) {
+		DP_NOTICE(p_hwfn, true,
+			 "Right isle %d is not found(cid %d)\n",
+			 left_isle + 1, cid);
+		return;
+	}
+	p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
+	OSAL_LIST_REMOVE_ENTRY(&p_right_isle->list_entry,
+			      &p_archipelago->isles_list);
+	p_ooo_info->cur_isles_number--;
+	if (left_isle) {
+		p_left_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
+						 left_isle);
+		if (!p_left_isle) {
+			DP_NOTICE(p_hwfn, true,
+				 "Left isle %d is not found(cid %d)\n",
+				 left_isle, cid);
+			return;
+		}
+		OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list,
+					  &p_left_isle->buffers_list);
+	} else {
+		OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list,
+					  &p_ooo_info->ready_buffers_list);
+	}
+	OSAL_LIST_PUSH_TAIL(&p_right_isle->list_entry,
+			   &p_ooo_info->free_isles_list);
+}
+
+void ecore_ooo_dump_rx_event(struct ecore_hwfn	*p_hwfn,
+			     struct ooo_opaque *iscsi_ooo,
+			     struct ecore_ooo_buffer *p_buffer)
+{
+	int i;
+	u32 dp_module = ECORE_MSG_OOO;
+	u32 ph_hi, ph_lo;
+	u8 *packet_buffer = 0;
+
+	if (p_hwfn->dp_level > ECORE_LEVEL_VERBOSE)
+		return;
+	if (!(p_hwfn->dp_module & dp_module))
+		return;
+
+	packet_buffer = (u8 *)p_buffer->rx_buffer_virt_addr +
+		p_buffer->placement_offset;
+	DP_VERBOSE(p_hwfn, dp_module,
+		   "******************************************************\n");
+	ph_hi = DMA_HI(p_buffer->rx_buffer_phys_addr);
+	ph_lo = DMA_LO(p_buffer->rx_buffer_phys_addr);
+	DP_VERBOSE(p_hwfn, dp_module,
+		   "0x%x-%x: CID 0x%x, OP 0x%x, ISLE 0x%x\n",
+		   ph_hi, ph_lo,
+		   iscsi_ooo->cid, iscsi_ooo->ooo_opcode, iscsi_ooo->ooo_isle);
+	for (i = 0; i < 64; i = i + 8) {
+		DP_VERBOSE(p_hwfn, dp_module,
+			   "0x%x-%x:  0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+			   ph_hi, ph_lo,
+			   packet_buffer[i],
+			   packet_buffer[i + 1],
+			   packet_buffer[i + 2],
+			   packet_buffer[i + 3],
+			   packet_buffer[i + 4],
+			   packet_buffer[i + 5],
+			   packet_buffer[i + 6],
+			   packet_buffer[i + 7]);
+	}
+}
diff --git a/sys/dev/qlnx/qlnxe/ecore_rdma.c b/sys/dev/qlnx/qlnxe/ecore_rdma.c
new file mode 100644
index 000000000000..eb23aeb5cbfe
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/ecore_rdma.c
@@ -0,0 +1,2697 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : ecore_rdma.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "bcm_osal.h"
+#include "ecore.h"
+#include "ecore_status.h"
+#include "ecore_sp_commands.h"
+#include "ecore_cxt.h"
+#include "ecore_rdma.h"
+#include "reg_addr.h"
+#include "ecore_rt_defs.h"
+#include "ecore_init_ops.h"
+#include "ecore_hw.h"
+#include "ecore_mcp.h"
+#include "ecore_init_fw_funcs.h"
+#include "ecore_int.h"
+#include "pcics_reg_driver.h"
+#include "ecore_iro.h"
+#include "ecore_gtt_reg_addr.h"
+#include "ecore_hsi_iwarp.h"
+#include "ecore_ll2.h"
+#include "ecore_ooo.h"
+#ifndef LINUX_REMOVE
+#include "ecore_tcp_ip.h"
+#endif
+
+enum _ecore_status_t ecore_rdma_bmap_alloc(struct ecore_hwfn *p_hwfn,
+					   struct ecore_bmap *bmap,
+					   u32		    max_count,
+					   char              *name)
+{
+	u32 size_in_bytes;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "max_count = %08x\n", max_count);
+
+	bmap->max_count = max_count;
+
+	if (!max_count) {
+		bmap->bitmap = OSAL_NULL;
+		return ECORE_SUCCESS;
+	}
+
+	size_in_bytes = sizeof(unsigned long) *
+		DIV_ROUND_UP(max_count, (sizeof(unsigned long) * 8));
+
+	bmap->bitmap = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, size_in_bytes);
+	if (!bmap->bitmap)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore bmap alloc failed: cannot allocate memory (bitmap). rc = %d\n",
+			  ECORE_NOMEM);
+		return ECORE_NOMEM;
+	}
+
+	OSAL_SNPRINTF(bmap->name, QEDR_MAX_BMAP_NAME, "%s", name);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_SUCCESS\n");
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_rdma_bmap_alloc_id(struct ecore_hwfn *p_hwfn,
+					      struct ecore_bmap *bmap,
+					      u32	       *id_num)
+{
+	*id_num = OSAL_FIND_FIRST_ZERO_BIT(bmap->bitmap, bmap->max_count);
+	if (*id_num >= bmap->max_count)
+		return ECORE_INVAL;
+
+	OSAL_SET_BIT(*id_num, bmap->bitmap);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: allocated id %d\n",
+		   bmap->name, *id_num);
+
+	return ECORE_SUCCESS;
+}
+
+void ecore_bmap_set_id(struct ecore_hwfn *p_hwfn,
+		       struct ecore_bmap *bmap,
+		       u32		id_num)
+{
+	if (id_num >= bmap->max_count) {
+		DP_NOTICE(p_hwfn, true,
+			  "%s bitmap: cannot set id %d max is %d\n",
+			  bmap->name, id_num, bmap->max_count);
+
+		return;
+	}
+
+	OSAL_SET_BIT(id_num, bmap->bitmap);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: set id %d\n",
+		   bmap->name, id_num);
+}
+
+void ecore_bmap_release_id(struct ecore_hwfn *p_hwfn,
+			   struct ecore_bmap *bmap,
+			   u32		    id_num)
+{
+	bool b_acquired;
+
+	if (id_num >= bmap->max_count)
+		return;
+
+	b_acquired = OSAL_TEST_AND_CLEAR_BIT(id_num, bmap->bitmap);
+	if (!b_acquired)
+	{
+		DP_NOTICE(p_hwfn, false, "%s bitmap: id %d already released\n",
+			  bmap->name, id_num);
+		return;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: released id %d\n",
+		   bmap->name, id_num);
+}
+
+int ecore_bmap_test_id(struct ecore_hwfn *p_hwfn,
+		       struct ecore_bmap *bmap,
+		       u32		  id_num)
+{
+	if (id_num >= bmap->max_count) {
+		DP_NOTICE(p_hwfn, true,
+			  "%s bitmap: id %d too high. max is %d\n",
+			  bmap->name, id_num, bmap->max_count);
+		return -1;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: tested id %d\n",
+		   bmap->name, id_num);
+
+	return OSAL_TEST_BIT(id_num, bmap->bitmap);
+}
+
+static bool ecore_bmap_is_empty(struct ecore_bmap *bmap)
+{
+	return (bmap->max_count ==
+		OSAL_FIND_FIRST_BIT(bmap->bitmap, bmap->max_count));
+}
+
+#ifndef LINUX_REMOVE
+u32 ecore_rdma_get_sb_id(struct ecore_hwfn *p_hwfn, u32 rel_sb_id)
+{
+	/* first sb id for RoCE is after all the l2 sb */
+	return FEAT_NUM(p_hwfn, ECORE_PF_L2_QUE) + rel_sb_id;
+}
+
+u32 ecore_rdma_query_cau_timer_res(void)
+{
+	return ECORE_CAU_DEF_RX_TIMER_RES;
+}
+#endif
+
+enum _ecore_status_t ecore_rdma_info_alloc(struct ecore_hwfn    *p_hwfn)
+{
+	struct ecore_rdma_info *p_rdma_info;
+
+	p_rdma_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info));
+	if (!p_rdma_info) {
+		DP_NOTICE(p_hwfn, false,
+			  "ecore rdma alloc failed: cannot allocate memory (rdma info).\n");
+		return ECORE_NOMEM;
+	}
+	p_hwfn->p_rdma_info = p_rdma_info;
+
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_rdma_info->lock)) {
+		ecore_rdma_info_free(p_hwfn);
+		return ECORE_NOMEM;
+	}
+#endif
+	OSAL_SPIN_LOCK_INIT(&p_rdma_info->lock);
+
+	return ECORE_SUCCESS;
+}
+
+void ecore_rdma_info_free(struct ecore_hwfn *p_hwfn)
+{
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+	OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_rdma_info->lock);
+#endif
+	OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info);
+	p_hwfn->p_rdma_info = OSAL_NULL;
+}
+
+static enum _ecore_status_t ecore_rdma_inc_ref_cnt(struct ecore_hwfn *p_hwfn)
+{
+	enum _ecore_status_t rc = ECORE_INVAL;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	if (p_hwfn->p_rdma_info->active) {
+		p_hwfn->p_rdma_info->ref_cnt++;
+		rc = ECORE_SUCCESS;
+	} else {
+		DP_INFO(p_hwfn, "Ref cnt requested for inactive rdma\n");
+	}
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	return rc;
+}
+
+static void ecore_rdma_dec_ref_cnt(struct ecore_hwfn *p_hwfn)
+{
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	p_hwfn->p_rdma_info->ref_cnt--;
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+static void ecore_rdma_activate(struct ecore_hwfn *p_hwfn)
+{
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	p_hwfn->p_rdma_info->active = true;
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+/* Part of deactivating rdma is letting all the relevant flows complete before
+ * we start shutting down: Currently query-stats which can be called from MCP
+ * context.
+ */
+/* The longest time it can take a rdma flow to complete */
+#define ECORE_RDMA_MAX_FLOW_TIME (100)
+static enum _ecore_status_t ecore_rdma_deactivate(struct ecore_hwfn *p_hwfn)
+{
+	int wait_count;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	p_hwfn->p_rdma_info->active = false;
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	/* We'll give each flow it's time to complete... */
+	wait_count = p_hwfn->p_rdma_info->ref_cnt;
+
+	while (p_hwfn->p_rdma_info->ref_cnt) {
+		OSAL_MSLEEP(ECORE_RDMA_MAX_FLOW_TIME);
+		if (--wait_count == 0) {
+			DP_NOTICE(p_hwfn, false,
+				  "Timeout on refcnt=%d\n",
+				  p_hwfn->p_rdma_info->ref_cnt);
+			return ECORE_TIMEOUT;
+		}
+	}
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_rdma_alloc(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+	u32 num_cons, num_tasks;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocating RDMA\n");
+
+	if (!p_rdma_info)
+		return ECORE_INVAL;
+
+	if (p_hwfn->hw_info.personality == ECORE_PCI_ETH_IWARP)
+		p_rdma_info->proto = PROTOCOLID_IWARP;
+	else
+		p_rdma_info->proto = PROTOCOLID_ROCE;
+
+	num_cons = ecore_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto,
+						 OSAL_NULL);
+
+	if (IS_IWARP(p_hwfn))
+		p_rdma_info->num_qps = num_cons;
+	else
+		p_rdma_info->num_qps = num_cons / 2;
+
+	/* INTERNAL: RoCE & iWARP use the same taskid */
+	num_tasks = ecore_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE);
+
+	/* Each MR uses a single task */
+	p_rdma_info->num_mrs = num_tasks;
+
+	/* Queue zone lines are shared between RoCE and L2 in such a way that
+	 * they can be used by each without obstructing the other.
+	 */
+	p_rdma_info->queue_zone_base = (u16) RESC_START(p_hwfn, ECORE_L2_QUEUE);
+	p_rdma_info->max_queue_zones = (u16) RESC_NUM(p_hwfn, ECORE_L2_QUEUE);
+
+	/* Allocate a struct with device params and fill it */
+	p_rdma_info->dev = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info->dev));
+	if (!p_rdma_info->dev)
+	{
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "ecore rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Allocate a struct with port params and fill it */
+	p_rdma_info->port = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info->port));
+	if (!p_rdma_info->port)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore rdma alloc failed: cannot allocate memory (rdma info port)\n");
+		return ECORE_NOMEM;
+	}
+
+	/* Allocate bit map for pd's */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS,
+				   "PD");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate pd_map,rc = %d\n",
+			   rc);
+		return rc;
+	}
+
+	/* Allocate bit map for XRC Domains */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->xrcd_map,
+				   ECORE_RDMA_MAX_XRCDS, "XRCD");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate xrcd_map,rc = %d\n",
+			   rc);
+		return rc;
+	}
+
+	/* Allocate DPI bitmap */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map,
+				   p_hwfn->dpi_count, "DPI");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate DPI bitmap, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for cq's. The maximum number of CQs is bounded to
+	 * twice the number of QPs.
+	 */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map,
+				   num_cons, "CQ");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate cq bitmap, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for toggle bit for cq icids
+	 * We toggle the bit every time we create or resize cq for a given icid.
+	 * The maximum number of CQs is bounded to the number of connections we
+	 * support. (num_qps in iWARP or num_qps/2 in RoCE).
+	 */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits,
+				   num_cons, "Toggle");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate toogle bits, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for itids */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map,
+				   p_rdma_info->num_mrs, "MR");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate itids bitmaps, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for qps. */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->qp_map,
+				   p_rdma_info->num_qps, "QP");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate qp bitmap, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for cids used for responders/requesters. */
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons,
+				   "REAL CID");
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate cid bitmap, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* The first SRQ follows the last XRC SRQ. This means that the
+	 * SRQ IDs start from an offset equals to max_xrc_srqs.
+	 */
+	p_rdma_info->srq_id_offset = (u16)ecore_cxt_get_xrc_srq_count(p_hwfn);
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->xrc_srq_map,
+				   p_rdma_info->srq_id_offset, "XRC SRQ");
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate xrc srq bitmap, rc = %d\n", rc);
+		return rc;
+	}
+
+	/* Allocate bitmap for srqs */
+	p_rdma_info->num_srqs = ecore_cxt_get_srq_count(p_hwfn);
+	rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->srq_map,
+				   p_rdma_info->num_srqs,
+				   "SRQ");
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Failed to allocate srq bitmap, rc = %d\n", rc);
+
+		return rc;
+	}
+
+	if (IS_IWARP(p_hwfn))
+		rc = ecore_iwarp_alloc(p_hwfn);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+
+	return rc;
+}
+
+void ecore_rdma_bmap_free(struct ecore_hwfn *p_hwfn,
+			  struct ecore_bmap *bmap,
+			  bool check)
+{
+	int weight, line, item, last_line, last_item;
+	u64 *pmap;
+
+	if (!bmap || !bmap->bitmap)
+		return;
+
+	if (!check)
+		goto end;
+
+	weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count);
+	if (!weight)
+		goto end;
+
+	DP_NOTICE(p_hwfn, false,
+		  "%s bitmap not free - size=%d, weight=%d, 512 bits per line\n",
+		  bmap->name, bmap->max_count, weight);
+
+	pmap = (u64 *)bmap->bitmap;
+	last_line = bmap->max_count / (64*8);
+	last_item = last_line * 8 + (((bmap->max_count % (64*8)) + 63) / 64);
+
+	/* print aligned non-zero lines, if any */
+	for (item = 0, line = 0; line < last_line; line++, item += 8) {
+		if (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item], 64*8))
+			DP_NOTICE(p_hwfn, false,
+				  "line 0x%04x: 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n",
+				  line, (unsigned long long)pmap[item],
+				(unsigned long long)pmap[item+1],
+				(unsigned long long)pmap[item+2],
+				  (unsigned long long)pmap[item+3],
+				(unsigned long long)pmap[item+4],
+				(unsigned long long)pmap[item+5],
+				  (unsigned long long)pmap[item+6],
+				(unsigned long long)pmap[item+7]);
+	}
+
+	/* print last unaligned non-zero line, if any */
+	if ((bmap->max_count % (64*8)) &&
+	    (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item],
+				bmap->max_count-item*64))) {
+		u8 str_last_line[200] = { 0 };
+		int  offset;
+
+		offset = OSAL_SPRINTF(str_last_line, "line 0x%04x: ", line);
+		for (; item < last_item; item++) {
+			offset += OSAL_SPRINTF(str_last_line+offset,
+					       "0x%016llx ",
+				(unsigned long long)pmap[item]);
+		}
+		DP_NOTICE(p_hwfn, false, "%s\n", str_last_line);
+	}
+
+end:
+	OSAL_FREE(p_hwfn->p_dev, bmap->bitmap);
+	bmap->bitmap = OSAL_NULL;
+}
+
+
+void ecore_rdma_resc_free(struct ecore_hwfn *p_hwfn)
+{
+	if (IS_IWARP(p_hwfn))
+		ecore_iwarp_resc_free(p_hwfn);
+
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->qp_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->pd_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->xrcd_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cq_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->toggle_bits, 0);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tid_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->srq_map, 1);
+	ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->xrc_srq_map, 1);
+
+	OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info->port);
+	p_hwfn->p_rdma_info->port = OSAL_NULL;
+
+	OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info->dev);
+	p_hwfn->p_rdma_info->dev = OSAL_NULL;
+}
+
+static OSAL_INLINE void ecore_rdma_free_reserved_lkey(struct ecore_hwfn *p_hwfn)
+{
+	ecore_rdma_free_tid(p_hwfn, p_hwfn->p_rdma_info->dev->reserved_lkey);
+}
+
+static void ecore_rdma_free_ilt(struct ecore_hwfn *p_hwfn)
+{
+	/* Free Connection CXT */
+	ecore_cxt_free_ilt_range(
+		p_hwfn, ECORE_ELEM_CXT,
+		ecore_cxt_get_proto_cid_start(p_hwfn,
+					      p_hwfn->p_rdma_info->proto),
+		ecore_cxt_get_proto_cid_count(p_hwfn,
+					      p_hwfn->p_rdma_info->proto,
+					      OSAL_NULL));
+
+	/* Free Task CXT ( Intentionally RoCE as task-id is shared between
+	 * RoCE and iWARP
+	 */
+	ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_TASK, 0,
+				 ecore_cxt_get_proto_tid_count(
+					 p_hwfn, PROTOCOLID_ROCE));
+
+	/* Free TSDM CXT */
+	ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_SRQ, 0,
+				 ecore_cxt_get_srq_count(p_hwfn));
+}
+
+static void ecore_rdma_free(struct ecore_hwfn *p_hwfn)
+{
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n");
+
+	ecore_rdma_free_reserved_lkey(p_hwfn);
+
+	ecore_rdma_resc_free(p_hwfn);
+
+	ecore_rdma_free_ilt(p_hwfn);
+}
+
+static void ecore_rdma_get_guid(struct ecore_hwfn *p_hwfn, u8 *guid)
+{
+	u8 mac_addr[6];
+
+	OSAL_MEMCPY(&mac_addr[0], &p_hwfn->hw_info.hw_mac_addr[0], ETH_ALEN);
+	guid[0] = mac_addr[0] ^ 2;
+	guid[1] = mac_addr[1];
+	guid[2] = mac_addr[2];
+	guid[3] = 0xff;
+	guid[4] = 0xfe;
+	guid[5] = mac_addr[3];
+	guid[6] = mac_addr[4];
+	guid[7] = mac_addr[5];
+}
+
+
+static void ecore_rdma_init_events(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_start_in_params *params)
+{
+	struct ecore_rdma_events *events;
+
+	events = &p_hwfn->p_rdma_info->events;
+
+	events->unaffiliated_event = params->events->unaffiliated_event;
+	events->affiliated_event = params->events->affiliated_event;
+	events->context = params->events->context;
+}
+
+static void ecore_rdma_init_devinfo(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_start_in_params *params)
+{
+	struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+	u32 pci_status_control;
+
+	/* Vendor specific information */
+	dev->vendor_id = p_hwfn->p_dev->vendor_id;
+	dev->vendor_part_id = p_hwfn->p_dev->device_id;
+	dev->hw_ver = 0;
+	dev->fw_ver = STORM_FW_VERSION;
+
+	ecore_rdma_get_guid(p_hwfn, (u8 *)(&dev->sys_image_guid));
+	dev->node_guid = dev->sys_image_guid;
+
+	dev->max_sge = OSAL_MIN_T(u32, RDMA_MAX_SGE_PER_SQ_WQE,
+				  RDMA_MAX_SGE_PER_RQ_WQE);
+
+	if (p_hwfn->p_dev->rdma_max_sge) {
+		dev->max_sge = OSAL_MIN_T(u32,
+				     p_hwfn->p_dev->rdma_max_sge,
+				     dev->max_sge);
+	}
+
+	/* Set these values according to configuration
+	 * MAX SGE for SRQ is not defined by FW for now
+	 * define it in driver.
+	 * TODO: Get this value from FW.
+	 */
+	dev->max_srq_sge = ECORE_RDMA_MAX_SGE_PER_SRQ_WQE;
+	if (p_hwfn->p_dev->rdma_max_srq_sge) {
+		dev->max_srq_sge = OSAL_MIN_T(u32,
+				     p_hwfn->p_dev->rdma_max_srq_sge,
+				     dev->max_srq_sge);
+	}
+
+	dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE;
+	dev->max_inline = (p_hwfn->p_dev->rdma_max_inline) ?
+		OSAL_MIN_T(u32,
+			   p_hwfn->p_dev->rdma_max_inline,
+			   dev->max_inline) :
+			dev->max_inline;
+
+	dev->max_wqe = ECORE_RDMA_MAX_WQE;
+	dev->max_cnq = (u8)FEAT_NUM(p_hwfn, ECORE_RDMA_CNQ);
+
+	/* The number of QPs may be higher than ECORE_ROCE_MAX_QPS. because
+	 * it is up-aligned to 16 and then to ILT page size within ecore cxt.
+	 * This is OK in terms of ILT but we don't want to configure the FW
+	 * above its abilities
+	 */
+	dev->max_qp = OSAL_MIN_T(u64, ROCE_MAX_QPS,
+			     p_hwfn->p_rdma_info->num_qps);
+
+	/* CQs uses the same icids that QPs use hence they are limited by the
+	 * number of icids. There are two icids per QP.
+	 */
+	dev->max_cq = dev->max_qp * 2;
+
+	/* The number of mrs is smaller by 1 since the first is reserved */
+	dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1;
+	dev->max_mr_size = ECORE_RDMA_MAX_MR_SIZE;
+	/* The maximum CQE capacity per CQ supported */
+	/* max number of cqes will be in two layer pbl,
+	 * 8 is the pointer size in bytes
+	 * 32 is the size of cq element in bytes
+	 */
+	if (params->roce.cq_mode == ECORE_RDMA_CQ_MODE_32_BITS)
+		dev->max_cqe = ECORE_RDMA_MAX_CQE_32_BIT;
+	else
+		dev->max_cqe = ECORE_RDMA_MAX_CQE_16_BIT;
+
+	dev->max_mw = 0;
+	dev->max_fmr = ECORE_RDMA_MAX_FMR;
+	dev->max_mr_mw_fmr_pbl = (OSAL_PAGE_SIZE/8) * (OSAL_PAGE_SIZE/8);
+	dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * OSAL_PAGE_SIZE;
+	dev->max_pkey = ECORE_RDMA_MAX_P_KEY;
+	/* Right now we dont take any parameters from user
+	 * So assign predefined max_srq to num_srqs.
+	 */
+	dev->max_srq = p_hwfn->p_rdma_info->num_srqs;
+
+	/* SRQ WQE size */
+	dev->max_srq_wr = ECORE_RDMA_MAX_SRQ_WQE_ELEM;
+
+	dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
+					  (RDMA_RESP_RD_ATOMIC_ELM_SIZE*2);
+	dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
+					 RDMA_REQ_RD_ATOMIC_ELM_SIZE;
+
+	dev->max_dev_resp_rd_atomic_resc =
+		dev->max_qp_resp_rd_atomic_resc * p_hwfn->p_rdma_info->num_qps;
+	dev->page_size_caps = ECORE_RDMA_PAGE_SIZE_CAPS;
+	dev->dev_ack_delay = ECORE_RDMA_ACK_DELAY;
+	dev->max_pd = RDMA_MAX_PDS;
+	dev->max_ah = dev->max_qp;
+	dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, ECORE_RDMA_STATS_QUEUE);
+
+	/* Set capablities */
+	dev->dev_caps = 0;
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_RNR_NAK, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_RESIZE_CQ, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_ZBVA, 1);
+	SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1);
+
+	/* Check atomic operations support in PCI configuration space. */
+	OSAL_PCI_READ_CONFIG_DWORD(p_hwfn->p_dev,
+				   PCICFG_DEVICE_STATUS_CONTROL_2,
+				   &pci_status_control);
+
+	if (pci_status_control &
+	    PCICFG_DEVICE_STATUS_CONTROL_2_ATOMIC_REQ_ENABLE)
+		SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_ATOMIC_OP, 1);
+
+	if (IS_IWARP(p_hwfn))
+		ecore_iwarp_init_devinfo(p_hwfn);
+}
+
+static void ecore_rdma_init_port(
+	struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_rdma_port *port = p_hwfn->p_rdma_info->port;
+	struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+
+	port->port_state = p_hwfn->mcp_info->link_output.link_up ?
+		ECORE_RDMA_PORT_UP : ECORE_RDMA_PORT_DOWN;
+
+	port->max_msg_size = OSAL_MIN_T(u64,
+				   (dev->max_mr_mw_fmr_size *
+				    p_hwfn->p_dev->rdma_max_sge),
+					((u64)1 << 31));
+
+	port->pkey_bad_counter = 0;
+}
+
+static enum _ecore_status_t ecore_rdma_init_hw(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_ptt *p_ptt)
+{
+	u32 ll2_ethertype_en;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Initializing HW\n");
+	p_hwfn->b_rdma_enabled_in_prs = false;
+
+	if (IS_IWARP(p_hwfn))
+		return ecore_iwarp_init_hw(p_hwfn, p_ptt);
+
+	ecore_wr(p_hwfn,
+		 p_ptt,
+		 PRS_REG_ROCE_DEST_QP_MAX_PF,
+		 0);
+
+	p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE;
+
+	/* We delay writing to this reg until first cid is allocated. See
+	 * ecore_cxt_dynamic_ilt_alloc function for more details
+	 */
+
+	ll2_ethertype_en = ecore_rd(p_hwfn,
+			     p_ptt,
+			     PRS_REG_LIGHT_L2_ETHERTYPE_EN);
+	ecore_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
+		 (ll2_ethertype_en | 0x01));
+
+#ifndef REAL_ASIC_ONLY
+	if (ECORE_IS_BB_A0(p_hwfn->p_dev) && ECORE_IS_CMT(p_hwfn->p_dev)) {
+		ecore_wr(p_hwfn,
+			 p_ptt,
+			 NIG_REG_LLH_ENG_CLS_ENG_ID_TBL,
+			 0);
+		ecore_wr(p_hwfn,
+			 p_ptt,
+			 NIG_REG_LLH_ENG_CLS_ENG_ID_TBL + 4,
+			 0);
+	}
+#endif
+
+	if (ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2)
+	{
+		DP_NOTICE(p_hwfn,
+			  true,
+			  "The first RoCE's cid should be even\n");
+		return ECORE_UNKNOWN_ERROR;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Initializing HW - Done\n");
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_rdma_start_fw(struct ecore_hwfn *p_hwfn,
+#ifdef CONFIG_DCQCN
+		    struct ecore_ptt *p_ptt,
+#else
+		    struct ecore_ptt OSAL_UNUSED *p_ptt,
+#endif
+		    struct ecore_rdma_start_in_params *params)
+{
+	struct rdma_init_func_ramrod_data *p_ramrod;
+	struct rdma_init_func_hdr *pheader;
+	struct ecore_rdma_info *p_rdma_info;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	u16 igu_sb_id, sb_id;
+	u8 ll2_queue_id;
+	u32 cnq_id;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Starting FW\n");
+
+	p_rdma_info = p_hwfn->p_rdma_info;
+
+	/* Save the number of cnqs for the function close ramrod */
+	p_rdma_info->num_cnqs = params->desired_cnq;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT,
+				   p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (IS_IWARP(p_hwfn)) {
+		ecore_iwarp_init_fw_ramrod(p_hwfn,
+					   &p_ent->ramrod.iwarp_init_func);
+		p_ramrod = &p_ent->ramrod.iwarp_init_func.rdma;
+	} else {
+
+#ifdef CONFIG_DCQCN
+		rc = ecore_roce_dcqcn_cfg(p_hwfn, &params->roce.dcqcn_params,
+					  &p_ent->ramrod.roce_init_func, p_ptt);
+		if (rc != ECORE_SUCCESS) {
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to configure DCQCN. rc = %d.\n", rc);
+			return rc;
+		}
+#endif
+		p_ramrod = &p_ent->ramrod.roce_init_func.rdma;
+
+		/* The ll2_queue_id is used only for UD QPs */
+		ll2_queue_id = ecore_ll2_handle_to_queue_id(
+			p_hwfn, params->roce.ll2_handle);
+		p_ent->ramrod.roce_init_func.roce.ll2_queue_id = ll2_queue_id;
+
+	}
+
+	pheader = &p_ramrod->params_header;
+	pheader->cnq_start_offset = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM);
+	pheader->num_cnqs = params->desired_cnq;
+
+	/* The first SRQ ILT page is used for XRC SRQs and all the following
+	 * pages contain regular SRQs. Hence the first regular SRQ ID is the
+	 * maximum number XRC SRQs.
+	 */
+	pheader->first_reg_srq_id = p_rdma_info->srq_id_offset;
+	pheader->reg_srq_base_addr =
+		ecore_cxt_get_ilt_page_size(p_hwfn, ILT_CLI_TSDM);
+
+	if (params->roce.cq_mode == ECORE_RDMA_CQ_MODE_16_BITS)
+		pheader->cq_ring_mode = 1; /* 1=16 bits */
+	else
+		pheader->cq_ring_mode = 0; /* 0=32 bits */
+
+	for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++)
+	{
+		sb_id = (u16)OSAL_GET_RDMA_SB_ID(p_hwfn, cnq_id);
+		igu_sb_id = ecore_get_igu_sb_id(p_hwfn, sb_id);
+		p_ramrod->cnq_params[cnq_id].sb_num =
+			OSAL_CPU_TO_LE16(igu_sb_id);
+
+		p_ramrod->cnq_params[cnq_id].sb_index =
+			p_hwfn->pf_params.rdma_pf_params.gl_pi;
+
+		p_ramrod->cnq_params[cnq_id].num_pbl_pages =
+			params->cnq_pbl_list[cnq_id].num_pbl_pages;
+
+		p_ramrod->cnq_params[cnq_id].pbl_base_addr.hi =
+			DMA_HI_LE(params->cnq_pbl_list[cnq_id].pbl_ptr);
+		p_ramrod->cnq_params[cnq_id].pbl_base_addr.lo =
+			DMA_LO_LE(params->cnq_pbl_list[cnq_id].pbl_ptr);
+
+		/* we arbitrarily decide that cnq_id will be as qz_offset */
+		p_ramrod->cnq_params[cnq_id].queue_zone_num =
+			OSAL_CPU_TO_LE16(p_rdma_info->queue_zone_base + cnq_id);
+	}
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_alloc_tid(void	*rdma_cxt,
+					  u32	*itid)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocate TID\n");
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->tid_map,
+				      itid);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "Failed in allocating tid\n");
+		goto out;
+	}
+
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_TASK, *itid);
+out:
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc);
+	return rc;
+}
+
+static OSAL_INLINE enum _ecore_status_t ecore_rdma_reserve_lkey(
+		struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+
+	/* Tid 0 will be used as the key for "reserved MR".
+	 * The driver should allocate memory for it so it can be loaded but no
+	 * ramrod should be passed on it.
+	 */
+	ecore_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey);
+	if (dev->reserved_lkey != RDMA_RESERVED_LKEY)
+	{
+		DP_NOTICE(p_hwfn, true,
+			  "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n");
+		return ECORE_INVAL;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_rdma_setup(struct ecore_hwfn    *p_hwfn,
+				struct ecore_ptt                  *p_ptt,
+				struct ecore_rdma_start_in_params *params)
+{
+	enum _ecore_status_t rc = 0;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA setup\n");
+
+	ecore_rdma_init_devinfo(p_hwfn, params);
+	ecore_rdma_init_port(p_hwfn);
+	ecore_rdma_init_events(p_hwfn, params);
+
+	rc = ecore_rdma_reserve_lkey(p_hwfn);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	rc = ecore_rdma_init_hw(p_hwfn, p_ptt);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (IS_IWARP(p_hwfn)) {
+		rc = ecore_iwarp_setup(p_hwfn, params);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	} else {
+		rc = ecore_roce_setup(p_hwfn);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
+	return ecore_rdma_start_fw(p_hwfn, p_ptt, params);
+}
+
+
+enum _ecore_status_t ecore_rdma_stop(void *rdma_cxt)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_close_func_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	struct ecore_ptt *p_ptt;
+	u32 ll2_ethertype_en;
+	enum _ecore_status_t rc = ECORE_TIMEOUT;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA stop\n");
+
+	rc = ecore_rdma_deactivate(p_hwfn);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Failed to acquire PTT\n");
+		return rc;
+	}
+
+#ifdef CONFIG_DCQCN
+	ecore_roce_stop_rl(p_hwfn);
+#endif
+
+	/* Disable RoCE search */
+	ecore_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0);
+	p_hwfn->b_rdma_enabled_in_prs = false;
+
+	ecore_wr(p_hwfn,
+		 p_ptt,
+		 PRS_REG_ROCE_DEST_QP_MAX_PF,
+		 0);
+
+	ll2_ethertype_en = ecore_rd(p_hwfn,
+				    p_ptt,
+				    PRS_REG_LIGHT_L2_ETHERTYPE_EN);
+
+	ecore_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
+		 (ll2_ethertype_en & 0xFFFE));
+
+#ifndef REAL_ASIC_ONLY
+	/* INTERNAL: In CMT mode, re-initialize nig to direct packets to both
+	 * enginesfor L2 performance, Roce requires all traffic to go just to
+	 * engine 0.
+	 */
+	if (ECORE_IS_BB_A0(p_hwfn->p_dev) && ECORE_IS_CMT(p_hwfn->p_dev)) {
+		DP_ERR(p_hwfn->p_dev,
+		       "On Everest 4 Big Bear Board revision A0 when RoCE driver is loaded L2 performance is sub-optimal (all traffic is routed to engine 0). For optimal L2 results either remove RoCE driver or use board revision B0\n");
+
+		ecore_wr(p_hwfn,
+			 p_ptt,
+			 NIG_REG_LLH_ENG_CLS_ENG_ID_TBL,
+			 0x55555555);
+		ecore_wr(p_hwfn,
+			 p_ptt,
+			 NIG_REG_LLH_ENG_CLS_ENG_ID_TBL + 0x4,
+			 0x55555555);
+	}
+#endif
+
+	if (IS_IWARP(p_hwfn)) {
+		rc = ecore_iwarp_stop(p_hwfn);
+		if (rc != ECORE_SUCCESS) {
+			ecore_ptt_release(p_hwfn, p_ptt);
+			return 0;
+		}
+	} else {
+		rc = ecore_roce_stop(p_hwfn);
+		if (rc != ECORE_SUCCESS) {
+			ecore_ptt_release(p_hwfn, p_ptt);
+			return 0;
+		}
+	}
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	/* Stop RoCE */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto out;
+
+	p_ramrod = &p_ent->ramrod.rdma_close_func;
+
+	p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs;
+	p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+out:
+	ecore_rdma_free(p_hwfn);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA stop done, rc = %d\n", rc);
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_add_user(void		      *rdma_cxt,
+			struct ecore_rdma_add_user_out_params *out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	u32 dpi_start_offset;
+	u32 returned_id = 0;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Adding User\n");
+
+	/* Allocate DPI */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map,
+				      &returned_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	if (rc != ECORE_SUCCESS)
+		DP_NOTICE(p_hwfn, false, "Failed in allocating dpi\n");
+
+	out_params->dpi = (u16)returned_id;
+
+	/* Calculate the corresponding DPI address */
+	dpi_start_offset = p_hwfn->dpi_start_offset;
+
+	out_params->dpi_addr = (u64)(osal_int_ptr_t)((u8 OSAL_IOMEM*)p_hwfn->doorbells +
+						     dpi_start_offset +
+						     ((out_params->dpi) * p_hwfn->dpi_size));
+
+	out_params->dpi_phys_addr = p_hwfn->db_phys_addr + dpi_start_offset +
+				    out_params->dpi * p_hwfn->dpi_size;
+
+	out_params->dpi_size = p_hwfn->dpi_size;
+	out_params->wid_count = p_hwfn->wid_count;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Adding user - done, rc = %d\n", rc);
+	return rc;
+}
+
+struct ecore_rdma_port *ecore_rdma_query_port(void	*rdma_cxt)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_rdma_port *p_port = p_hwfn->p_rdma_info->port;
+	struct ecore_mcp_link_state *p_link_output;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA Query port\n");
+
+	/* The link state is saved only for the leading hwfn */
+	p_link_output =
+		&ECORE_LEADING_HWFN(p_hwfn->p_dev)->mcp_info->link_output;
+
+	/* Link may have changed... */
+	p_port->port_state = p_link_output->link_up ? ECORE_RDMA_PORT_UP
+						    : ECORE_RDMA_PORT_DOWN;
+
+	p_port->link_speed = p_link_output->speed;
+
+	p_port->max_msg_size = RDMA_MAX_DATA_SIZE_IN_WQE;
+
+	return p_port;
+}
+
+struct ecore_rdma_device *ecore_rdma_query_device(void	*rdma_cxt)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Query device\n");
+
+	/* Return struct with device parameters */
+	return p_hwfn->p_rdma_info->dev;
+}
+
+void ecore_rdma_free_tid(void	*rdma_cxt,
+			 u32	itid)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "itid = %08x\n", itid);
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn,
+			      &p_hwfn->p_rdma_info->tid_map,
+			      itid);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+void ecore_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod)
+{
+	struct ecore_hwfn *p_hwfn;
+	u16 qz_num;
+	u32 addr;
+
+	p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	if (qz_offset > p_hwfn->p_rdma_info->max_queue_zones) {
+		DP_NOTICE(p_hwfn, false,
+			  "queue zone offset %d is too large (max is %d)\n",
+			  qz_offset, p_hwfn->p_rdma_info->max_queue_zones);
+		return;
+	}
+
+	qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset;
+	addr = GTT_BAR0_MAP_REG_USDM_RAM +
+	       USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num);
+
+	REG_WR16(p_hwfn, addr, prod);
+
+	/* keep prod updates ordered */
+	OSAL_WMB(p_hwfn->p_dev);
+}
+
+enum _ecore_status_t ecore_rdma_alloc_pd(void	*rdma_cxt,
+					 u16	*pd)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	u32                  returned_id;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc PD\n");
+
+	/* Allocates an unused protection domain */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->pd_map,
+				      &returned_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	if (rc != ECORE_SUCCESS)
+		DP_NOTICE(p_hwfn, false, "Failed in allocating pd id\n");
+
+	*pd = (u16)returned_id;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc);
+	return rc;
+}
+
+void ecore_rdma_free_pd(void	*rdma_cxt,
+			u16	pd)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "pd = %08x\n", pd);
+
+	/* Returns a previously allocated protection domain for reuse */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+enum _ecore_status_t ecore_rdma_alloc_xrcd(void	*rdma_cxt,
+					   u16	*xrcd_id)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	u32                  returned_id;
+	enum _ecore_status_t rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc XRCD\n");
+
+	/* Allocates an unused XRC domain */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn,
+				      &p_hwfn->p_rdma_info->xrcd_map,
+				      &returned_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	if (rc != ECORE_SUCCESS)
+		DP_NOTICE(p_hwfn, false, "Failed in allocating xrcd id\n");
+
+	*xrcd_id = (u16)returned_id;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc XRCD - done, rc = %d\n", rc);
+	return rc;
+}
+
+void ecore_rdma_free_xrcd(void	*rdma_cxt,
+			  u16	xrcd_id)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "xrcd_id = %08x\n", xrcd_id);
+
+	/* Returns a previously allocated protection domain for reuse */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->xrcd_map, xrcd_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+static enum ecore_rdma_toggle_bit
+ecore_rdma_toggle_bit_create_resize_cq(struct ecore_hwfn *p_hwfn,
+				       u16 icid)
+{
+	struct ecore_rdma_info *p_info = p_hwfn->p_rdma_info;
+	enum ecore_rdma_toggle_bit toggle_bit;
+	u32 bmap_id;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", icid);
+
+	/* the function toggle the bit that is related to a given icid
+	 * and returns the new toggle bit's value
+	 */
+	bmap_id = icid - ecore_cxt_get_proto_cid_start(p_hwfn, p_info->proto);
+
+	OSAL_SPIN_LOCK(&p_info->lock);
+	toggle_bit = !OSAL_TEST_AND_FLIP_BIT(bmap_id, p_info->toggle_bits.bitmap);
+	OSAL_SPIN_UNLOCK(&p_info->lock);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_RDMA_TOGGLE_BIT_= %d\n",
+		   toggle_bit);
+
+	return toggle_bit;
+}
+
+enum _ecore_status_t ecore_rdma_create_cq(void			      *rdma_cxt,
+				struct ecore_rdma_create_cq_in_params *params,
+				u16                                   *icid)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_rdma_info *p_info = p_hwfn->p_rdma_info;
+	struct rdma_create_cq_ramrod_data	*p_ramrod;
+	enum ecore_rdma_toggle_bit		toggle_bit;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	enum _ecore_status_t			rc;
+	u32					returned_id;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "cq_handle = %08x%08x\n",
+		   params->cq_handle_hi, params->cq_handle_lo);
+
+	/* Allocate icid */
+	OSAL_SPIN_LOCK(&p_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_info->cq_map, &returned_id);
+	OSAL_SPIN_UNLOCK(&p_info->lock);
+
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_NOTICE(p_hwfn, false, "Can't create CQ, rc = %d\n", rc);
+		return rc;
+	}
+
+	*icid = (u16)(returned_id +
+		      ecore_cxt_get_proto_cid_start(
+			      p_hwfn, p_info->proto));
+
+	/* Check if icid requires a page allocation */
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, *icid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = *icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	/* Send create CQ ramrod */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_CREATE_CQ,
+				   p_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_create_cq;
+
+	p_ramrod->cq_handle.hi = OSAL_CPU_TO_LE32(params->cq_handle_hi);
+	p_ramrod->cq_handle.lo = OSAL_CPU_TO_LE32(params->cq_handle_lo);
+	p_ramrod->dpi = OSAL_CPU_TO_LE16(params->dpi);
+	p_ramrod->is_two_level_pbl = params->pbl_two_level;
+	p_ramrod->max_cqes = OSAL_CPU_TO_LE32(params->cq_size);
+	DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr);
+	p_ramrod->pbl_num_pages = OSAL_CPU_TO_LE16(params->pbl_num_pages);
+	p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM)
+			+ params->cnq_id;
+	p_ramrod->int_timeout = params->int_timeout;
+	/* INTERNAL: Two layer PBL is currently not supported, ignoring next line */
+	/* INTERNAL: p_ramrod->pbl_log_page_size = params->pbl_page_size_log - 12; */
+
+	/* toggle the bit for every resize or create cq for a given icid */
+	toggle_bit = ecore_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
+
+	p_ramrod->toggle_bit = toggle_bit;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS) {
+		/* restore toggle bit */
+		ecore_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
+		goto err;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Created CQ, rc = %d\n", rc);
+	return rc;
+
+err:
+	/* release allocated icid */
+	OSAL_SPIN_LOCK(&p_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id);
+	OSAL_SPIN_UNLOCK(&p_info->lock);
+
+	DP_NOTICE(p_hwfn, false, "Create CQ failed, rc = %d\n", rc);
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_destroy_cq(void			*rdma_cxt,
+			struct ecore_rdma_destroy_cq_in_params	*in_params,
+			struct ecore_rdma_destroy_cq_out_params	*out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_destroy_cq_output_params *p_ramrod_res;
+	struct rdma_destroy_cq_ramrod_data	*p_ramrod;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	dma_addr_t				ramrod_res_phys;
+	enum _ecore_status_t			rc = ECORE_NOMEM;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", in_params->icid);
+
+	p_ramrod_res = (struct rdma_destroy_cq_output_params *)
+			OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys,
+				sizeof(struct rdma_destroy_cq_output_params));
+	if (!p_ramrod_res)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore destroy cq failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid =  in_params->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	/* Send destroy CQ ramrod */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_DESTROY_CQ,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_destroy_cq;
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	out_params->num_cq_notif =
+		OSAL_LE16_TO_CPU(p_ramrod_res->cnq_num);
+
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+			       sizeof(struct rdma_destroy_cq_output_params));
+
+	/* Free icid */
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+
+	ecore_bmap_release_id(p_hwfn,
+			      &p_hwfn->p_rdma_info->cq_map,
+		(in_params->icid - ecore_cxt_get_proto_cid_start(
+			p_hwfn, p_hwfn->p_rdma_info->proto)));
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc);
+	return rc;
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+			       sizeof(struct rdma_destroy_cq_output_params));
+
+	return rc;
+}
+
+void ecore_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_ecore_mac)
+{
+	p_fw_mac[0] = OSAL_CPU_TO_LE16((p_ecore_mac[0] << 8) + p_ecore_mac[1]);
+	p_fw_mac[1] = OSAL_CPU_TO_LE16((p_ecore_mac[2] << 8) + p_ecore_mac[3]);
+	p_fw_mac[2] = OSAL_CPU_TO_LE16((p_ecore_mac[4] << 8) + p_ecore_mac[5]);
+}
+
+
+enum _ecore_status_t ecore_rdma_query_qp(void			*rdma_cxt,
+			struct ecore_rdma_qp			*qp,
+			struct ecore_rdma_query_qp_out_params	*out_params)
+
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	/* The following fields are filled in from qp and not FW as they can't
+	 * be modified by FW
+	 */
+	out_params->mtu = qp->mtu;
+	out_params->dest_qp = qp->dest_qp;
+	out_params->incoming_atomic_en = qp->incoming_atomic_en;
+	out_params->e2e_flow_control_en = qp->e2e_flow_control_en;
+	out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en;
+	out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en;
+	out_params->dgid = qp->dgid;
+	out_params->flow_label = qp->flow_label;
+	out_params->hop_limit_ttl = qp->hop_limit_ttl;
+	out_params->traffic_class_tos = qp->traffic_class_tos;
+	out_params->timeout = qp->ack_timeout;
+	out_params->rnr_retry = qp->rnr_retry_cnt;
+	out_params->retry_cnt = qp->retry_cnt;
+	out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer;
+	out_params->pkey_index = 0;
+	out_params->max_rd_atomic = qp->max_rd_atomic_req;
+	out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp;
+	out_params->sqd_async = qp->sqd_async;
+
+	if (IS_IWARP(p_hwfn))
+		rc = ecore_iwarp_query_qp(qp, out_params);
+	else
+		rc = ecore_roce_query_qp(p_hwfn, qp, out_params);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Query QP, rc = %d\n", rc);
+	return rc;
+}
+
+
+enum _ecore_status_t ecore_rdma_destroy_qp(void *rdma_cxt,
+					   struct ecore_rdma_qp *qp,
+					   struct ecore_rdma_destroy_qp_out_params *out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (!rdma_cxt || !qp) {
+		DP_ERR(p_hwfn,
+		       "ecore rdma destroy qp failed: invalid NULL input. rdma_cxt=%p, qp=%p\n",
+		       rdma_cxt, qp);
+		return ECORE_INVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x)\n", qp->icid);
+
+	if (IS_IWARP(p_hwfn))
+		rc = ecore_iwarp_destroy_qp(p_hwfn, qp);
+	else
+		rc = ecore_roce_destroy_qp(p_hwfn, qp, out_params);
+
+	/* free qp params struct */
+	OSAL_FREE(p_hwfn->p_dev, qp);
+
+	return rc;
+}
+
+
+struct ecore_rdma_qp *ecore_rdma_create_qp(void			*rdma_cxt,
+			struct ecore_rdma_create_qp_in_params	*in_params,
+			struct ecore_rdma_create_qp_out_params	*out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_rdma_qp *qp;
+	u8 max_stats_queues;
+	enum _ecore_status_t rc = 0;
+
+	if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) {
+		DP_ERR(p_hwfn->p_dev,
+		       "ecore roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n",
+		       rdma_cxt,
+		       in_params,
+		       out_params);
+		return OSAL_NULL;
+	}
+
+	/* Some sanity checks... */
+	max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues;
+	if (in_params->stats_queue >= max_stats_queues) {
+		DP_ERR(p_hwfn->p_dev,
+		       "ecore rdma create qp failed due to invalid statistics queue %d. maximum is %d\n",
+		       in_params->stats_queue, max_stats_queues);
+		return OSAL_NULL;
+	}
+
+	if (IS_IWARP(p_hwfn)) {
+		if (in_params->sq_num_pages*sizeof(struct regpair) >
+		    IWARP_SHARED_QUEUE_PAGE_SQ_PBL_MAX_SIZE) {
+			DP_NOTICE(p_hwfn->p_dev, true, "Sq num pages: %d exceeds maximum\n",
+				  in_params->sq_num_pages);
+			return OSAL_NULL;
+		}
+		if (in_params->rq_num_pages*sizeof(struct regpair) >
+		    IWARP_SHARED_QUEUE_PAGE_RQ_PBL_MAX_SIZE) {
+			DP_NOTICE(p_hwfn->p_dev, true,
+				  "Rq num pages: %d exceeds maximum\n",
+				  in_params->rq_num_pages);
+			return OSAL_NULL;
+		}
+	}
+
+	qp = OSAL_ZALLOC(p_hwfn->p_dev,
+			 GFP_KERNEL,
+			 sizeof(struct ecore_rdma_qp));
+	if (!qp)
+	{
+		DP_NOTICE(p_hwfn, false, "Failed to allocate ecore_rdma_qp\n");
+		return OSAL_NULL;
+	}
+
+	qp->cur_state = ECORE_ROCE_QP_STATE_RESET;
+#ifdef CONFIG_ECORE_IWARP
+	qp->iwarp_state = ECORE_IWARP_QP_STATE_IDLE;
+#endif
+	qp->qp_handle.hi = OSAL_CPU_TO_LE32(in_params->qp_handle_hi);
+	qp->qp_handle.lo = OSAL_CPU_TO_LE32(in_params->qp_handle_lo);
+	qp->qp_handle_async.hi = OSAL_CPU_TO_LE32(in_params->qp_handle_async_hi);
+	qp->qp_handle_async.lo = OSAL_CPU_TO_LE32(in_params->qp_handle_async_lo);
+	qp->use_srq = in_params->use_srq;
+	qp->signal_all = in_params->signal_all;
+	qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey;
+	qp->pd = in_params->pd;
+	qp->dpi = in_params->dpi;
+	qp->sq_cq_id = in_params->sq_cq_id;
+	qp->sq_num_pages = in_params->sq_num_pages;
+	qp->sq_pbl_ptr = in_params->sq_pbl_ptr;
+	qp->rq_cq_id = in_params->rq_cq_id;
+	qp->rq_num_pages = in_params->rq_num_pages;
+	qp->rq_pbl_ptr = in_params->rq_pbl_ptr;
+	qp->srq_id = in_params->srq_id;
+	qp->req_offloaded = false;
+	qp->resp_offloaded = false;
+	/* e2e_flow_control cannot be done in case of S-RQ.
+	 * Refer to 9.7.7.2 End-to-End Flow Control section of IB spec
+	 */
+	qp->e2e_flow_control_en = qp->use_srq ? false : true;
+	qp->stats_queue = in_params->stats_queue;
+	qp->qp_type = in_params->qp_type;
+	qp->xrcd_id = in_params->xrcd_id;
+
+	if (IS_IWARP(p_hwfn)) {
+		rc = ecore_iwarp_create_qp(p_hwfn, qp, out_params);
+		qp->qpid = qp->icid;
+	} else {
+		rc = ecore_roce_alloc_qp_idx(p_hwfn, &qp->qp_idx);
+		qp->icid = ECORE_ROCE_QP_TO_ICID(qp->qp_idx);
+		qp->qpid = ((0xFF << 16) | qp->icid);
+	}
+
+	if (rc != ECORE_SUCCESS) {
+		OSAL_FREE(p_hwfn->p_dev, qp);
+		return OSAL_NULL;
+	}
+
+	out_params->icid = qp->icid;
+	out_params->qp_id = qp->qpid;
+
+	/* INTERNAL: max_sq_sges future use only*/
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Create QP, rc = %d\n", rc);
+	return qp;
+}
+
+#define ECORE_RDMA_ECN_SHIFT 0
+#define ECORE_RDMA_ECN_MASK 0x3
+#define ECORE_RDMA_DSCP_SHIFT 2
+#define ECORE_RDMA_DSCP_MASK 0x3f
+#define ECORE_RDMA_VLAN_PRIO_SHIFT 13
+#define ECORE_RDMA_VLAN_PRIO_MASK 0x7
+enum _ecore_status_t ecore_rdma_modify_qp(
+	void *rdma_cxt,
+	struct ecore_rdma_qp *qp,
+	struct ecore_rdma_modify_qp_in_params *params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	enum ecore_roce_qp_state prev_state;
+	enum _ecore_status_t     rc = ECORE_SUCCESS;
+
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN))
+	{
+		qp->incoming_rdma_read_en = params->incoming_rdma_read_en;
+		qp->incoming_rdma_write_en = params->incoming_rdma_write_en;
+		qp->incoming_atomic_en = params->incoming_atomic_en;
+	}
+
+	/* Update QP structure with the updated values */
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_ROCE_MODE))
+	{
+		qp->roce_mode = params->roce_mode;
+	}
+	if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY))
+	{
+		qp->pkey = params->pkey;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN))
+	{
+		qp->e2e_flow_control_en = params->e2e_flow_control_en;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_DEST_QP))
+	{
+		qp->dest_qp = params->dest_qp;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR))
+	{
+		/* Indicates that the following parameters have changed:
+		 * Traffic class, flow label, hop limit, source GID,
+		 * destination GID, loopback indicator
+		 */
+		qp->flow_label = params->flow_label;
+		qp->hop_limit_ttl = params->hop_limit_ttl;
+
+		qp->sgid = params->sgid;
+		qp->dgid = params->dgid;
+		qp->udp_src_port = params->udp_src_port;
+		qp->vlan_id = params->vlan_id;
+		qp->traffic_class_tos = params->traffic_class_tos;
+
+		/* apply global override values */
+		if (p_hwfn->p_rdma_info->glob_cfg.vlan_pri_en)
+			SET_FIELD(qp->vlan_id, ECORE_RDMA_VLAN_PRIO,
+				  p_hwfn->p_rdma_info->glob_cfg.vlan_pri);
+
+		if (p_hwfn->p_rdma_info->glob_cfg.ecn_en)
+			SET_FIELD(qp->traffic_class_tos, ECORE_RDMA_ECN,
+				  p_hwfn->p_rdma_info->glob_cfg.ecn);
+
+		if (p_hwfn->p_rdma_info->glob_cfg.dscp_en)
+			SET_FIELD(qp->traffic_class_tos, ECORE_RDMA_DSCP,
+				  p_hwfn->p_rdma_info->glob_cfg.dscp);
+
+		qp->mtu = params->mtu;
+
+		OSAL_MEMCPY((u8 *)&qp->remote_mac_addr[0],
+			    (u8 *)&params->remote_mac_addr[0], ETH_ALEN);
+		if (params->use_local_mac) {
+			OSAL_MEMCPY((u8 *)&qp->local_mac_addr[0],
+				    (u8 *)&params->local_mac_addr[0],
+				    ETH_ALEN);
+		} else {
+			OSAL_MEMCPY((u8 *)&qp->local_mac_addr[0],
+				    (u8 *)&p_hwfn->hw_info.hw_mac_addr,
+				    ETH_ALEN);
+		}
+	}
+	if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN))
+	{
+		qp->rq_psn = params->rq_psn;
+	}
+	if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN))
+	{
+		qp->sq_psn = params->sq_psn;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ))
+	{
+		qp->max_rd_atomic_req = params->max_rd_atomic_req;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP))
+	{
+		qp->max_rd_atomic_resp = params->max_rd_atomic_resp;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT))
+	{
+		qp->ack_timeout = params->ack_timeout;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT))
+	{
+		qp->retry_cnt = params->retry_cnt;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT))
+	{
+		qp->rnr_retry_cnt = params->rnr_retry_cnt;
+	}
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER))
+	{
+		qp->min_rnr_nak_timer = params->min_rnr_nak_timer;
+	}
+
+	qp->sqd_async = params->sqd_async;
+
+	prev_state = qp->cur_state;
+	if (GET_FIELD(params->modify_flags,
+		      ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE))
+	{
+		qp->cur_state = params->new_state;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "qp->cur_state=%d\n",
+			   qp->cur_state);
+	}
+
+	if (qp->qp_type == ECORE_RDMA_QP_TYPE_XRC_INI) {
+		qp->has_req = 1;
+	} else if (qp->qp_type == ECORE_RDMA_QP_TYPE_XRC_TGT)
+	{
+		qp->has_resp = 1;
+	} else {
+		qp->has_req = 1;
+		qp->has_resp = 1;
+	}
+
+	if (IS_IWARP(p_hwfn)) {
+		enum ecore_iwarp_qp_state new_state =
+			ecore_roce2iwarp_state(qp->cur_state);
+
+		rc = ecore_iwarp_modify_qp(p_hwfn, qp, new_state, 0);
+	} else {
+		rc = ecore_roce_modify_qp(p_hwfn, qp, prev_state, params);
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify QP, rc = %d\n", rc);
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_register_tid(void		 *rdma_cxt,
+			struct ecore_rdma_register_tid_in_params *params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_register_tid_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data	     init_data;
+	struct ecore_spq_entry               *p_ent;
+	enum rdma_tid_type                   tid_type;
+	u8                                   fw_return_code;
+	enum _ecore_status_t                 rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "itid = %08x\n", params->itid);
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	if (p_hwfn->p_rdma_info->last_tid < params->itid) {
+		p_hwfn->p_rdma_info->last_tid = params->itid;
+	}
+
+	p_ramrod = &p_ent->ramrod.rdma_register_tid;
+
+	p_ramrod->flags = 0;
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL,
+		  params->pbl_two_level);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED,
+		  params->zbva);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR,
+		  params->phy_mr);
+
+	/* Don't initialize D/C field, as it may override other bits. */
+	if (!(params->tid_type == ECORE_RDMA_TID_FMR) &&
+	    !(params->dma_mr))
+		SET_FIELD(p_ramrod->flags,
+			  RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG,
+			  params->page_size_log - 12);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ,
+		  params->remote_read);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE,
+		  params->remote_write);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC,
+		  params->remote_atomic);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE,
+		  params->local_write);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ,
+		  params->local_read);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND,
+		  params->mw_bind);
+
+	SET_FIELD(p_ramrod->flags1,
+		  RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG,
+		  params->pbl_page_size_log - 12);
+
+	SET_FIELD(p_ramrod->flags2,
+		  RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR,
+		  params->dma_mr);
+
+	switch (params->tid_type)
+	{
+	case ECORE_RDMA_TID_REGISTERED_MR:
+		tid_type = RDMA_TID_REGISTERED_MR;
+		break;
+	case ECORE_RDMA_TID_FMR:
+		tid_type = RDMA_TID_FMR;
+		break;
+	case ECORE_RDMA_TID_MW_TYPE1:
+		tid_type = RDMA_TID_MW_TYPE1;
+		break;
+	case ECORE_RDMA_TID_MW_TYPE2A:
+		tid_type = RDMA_TID_MW_TYPE2A;
+		break;
+	default:
+		rc = ECORE_INVAL;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+	SET_FIELD(p_ramrod->flags1,
+		  RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE,
+		  tid_type);
+
+	p_ramrod->itid = OSAL_CPU_TO_LE32(params->itid);
+	p_ramrod->key = params->key;
+	p_ramrod->pd = OSAL_CPU_TO_LE16(params->pd);
+	p_ramrod->length_hi = (u8)(params->length >> 32);
+	p_ramrod->length_lo = DMA_LO_LE(params->length);
+	if (params->zbva)
+	{
+		/* Lower 32 bits of the registered MR address.
+		 * In case of zero based MR, will hold FBO
+		 */
+		p_ramrod->va.hi = 0;
+		p_ramrod->va.lo = OSAL_CPU_TO_LE32(params->fbo);
+	} else {
+		DMA_REGPAIR_LE(p_ramrod->va, params->vaddr);
+	}
+	DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr);
+
+	/* DIF */
+	if (params->dif_enabled) {
+		SET_FIELD(p_ramrod->flags2,
+			  RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1);
+		DMA_REGPAIR_LE(p_ramrod->dif_error_addr,
+			       params->dif_error_addr);
+		DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr);
+	}
+
+	rc = ecore_spq_post(p_hwfn, p_ent, &fw_return_code);
+	if (rc)
+		return rc;
+
+	if (fw_return_code != RDMA_RETURN_OK) {
+		DP_NOTICE(p_hwfn, true, "fw_return_code = %d\n", fw_return_code);
+		return ECORE_UNKNOWN_ERROR;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Register TID, rc = %d\n", rc);
+	return rc;
+}
+
+static OSAL_INLINE int ecore_rdma_send_deregister_tid_ramrod(
+		struct ecore_hwfn *p_hwfn,
+		u32 itid,
+		u8 *fw_return_code)
+{
+	struct ecore_sp_init_data              init_data;
+	struct rdma_deregister_tid_ramrod_data *p_ramrod;
+	struct ecore_spq_entry                 *p_ent;
+	enum _ecore_status_t                   rc;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_DEREGISTER_MR,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.rdma_deregister_tid;
+	p_ramrod->itid = OSAL_CPU_TO_LE32(itid);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, fw_return_code);
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	return rc;
+}
+
+#define ECORE_RDMA_DEREGISTER_TIMEOUT_MSEC	(1)
+
+enum _ecore_status_t ecore_rdma_deregister_tid(void	*rdma_cxt,
+					       u32	itid)
+{
+	enum _ecore_status_t                   rc;
+	u8                                     fw_ret_code;
+	struct ecore_ptt                       *p_ptt;
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	/* First attempt */
+	rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (fw_ret_code != RDMA_RETURN_NIG_DRAIN_REQ)
+		goto done;
+
+	/* Second attempt, after 1msec, if device still holds data.
+	 * This can occur since 'destroy QP' returns to the caller rather fast.
+	 * The synchronous part of it returns after freeing a few of the
+	 * resources but not all of them, allowing the consumer to continue its
+	 * flow. All of the resources will be freed after the asynchronous part
+	 * of the destroy QP is complete.
+	 */
+	OSAL_MSLEEP(ECORE_RDMA_DEREGISTER_TIMEOUT_MSEC);
+	rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (fw_ret_code != RDMA_RETURN_NIG_DRAIN_REQ)
+		goto done;
+
+	/* Third and last attempt, perform NIG drain and resend the ramrod */
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return ECORE_TIMEOUT;
+
+	rc = ecore_mcp_drain(p_hwfn, p_ptt);
+	if (rc != ECORE_SUCCESS) {
+		ecore_ptt_release(p_hwfn, p_ptt);
+		return rc;
+	}
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+done:
+	if (fw_ret_code == RDMA_RETURN_OK) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "De-registered itid=%d\n",
+			   itid);
+		return ECORE_SUCCESS;
+	} else if (fw_ret_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) {
+		/* INTERNAL: This error is returned in case trying to deregister
+		 * a MR that is not allocated. We define "allocated" as either:
+		 * 1. Registered.
+		 * 2. This is an FMR MR type, which is not currently registered
+		 *    but can accept FMR WQEs on SQ.
+		 */
+		DP_NOTICE(p_hwfn, false, "itid=%d, fw_ret_code=%d\n", itid,
+			  fw_ret_code);
+		return ECORE_INVAL;
+	} else { /* fw_ret_code == RDMA_RETURN_NIG_DRAIN_REQ */
+		DP_NOTICE(p_hwfn, true,
+			  "deregister failed after three attempts. itid=%d, fw_ret_code=%d\n",
+			  itid, fw_ret_code);
+		return ECORE_UNKNOWN_ERROR;
+	}
+}
+
+static struct ecore_bmap *ecore_rdma_get_srq_bmap(struct ecore_hwfn *p_hwfn, bool is_xrc)
+{
+	if (is_xrc)
+		return &p_hwfn->p_rdma_info->xrc_srq_map;
+
+	return &p_hwfn->p_rdma_info->srq_map;
+}
+
+u16 ecore_rdma_get_fw_srq_id(struct ecore_hwfn *p_hwfn, u16 id, bool is_xrc)
+{
+	if (is_xrc)
+		return id;
+
+	return id + p_hwfn->p_rdma_info->srq_id_offset;
+}
+
+enum _ecore_status_t
+ecore_rdma_modify_srq(void *rdma_cxt,
+		      struct ecore_rdma_modify_srq_in_params *in_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_srq_modify_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	u16 opaque_fid, fw_srq_id;
+	enum _ecore_status_t rc;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+	/* Send modify SRQ ramrod */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_MODIFY_SRQ,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.rdma_modify_srq;
+
+	fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, in_params->srq_id,
+					     in_params->is_xrc);
+	p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id);
+	opaque_fid = p_hwfn->hw_info.opaque_fid;
+	p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid);
+	p_ramrod->wqe_limit = OSAL_CPU_TO_LE16(in_params->wqe_limit);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "modified SRQ id = %x, is_xrc=%u\n",
+		   in_params->srq_id, in_params->is_xrc);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_rdma_destroy_srq(void *rdma_cxt,
+		       struct ecore_rdma_destroy_srq_in_params *in_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_srq_destroy_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	u16 opaque_fid, fw_srq_id;
+	struct ecore_bmap *bmap;
+	enum _ecore_status_t rc;
+
+	opaque_fid = p_hwfn->hw_info.opaque_fid;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	/* Send destroy SRQ ramrod */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_DESTROY_SRQ,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.rdma_destroy_srq;
+
+	fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, in_params->srq_id,
+					     in_params->is_xrc);
+	p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id);
+	p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	bmap = ecore_rdma_get_srq_bmap(p_hwfn, in_params->is_xrc);
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, bmap, in_params->srq_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "XRC/SRQ destroyed Id = %x, is_xrc=%u\n",
+		   in_params->srq_id, in_params->is_xrc);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_rdma_create_srq(void *rdma_cxt,
+		      struct ecore_rdma_create_srq_in_params *in_params,
+		      struct ecore_rdma_create_srq_out_params *out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct rdma_srq_create_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data init_data;
+	enum ecore_cxt_elem_type elem_type;
+	struct ecore_spq_entry *p_ent;
+	u16 opaque_fid, fw_srq_id;
+	struct ecore_bmap *bmap;
+	u32 returned_id;
+	enum _ecore_status_t rc;
+
+	/* Allocate XRC/SRQ ID */
+	bmap = ecore_rdma_get_srq_bmap(p_hwfn, in_params->is_xrc);
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn, bmap, &returned_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false,
+			  "failed to allocate xrc/srq id (is_xrc=%u)\n",
+			  in_params->is_xrc);
+		return rc;
+	}
+	/* Allocate XRC/SRQ ILT page */
+	elem_type = (in_params->is_xrc) ? (ECORE_ELEM_XRC_SRQ) : (ECORE_ELEM_SRQ);
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, elem_type, returned_id);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.opaque_fid = opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	/* Create XRC/SRQ ramrod */
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_CREATE_SRQ,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_create_srq;
+
+	p_ramrod->pbl_base_addr.hi = DMA_HI_LE(in_params->pbl_base_addr);
+	p_ramrod->pbl_base_addr.lo = DMA_LO_LE(in_params->pbl_base_addr);
+	p_ramrod->pages_in_srq_pbl = OSAL_CPU_TO_LE16(in_params->num_pages);
+	p_ramrod->pd_id = OSAL_CPU_TO_LE16(in_params->pd_id);
+	p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid);
+	p_ramrod->page_size = OSAL_CPU_TO_LE16(in_params->page_size);
+	p_ramrod->producers_addr.hi = DMA_HI_LE(in_params->prod_pair_addr);
+	p_ramrod->producers_addr.lo = DMA_LO_LE(in_params->prod_pair_addr);
+	fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, (u16) returned_id,
+					     in_params->is_xrc);
+	p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id);
+
+	if (in_params->is_xrc) {
+		SET_FIELD(p_ramrod->flags,
+			  RDMA_SRQ_CREATE_RAMROD_DATA_XRC_FLAG,
+			  1);
+		SET_FIELD(p_ramrod->flags,
+			  RDMA_SRQ_CREATE_RAMROD_DATA_RESERVED_KEY_EN,
+			  in_params->reserved_key_en);
+		p_ramrod->xrc_srq_cq_cid = OSAL_CPU_TO_LE32(in_params->cq_cid);
+		p_ramrod->xrc_domain = OSAL_CPU_TO_LE16(in_params->xrcd_id);
+	}
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	out_params->srq_id = (u16)returned_id;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "XRC/SRQ created Id = %x (is_xrc=%u)\n",
+		   out_params->srq_id, in_params->is_xrc);
+	return rc;
+
+err:
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, bmap, returned_id);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+
+	return rc;
+}
+
+bool ecore_rdma_allocated_qps(struct ecore_hwfn *p_hwfn)
+{
+	bool result;
+
+	/* if rdma info has not been allocated, naturally there are no qps */
+	if (!p_hwfn->p_rdma_info)
+		return false;
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	if (!p_hwfn->p_rdma_info->qp_map.bitmap)
+		result = false;
+	else
+		result = !ecore_bmap_is_empty(&p_hwfn->p_rdma_info->qp_map);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+	return result;
+}
+
+enum _ecore_status_t ecore_rdma_resize_cq(void			*rdma_cxt,
+			struct ecore_rdma_resize_cq_in_params	*in_params,
+			struct ecore_rdma_resize_cq_out_params	*out_params)
+{
+	enum _ecore_status_t			rc;
+	enum ecore_rdma_toggle_bit		toggle_bit;
+	struct ecore_spq_entry			*p_ent;
+	struct rdma_resize_cq_ramrod_data	*p_ramrod;
+	u8                                      fw_return_code;
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	dma_addr_t							ramrod_res_phys;
+	struct rdma_resize_cq_output_params	*p_ramrod_res;
+	struct ecore_sp_init_data		init_data;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", in_params->icid);
+
+	/* Send resize CQ ramrod */
+
+	p_ramrod_res = (struct rdma_resize_cq_output_params *)
+			OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys,
+				sizeof(*p_ramrod_res));
+	if (!p_ramrod_res)
+	{
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "ecore resize cq failed: cannot allocate memory (ramrod). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = in_params->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   RDMA_RAMROD_RESIZE_CQ,
+				   p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_resize_cq;
+
+	p_ramrod->flags = 0;
+
+	/* toggle the bit for every resize or create cq for a given icid */
+	toggle_bit = ecore_rdma_toggle_bit_create_resize_cq(p_hwfn,
+							    in_params->icid);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT,
+		  toggle_bit);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL,
+		  in_params->pbl_two_level);
+
+	p_ramrod->pbl_log_page_size = in_params->pbl_page_size_log - 12;
+	p_ramrod->pbl_num_pages = OSAL_CPU_TO_LE16(in_params->pbl_num_pages);
+	p_ramrod->max_cqes = OSAL_CPU_TO_LE32(in_params->cq_size);
+	p_ramrod->pbl_addr.hi = DMA_HI_LE(in_params->pbl_ptr);
+	p_ramrod->pbl_addr.lo = DMA_LO_LE(in_params->pbl_ptr);
+
+	p_ramrod->output_params_addr.hi = DMA_HI_LE(ramrod_res_phys);
+	p_ramrod->output_params_addr.lo = DMA_LO_LE(ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, &fw_return_code);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	if (fw_return_code != RDMA_RETURN_OK)
+	{
+		DP_NOTICE(p_hwfn, fw_return_code != RDMA_RETURN_RESIZE_CQ_ERR,
+			  "fw_return_code = %d\n", fw_return_code);
+		DP_NOTICE(p_hwfn,
+			  true, "fw_return_code = %d\n", fw_return_code);
+		rc = ECORE_UNKNOWN_ERROR;
+		goto err;
+	}
+
+	out_params->prod = OSAL_LE32_TO_CPU(p_ramrod_res->old_cq_prod);
+	out_params->cons = OSAL_LE32_TO_CPU(p_ramrod_res->old_cq_cons);
+
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+			       sizeof(*p_ramrod_res));
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+
+	return rc;
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+			       sizeof(*p_ramrod_res));
+	DP_NOTICE(p_hwfn, false, "rc = %d\n", rc);
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_start(void *rdma_cxt,
+				struct ecore_rdma_start_in_params *params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_ptt *p_ptt;
+	enum _ecore_status_t rc = ECORE_TIMEOUT;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+		   "desired_cnq = %08x\n", params->desired_cnq);
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		goto err;
+
+	rc = ecore_rdma_alloc(p_hwfn);
+	if (rc)
+		goto err1;
+
+	rc = ecore_rdma_setup(p_hwfn, p_ptt, params);
+	if (rc)
+		goto err2;
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	ecore_rdma_activate(p_hwfn);
+	return rc;
+
+err2:
+	ecore_rdma_free(p_hwfn);
+err1:
+	ecore_ptt_release(p_hwfn, p_ptt);
+err:
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA start - error, rc = %d\n", rc);
+	return rc;
+}
+
+enum _ecore_status_t ecore_rdma_query_stats(void *rdma_cxt, u8 stats_queue,
+				struct ecore_rdma_stats_out_params *out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	u8 abs_stats_queue, max_stats_queues;
+	u32 pstats_addr, tstats_addr, addr;
+	struct ecore_rdma_info *info;
+	struct ecore_ptt *p_ptt;
+#ifdef CONFIG_ECORE_IWARP
+	u32 xstats_addr;
+#endif
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (!p_hwfn)
+		return ECORE_INVAL;
+
+	if (!p_hwfn->p_rdma_info) {
+		DP_INFO(p_hwfn->p_dev, "ecore rdma query stats failed due to NULL rdma_info\n");
+		return ECORE_INVAL;
+	}
+
+	info = p_hwfn->p_rdma_info;
+
+	rc = ecore_rdma_inc_ref_cnt(p_hwfn);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues;
+	if (stats_queue >= max_stats_queues) {
+		DP_ERR(p_hwfn->p_dev,
+		       "ecore rdma query stats failed due to invalid statistics queue %d. maximum is %d\n",
+		       stats_queue, max_stats_queues);
+		rc = ECORE_INVAL;
+		goto err;
+	}
+
+	/* Statistics collected in statistics queues (for PF/VF) */
+	abs_stats_queue = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) +
+			    stats_queue;
+	pstats_addr = BAR0_MAP_REG_PSDM_RAM +
+		      PSTORM_RDMA_QUEUE_STAT_OFFSET(abs_stats_queue);
+	tstats_addr = BAR0_MAP_REG_TSDM_RAM +
+		      TSTORM_RDMA_QUEUE_STAT_OFFSET(abs_stats_queue);
+
+#ifdef CONFIG_ECORE_IWARP
+	/* Statistics per PF ID */
+	xstats_addr = BAR0_MAP_REG_XSDM_RAM +
+		      XSTORM_IWARP_RXMIT_STATS_OFFSET(p_hwfn->rel_pf_id);
+#endif
+
+	OSAL_MEMSET(&info->rdma_sent_pstats, 0, sizeof(info->rdma_sent_pstats));
+	OSAL_MEMSET(&info->rdma_rcv_tstats, 0, sizeof(info->rdma_rcv_tstats));
+	OSAL_MEMSET(&info->roce.event_stats, 0, sizeof(info->roce.event_stats));
+	OSAL_MEMSET(&info->roce.dcqcn_rx_stats, 0,sizeof(info->roce.dcqcn_rx_stats));
+	OSAL_MEMSET(&info->roce.dcqcn_tx_stats, 0,sizeof(info->roce.dcqcn_tx_stats));
+#ifdef CONFIG_ECORE_IWARP
+	OSAL_MEMSET(&info->iwarp.stats, 0, sizeof(info->iwarp.stats));
+#endif
+
+	p_ptt = ecore_ptt_acquire(p_hwfn);
+
+	if (!p_ptt) {
+		rc = ECORE_TIMEOUT;
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+		goto err;
+	}
+
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->rdma_sent_pstats,
+			  pstats_addr, sizeof(struct rdma_sent_stats));
+
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->rdma_rcv_tstats,
+			  tstats_addr, sizeof(struct rdma_rcv_stats));
+
+	addr = BAR0_MAP_REG_TSDM_RAM +
+	       TSTORM_ROCE_EVENTS_STAT_OFFSET(p_hwfn->rel_pf_id);
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.event_stats, addr,
+			  sizeof(struct roce_events_stats));
+
+	addr = BAR0_MAP_REG_YSDM_RAM +
+		YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(p_hwfn->rel_pf_id);
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.dcqcn_rx_stats, addr,
+			  sizeof(struct roce_dcqcn_received_stats));
+
+	addr = BAR0_MAP_REG_PSDM_RAM +
+	       PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(p_hwfn->rel_pf_id);
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.dcqcn_tx_stats, addr,
+			  sizeof(struct roce_dcqcn_sent_stats));
+
+#ifdef CONFIG_ECORE_IWARP
+	ecore_memcpy_from(p_hwfn, p_ptt, &info->iwarp.stats,
+			  xstats_addr, sizeof(struct iwarp_rxmit_stats_drv));
+#endif
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	OSAL_MEMSET(out_params, 0, sizeof(*out_params));
+
+	out_params->sent_bytes =
+		HILO_64_REGPAIR(info->rdma_sent_pstats.sent_bytes);
+	out_params->sent_pkts =
+		HILO_64_REGPAIR(info->rdma_sent_pstats.sent_pkts);
+	out_params->rcv_bytes =
+		HILO_64_REGPAIR(info->rdma_rcv_tstats.rcv_bytes);
+	out_params->rcv_pkts =
+		HILO_64_REGPAIR(info->rdma_rcv_tstats.rcv_pkts);
+
+	out_params->silent_drops =
+		OSAL_LE16_TO_CPU(info->roce.event_stats.silent_drops);
+	out_params->rnr_nacks_sent =
+		OSAL_LE16_TO_CPU(info->roce.event_stats.rnr_naks_sent);
+	out_params->icrc_errors =
+		OSAL_LE32_TO_CPU(info->roce.event_stats.icrc_error_count);
+	out_params->retransmit_events =
+		OSAL_LE32_TO_CPU(info->roce.event_stats.retransmit_count);
+	out_params->ecn_pkt_rcv =
+		HILO_64_REGPAIR(info->roce.dcqcn_rx_stats.ecn_pkt_rcv);
+	out_params->cnp_pkt_rcv =
+		HILO_64_REGPAIR(info->roce.dcqcn_rx_stats.cnp_pkt_rcv);
+	out_params->cnp_pkt_sent =
+		HILO_64_REGPAIR(info->roce.dcqcn_tx_stats.cnp_pkt_sent);
+
+#ifdef CONFIG_ECORE_IWARP
+	out_params->iwarp_tx_fast_rxmit_cnt =
+		HILO_64_REGPAIR(info->iwarp.stats.tx_fast_retransmit_event_cnt);
+	out_params->iwarp_tx_slow_start_cnt =
+		HILO_64_REGPAIR(
+			info->iwarp.stats.tx_go_to_slow_start_event_cnt);
+	out_params->unalign_rx_comp = info->iwarp.unalign_rx_comp;
+#endif
+
+err:
+	ecore_rdma_dec_ref_cnt(p_hwfn);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_rdma_query_counters(void *rdma_cxt,
+			  struct ecore_rdma_counters_out_params *out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	unsigned long *bitmap;
+	unsigned int nbits;
+
+	if (!p_hwfn->p_rdma_info)
+		return ECORE_INVAL;
+
+	OSAL_MEMSET(out_params, 0, sizeof(*out_params));
+
+	bitmap = p_hwfn->p_rdma_info->pd_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->pd_map.max_count;
+	out_params->pd_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_pd = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->dpi_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->dpi_map.max_count;
+	out_params->dpi_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_dpi = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->cq_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->cq_map.max_count;
+	out_params->cq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_cq = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->qp_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->qp_map.max_count;
+	out_params->qp_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_qp = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->tid_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->tid_map.max_count;
+	out_params->tid_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_tid = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->srq_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->srq_map.max_count;
+	out_params->srq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_srq = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->xrc_srq_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->xrc_srq_map.max_count;
+	out_params->xrc_srq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_xrc_srq = nbits;
+
+	bitmap = p_hwfn->p_rdma_info->xrcd_map.bitmap;
+	nbits = p_hwfn->p_rdma_info->xrcd_map.max_count;
+	out_params->xrcd_count = OSAL_BITMAP_WEIGHT(bitmap, nbits);
+	out_params->max_xrcd = nbits;
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_rdma_resize_cnq(void			      *rdma_cxt,
+				struct ecore_rdma_resize_cnq_in_params *params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "cnq_id = %08x\n", params->cnq_id);
+
+	/* @@@TBD: waiting for fw (there is no ramrod yet) */
+	return ECORE_NOTIMPL;
+}
+
+void ecore_rdma_remove_user(void	*rdma_cxt,
+			    u16		dpi)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "dpi = %08x\n", dpi);
+
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+#ifndef LINUX_REMOVE
+enum _ecore_status_t
+ecore_rdma_set_glob_cfg(struct ecore_hwfn *p_hwfn,
+			struct ecore_rdma_glob_cfg *in_params,
+			u32 glob_cfg_bits)
+{
+	struct ecore_rdma_glob_cfg glob_cfg;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn->p_dev, ECORE_MSG_RDMA,
+		   "dscp %d dscp en %d ecn %d ecn en %d vlan pri %d vlan_pri_en %d\n",
+		   in_params->dscp, in_params->dscp_en,
+		   in_params->ecn, in_params->ecn_en, in_params->vlan_pri,
+		   in_params->vlan_pri_en);
+
+	/* Read global cfg to local */
+	OSAL_MEMCPY(&glob_cfg, &p_hwfn->p_rdma_info->glob_cfg,
+		    sizeof(glob_cfg));
+
+	if (glob_cfg_bits & ECORE_RDMA_DCSP_BIT_MASK) {
+		if (in_params->dscp > MAX_DSCP) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob dscp %d\n",
+			       in_params->dscp);
+			return ECORE_INVAL;
+		}
+		glob_cfg.dscp = in_params->dscp;
+	}
+
+	if (glob_cfg_bits & ECORE_RDMA_DCSP_EN_BIT_MASK) {
+		if (in_params->dscp_en > 1) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob_dscp_en %d\n",
+			       in_params->dscp_en);
+			return ECORE_INVAL;
+		}
+		glob_cfg.dscp_en = in_params->dscp_en;
+	}
+
+	if (glob_cfg_bits & ECORE_RDMA_ECN_BIT_MASK) {
+		if (in_params->ecn > INET_ECN_ECT_0) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob ecn %d\n",
+			       in_params->ecn);
+			return ECORE_INVAL;
+		}
+		glob_cfg.ecn = in_params->ecn;
+	}
+
+	if (glob_cfg_bits & ECORE_RDMA_ECN_EN_BIT_MASK) {
+		if (in_params->ecn_en > 1) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob ecn en %d\n",
+			       in_params->ecn_en);
+			return ECORE_INVAL;
+		}
+		glob_cfg.ecn_en = in_params->ecn_en;
+	}
+
+	if (glob_cfg_bits & ECORE_RDMA_VLAN_PRIO_BIT_MASK) {
+		if (in_params->vlan_pri > MAX_VLAN_PRIO) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob vlan pri %d\n",
+			       in_params->vlan_pri);
+			return ECORE_INVAL;
+		}
+		glob_cfg.vlan_pri = in_params->vlan_pri;
+	}
+
+	if (glob_cfg_bits & ECORE_RDMA_VLAN_PRIO_EN_BIT_MASK) {
+		if (in_params->vlan_pri_en > 1) {
+			DP_ERR(p_hwfn->p_dev, "invalid glob vlan pri en %d\n",
+			       in_params->vlan_pri_en);
+			return ECORE_INVAL;
+		}
+		glob_cfg.vlan_pri_en = in_params->vlan_pri_en;
+	}
+
+	/* Write back local cfg to global */
+	OSAL_MEMCPY(&p_hwfn->p_rdma_info->glob_cfg, &glob_cfg,
+		    sizeof(glob_cfg));
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_rdma_get_glob_cfg(struct ecore_hwfn *p_hwfn,
+			struct ecore_rdma_glob_cfg *out_params)
+{
+	OSAL_MEMCPY(out_params, &p_hwfn->p_rdma_info->glob_cfg,
+		    sizeof(struct ecore_rdma_glob_cfg));
+
+	return ECORE_SUCCESS;
+}
+#endif /* LINUX_REMOVE */
diff --git a/sys/dev/qlnx/qlnxe/ecore_roce.c b/sys/dev/qlnx/qlnxe/ecore_roce.c
new file mode 100644
index 000000000000..7a5d1f6c38e9
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/ecore_roce.c
@@ -0,0 +1,1579 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : ecore_roce.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "bcm_osal.h"
+#include "ecore.h"
+#include "ecore_status.h"
+#include "ecore_sp_commands.h"
+#include "ecore_cxt.h"
+#include "ecore_rdma.h"
+#include "reg_addr.h"
+#include "ecore_rt_defs.h"
+#include "ecore_init_ops.h"
+#include "ecore_hw.h"
+#include "ecore_mcp.h"
+#include "ecore_init_fw_funcs.h"
+#include "ecore_int.h"
+#include "pcics_reg_driver.h"
+#include "ecore_iro.h"
+#include "ecore_gtt_reg_addr.h"
+#ifndef LINUX_REMOVE
+#include "ecore_tcp_ip.h"
+#endif
+
+#ifdef _NTDDK_
+#pragma warning(push)
+#pragma warning(disable : 28167)
+#pragma warning(disable : 28123)
+#pragma warning(disable : 28182)
+#pragma warning(disable : 6011)
+#endif
+
+static void ecore_roce_free_icid(struct ecore_hwfn *p_hwfn, u16 icid);
+
+static enum _ecore_status_t
+ecore_roce_async_event(struct ecore_hwfn *p_hwfn,
+		       u8 fw_event_code,
+		       u16 OSAL_UNUSED echo,
+		       union event_ring_data *data,
+		       u8 OSAL_UNUSED fw_return_code)
+{
+	if (fw_event_code == ROCE_ASYNC_EVENT_DESTROY_QP_DONE) {
+		u16 icid = (u16)OSAL_LE32_TO_CPU(
+				data->rdma_data.rdma_destroy_qp_data.cid);
+
+		/* icid release in this async event can occur only if the icid
+		 * was offloaded to the FW. In case it wasn't offloaded this is
+		 * handled in ecore_roce_sp_destroy_qp.
+		 */
+		ecore_roce_free_icid(p_hwfn, icid);
+	} else
+		p_hwfn->p_rdma_info->events.affiliated_event(
+			p_hwfn->p_rdma_info->events.context,
+			fw_event_code,
+			(void *)&data->rdma_data.async_handle);
+
+	return ECORE_SUCCESS;
+}
+
+
+
+#ifdef CONFIG_DCQCN
+static enum _ecore_status_t ecore_roce_start_rl(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_roce_dcqcn_params *dcqcn_params)
+{
+	struct ecore_rl_update_params params;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n");
+	OSAL_MEMSET(&params, 0, sizeof(params));
+
+	params.rl_id_first = (u8)RESC_START(p_hwfn, ECORE_RL);
+	params.rl_id_last = RESC_START(p_hwfn, ECORE_RL) +
+		ecore_init_qm_get_num_pf_rls(p_hwfn);
+	params.dcqcn_update_param_flg = 1;
+	params.rl_init_flg = 1;
+	params.rl_start_flg = 1;
+	params.rl_stop_flg = 0;
+	params.rl_dc_qcn_flg = 1;
+
+	params.rl_bc_rate = dcqcn_params->rl_bc_rate;
+	params.rl_max_rate = dcqcn_params->rl_max_rate;
+	params.rl_r_ai = dcqcn_params->rl_r_ai;
+	params.rl_r_hai = dcqcn_params->rl_r_hai;
+	params.dcqcn_gd = dcqcn_params->dcqcn_gd;
+	params.dcqcn_k_us = dcqcn_params->dcqcn_k_us;
+	params.dcqcn_timeuot_us = dcqcn_params->dcqcn_timeout_us;
+
+	return ecore_sp_rl_update(p_hwfn, &params);
+}
+
+enum _ecore_status_t ecore_roce_stop_rl(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_rl_update_params params;
+
+	if (!p_hwfn->p_rdma_info->roce.dcqcn_reaction_point)
+		return ECORE_SUCCESS;
+
+	OSAL_MEMSET(&params, 0, sizeof(params));
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n");
+
+	params.rl_id_first = (u8)RESC_START(p_hwfn, ECORE_RL);
+	params.rl_id_last = RESC_START(p_hwfn, ECORE_RL) +
+		ecore_init_qm_get_num_pf_rls(p_hwfn);
+	params.rl_stop_flg = 1;
+
+	return ecore_sp_rl_update(p_hwfn, &params);
+}
+
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST_BTH 2
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST_ECN 1
+
+enum _ecore_status_t ecore_roce_dcqcn_cfg(
+	struct ecore_hwfn			*p_hwfn,
+	struct ecore_roce_dcqcn_params		*params,
+	struct roce_init_func_ramrod_data	*p_ramrod,
+	struct ecore_ptt			*p_ptt)
+{
+	u32 val = 0;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (!p_hwfn->pf_params.rdma_pf_params.enable_dcqcn ||
+	    p_hwfn->p_rdma_info->proto == PROTOCOLID_IWARP)
+		return rc;
+
+	p_hwfn->p_rdma_info->roce.dcqcn_enabled = 0;
+	if (params->notification_point) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Configuring dcqcn notification point: timeout = 0x%x\n",
+			   params->cnp_send_timeout);
+		p_ramrod->roce.cnp_send_timeout = params->cnp_send_timeout;
+		p_hwfn->p_rdma_info->roce.dcqcn_enabled = 1;
+		/* Configure NIG to duplicate to host and storm when:
+		 *  - (ECN == 2'b11 (notification point)
+		 */
+		val |= 1 << NIG_REG_ROCE_DUPLICATE_TO_HOST_ECN;
+	}
+
+	if (params->reaction_point) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA,
+			   "Configuring dcqcn reaction point\n");
+		p_hwfn->p_rdma_info->roce.dcqcn_enabled = 1;
+		p_hwfn->p_rdma_info->roce.dcqcn_reaction_point = 1;
+		/* Configure NIG to duplicate to host and storm when:
+		 * - BTH opcode equals bth_hdr_flow_ctrl_opcode_2
+		 * (reaction point)
+		 */
+		val |= 1 << NIG_REG_ROCE_DUPLICATE_TO_HOST_BTH;
+
+		rc = ecore_roce_start_rl(p_hwfn, params);
+	}
+
+	if (rc)
+		return rc;
+
+	p_ramrod->roce.cnp_dscp = params->cnp_dscp;
+	p_ramrod->roce.cnp_vlan_priority = params->cnp_vlan_priority;
+
+	ecore_wr(p_hwfn,
+		 p_ptt,
+		 NIG_REG_ROCE_DUPLICATE_TO_HOST,
+		 val);
+
+	return rc;
+}
+#endif
+
+
+enum _ecore_status_t ecore_roce_stop(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_bmap *cid_map = &p_hwfn->p_rdma_info->cid_map;
+	int wait_count = 0;
+
+	/* when destroying a_RoCE QP the control is returned to the
+	 * user after the synchronous part. The asynchronous part may
+	 * take a little longer. We delay for a short while if an
+	 * asyn destroy QP is still expected. Beyond the added delay
+	 * we clear the bitmap anyway.
+	 */
+	while (OSAL_BITMAP_WEIGHT(cid_map->bitmap, cid_map->max_count)) {
+		OSAL_MSLEEP(100);
+		if (wait_count++ > 20) {
+			DP_NOTICE(p_hwfn, false,
+				  "cid bitmap wait timed out\n");
+			break;
+		}
+	}
+
+	ecore_spq_unregister_async_cb(p_hwfn, PROTOCOLID_ROCE);
+
+	return ECORE_SUCCESS;
+}
+
+
+static void ecore_rdma_copy_gids(struct ecore_rdma_qp *qp, __le32 *src_gid,
+				 __le32 *dst_gid) {
+	u32 i;
+
+	if (qp->roce_mode == ROCE_V2_IPV4) {
+		/* The IPv4 addresses shall be aligned to the highest word.
+		 * The lower words must be zero.
+		 */
+		OSAL_MEMSET(src_gid, 0, sizeof(union ecore_gid));
+		OSAL_MEMSET(dst_gid, 0, sizeof(union ecore_gid));
+		src_gid[3] = OSAL_CPU_TO_LE32(qp->sgid.ipv4_addr);
+		dst_gid[3] = OSAL_CPU_TO_LE32(qp->dgid.ipv4_addr);
+	} else {
+		/* RoCE, and RoCE v2 - IPv6: GIDs and IPv6 addresses coincide in
+		 * location and size
+		 */
+		for (i = 0; i < OSAL_ARRAY_SIZE(qp->sgid.dwords); i++) {
+			src_gid[i] = OSAL_CPU_TO_LE32(qp->sgid.dwords[i]);
+			dst_gid[i] = OSAL_CPU_TO_LE32(qp->dgid.dwords[i]);
+		}
+	}
+}
+
+static enum roce_flavor ecore_roce_mode_to_flavor(enum roce_mode roce_mode)
+{
+	enum roce_flavor flavor;
+
+	switch (roce_mode) {
+	case ROCE_V1:
+		flavor = PLAIN_ROCE;
+		break;
+	case ROCE_V2_IPV4:
+		flavor = RROCE_IPV4;
+		break;
+	case ROCE_V2_IPV6:
+		flavor = (enum roce_flavor)ROCE_V2_IPV6;
+		break;
+	default:
+		flavor = (enum roce_flavor)MAX_ROCE_MODE;
+		break;
+	}
+	return flavor;
+}
+
+#if 0
+static void ecore_roce_free_cid_pair(struct ecore_hwfn *p_hwfn, u16 cid)
+{
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, cid);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, cid + 1);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+#endif
+
+static void ecore_roce_free_qp(struct ecore_hwfn *p_hwfn, u16 qp_idx)
+{
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, qp_idx);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+#define ECORE_ROCE_CREATE_QP_ATTEMPTS		(20)
+#define ECORE_ROCE_CREATE_QP_MSLEEP		(10)
+
+static enum _ecore_status_t ecore_roce_wait_free_cids(struct ecore_hwfn *p_hwfn, u32 qp_idx)
+{
+	struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+	bool cids_free = false;
+	u32 icid, iter = 0;
+	int req, resp;
+
+	icid = ECORE_ROCE_QP_TO_ICID(qp_idx);
+
+	/* Make sure that the cids that were used by the QP index are free.
+	 * This is necessary because the destroy flow returns to the user before
+	 * the device finishes clean up.
+	 * It can happen in the following flows:
+	 * (1) ib_destroy_qp followed by an ib_create_qp
+	 * (2) ib_modify_qp to RESET followed (not immediately), by an
+	 *     ib_modify_qp to RTR
+	 */
+
+	do {
+		OSAL_SPIN_LOCK(&p_rdma_info->lock);
+		resp = ecore_bmap_test_id(p_hwfn, &p_rdma_info->cid_map, icid);
+		req = ecore_bmap_test_id(p_hwfn, &p_rdma_info->cid_map, icid + 1);
+		if (!resp && !req)
+			cids_free = true;
+
+		OSAL_SPIN_UNLOCK(&p_rdma_info->lock);
+
+		if (!cids_free) {
+			OSAL_MSLEEP(ECORE_ROCE_CREATE_QP_MSLEEP);
+			iter++;
+		}
+	} while (!cids_free && iter < ECORE_ROCE_CREATE_QP_ATTEMPTS);
+
+	if (!cids_free) {
+		DP_ERR(p_hwfn->p_dev,
+		       "responder and/or requester CIDs are still in use. resp=%d, req=%d\n",
+		       resp, req);
+		return ECORE_AGAIN;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_roce_alloc_qp_idx(
+		struct ecore_hwfn *p_hwfn, u16 *qp_idx16)
+{
+	struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+	u32 start_cid, icid, cid, qp_idx;
+	enum _ecore_status_t rc;
+
+	OSAL_SPIN_LOCK(&p_rdma_info->lock);
+	rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->qp_map, &qp_idx);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "failed to allocate qp\n");
+		OSAL_SPIN_UNLOCK(&p_rdma_info->lock);
+		return rc;
+	}
+
+	OSAL_SPIN_UNLOCK(&p_rdma_info->lock);
+
+	/* Verify the cid bits that of this qp index are clear */
+	rc = ecore_roce_wait_free_cids(p_hwfn, qp_idx);
+	if (rc) {
+		rc = ECORE_UNKNOWN_ERROR;
+		goto err;
+	}
+
+	/* Allocate a DMA-able context for an ILT page, if not existing, for the
+	 * associated iids.
+	 * Note: If second allocation fails there's no need to free the first as
+	 *       it will be used in the future.
+	 */
+	icid = ECORE_ROCE_QP_TO_ICID(qp_idx);
+	start_cid = ecore_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto);
+	cid = start_cid + icid;
+
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, cid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, cid + 1);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* qp index is under 2^16 */
+	*qp_idx16 = (u16)qp_idx;
+
+	return ECORE_SUCCESS;
+
+err:
+	ecore_roce_free_qp(p_hwfn, (u16)qp_idx);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+
+	return rc;
+}
+
+static void ecore_roce_set_cid(struct ecore_hwfn *p_hwfn,
+			     u32 cid)
+{
+	OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock);
+	ecore_bmap_set_id(p_hwfn,
+			  &p_hwfn->p_rdma_info->cid_map,
+			  cid);
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+static enum _ecore_status_t ecore_roce_sp_create_responder(
+	struct ecore_hwfn    *p_hwfn,
+	struct ecore_rdma_qp *qp)
+{
+	struct roce_create_qp_resp_ramrod_data *p_ramrod;
+	u16 regular_latency_queue, low_latency_queue;
+	struct ecore_sp_init_data init_data;
+	enum roce_flavor roce_flavor;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+	u32 cid_start;
+	u16 fw_srq_id;
+	bool is_xrc;
+
+	if (!qp->has_resp)
+		return ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "qp_idx = %08x\n", qp->qp_idx);
+
+	/* Allocate DMA-able memory for IRQ */
+	qp->irq_num_pages = 1;
+	qp->irq = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+					  &qp->irq_phys_addr,
+					  RDMA_RING_PAGE_SIZE);
+	if (!qp->irq) {
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "ecore create responder failed: cannot allocate memory (irq). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_create_qp_resp;
+
+	p_ramrod->flags = 0;
+
+	roce_flavor = ecore_roce_mode_to_flavor(qp->roce_mode);
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR,
+		  roce_flavor);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
+		  qp->incoming_rdma_read_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
+		  qp->incoming_rdma_write_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN,
+		  qp->incoming_atomic_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
+		  qp->e2e_flow_control_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG,
+		  qp->use_srq);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN,
+		  qp->fmr_and_reserved_lkey);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_XRC_FLAG,
+		  ecore_rdma_is_xrc_qp(qp));
+
+	/* TBD: future use only
+	 * #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK
+	 * #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT
+	 */
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
+		  qp->min_rnr_nak_timer);
+
+	p_ramrod->max_ird =
+		qp->max_rd_atomic_resp;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->irq_num_pages = qp->irq_num_pages;
+	p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey);
+	p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label);
+	p_ramrod->dst_qp_id = OSAL_CPU_TO_LE32(qp->dest_qp);
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu);
+	p_ramrod->initial_psn = OSAL_CPU_TO_LE32(qp->rq_psn);
+	p_ramrod->pd = OSAL_CPU_TO_LE16(qp->pd);
+	p_ramrod->rq_num_pages = OSAL_CPU_TO_LE16(qp->rq_num_pages);
+	DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr);
+	DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr);
+	ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	p_ramrod->qp_handle_for_async.hi =
+			OSAL_CPU_TO_LE32(qp->qp_handle_async.hi);
+	p_ramrod->qp_handle_for_async.lo =
+			OSAL_CPU_TO_LE32(qp->qp_handle_async.lo);
+	p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi);
+	p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo);
+	p_ramrod->cq_cid = OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id);
+	p_ramrod->xrc_domain = OSAL_CPU_TO_LE16(qp->xrcd_id);
+
+#ifdef CONFIG_DCQCN
+	/* when dcqcn is enabled physical queues are determined accoridng to qp id */
+	if (p_hwfn->p_rdma_info->roce.dcqcn_enabled)
+		regular_latency_queue =
+			ecore_get_cm_pq_idx_rl(p_hwfn,
+					       (qp->icid >> 1) %
+							ROCE_DCQCN_RP_MAX_QPS);
+	else
+#endif
+		regular_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
+	low_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LLT);
+
+	p_ramrod->regular_latency_phy_queue = OSAL_CPU_TO_LE16(regular_latency_queue);
+	p_ramrod->low_latency_phy_queue = OSAL_CPU_TO_LE16(low_latency_queue);
+	p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi);
+
+	ecore_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
+	ecore_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
+
+	p_ramrod->udp_src_port = qp->udp_src_port;
+	p_ramrod->vlan_id = OSAL_CPU_TO_LE16(qp->vlan_id);
+	is_xrc = ecore_rdma_is_xrc_qp(qp);
+	fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, qp->srq_id, is_xrc);
+	p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id);
+	p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(p_hwfn->hw_info.opaque_fid);
+
+	p_ramrod->stats_counter_id = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) +
+				     qp->stats_queue;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d regular physical queue = 0x%x, low latency physical queue 0x%x\n",
+		   rc, regular_latency_queue, low_latency_queue);
+
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	qp->resp_offloaded = true;
+	qp->cq_prod.resp = 0;
+
+	cid_start = ecore_cxt_get_proto_cid_start(p_hwfn,
+						  p_hwfn->p_rdma_info->proto);
+	ecore_roce_set_cid(p_hwfn, qp->icid - cid_start);
+
+	return rc;
+
+err:
+	DP_NOTICE(p_hwfn, false, "create responder - failed, rc = %d\n", rc);
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       qp->irq,
+			       qp->irq_phys_addr,
+			       qp->irq_num_pages *
+			       RDMA_RING_PAGE_SIZE);
+
+	return rc;
+}
+
+static enum _ecore_status_t ecore_roce_sp_create_requester(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_qp *qp)
+{
+	struct roce_create_qp_req_ramrod_data *p_ramrod;
+	u16 regular_latency_queue, low_latency_queue;
+	struct ecore_sp_init_data init_data;
+	enum roce_flavor roce_flavor;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+	u32 cid_start;
+
+	if (!qp->has_req)
+		return ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	/* Allocate DMA-able memory for ORQ */
+	qp->orq_num_pages = 1;
+	qp->orq = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+					  &qp->orq_phys_addr,
+					  RDMA_RING_PAGE_SIZE);
+	if (!qp->orq)
+	{
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "ecore create requester failed: cannot allocate memory (orq). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   ROCE_RAMROD_CREATE_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_create_qp_req;
+
+	p_ramrod->flags = 0;
+
+	roce_flavor = ecore_roce_mode_to_flavor(qp->roce_mode);
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR,
+		  roce_flavor);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN,
+		  qp->fmr_and_reserved_lkey);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP,
+		  qp->signal_all);
+
+	/* TBD:
+	 * future use only
+	 * #define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_MASK
+	 * #define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_SHIFT
+	 */
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT,
+		  qp->retry_cnt);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
+		  qp->rnr_retry_cnt);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_XRC_FLAG,
+		  ecore_rdma_is_xrc_qp(qp));
+
+	p_ramrod->max_ord = qp->max_rd_atomic_req;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->orq_num_pages = qp->orq_num_pages;
+	p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey);
+	p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label);
+	p_ramrod->dst_qp_id = OSAL_CPU_TO_LE32(qp->dest_qp);
+	p_ramrod->ack_timeout_val = OSAL_CPU_TO_LE32(qp->ack_timeout);
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu);
+	p_ramrod->initial_psn = OSAL_CPU_TO_LE32(qp->sq_psn);
+	p_ramrod->pd = OSAL_CPU_TO_LE16(qp->pd);
+	p_ramrod->sq_num_pages = OSAL_CPU_TO_LE16(qp->sq_num_pages);
+	DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr);
+	DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr);
+	ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	p_ramrod->qp_handle_for_async.hi =
+			OSAL_CPU_TO_LE32(qp->qp_handle_async.hi);
+	p_ramrod->qp_handle_for_async.lo =
+			OSAL_CPU_TO_LE32(qp->qp_handle_async.lo);
+	p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi);
+	p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo);
+	p_ramrod->cq_cid = OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) |
+				       qp->sq_cq_id);
+
+#ifdef CONFIG_DCQCN
+	/* when dcqcn is enabled physical queues are determined accoridng to qp id */
+	if (p_hwfn->p_rdma_info->roce.dcqcn_enabled)
+		regular_latency_queue =
+			ecore_get_cm_pq_idx_rl(p_hwfn,
+					       (qp->icid >> 1) %
+							ROCE_DCQCN_RP_MAX_QPS);
+	else
+#endif
+		regular_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
+	low_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LLT);
+
+	p_ramrod->regular_latency_phy_queue = OSAL_CPU_TO_LE16(regular_latency_queue);
+	p_ramrod->low_latency_phy_queue = OSAL_CPU_TO_LE16(low_latency_queue);
+	p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi);
+
+	ecore_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
+	ecore_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
+
+	p_ramrod->udp_src_port = qp->udp_src_port;
+	p_ramrod->vlan_id = OSAL_CPU_TO_LE16(qp->vlan_id);
+	p_ramrod->stats_counter_id = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) +
+				     qp->stats_queue;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc);
+
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	qp->req_offloaded = true;
+	qp->cq_prod.req = 0;
+
+	cid_start = ecore_cxt_get_proto_cid_start(p_hwfn,
+						  p_hwfn->p_rdma_info->proto);
+	ecore_roce_set_cid(p_hwfn, qp->icid + 1 - cid_start);
+
+	return rc;
+
+err:
+	DP_NOTICE(p_hwfn, false, "Create requested - failed, rc = %d\n", rc);
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       qp->orq,
+			       qp->orq_phys_addr,
+			       qp->orq_num_pages *
+			       RDMA_RING_PAGE_SIZE);
+	return rc;
+}
+
+static enum _ecore_status_t ecore_roce_sp_modify_responder(
+	struct ecore_hwfn	*p_hwfn,
+	struct ecore_rdma_qp	*qp,
+	bool			move_to_err,
+	u32			modify_flags)
+{
+	struct roce_modify_qp_resp_ramrod_data	*p_ramrod;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	enum _ecore_status_t			rc;
+
+	if (!qp->has_resp)
+		return ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (move_to_err && !qp->resp_offloaded)
+		return ECORE_SUCCESS;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   ROCE_EVENT_MODIFY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+	{
+		DP_NOTICE(p_hwfn, false, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.roce_modify_qp_resp;
+
+	p_ramrod->flags = 0;
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG,
+		  move_to_err);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
+		  qp->incoming_rdma_read_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
+		  qp->incoming_rdma_write_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN,
+		  qp->incoming_atomic_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
+		  qp->e2e_flow_control_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG,
+		  GET_FIELD(modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP));
+
+	/* TBD: future use only
+	 * #define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_MASK
+	 * #define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_SHIFT
+	 */
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER));
+
+	p_ramrod->fields = 0;
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
+		  qp->min_rnr_nak_timer);
+
+	p_ramrod->max_ird = qp->max_rd_atomic_resp;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey);
+	p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label);
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu);
+	ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify responder, rc = %d\n", rc);
+	return rc;
+}
+
+static enum _ecore_status_t ecore_roce_sp_modify_requester(
+	struct ecore_hwfn	*p_hwfn,
+	struct ecore_rdma_qp	*qp,
+	bool			move_to_sqd,
+	bool			move_to_err,
+	u32			modify_flags)
+{
+	struct roce_modify_qp_req_ramrod_data	*p_ramrod;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	enum _ecore_status_t			rc;
+
+	if (!qp->has_req)
+		return ECORE_SUCCESS;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (move_to_err && !(qp->req_offloaded))
+		return ECORE_SUCCESS;
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   ROCE_EVENT_MODIFY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_hwfn, false, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.roce_modify_qp_req;
+
+	p_ramrod->flags = 0;
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG,
+		  move_to_err);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG,
+		  move_to_sqd);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY,
+		  qp->sqd_async);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG,
+		  GET_FIELD(modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG,
+		  GET_FIELD(modify_flags,
+			    ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT));
+
+	/* TBD: future use only
+	 * #define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_MASK
+	 * #define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_SHIFT
+	 */
+
+	p_ramrod->fields = 0;
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT,
+		  qp->retry_cnt);
+
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
+		  qp->rnr_retry_cnt);
+
+	p_ramrod->max_ord = qp->max_rd_atomic_req;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey);
+	p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label);
+	p_ramrod->ack_timeout_val = OSAL_CPU_TO_LE32(qp->ack_timeout);
+	p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu);
+	ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify requester, rc = %d\n", rc);
+	return rc;
+}
+
+static enum _ecore_status_t ecore_roce_sp_destroy_qp_responder(
+	struct ecore_hwfn    *p_hwfn,
+	struct ecore_rdma_qp *qp,
+	u32                  *num_invalidated_mw,
+	u32                  *cq_prod)
+{
+	struct roce_destroy_qp_resp_output_params	*p_ramrod_res;
+	struct roce_destroy_qp_resp_ramrod_data *p_ramrod;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	dma_addr_t				ramrod_res_phys;
+	enum _ecore_status_t			rc;
+
+	if (!qp->has_resp) {
+		*num_invalidated_mw = 0;
+		*cq_prod = 0;
+		return ECORE_SUCCESS;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	*num_invalidated_mw = 0;
+
+	if (!qp->resp_offloaded) {
+		*cq_prod = qp->cq_prod.resp;
+		return ECORE_SUCCESS;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent,
+				   ROCE_RAMROD_DESTROY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp;
+
+	p_ramrod_res = (struct roce_destroy_qp_resp_output_params *)OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev,
+		&ramrod_res_phys, sizeof(*p_ramrod_res));
+
+	if (!p_ramrod_res)
+	{
+		rc = ECORE_NOMEM;
+		DP_NOTICE(p_hwfn, false,
+			  "ecore destroy responder failed: cannot allocate memory (ramrod). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	*num_invalidated_mw
+		= OSAL_LE32_TO_CPU(p_ramrod_res->num_invalidated_mw);
+	*cq_prod = OSAL_LE32_TO_CPU(p_ramrod_res->cq_prod);
+	qp->cq_prod.resp = *cq_prod;
+
+	/* Free IRQ - only if ramrod succeeded, in case FW is still using it */
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       qp->irq,
+			       qp->irq_phys_addr,
+			       qp->irq_num_pages *
+			       RDMA_RING_PAGE_SIZE);
+
+	qp->resp_offloaded = false;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroy responder, rc = %d\n", rc);
+
+	/* "fall through" */
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+		sizeof(*p_ramrod_res));
+
+	return rc;
+}
+
+static enum _ecore_status_t ecore_roce_sp_destroy_qp_requester(
+	struct ecore_hwfn    *p_hwfn,
+	struct ecore_rdma_qp *qp,
+	u32                  *num_bound_mw,
+	u32                  *cq_prod)
+{
+	struct roce_destroy_qp_req_output_params	*p_ramrod_res;
+	struct roce_destroy_qp_req_ramrod_data	*p_ramrod;
+	struct ecore_sp_init_data		init_data;
+	struct ecore_spq_entry			*p_ent;
+	dma_addr_t				ramrod_res_phys;
+	enum _ecore_status_t			rc;
+
+	if (!qp->has_req) {
+		*num_bound_mw = 0;
+		*cq_prod = 0;
+		return ECORE_SUCCESS;
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (!qp->req_offloaded) {
+		*cq_prod = qp->cq_prod.req;
+		return ECORE_SUCCESS;
+	}
+
+	p_ramrod_res = (struct roce_destroy_qp_req_output_params *)
+			OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys,
+				sizeof(*p_ramrod_res));
+	if (!p_ramrod_res)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore destroy requester failed: cannot allocate memory (ramrod)\n");
+		return ECORE_NOMEM;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_destroy_qp_req;
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	*num_bound_mw = OSAL_LE32_TO_CPU(p_ramrod_res->num_bound_mw);
+	*cq_prod = OSAL_LE32_TO_CPU(p_ramrod_res->cq_prod);
+	qp->cq_prod.req = *cq_prod;
+
+	/* Free ORQ - only if ramrod succeeded, in case FW is still using it */
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
+			       qp->orq,
+			       qp->orq_phys_addr,
+			       qp->orq_num_pages *
+			       RDMA_RING_PAGE_SIZE);
+
+	qp->req_offloaded = false;
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroy requester, rc = %d\n", rc);
+
+	/* "fall through" */
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys,
+			       sizeof(*p_ramrod_res));
+
+	return rc;
+}
+
+static OSAL_INLINE enum _ecore_status_t ecore_roce_sp_query_responder(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_qp *qp,
+	struct ecore_rdma_query_qp_out_params *out_params)
+{
+	struct roce_query_qp_resp_output_params	*p_resp_ramrod_res;
+	struct roce_query_qp_resp_ramrod_data	*p_resp_ramrod;
+	struct ecore_sp_init_data		init_data;
+	dma_addr_t				resp_ramrod_res_phys;
+	struct ecore_spq_entry			*p_ent;
+	enum _ecore_status_t			rc = ECORE_SUCCESS;
+	bool					error_flag;
+
+	if (!qp->resp_offloaded) {
+		/* Don't send query qp for the responder */
+		out_params->rq_psn = qp->rq_psn;
+
+		return  ECORE_SUCCESS;
+	}
+
+	/* Send a query responder ramrod to the FW */
+	p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *)
+		OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &resp_ramrod_res_phys,
+					sizeof(*p_resp_ramrod_res));
+	if (!p_resp_ramrod_res)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore query qp failed: cannot allocate memory (ramrod)\n");
+		return ECORE_NOMEM;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp;
+	DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	out_params->rq_psn = OSAL_LE32_TO_CPU(p_resp_ramrod_res->psn);
+	error_flag = GET_FIELD(
+			OSAL_LE32_TO_CPU(p_resp_ramrod_res->err_flag),
+			ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG);
+	if (error_flag)
+		qp->cur_state = ECORE_ROCE_QP_STATE_ERR;
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_resp_ramrod_res,
+			       resp_ramrod_res_phys,
+			       sizeof(*p_resp_ramrod_res));
+
+	return rc;
+}
+
+static OSAL_INLINE enum _ecore_status_t ecore_roce_sp_query_requester(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_qp *qp,
+	struct ecore_rdma_query_qp_out_params *out_params,
+	bool *sq_draining)
+{
+	struct roce_query_qp_req_output_params	*p_req_ramrod_res;
+	struct roce_query_qp_req_ramrod_data	*p_req_ramrod;
+	struct ecore_sp_init_data		init_data;
+	dma_addr_t				req_ramrod_res_phys;
+	struct ecore_spq_entry			*p_ent;
+	enum _ecore_status_t			rc = ECORE_SUCCESS;
+	bool					error_flag;
+
+	if (!qp->req_offloaded)
+	{
+		/* Don't send query qp for the requester */
+		out_params->sq_psn = qp->sq_psn;
+		out_params->draining = false;
+
+		*sq_draining = 0;
+
+		return ECORE_SUCCESS;
+	}
+
+	/* Send a query requester ramrod to the FW */
+	p_req_ramrod_res = (struct roce_query_qp_req_output_params *)
+		OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &req_ramrod_res_phys,
+					sizeof(*p_req_ramrod_res));
+	if (!p_req_ramrod_res)
+	{
+		DP_NOTICE(p_hwfn, false,
+			  "ecore query qp failed: cannot allocate memory (ramrod). rc = %d\n",
+			  rc);
+		return ECORE_NOMEM;
+	}
+
+	/* Get SPQ entry */
+	init_data.cid = qp->icid + 1;
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	p_req_ramrod = &p_ent->ramrod.roce_query_qp_req;
+	DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys);
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	out_params->sq_psn = OSAL_LE32_TO_CPU(p_req_ramrod_res->psn);
+	error_flag = GET_FIELD(OSAL_LE32_TO_CPU(p_req_ramrod_res->flags),
+			       ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG);
+	if (error_flag)
+		qp->cur_state = ECORE_ROCE_QP_STATE_ERR;
+	else
+		*sq_draining = GET_FIELD(
+			OSAL_LE32_TO_CPU(p_req_ramrod_res->flags),
+			ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG);
+
+err:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_req_ramrod_res,
+			       req_ramrod_res_phys, sizeof(*p_req_ramrod_res));
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_roce_query_qp(
+	struct ecore_hwfn *p_hwfn,
+	struct ecore_rdma_qp *qp,
+	struct ecore_rdma_query_qp_out_params *out_params)
+{
+	enum _ecore_status_t	rc;
+
+	rc = ecore_roce_sp_query_responder(p_hwfn, qp, out_params);
+	if (rc)
+		return rc;
+
+	rc = ecore_roce_sp_query_requester(p_hwfn, qp, out_params,
+					   &out_params->draining);
+	if (rc)
+		return rc;
+
+	out_params->state = qp->cur_state;
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_roce_destroy_qp(struct ecore_hwfn *p_hwfn,
+					   struct ecore_rdma_qp *qp,
+					   struct ecore_rdma_destroy_qp_out_params *out_params)
+{
+	u32 cq_prod_resp = qp->cq_prod.resp, cq_prod_req = qp->cq_prod.req;
+	u32 num_invalidated_mw = 0;
+	u32 num_bound_mw = 0;
+	enum _ecore_status_t rc;
+
+	/* Destroys the specified QP
+	 * Note: if qp state != RESET/ERR/INIT then upper driver first need to
+	 * call modify qp to move the qp to ERR state
+	 */
+	if ((qp->cur_state != ECORE_ROCE_QP_STATE_RESET) &&
+	    (qp->cur_state != ECORE_ROCE_QP_STATE_ERR) &&
+	    (qp->cur_state != ECORE_ROCE_QP_STATE_INIT))
+	{
+		DP_NOTICE(p_hwfn,
+			  true,
+		   "QP must be in error, reset or init state before destroying it\n");
+		return ECORE_INVAL;
+	}
+
+	if (qp->cur_state != ECORE_ROCE_QP_STATE_RESET) {
+		rc = ecore_roce_sp_destroy_qp_responder(p_hwfn,
+							qp,
+							&num_invalidated_mw,
+							&cq_prod_resp);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		/* Send destroy requester ramrod */
+		rc = ecore_roce_sp_destroy_qp_requester(p_hwfn, qp,
+							&num_bound_mw,
+							&cq_prod_req);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		/* resp_ofload was true, num_invalidated_mw is valid */
+		if (num_invalidated_mw != num_bound_mw) {
+			DP_NOTICE(p_hwfn,
+				  true,
+				  "number of invalidate memory windows is different from bounded ones\n");
+			return ECORE_INVAL;
+		}
+	}
+
+	ecore_roce_free_qp(p_hwfn, qp->qp_idx);
+
+	out_params->rq_cq_prod = cq_prod_resp;
+	out_params->sq_cq_prod = cq_prod_req;
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_roce_destroy_ud_qp(void *rdma_cxt, u16 cid)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+
+	if (!rdma_cxt) {
+		DP_ERR(p_hwfn->p_dev,
+		       "destroy ud qp failed due to NULL rdma_cxt\n");
+		return ECORE_INVAL;
+	}
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_UD_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	ecore_roce_free_qp(p_hwfn, ECORE_ROCE_ICID_TO_QP(cid));
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "freed a ud qp with cid=%d\n", cid);
+
+	return ECORE_SUCCESS;
+
+err:
+	DP_ERR(p_hwfn, "failed destroying a ud qp with cid=%d\n", cid);
+
+	return rc;
+}
+
+
+enum _ecore_status_t ecore_roce_create_ud_qp(void		*rdma_cxt,
+			struct ecore_rdma_create_qp_out_params	*out_params)
+{
+	struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt;
+	struct ecore_sp_init_data init_data;
+	struct ecore_spq_entry *p_ent;
+	enum _ecore_status_t rc;
+	u16 icid, qp_idx;
+
+	if (!rdma_cxt || !out_params) {
+		DP_ERR(p_hwfn->p_dev,
+		       "ecore roce create ud qp failed due to NULL entry (rdma_cxt=%p, out=%p)\n",
+		       rdma_cxt, out_params);
+		return ECORE_INVAL;
+	}
+
+	rc = ecore_roce_alloc_qp_idx(p_hwfn, &qp_idx);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	icid = ECORE_ROCE_QP_TO_ICID(qp_idx);
+
+	/* Get SPQ entry */
+	OSAL_MEMSET(&init_data, 0, sizeof(init_data));
+	init_data.cid = icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK;
+	rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_UD_QP,
+				   PROTOCOLID_ROCE, &init_data);
+	if (rc != ECORE_SUCCESS)
+		goto err1;
+
+	rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
+	if (rc != ECORE_SUCCESS)
+		goto err1;
+
+	out_params->icid = icid;
+	out_params->qp_id = ((0xFF << 16) | icid);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "created a ud qp with icid=%d\n",
+		   icid);
+
+	return ECORE_SUCCESS;
+
+err1:
+	ecore_roce_free_qp(p_hwfn, qp_idx);
+
+err:
+	DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "failed creating a ud qp\n");
+
+	return rc;
+}
+
+
+enum _ecore_status_t
+ecore_roce_modify_qp(struct ecore_hwfn *p_hwfn,
+		     struct ecore_rdma_qp *qp,
+		     enum ecore_roce_qp_state prev_state,
+		     struct ecore_rdma_modify_qp_in_params *params)
+{
+	u32 num_invalidated_mw = 0, num_bound_mw = 0;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	/* Perform additional operations according to the current state and the
+	 * next state
+	 */
+	if (((prev_state == ECORE_ROCE_QP_STATE_INIT) ||
+	     (prev_state == ECORE_ROCE_QP_STATE_RESET)) &&
+	    (qp->cur_state == ECORE_ROCE_QP_STATE_RTR))
+	{
+		/* Init->RTR or Reset->RTR */
+
+		/* Verify the cid bits that of this qp index are clear */
+		rc = ecore_roce_wait_free_cids(p_hwfn, qp->qp_idx);
+		if (rc)
+			return rc;
+
+		rc = ecore_roce_sp_create_responder(p_hwfn, qp);
+		return rc;
+
+	} else if ((prev_state == ECORE_ROCE_QP_STATE_RTR) &&
+		   (qp->cur_state == ECORE_ROCE_QP_STATE_RTS))
+	{
+		/* RTR-> RTS */
+		rc = ecore_roce_sp_create_requester(p_hwfn, qp);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		/* Send modify responder ramrod */
+		rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false,
+						    params->modify_flags);
+		return rc;
+
+	} else if ((prev_state == ECORE_ROCE_QP_STATE_RTS) &&
+		   (qp->cur_state == ECORE_ROCE_QP_STATE_RTS))
+	{
+		/* RTS->RTS */
+		rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false,
+						    params->modify_flags);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, false,
+						    params->modify_flags);
+		return rc;
+
+	} else if ((prev_state == ECORE_ROCE_QP_STATE_RTS) &&
+		   (qp->cur_state == ECORE_ROCE_QP_STATE_SQD))
+	{
+		/* RTS->SQD */
+		rc = ecore_roce_sp_modify_requester(p_hwfn, qp, true, false,
+						    params->modify_flags);
+		return rc;
+
+	} else if ((prev_state == ECORE_ROCE_QP_STATE_SQD) &&
+		   (qp->cur_state == ECORE_ROCE_QP_STATE_SQD))
+	{
+		/* SQD->SQD */
+		rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false,
+						    params->modify_flags);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		rc = ecore_roce_sp_modify_requester(p_hwfn,  qp, false, false,
+						    params->modify_flags);
+		return rc;
+
+	} else if ((prev_state == ECORE_ROCE_QP_STATE_SQD) &&
+		 (qp->cur_state == ECORE_ROCE_QP_STATE_RTS))
+	{
+		/* SQD->RTS */
+		rc = ecore_roce_sp_modify_responder(p_hwfn,  qp, false,
+						    params->modify_flags);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		rc = ecore_roce_sp_modify_requester(p_hwfn,  qp, false, false,
+						    params->modify_flags);
+
+		return rc;
+	} else if (qp->cur_state == ECORE_ROCE_QP_STATE_ERR) {
+		/* ->ERR */
+		rc = ecore_roce_sp_modify_responder(p_hwfn, qp, true,
+						    params->modify_flags);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, true,
+						    params->modify_flags);
+		return rc;
+
+	} else if (qp->cur_state == ECORE_ROCE_QP_STATE_RESET) {
+		/* Any state -> RESET */
+
+		/* Send destroy responder ramrod */
+		rc = ecore_roce_sp_destroy_qp_responder(p_hwfn, qp,
+							&num_invalidated_mw,
+							&qp->cq_prod.resp);
+
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		rc = ecore_roce_sp_destroy_qp_requester(p_hwfn, qp,
+							&num_bound_mw,
+							&qp->cq_prod.req);
+
+
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		if (num_invalidated_mw != num_bound_mw) {
+			DP_NOTICE(p_hwfn,
+				  true,
+				  "number of invalidate memory windows is different from bounded ones\n");
+			return ECORE_INVAL;
+		}
+	} else {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_SUCCESS\n");
+	}
+
+	return rc;
+}
+
+static void ecore_roce_free_icid(struct ecore_hwfn *p_hwfn, u16 icid)
+{
+	struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+	u32 start_cid, cid;
+
+	start_cid = ecore_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto);
+	cid = icid - start_cid;
+
+	OSAL_SPIN_LOCK(&p_rdma_info->lock);
+
+	ecore_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, cid);
+
+	OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock);
+}
+
+static void ecore_rdma_dpm_conf(struct ecore_hwfn *p_hwfn,
+				struct ecore_ptt *p_ptt)
+{
+	u32 val;
+
+	val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1;
+
+	ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val);
+	DP_VERBOSE(p_hwfn, (ECORE_MSG_DCB | ECORE_MSG_RDMA),
+		   "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n",
+		   val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm);
+}
+
+/* This function disables EDPM due to DCBx considerations */
+void ecore_roce_dpm_dcbx(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
+{
+	u8 val;
+
+	/* if any QPs are already active, we want to disable DPM, since their
+	 * context information contains information from before the latest DCBx
+	 * update. Otherwise enable it.
+	 */
+	val = (ecore_rdma_allocated_qps(p_hwfn)) ? true : false;
+	p_hwfn->dcbx_no_edpm = (u8)val;
+
+	ecore_rdma_dpm_conf(p_hwfn, p_ptt);
+}
+
+/* This function disables EDPM due to doorbell bar considerations */
+void ecore_rdma_dpm_bar(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
+{
+	p_hwfn->db_bar_no_edpm = true;
+
+	ecore_rdma_dpm_conf(p_hwfn, p_ptt);
+}
+
+enum _ecore_status_t ecore_roce_setup(struct ecore_hwfn *p_hwfn)
+{
+	return ecore_spq_register_async_cb(p_hwfn, PROTOCOLID_ROCE,
+					   ecore_roce_async_event);
+}
+
+#ifdef _NTDDK_
+#pragma warning(pop)
+#endif
diff --git a/sys/dev/qlnx/qlnxe/qlnx_rdma.c b/sys/dev/qlnx/qlnxe/qlnx_rdma.c
new file mode 100644
index 000000000000..dc105e1e9e45
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/qlnx_rdma.c
@@ -0,0 +1,347 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * File : qlnx_rdma.c
+ * Author: David C Somayajulu
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+
+#include "qlnx_os.h"
+#include "bcm_osal.h"
+
+#include "reg_addr.h"
+#include "ecore_gtt_reg_addr.h"
+#include "ecore.h"
+#include "ecore_chain.h"
+#include "ecore_status.h"
+#include "ecore_hw.h"
+#include "ecore_rt_defs.h"
+#include "ecore_init_ops.h"
+#include "ecore_int.h"
+#include "ecore_cxt.h"
+#include "ecore_spq.h"
+#include "ecore_init_fw_funcs.h"
+#include "ecore_sp_commands.h"
+#include "ecore_dev_api.h"
+#include "ecore_l2_api.h"
+#ifdef CONFIG_ECORE_SRIOV
+#include "ecore_sriov.h"
+#include "ecore_vf.h"
+#endif
+#ifdef CONFIG_ECORE_LL2
+#include "ecore_ll2.h"
+#endif
+#ifdef CONFIG_ECORE_FCOE
+#include "ecore_fcoe.h"
+#endif
+#ifdef CONFIG_ECORE_ISCSI
+#include "ecore_iscsi.h"
+#endif
+#include "ecore_mcp.h"
+#include "ecore_hw_defs.h"
+#include "mcp_public.h"
+
+#ifdef CONFIG_ECORE_RDMA
+#include "ecore_rdma.h"
+#endif
+
+#ifdef CONFIG_ECORE_ROCE
+#include "ecore_roce.h"
+#endif
+
+#ifdef CONFIG_ECORE_IWARP
+#include "ecore_iwarp.h"
+#endif
+
+#include "ecore_iro.h"
+#include "nvm_cfg.h"
+#include "ecore_dev_api.h"
+#include "ecore_dbg_fw_funcs.h"
+
+#include "qlnx_ioctl.h"
+#include "qlnx_def.h"
+#include "qlnx_rdma.h"
+#include "qlnx_ver.h"
+#include <sys/smp.h>
+
+struct mtx qlnx_rdma_dev_lock;
+struct qlnx_rdma_if *qlnx_rdma_if = NULL;
+
+qlnx_host_t *qlnx_host_list = NULL;
+
+void
+qlnx_rdma_init(void)
+{
+	if (!mtx_initialized(&qlnx_rdma_dev_lock)) {
+		mtx_init(&qlnx_rdma_dev_lock, "qlnx_rdma_dev_lock", NULL, MTX_DEF);
+	}
+	return;
+}
+
+void
+qlnx_rdma_deinit(void)
+{
+	if (mtx_initialized(&qlnx_rdma_dev_lock) && (qlnx_host_list == NULL)) {
+		mtx_destroy(&qlnx_rdma_dev_lock);
+	}
+	return;
+}
+
+static void
+_qlnx_rdma_dev_add(struct qlnx_host *ha)
+{
+	QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+
+	if (qlnx_rdma_if == NULL) 
+		return;
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP && 
+		ha->personality != ECORE_PCI_ETH_ROCE)
+		return;
+
+	ha->qlnx_rdma = qlnx_rdma_if->add(ha);
+
+	QL_DPRINT12(ha, "exit (ha = %p, qlnx_rdma = %p)\n", ha, ha->qlnx_rdma);
+	return;
+}
+
+void
+qlnx_rdma_dev_add(struct qlnx_host *ha)
+{
+	QL_DPRINT12(ha, "enter ha = %p\n", ha);
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP &&
+			ha->personality != ECORE_PCI_ETH_ROCE)
+		return;
+
+	mtx_lock(&qlnx_rdma_dev_lock);
+
+	if (qlnx_host_list == NULL) {
+		qlnx_host_list = ha;
+		ha->next = NULL;
+	} else {
+		ha->next = qlnx_host_list;
+		qlnx_host_list = ha;
+	}
+
+	mtx_unlock(&qlnx_rdma_dev_lock);
+
+	_qlnx_rdma_dev_add(ha);
+	
+	QL_DPRINT12(ha, "exit (%p)\n", ha);
+
+	return;
+}
+
+static int
+_qlnx_rdma_dev_remove(struct qlnx_host *ha)
+{
+	int ret = 0;
+
+	QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+
+	if (qlnx_rdma_if == NULL)
+		return (ret);
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP &&
+		ha->personality != ECORE_PCI_ETH_ROCE)
+		return (ret);
+
+	ret = qlnx_rdma_if->remove(ha, ha->qlnx_rdma);
+
+	QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+	return (ret);
+}
+
+int
+qlnx_rdma_dev_remove(struct qlnx_host *ha)
+{
+	int ret = 0;
+	qlnx_host_t *ha_prev;
+	qlnx_host_t *ha_cur;
+
+	QL_DPRINT12(ha, "enter ha = %p\n", ha);
+
+	if ((qlnx_host_list == NULL) || (ha == NULL))
+		return (ret);
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP &&
+		ha->personality != ECORE_PCI_ETH_ROCE)
+		return (ret);
+
+	ret = _qlnx_rdma_dev_remove(ha);
+
+	if (ret)
+		return (ret);
+
+	mtx_lock(&qlnx_rdma_dev_lock);
+
+	if (qlnx_host_list == ha) {
+		qlnx_host_list = ha->next;
+		ha->next = NULL;
+		mtx_unlock(&qlnx_rdma_dev_lock);
+		QL_DPRINT12(ha, "exit0 ha = %p\n", ha);
+		return (ret);
+	}
+
+	ha_prev = ha_cur = qlnx_host_list;
+
+	while ((ha_cur != ha) && (ha_cur != NULL)) {
+		ha_prev = ha_cur;
+		ha_cur = ha_cur->next;
+	}
+
+	if (ha_cur == ha) {
+		ha_prev = ha->next;
+		ha->next = NULL;
+	}
+
+	mtx_unlock(&qlnx_rdma_dev_lock);
+
+	QL_DPRINT12(ha, "exit1 ha = %p\n", ha);
+	return (ret);
+}
+
+int
+qlnx_rdma_register_if(qlnx_rdma_if_t *rdma_if)
+{
+	qlnx_host_t *ha;
+
+	if (mtx_initialized(&qlnx_rdma_dev_lock)) {
+
+		mtx_lock(&qlnx_rdma_dev_lock);
+		qlnx_rdma_if = rdma_if;
+
+		ha = qlnx_host_list;
+
+		while (ha != NULL) {
+			_qlnx_rdma_dev_add(ha);
+			ha = ha->next;
+		}
+
+		mtx_unlock(&qlnx_rdma_dev_lock);
+
+		return (0);
+	}
+
+	return (-1);
+}
+
+int
+qlnx_rdma_deregister_if(qlnx_rdma_if_t *rdma_if)
+{
+	int ret = 0;
+	qlnx_host_t *ha;
+
+	printf("%s: enter rdma_if = %p\n", __func__, rdma_if);
+
+        if (mtx_initialized(&qlnx_rdma_dev_lock)) {
+
+                mtx_lock(&qlnx_rdma_dev_lock);
+
+		ha = qlnx_host_list;
+
+		while (ha != NULL) {
+
+                	mtx_unlock(&qlnx_rdma_dev_lock);
+
+			if (ha->dbg_level & 0xF000)
+				ret = EBUSY;
+			else
+				ret = _qlnx_rdma_dev_remove(ha);
+
+        		device_printf(ha->pci_dev, "%s [%d]: ret = 0x%x\n",
+				__func__, __LINE__, ret);
+			if (ret)
+				return (ret);
+			
+                	mtx_lock(&qlnx_rdma_dev_lock);
+
+			ha->qlnx_rdma = NULL;
+
+			ha = ha->next;
+		}
+
+		if (!ret)
+			qlnx_rdma_if = NULL;
+
+                mtx_unlock(&qlnx_rdma_dev_lock);
+
+        }
+	printf("%s: exit rdma_if = %p\n", __func__, rdma_if);
+
+        return (ret);
+}
+
+
+void
+qlnx_rdma_dev_open(struct qlnx_host *ha)
+{
+	QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+
+	if (qlnx_rdma_if == NULL)
+		return;
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP && 
+		ha->personality != ECORE_PCI_ETH_ROCE)
+		return;
+
+	qlnx_rdma_if->notify(ha, ha->qlnx_rdma, QLNX_ETHDEV_UP);
+
+	QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+	return;
+}
+
+
+void
+qlnx_rdma_dev_close(struct qlnx_host *ha)
+{
+	QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+
+	if (qlnx_rdma_if == NULL)
+		return;
+
+	if (ha->personality != ECORE_PCI_ETH_IWARP && 
+		ha->personality != ECORE_PCI_ETH_ROCE)
+		return;
+
+	qlnx_rdma_if->notify(ha, ha->qlnx_rdma, QLNX_ETHDEV_DOWN);
+
+	QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if);
+	return;
+}
+
+int
+qlnx_rdma_get_num_irqs(struct qlnx_host *ha)
+{
+        return (QLNX_NUM_CNQ + ecore_rdma_get_sb_id(&ha->cdev.hwfns[0], 0) + 2);
+}
+
+
diff --git a/sys/dev/qlnx/qlnxe/qlnx_rdma.h b/sys/dev/qlnx/qlnxe/qlnx_rdma.h
new file mode 100644
index 000000000000..9b3526a9e8d7
--- /dev/null
+++ b/sys/dev/qlnx/qlnxe/qlnx_rdma.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+/*
+ * File: qlnx_rdma.h
+ * Author: David C Somayajulu
+ */
+
+#ifndef _QLNX_RDMA_H_
+#define _QLNX_RDMA_H_
+
+enum qlnx_rdma_event {
+	QLNX_ETHDEV_UP = 0x10,
+	QLNX_ETHDEV_DOWN = 0x11,
+	QLNX_ETHDEV_CHANGE_ADDR = 0x12
+};
+
+struct qlnx_rdma_if {
+	void *	(*add)(void *ha);
+	int 	(*remove)(void *ha, void *qlnx_rdma_dev);
+	void 	(*notify)(void *ha, void *qlnx_rdma_dev, enum qlnx_rdma_event);
+};
+typedef struct qlnx_rdma_if qlnx_rdma_if_t;
+
+extern int qlnx_rdma_register_if(qlnx_rdma_if_t *rdma_if);
+extern int qlnx_rdma_deregister_if(qlnx_rdma_if_t *rdma_if);
+extern int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address,
+                uint8_t *new_mac_address);
+
+#define QLNX_NUM_CNQ	1
+
+extern int qlnx_rdma_get_num_irqs(struct qlnx_host *ha);
+extern void qlnx_rdma_dev_add(struct qlnx_host *ha);
+extern void qlnx_rdma_dev_open(struct qlnx_host *ha);
+extern void qlnx_rdma_dev_close(struct qlnx_host *ha);
+extern int qlnx_rdma_dev_remove(struct qlnx_host *ha);
+extern void qlnx_rdma_changeaddr(struct qlnx_host *ha);
+
+extern void qlnx_rdma_init(void);
+extern void qlnx_rdma_deinit(void);
+
+#endif /* #ifndef _QLNX_RDMA_H_ */
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_cm.c b/sys/dev/qlnx/qlnxr/qlnxr_cm.c
new file mode 100644
index 000000000000..23c8c3000765
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_cm.c
@@ -0,0 +1,887 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qlnxr_def.h"
+#include "rdma_common.h"
+#include "qlnxr_cm.h"
+
+void
+qlnxr_inc_sw_gsi_cons(struct qlnxr_qp_hwq_info *info)
+{
+	info->gsi_cons = (info->gsi_cons + 1) % info->max_wr;
+}
+
+void
+qlnxr_store_gsi_qp_cq(struct qlnxr_dev *dev,
+		struct qlnxr_qp *qp,
+		struct ib_qp_init_attr *attrs)
+{
+	QL_DPRINT12(dev->ha, "enter\n");
+		
+	dev->gsi_qp_created = 1;
+	dev->gsi_sqcq = get_qlnxr_cq((attrs->send_cq));
+	dev->gsi_rqcq = get_qlnxr_cq((attrs->recv_cq));
+	dev->gsi_qp = qp;
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return;
+}
+
+void
+qlnxr_ll2_complete_tx_packet(void *cxt,
+		uint8_t connection_handle,
+		void *cookie,
+		dma_addr_t first_frag_addr,
+		bool b_last_fragment,
+		bool b_last_packet)
+{
+	struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt;
+	struct ecore_roce_ll2_packet *pkt = cookie;
+	struct qlnxr_cq *cq = dev->gsi_sqcq;
+	struct qlnxr_qp *qp = dev->gsi_qp;
+	unsigned long flags;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	qlnx_dma_free_coherent(&dev->ha->cdev, pkt->header.vaddr,
+			pkt->header.baddr, pkt->header.len);
+	kfree(pkt);
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	qlnxr_inc_sw_gsi_cons(&qp->sq);
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	if (cq->ibcq.comp_handler)
+		(*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return;
+}
+
+void
+qlnxr_ll2_complete_rx_packet(void *cxt,
+		struct ecore_ll2_comp_rx_data *data)
+{
+	struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt;
+	struct qlnxr_cq *cq = dev->gsi_rqcq;
+	// struct qlnxr_qp *qp = dev->gsi_qp;
+	struct qlnxr_qp *qp = NULL;
+	unsigned long flags;
+	uint32_t qp_num = 0;
+	// uint32_t delay_count = 0, gsi_cons = 0;
+	//void * dest_va;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	if (data->u.data_length_error) {
+		/* TODO: add statistic */
+	}
+
+	if (data->cookie == NULL) {
+		QL_DPRINT12(dev->ha, "cookie is NULL, bad sign\n");
+	}
+
+	qp_num = (0xFF << 16) | data->qp_id;
+
+	if (data->qp_id == 1) {
+		qp = dev->gsi_qp;
+	} else {
+		/* TODO: This will be needed for UD QP support */
+		/* For RoCEv1 this is invalid */
+		QL_DPRINT12(dev->ha, "invalid QP\n");
+		return;
+	}
+	/* note: currently only one recv sg is supported */
+	QL_DPRINT12(dev->ha, "MAD received on QP : %x\n", data->rx_buf_addr);
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	qp->rqe_wr_id[qp->rq.gsi_cons].rc =
+		data->u.data_length_error ? -EINVAL : 0;
+	qp->rqe_wr_id[qp->rq.gsi_cons].vlan_id = data->vlan;
+	/* note: length stands for data length i.e. GRH is excluded */
+	qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length =
+		data->length.data_length;
+	*((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) =
+		ntohl(data->opaque_data_0);
+	*((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) =
+		ntohs((u16)data->opaque_data_1);
+
+	qlnxr_inc_sw_gsi_cons(&qp->rq);
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	if (cq->ibcq.comp_handler)
+		(*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return;
+}
+
+void qlnxr_ll2_release_rx_packet(void *cxt,
+		u8 connection_handle,
+		void *cookie,
+		dma_addr_t rx_buf_addr,
+		bool b_last_packet)
+{
+	/* Do nothing... */
+}
+
+static void
+qlnxr_destroy_gsi_cq(struct qlnxr_dev *dev,
+		struct ib_qp_init_attr *attrs)
+{
+	struct ecore_rdma_destroy_cq_in_params iparams;
+	struct ecore_rdma_destroy_cq_out_params oparams;
+	struct qlnxr_cq *cq;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	cq = get_qlnxr_cq((attrs->send_cq));
+	iparams.icid = cq->icid;
+	ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
+	ecore_chain_free(&dev->ha->cdev, &cq->pbl);
+
+	cq = get_qlnxr_cq((attrs->recv_cq));
+	/* if a dedicated recv_cq was used, delete it too */
+	if (iparams.icid != cq->icid) {
+		iparams.icid = cq->icid;
+		ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
+		ecore_chain_free(&dev->ha->cdev, &cq->pbl);
+	}
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return;
+}
+
+static inline int
+qlnxr_check_gsi_qp_attrs(struct qlnxr_dev *dev,
+		struct ib_qp_init_attr *attrs)
+{
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	if (attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE) {
+		QL_DPRINT11(dev->ha,
+			"(attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE)\n");
+		return -EINVAL;
+	}
+
+	if (attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR) {
+		QL_DPRINT11(dev->ha,
+			"(attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR)\n");
+		return -EINVAL;
+	}
+
+	if (attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR) {
+		QL_DPRINT11(dev->ha,
+			"(attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR)\n");
+		return -EINVAL;
+	}
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return 0;
+}
+
+
+static int
+qlnxr_ll2_post_tx(struct qlnxr_dev *dev, struct ecore_roce_ll2_packet *pkt)
+{
+	enum ecore_ll2_roce_flavor_type roce_flavor;
+	struct ecore_ll2_tx_pkt_info ll2_tx_pkt;
+	int rc;
+	int i;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	memset(&ll2_tx_pkt, 0, sizeof(ll2_tx_pkt));
+
+	if (pkt->roce_mode != ROCE_V1) {
+		QL_DPRINT11(dev->ha, "roce_mode != ROCE_V1\n");
+		return (-1);
+	}
+
+	roce_flavor = (pkt->roce_mode == ROCE_V1) ?
+		ECORE_LL2_ROCE : ECORE_LL2_RROCE;
+
+	ll2_tx_pkt.num_of_bds = 1 /* hdr */ +  pkt->n_seg;
+	ll2_tx_pkt.vlan = 0; /* ??? */
+	ll2_tx_pkt.tx_dest = ECORE_LL2_TX_DEST_NW;
+	ll2_tx_pkt.ecore_roce_flavor = roce_flavor;
+	ll2_tx_pkt.first_frag = pkt->header.baddr;
+	ll2_tx_pkt.first_frag_len = pkt->header.len;
+	ll2_tx_pkt.cookie = pkt;
+	ll2_tx_pkt.enable_ip_cksum = 1; // Only for RoCEv2:IPv4
+
+	/* tx header */
+	rc = ecore_ll2_prepare_tx_packet(dev->rdma_ctx,
+			dev->gsi_ll2_handle,
+			&ll2_tx_pkt,
+			1);
+	if (rc) {
+
+		QL_DPRINT11(dev->ha, "ecore_ll2_prepare_tx_packet failed\n");
+
+		/* TX failed while posting header - release resources*/
+                qlnx_dma_free_coherent(&dev->ha->cdev,
+			pkt->header.vaddr,
+			pkt->header.baddr,
+                        pkt->header.len);
+
+		kfree(pkt);
+
+		return rc;
+	}
+
+	/* tx payload */
+	for (i = 0; i < pkt->n_seg; i++) {
+		rc = ecore_ll2_set_fragment_of_tx_packet(dev->rdma_ctx,
+						       dev->gsi_ll2_handle,
+						       pkt->payload[i].baddr,
+						       pkt->payload[i].len);
+		if (rc) {
+			/* if failed not much to do here, partial packet has
+			 * been posted we can't free memory, will need to wait
+			 * for completion
+			 */
+			QL_DPRINT11(dev->ha,
+				"ecore_ll2_set_fragment_of_tx_packet failed\n");
+			return rc;
+		}
+	}
+	struct ecore_ll2_stats stats = {0};
+	rc = ecore_ll2_get_stats(dev->rdma_ctx, dev->gsi_ll2_handle, &stats);
+	if (rc) {
+		QL_DPRINT11(dev->ha, "failed to obtain ll2 stats\n");
+	}
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	return 0;
+}
+
+int
+qlnxr_ll2_stop(struct qlnxr_dev *dev)
+{
+	int rc;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	if (dev->gsi_ll2_handle == 0xFF)
+		return 0;
+
+	/* remove LL2 MAC address filter */
+	rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx,
+			  dev->gsi_ll2_mac_address, NULL);
+
+	rc = ecore_ll2_terminate_connection(dev->rdma_ctx,
+			dev->gsi_ll2_handle);
+
+	ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle);
+
+	dev->gsi_ll2_handle = 0xFF;
+
+	QL_DPRINT12(dev->ha, "exit rc = %d\n", rc);
+	return rc;
+}
+
+int qlnxr_ll2_start(struct qlnxr_dev *dev,
+		   struct ib_qp_init_attr *attrs,
+		   struct qlnxr_qp *qp)
+{
+	struct ecore_ll2_acquire_data data;
+	struct ecore_ll2_cbs cbs;
+	int rc;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	/* configure and start LL2 */
+	cbs.rx_comp_cb = qlnxr_ll2_complete_rx_packet;
+	cbs.tx_comp_cb = qlnxr_ll2_complete_tx_packet;
+	cbs.rx_release_cb = qlnxr_ll2_release_rx_packet;
+	cbs.tx_release_cb = qlnxr_ll2_complete_tx_packet;
+	cbs.cookie = dev;
+	dev->gsi_ll2_handle = 0xFF;
+
+	memset(&data, 0, sizeof(data));
+	data.input.conn_type = ECORE_LL2_TYPE_ROCE;
+	data.input.mtu = dev->ha->ifp->if_mtu;
+	data.input.rx_num_desc = 8 * 1024;
+	data.input.rx_drop_ttl0_flg = 1;
+	data.input.rx_vlan_removal_en = 0;
+	data.input.tx_num_desc = 8 * 1024;
+	data.input.tx_tc = 0;
+	data.input.tx_dest = ECORE_LL2_TX_DEST_NW;
+	data.input.ai_err_packet_too_big = ECORE_LL2_DROP_PACKET;
+	data.input.ai_err_no_buf = ECORE_LL2_DROP_PACKET;
+	data.input.gsi_enable = 1;
+	data.p_connection_handle = &dev->gsi_ll2_handle;
+	data.cbs = &cbs;
+
+	rc = ecore_ll2_acquire_connection(dev->rdma_ctx, &data);
+
+	if (rc) {
+		QL_DPRINT11(dev->ha,
+			"ecore_ll2_acquire_connection failed: %d\n",
+			rc);
+		return rc;
+	}
+
+	QL_DPRINT11(dev->ha,
+		"ll2 connection acquired successfully\n");
+	rc = ecore_ll2_establish_connection(dev->rdma_ctx,
+		dev->gsi_ll2_handle);
+
+	if (rc) {
+		QL_DPRINT11(dev->ha,
+			"ecore_ll2_establish_connection failed\n", rc);
+		goto err1;
+	}
+
+	QL_DPRINT11(dev->ha,
+		"ll2 connection established successfully\n");
+	rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx, NULL,
+			dev->ha->primary_mac);
+	if (rc) {
+		QL_DPRINT11(dev->ha, "qlnx_rdma_ll2_set_mac_filter failed\n", rc);
+		goto err2;
+	}
+
+	QL_DPRINT12(dev->ha, "exit rc = %d\n", rc);
+	return 0;
+
+err2:
+	ecore_ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle);
+err1:
+	ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle);
+
+	QL_DPRINT12(dev->ha, "exit rc = %d\n", rc);
+	return rc;
+}
+
+struct ib_qp*
+qlnxr_create_gsi_qp(struct qlnxr_dev *dev,
+		 struct ib_qp_init_attr *attrs,
+		 struct qlnxr_qp *qp)
+{
+	int rc;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	rc = qlnxr_check_gsi_qp_attrs(dev, attrs);
+
+	if (rc) {
+		QL_DPRINT11(dev->ha, "qlnxr_check_gsi_qp_attrs failed\n");
+		return ERR_PTR(rc);
+	}
+
+	rc = qlnxr_ll2_start(dev, attrs, qp);
+	if (rc) {
+		QL_DPRINT11(dev->ha, "qlnxr_ll2_start failed\n");
+		return ERR_PTR(rc);
+	}
+
+	/* create QP */
+	qp->ibqp.qp_num = 1;
+	qp->rq.max_wr = attrs->cap.max_recv_wr;
+	qp->sq.max_wr = attrs->cap.max_send_wr;
+
+	qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
+				GFP_KERNEL);
+	if (!qp->rqe_wr_id) {
+		QL_DPRINT11(dev->ha, "(!qp->rqe_wr_id)\n");
+		goto err;
+	}
+
+	qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
+				GFP_KERNEL);
+	if (!qp->wqe_wr_id) {
+		QL_DPRINT11(dev->ha, "(!qp->wqe_wr_id)\n");
+		goto err;
+	}
+
+	qlnxr_store_gsi_qp_cq(dev, qp, attrs);
+	memcpy(dev->gsi_ll2_mac_address, dev->ha->primary_mac, ETH_ALEN);
+
+	/* the GSI CQ is handled by the driver so remove it from the FW */
+	qlnxr_destroy_gsi_cq(dev, attrs);
+	dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI;
+	dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI;
+
+	QL_DPRINT12(dev->ha, "exit &qp->ibqp = %p\n", &qp->ibqp);
+
+	return &qp->ibqp;
+err:
+	kfree(qp->rqe_wr_id);
+
+	rc = qlnxr_ll2_stop(dev);
+
+	QL_DPRINT12(dev->ha, "exit with error\n");
+
+	return ERR_PTR(-ENOMEM);
+}
+
+int
+qlnxr_destroy_gsi_qp(struct qlnxr_dev *dev)
+{
+	int rc = 0;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	rc = qlnxr_ll2_stop(dev);
+
+	QL_DPRINT12(dev->ha, "exit rc = %d\n", rc);
+	return (rc);
+}
+
+
+static inline bool
+qlnxr_get_vlan_id_gsi(struct ib_ah_attr *ah_attr, u16 *vlan_id)
+{
+	u16 tmp_vlan_id;
+	union ib_gid *dgid = &ah_attr->grh.dgid;
+
+	tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
+	if (tmp_vlan_id < 0x1000) {
+		*vlan_id = tmp_vlan_id;
+		return true;
+	} else {
+		*vlan_id = 0;
+		return false;
+	}
+}
+
+#define QLNXR_MAX_UD_HEADER_SIZE	(100)
+#define QLNXR_GSI_QPN		(1)
+static inline int
+qlnxr_gsi_build_header(struct qlnxr_dev *dev,
+		struct qlnxr_qp *qp,
+		struct ib_send_wr *swr,
+		struct ib_ud_header *udh,
+		int *roce_mode)
+{
+	bool has_vlan = false, has_grh_ipv6 = true;
+	struct ib_ah_attr *ah_attr = &get_qlnxr_ah((ud_wr(swr)->ah))->attr;
+	struct ib_global_route *grh = &ah_attr->grh;
+	union ib_gid sgid;
+	int send_size = 0;
+	u16 vlan_id = 0;
+	u16 ether_type;
+
+#if __FreeBSD_version >= 1102000
+	int rc = 0;
+	int ip_ver = 0;
+	bool has_udp = false;
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+
+#if !DEFINE_IB_AH_ATTR_WITH_DMAC
+	u8 mac[ETH_ALEN];
+#endif
+	int i;
+
+	send_size = 0;
+	for (i = 0; i < swr->num_sge; ++i)
+		send_size += swr->sg_list[i].length;
+
+	has_vlan = qlnxr_get_vlan_id_gsi(ah_attr, &vlan_id);
+	ether_type = ETH_P_ROCE;
+	*roce_mode = ROCE_V1;
+	if (grh->sgid_index < QLNXR_MAX_SGID)
+		sgid = dev->sgid_tbl[grh->sgid_index];
+	else
+		sgid = dev->sgid_tbl[0];
+
+#if __FreeBSD_version >= 1102000
+
+	rc = ib_ud_header_init(send_size, false /* LRH */, true /* ETH */,
+			has_vlan, has_grh_ipv6, ip_ver, has_udp,
+			0 /* immediate */, udh);
+
+	if (rc) {
+		QL_DPRINT11(dev->ha, "gsi post send: failed to init header\n");
+		return rc;
+	}
+
+#else
+	ib_ud_header_init(send_size, false /* LRH */, true /* ETH */,
+			  has_vlan, has_grh_ipv6, 0 /* immediate */, udh);
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+	/* ENET + VLAN headers*/
+#if DEFINE_IB_AH_ATTR_WITH_DMAC
+	memcpy(udh->eth.dmac_h, ah_attr->dmac, ETH_ALEN);
+#else
+	qlnxr_get_dmac(dev, ah_attr, mac);
+	memcpy(udh->eth.dmac_h, mac, ETH_ALEN);
+#endif
+	memcpy(udh->eth.smac_h, dev->ha->primary_mac, ETH_ALEN);
+	if (has_vlan) {
+		udh->eth.type = htons(ETH_P_8021Q);
+		udh->vlan.tag = htons(vlan_id);
+		udh->vlan.type = htons(ether_type);
+	} else {
+		udh->eth.type = htons(ether_type);
+	}
+
+	for (int j = 0; j < 4; j++) {
+		QL_DPRINT12(dev->ha, "destination mac: %x\n",
+				udh->eth.dmac_h[j]);
+	}
+	for (int j = 0; j < 4; j++) {
+		QL_DPRINT12(dev->ha, "source mac: %x\n",
+				udh->eth.smac_h[j]);
+	}
+	
+	QL_DPRINT12(dev->ha, "QP: %p, opcode: %d, wq: %lx, roce: %x, hops:%d,"
+			"imm : %d, vlan :%d, AH: %p\n",
+			qp, swr->opcode, swr->wr_id, *roce_mode, grh->hop_limit,
+			0, has_vlan, get_qlnxr_ah((ud_wr(swr)->ah)));
+
+	if (has_grh_ipv6) {
+		/* GRH / IPv6 header */
+		udh->grh.traffic_class = grh->traffic_class;
+		udh->grh.flow_label = grh->flow_label;
+		udh->grh.hop_limit = grh->hop_limit;
+		udh->grh.destination_gid = grh->dgid;
+		memcpy(&udh->grh.source_gid.raw, &sgid.raw,
+		       sizeof(udh->grh.source_gid.raw));
+		QL_DPRINT12(dev->ha, "header: tc: %x, flow_label : %x, "
+			"hop_limit: %x \n", udh->grh.traffic_class,
+			udh->grh.flow_label, udh->grh.hop_limit);
+		for (i = 0; i < 16; i++) {
+			QL_DPRINT12(dev->ha, "udh dgid = %x\n", udh->grh.destination_gid.raw[i]);
+		}
+		for (i = 0; i < 16; i++) {
+			QL_DPRINT12(dev->ha, "udh sgid = %x\n", udh->grh.source_gid.raw[i]);
+		}
+		udh->grh.next_header = 0x1b;
+	}
+#ifdef DEFINE_IB_UD_HEADER_INIT_UDP_PRESENT 
+        /* This is for RoCEv2 */
+	else {
+                /* IPv4 header */
+                u32 ipv4_addr;
+
+                udh->ip4.protocol = IPPROTO_UDP;
+                udh->ip4.tos = htonl(grh->flow_label);
+                udh->ip4.frag_off = htons(IP_DF);
+                udh->ip4.ttl = grh->hop_limit;
+
+                ipv4_addr = qedr_get_ipv4_from_gid(sgid.raw);
+                udh->ip4.saddr = ipv4_addr;
+                ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw);
+                udh->ip4.daddr = ipv4_addr;
+                /* note: checksum is calculated by the device */
+        }
+#endif
+
+	/* BTH */
+	udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED);
+	udh->bth.pkey = QLNXR_ROCE_PKEY_DEFAULT;/* TODO: ib_get_cahced_pkey?! */
+	//udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn);
+	udh->bth.destination_qpn = OSAL_CPU_TO_BE32(ud_wr(swr)->remote_qpn);
+	//udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1));
+	udh->bth.psn = OSAL_CPU_TO_BE32((qp->sq_psn++) & ((1 << 24) - 1));
+	udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY;
+
+	/* DETH */
+	//udh->deth.qkey = htonl(0x80010000); /* qp->qkey */ /* TODO: what is?! */
+	//udh->deth.source_qpn = htonl(QLNXR_GSI_QPN);
+	udh->deth.qkey = OSAL_CPU_TO_BE32(0x80010000); /* qp->qkey */ /* TODO: what is?! */
+	udh->deth.source_qpn = OSAL_CPU_TO_BE32(QLNXR_GSI_QPN);
+	QL_DPRINT12(dev->ha, "exit\n");
+	return 0;
+}
+
+static inline int
+qlnxr_gsi_build_packet(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp, struct ib_send_wr *swr,
+	struct ecore_roce_ll2_packet **p_packet)
+{
+	u8 ud_header_buffer[QLNXR_MAX_UD_HEADER_SIZE];
+	struct ecore_roce_ll2_packet *packet;
+	int roce_mode, header_size;
+	struct ib_ud_header udh;
+	int i, rc;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	*p_packet = NULL;
+
+	rc = qlnxr_gsi_build_header(dev, qp, swr, &udh, &roce_mode);
+	if (rc) {
+		QL_DPRINT11(dev->ha,
+			"qlnxr_gsi_build_header failed rc = %d\n", rc);
+		return rc;
+	}
+
+	header_size = ib_ud_header_pack(&udh, &ud_header_buffer);
+
+	packet = kzalloc(sizeof(*packet), GFP_ATOMIC);
+	if (!packet) {
+		QL_DPRINT11(dev->ha, "packet == NULL\n");
+		return -ENOMEM;
+	}
+
+	packet->header.vaddr = qlnx_dma_alloc_coherent(&dev->ha->cdev,
+					&packet->header.baddr,
+					header_size);
+	if (!packet->header.vaddr) {
+		QL_DPRINT11(dev->ha, "packet->header.vaddr == NULL\n");
+		kfree(packet);
+		return -ENOMEM;
+	}
+
+	if (memcmp(udh.eth.smac_h, udh.eth.dmac_h, ETH_ALEN))
+		packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_NW;
+	else
+		packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_LB;
+
+	packet->roce_mode = roce_mode;
+	memcpy(packet->header.vaddr, ud_header_buffer, header_size);
+	packet->header.len = header_size;
+	packet->n_seg = swr->num_sge;
+	qp->wqe_wr_id[qp->sq.prod].bytes_len = IB_GRH_BYTES; //RDMA_GRH_BYTES
+	for (i = 0; i < packet->n_seg; i++) {
+		packet->payload[i].baddr = swr->sg_list[i].addr;
+		packet->payload[i].len = swr->sg_list[i].length;
+		qp->wqe_wr_id[qp->sq.prod].bytes_len +=
+			packet->payload[i].len;
+		QL_DPRINT11(dev->ha, "baddr: %p, len: %d\n",
+				packet->payload[i].baddr,
+				packet->payload[i].len);
+	}
+
+	*p_packet = packet;
+
+	QL_DPRINT12(dev->ha, "exit, packet->n_seg: %d\n", packet->n_seg);
+	return 0;
+}
+
+int
+qlnxr_gsi_post_send(struct ib_qp *ibqp,
+		struct ib_send_wr *wr,
+		struct ib_send_wr **bad_wr)
+{
+	struct ecore_roce_ll2_packet *pkt = NULL;
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	struct qlnxr_dev *dev = qp->dev;
+	unsigned long flags;
+	int rc;
+
+	QL_DPRINT12(dev->ha, "exit\n");
+
+	if (qp->state != ECORE_ROCE_QP_STATE_RTS) {
+		QL_DPRINT11(dev->ha,
+			"(qp->state != ECORE_ROCE_QP_STATE_RTS)\n");
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) {
+		QL_DPRINT11(dev->ha,
+			"(wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE)\n");
+		rc = -EINVAL;
+		goto err;
+	}
+
+	if (wr->opcode != IB_WR_SEND) {
+		QL_DPRINT11(dev->ha, "(wr->opcode > IB_WR_SEND)\n");
+		rc = -EINVAL;
+		goto err;
+	}
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	rc = qlnxr_gsi_build_packet(dev, qp, wr, &pkt);
+	if(rc) {
+		spin_unlock_irqrestore(&qp->q_lock, flags);
+		QL_DPRINT11(dev->ha, "qlnxr_gsi_build_packet failed\n");
+		goto err;
+	}
+
+	rc = qlnxr_ll2_post_tx(dev, pkt);
+
+	if (!rc) {
+		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
+		qp->wqe_wr_id[qp->sq.prod].signaled = 
+			!!(wr->send_flags & IB_SEND_SIGNALED);
+		qp->wqe_wr_id[qp->sq.prod].opcode = IB_WC_SEND;
+		qlnxr_inc_sw_prod(&qp->sq);
+		QL_DPRINT11(dev->ha, "packet sent over gsi qp\n");
+	} else {
+		QL_DPRINT11(dev->ha, "qlnxr_ll2_post_tx failed\n");
+		rc = -EAGAIN;
+		*bad_wr = wr;
+	}
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	if (wr->next != NULL) {
+		*bad_wr = wr->next;
+		rc=-EINVAL;
+	}
+
+	QL_DPRINT12(dev->ha, "exit\n");
+	return rc;
+
+err:
+	*bad_wr = wr;
+	QL_DPRINT12(dev->ha, "exit error\n");
+	return rc;
+}
+
+#define	QLNXR_LL2_RX_BUFFER_SIZE	(4 * 1024)
+int
+qlnxr_gsi_post_recv(struct ib_qp *ibqp,
+		struct ib_recv_wr *wr,
+		struct ib_recv_wr **bad_wr)
+{
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibqp->device));
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	unsigned long flags;
+	int rc = 0;
+
+	QL_DPRINT12(dev->ha, "enter, wr: %p\n", wr);
+
+	if ((qp->state != ECORE_ROCE_QP_STATE_RTR) &&
+	    (qp->state != ECORE_ROCE_QP_STATE_RTS)) {
+		*bad_wr = wr;
+		QL_DPRINT11(dev->ha, "exit 0\n");
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	while (wr) {
+		if (wr->num_sge > QLNXR_GSI_MAX_RECV_SGE) {
+			QL_DPRINT11(dev->ha, "exit 1\n");
+			goto err;
+		}
+
+		rc = ecore_ll2_post_rx_buffer(dev->rdma_ctx,
+				dev->gsi_ll2_handle,
+				wr->sg_list[0].addr,
+				wr->sg_list[0].length,
+				0 /* cookie */,
+				1 /* notify_fw */);
+		if (rc) {
+			QL_DPRINT11(dev->ha, "exit 2\n");
+			goto err;
+		}
+
+		memset(&qp->rqe_wr_id[qp->rq.prod], 0,
+			sizeof(qp->rqe_wr_id[qp->rq.prod]));
+		qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0];
+		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
+
+		qlnxr_inc_sw_prod(&qp->rq);
+
+		wr = wr->next;
+	}
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	QL_DPRINT12(dev->ha, "exit rc = %d\n", rc);
+	return rc;
+err:
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+	*bad_wr = wr;
+
+	QL_DPRINT12(dev->ha, "exit with -ENOMEM\n");
+	return -ENOMEM;
+}
+
+int
+qlnxr_gsi_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
+	struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
+	struct qlnxr_qp *qp = dev->gsi_qp;
+	unsigned long flags;
+	int i = 0;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+
+	while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) {
+		memset(&wc[i], 0, sizeof(*wc));
+
+		wc[i].qp = &qp->ibqp;
+		wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
+		wc[i].opcode = IB_WC_RECV;
+		wc[i].pkey_index = 0;
+		wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc)?
+			       IB_WC_GENERAL_ERR:IB_WC_SUCCESS;
+		/* 0 - currently only one recv sg is supported */
+		wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length;
+		wc[i].wc_flags |= IB_WC_GRH | IB_WC_IP_CSUM_OK;
+
+#if __FreeBSD_version >= 1100000
+		memcpy(&wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac, ETH_ALEN);
+		wc[i].wc_flags |= IB_WC_WITH_SMAC;
+
+		if (qp->rqe_wr_id[qp->rq.cons].vlan_id) {
+			wc[i].wc_flags |= IB_WC_WITH_VLAN;
+			wc[i].vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan_id;
+		}
+
+#endif
+		qlnxr_inc_sw_cons(&qp->rq);
+		i++;
+	}
+
+	while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) {
+		memset(&wc[i], 0, sizeof(*wc));
+
+		wc[i].qp = &qp->ibqp;
+		wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
+		wc[i].opcode = IB_WC_SEND;
+		wc[i].status = IB_WC_SUCCESS;
+
+		qlnxr_inc_sw_cons(&qp->sq);
+		i++;
+	}
+
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+	QL_DPRINT12(dev->ha, "exit i = %d\n", i);
+	return i;
+}
+
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_cm.h b/sys/dev/qlnx/qlnxr/qlnxr_cm.h
new file mode 100644
index 000000000000..79afc547362d
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_cm.h
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+
+#ifndef __QLNXR_CM_H__
+#define __QLNXR_CM_H__
+
+
+/* ECORE LL2 has a limit to the number of buffers it can handle.
+ * FYI, OFED used 512 and 128 for recv and send.
+ */
+#define QLNXR_GSI_MAX_RECV_WR	(4096)
+#define QLNXR_GSI_MAX_SEND_WR	(4096)
+
+#define QLNXR_GSI_MAX_RECV_SGE	(1)	/* LL2 FW limitation */
+
+/* future OFED/kernel will have these */
+#define ETH_P_ROCE		(0x8915)
+#define QLNXR_ROCE_V2_UDP_SPORT	(0000)
+
+#if __FreeBSD_version >= 1102000
+
+#define rdma_wr(_wr) rdma_wr(_wr)
+#define ud_wr(_wr) ud_wr(_wr)
+#define atomic_wr(_wr) atomic_wr(_wr)
+
+#else
+
+#define rdma_wr(_wr) (&(_wr->wr.rdma))
+#define ud_wr(_wr) (&(_wr->wr.ud))
+#define atomic_wr(_wr) (&(_wr->wr.atomic))
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+static inline u32 qlnxr_get_ipv4_from_gid(u8 *gid)
+{
+	return *(u32 *)(void *)&gid[12];
+}
+
+struct ecore_roce_ll2_header {
+        void *vaddr;
+        dma_addr_t baddr;
+        size_t len;
+};
+
+struct ecore_roce_ll2_buffer {
+        dma_addr_t baddr;
+        size_t len;
+};
+
+struct ecore_roce_ll2_packet {
+        struct ecore_roce_ll2_header header;
+        int n_seg;
+        struct ecore_roce_ll2_buffer payload[RDMA_MAX_SGE_PER_SQ_WQE];
+        int roce_mode;
+        enum ecore_roce_ll2_tx_dest tx_dest;
+};
+
+/* RDMA CM */
+
+extern int qlnxr_gsi_poll_cq(struct ib_cq *ibcq,
+			int num_entries,
+			struct ib_wc *wc);
+
+extern int qlnxr_gsi_post_recv(struct ib_qp *ibqp,
+			struct ib_recv_wr *wr,
+			struct ib_recv_wr **bad_wr);
+
+extern int qlnxr_gsi_post_send(struct ib_qp *ibqp,
+			struct ib_send_wr *wr,
+			struct ib_send_wr **bad_wr);
+
+extern struct ib_qp* qlnxr_create_gsi_qp(struct qlnxr_dev *dev,
+			struct ib_qp_init_attr *attrs,
+			struct qlnxr_qp *qp);
+
+extern void qlnxr_store_gsi_qp_cq(struct qlnxr_dev *dev,
+			struct qlnxr_qp *qp,
+			struct ib_qp_init_attr *attrs);
+
+extern void qlnxr_inc_sw_gsi_cons(struct qlnxr_qp_hwq_info *info);
+
+extern int qlnxr_destroy_gsi_qp(struct qlnxr_dev *dev);
+
+#endif /* #ifndef __QLNXR_CM_H__ */
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_def.h b/sys/dev/qlnx/qlnxr/qlnxr_def.h
new file mode 100644
index 000000000000..8e4394d4c43e
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_def.h
@@ -0,0 +1,924 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+/*
+ * File: qlnxr_def.h
+ * Author: David C Somayajulu
+ */
+
+#ifndef __QLNX_DEF_H_
+#define __QLNX_DEF_H_
+
+#include <sys/ktr.h>
+
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/idr.h>
+#include <linux/completion.h>
+#include <linux/netdevice.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/wait.h>
+#include <linux/kref.h>
+#include <linux/timer.h>
+#include <linux/io.h>
+#include <linux/fs.h>
+#include <sys/vmem.h>
+
+#include <asm/byteorder.h>
+
+#include <netinet/in.h>
+#include <net/ipv6.h>
+#include <netinet/toecore.h>
+
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/iw_cm.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_sa.h>
+
+#if __FreeBSD_version < 1100000
+#undef MODULE_VERSION
+#endif
+
+#include "qlnx_os.h"
+#include "bcm_osal.h"
+
+#include "reg_addr.h"
+#include "ecore_gtt_reg_addr.h"
+#include "ecore.h"
+#include "ecore_chain.h"
+#include "ecore_status.h"
+#include "ecore_hw.h"
+#include "ecore_rt_defs.h"
+#include "ecore_init_ops.h"
+#include "ecore_int.h"
+#include "ecore_cxt.h"
+#include "ecore_spq.h"
+#include "ecore_init_fw_funcs.h"
+#include "ecore_sp_commands.h"
+#include "ecore_dev_api.h"
+#include "ecore_l2_api.h"
+#ifdef CONFIG_ECORE_SRIOV
+#include "ecore_sriov.h"
+#include "ecore_vf.h"
+#endif
+#ifdef CONFIG_ECORE_LL2
+#include "ecore_ll2.h"
+#endif
+#ifdef CONFIG_ECORE_FCOE
+#include "ecore_fcoe.h"
+#endif
+#ifdef CONFIG_ECORE_ISCSI
+#include "ecore_iscsi.h"
+#endif
+#include "ecore_mcp.h"
+#include "ecore_hw_defs.h"
+#include "mcp_public.h"
+
+#ifdef CONFIG_ECORE_RDMA
+#include "ecore_rdma.h"
+#include "ecore_rdma_api.h"
+#endif
+
+#ifdef CONFIG_ECORE_ROCE
+#include "ecore_roce.h"
+#endif
+
+#ifdef CONFIG_ECORE_IWARP
+#include "ecore_iwarp.h"
+#endif
+
+#include "ecore_iro.h"
+#include "nvm_cfg.h"
+
+#include "ecore_dbg_fw_funcs.h"
+#include "rdma_common.h"
+
+#include "qlnx_ioctl.h"
+#include "qlnx_def.h"
+#include "qlnx_rdma.h"
+#include "qlnxr_verbs.h"
+#include "qlnxr_user.h"
+#include "qlnx_ver.h"
+#include <sys/smp.h>
+
+#define QLNXR_ROCE_INTERFACE_VERSION     1801
+
+#define QLNXR_MODULE_VERSION     "8.18.1.0"
+#define QLNXR_NODE_DESC "QLogic 579xx RoCE HCA"
+
+#define OC_SKH_DEVICE_PF 0x720
+#define OC_SKH_DEVICE_VF 0x728
+#define QLNXR_MAX_AH 512
+
+/* QLNXR Limitations */
+
+/* SQ/RQ Limitations
+ * An S/RQ PBL contains a list a pointers to pages. Each page contains S/RQE
+ * elements. Several S/RQE elements make an S/RQE, up to a certain maximum that
+ * is different between SQ and RQ. The size of the PBL was chosen such as not to
+ * limit the MAX_WR supported by ECORE, and rounded up to a power of two.
+ */
+/* SQ */
+#define QLNXR_MAX_SQ_PBL (0x8000) /* 2^15 bytes */
+#define QLNXR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */
+#define QLNXR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge)) /* bytes */
+#define QLNXR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
+                QLNXR_SQE_ELEMENT_SIZE) /* number */
+#define QLNXR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
+                QLNXR_SQE_ELEMENT_SIZE) /* number */
+#define QLNXR_MAX_SQE ((QLNXR_MAX_SQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \
+                (QLNXR_SQE_ELEMENT_SIZE) / (QLNXR_MAX_SQE_ELEMENTS_PER_SQE))
+/* RQ */
+#define QLNXR_MAX_RQ_PBL (0x2000) /* 2^13 bytes */
+#define QLNXR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */
+#define QLNXR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge)) /* bytes */
+#define QLNXR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE) /* number */
+#define QLNXR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
+                QLNXR_RQE_ELEMENT_SIZE) /* number */
+#define QLNXR_MAX_RQE ((QLNXR_MAX_RQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \
+                (QLNXR_RQE_ELEMENT_SIZE) / (QLNXR_MAX_RQE_ELEMENTS_PER_RQE))
+
+/* CQE Limitation
+ * Although FW supports two layer PBL we use single layer since it is more
+ * than enough. For that layer we use a maximum size of 512 kB, again, because
+ * it reaches the maximum number of page pointers. Notice is the '-1' in the
+ * calculation that comes from having a u16 for the number of pages i.e. 0xffff
+ * is the maximum number of pages (in single layer).
+ */
+#define QLNXR_CQE_SIZE   (sizeof(union rdma_cqe))
+#define QLNXR_MAX_CQE_PBL_SIZE (512*1024) /* 512kB */
+#define QLNXR_MAX_CQE_PBL_ENTRIES (((QLNXR_MAX_CQE_PBL_SIZE) / \
+                                  sizeof(u64)) - 1) /* 64k -1 */
+#define QLNXR_MAX_CQES ((u32)((QLNXR_MAX_CQE_PBL_ENTRIES) * (ECORE_CHAIN_PAGE_SIZE)\
+                             / QLNXR_CQE_SIZE)) /* 8M -4096/32 = 8,388,480 */
+
+/* CNQ size Limitation
+ * The maximum CNQ size is not reachable because the FW supports a chain of u16
+ * (specifically 64k-1). The FW can buffer CNQ elements avoiding an overflow, on
+ * the expense of performance. Hence we set it to an arbitrarily smaller value
+ * than the maximum.
+ */
+#define QLNXR_ROCE_MAX_CNQ_SIZE          (0x4000) /* 2^16 */
+
+#define QLNXR_MAX_PORT                   (1)
+#define QLNXR_PORT                       (1)
+
+#define QLNXR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
+
+#define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo)
+
+/* The following number is used to determine if a handle recevied from the FW
+ * actually point to a CQ/QP.
+ */
+#define QLNXR_CQ_MAGIC_NUMBER    (0x11223344)
+#define QLNXR_QP_MAGIC_NUMBER    (0x77889900)
+
+/* Fast path debug prints */
+#define FP_DP_VERBOSE(...)
+/* #define FP_DP_VERBOSE(...)   DP_VERBOSE(__VA_ARGS__) */
+
+#define FW_PAGE_SIZE    (RDMA_RING_PAGE_SIZE)
+
+#define QLNXR_MSG_INIT		0x10000,
+#define QLNXR_MSG_FAIL		0x10000,
+#define QLNXR_MSG_CQ		0x20000,
+#define QLNXR_MSG_RQ		0x40000,
+#define QLNXR_MSG_SQ		0x80000,
+#define QLNXR_MSG_QP		(QLNXR_MSG_SQ | QLNXR_MSG_RQ),
+#define QLNXR_MSG_MR		0x100000,
+#define QLNXR_MSG_GSI		0x200000,
+#define QLNXR_MSG_MISC		0x400000,
+#define QLNXR_MSG_SRQ		0x800000,
+#define QLNXR_MSG_IWARP		0x1000000,
+
+#define QLNXR_ROCE_PKEY_MAX		1
+#define QLNXR_ROCE_PKEY_TABLE_LEN	1
+#define QLNXR_ROCE_PKEY_DEFAULT		0xffff
+
+#define QLNXR_MAX_SGID			128 /* TBD - add more source gids... */
+
+#define QLNXR_ENET_STATE_BIT     (0)
+
+#define QLNXR_MAX_MSIX		(16)
+
+
+struct qlnxr_cnq {
+        struct qlnxr_dev	*dev;
+        struct ecore_chain	pbl;
+        struct ecore_sb_info	*sb;
+        char			name[32];
+        u64			n_comp;
+        __le16			*hw_cons_ptr;
+        u8			index;
+	int			irq_rid;
+	struct resource		*irq;
+	void			*irq_handle;
+};
+
+struct qlnxr_device_attr {
+        /* Vendor specific information */
+        u32     vendor_id;
+        u32     vendor_part_id;
+        u32     hw_ver;
+        u64     fw_ver;
+
+        u64     node_guid;      /* node GUID */
+        u64     sys_image_guid; /* System image GUID */
+
+        u8      max_cnq;
+        u8      max_sge;        /* Maximum # of scatter/gather entries
+                                 * per Work Request supported
+                                 */
+        u16     max_inline;
+        u32     max_sqe;        /* Maximum number of send outstanding send work
+                                 * requests on any Work Queue supported
+                                 */
+        u32     max_rqe;        /* Maximum number of receive outstanding receive
+                                 * work requests on any Work Queue supported
+                                 */
+        u8      max_qp_resp_rd_atomic_resc;     /* Maximum number of RDMA Reads
+                                                 * & atomic operation that can
+                                                 * be outstanding per QP
+                                                 */
+
+        u8      max_qp_req_rd_atomic_resc;      /* The maximum depth per QP for
+                                                 * initiation of RDMA Read
+                                                 * & atomic operations
+                                                 */
+        u64     max_dev_resp_rd_atomic_resc;
+        u32     max_cq;
+        u32     max_qp;
+        u32     max_mr;         /* Maximum # of MRs supported */
+        u64     max_mr_size;    /* Size (in bytes) of largest contiguous memory
+                                 * block that can be registered by this device
+                                 */
+        u32     max_cqe;
+        u32     max_mw;         /* Maximum # of memory windows supported */
+        u32     max_fmr;
+        u32     max_mr_mw_fmr_pbl;
+        u64     max_mr_mw_fmr_size;
+        u32     max_pd;         /* Maximum # of protection domains supported */
+        u32     max_ah;
+        u8      max_pkey;
+        u32     max_srq;        /* Maximum number of SRQs */
+        u32     max_srq_wr;     /* Maximum number of WRs per SRQ */
+        u8      max_srq_sge;     /* Maximum number of SGE per WQE */
+        u8      max_stats_queues; /* Maximum number of statistics queues */
+        u32     dev_caps;
+
+        /* Abilty to support RNR-NAK generation */
+
+#define QLNXR_ROCE_DEV_CAP_RNR_NAK_MASK                           0x1
+#define QLNXR_ROCE_DEV_CAP_RNR_NAK_SHIFT                  0
+        /* Abilty to support shutdown port */
+#define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_MASK                     0x1
+#define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_SHIFT                    1
+        /* Abilty to support port active event */
+#define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_MASK         0x1
+#define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_SHIFT                2
+        /* Abilty to support port change event */
+#define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_MASK         0x1
+#define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_SHIFT                3
+        /* Abilty to support system image GUID */
+#define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_MASK                 0x1
+#define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_SHIFT                        4
+        /* Abilty to support bad P_Key counter support */
+#define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_MASK                      0x1
+#define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_SHIFT                     5
+        /* Abilty to support atomic operations */
+#define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_MASK                 0x1
+#define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_SHIFT                        6
+#define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_MASK                 0x1
+#define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_SHIFT                        7
+        /* Abilty to support modifying the maximum number of
+         * outstanding work requests per QP
+         */
+#define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_MASK                     0x1
+#define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_SHIFT                    8
+
+                /* Abilty to support automatic path migration */
+#define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_MASK                     0x1
+#define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_SHIFT                    9
+        /* Abilty to support the base memory management extensions */
+#define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_MASK                   0x1
+#define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_SHIFT          10
+#define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_MASK                    0x1
+#define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_SHIFT                   11
+        /* Abilty to support multipile page sizes per memory region */
+#define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_MASK             0x1
+#define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_SHIFT            12
+        /* Abilty to support block list physical buffer list */
+#define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_MASK                        0x1
+#define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_SHIFT                       13
+        /* Abilty to support zero based virtual addresses */
+#define QLNXR_ROCE_DEV_CAP_ZBVA_MASK                              0x1
+#define QLNXR_ROCE_DEV_CAP_ZBVA_SHIFT                             14
+        /* Abilty to support local invalidate fencing */
+#define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_MASK                   0x1
+#define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_SHIFT          15
+        /* Abilty to support Loopback on QP */
+#define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_MASK                      0x1
+#define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_SHIFT                     16
+        u64                     page_size_caps;
+        u8                      dev_ack_delay;
+        u32                     reserved_lkey;   /* Value of reserved L_key */
+        u32                     bad_pkey_counter;/* Bad P_key counter support
+                                                  * indicator
+                                                  */
+        struct ecore_rdma_events  events;
+};
+
+struct qlnxr_dev {
+	struct ib_device	ibdev;
+	qlnx_host_t		*ha;
+	struct ecore_dev	*cdev;
+
+	/* Added to extend Applications Support */
+        struct pci_dev          *pdev;
+	uint32_t		dp_module;
+	uint8_t			dp_level;
+
+	void			*rdma_ctx;
+
+	struct mtx		idr_lock;
+	struct idr		qpidr;
+
+	uint32_t		wq_multiplier;
+	int			num_cnq;
+
+	struct ecore_sb_info	sb_array[QLNXR_MAX_MSIX];
+	struct qlnxr_cnq	cnq_array[QLNXR_MAX_MSIX];
+
+        int			sb_start;
+
+        int			gsi_qp_created;
+        struct qlnxr_cq		*gsi_sqcq;
+        struct qlnxr_cq		*gsi_rqcq;
+        struct qlnxr_qp		*gsi_qp;
+
+        /* TBD: we'll need an array of these probablly per DPI... */
+        void __iomem		*db_addr;
+        uint64_t		db_phys_addr;
+        uint32_t		db_size;
+        uint16_t		dpi;
+
+        uint64_t		guid;
+        enum ib_atomic_cap	atomic_cap;
+
+        union ib_gid		sgid_tbl[QLNXR_MAX_SGID];
+        struct mtx		sgid_lock;
+        struct notifier_block	nb_inet;
+        struct notifier_block	nb_inet6;
+
+        uint8_t			mr_key;
+        struct list_head	entry;
+
+        struct dentry		*dbgfs;
+
+        uint8_t			gsi_ll2_mac_address[ETH_ALEN];
+        uint8_t			gsi_ll2_handle;
+
+	unsigned long		enet_state;
+
+	struct workqueue_struct *iwarp_wq;
+
+	volatile uint32_t	pd_count;
+	struct                  qlnxr_device_attr attr;
+        uint8_t                 user_dpm_enabled;
+};
+
+typedef struct qlnxr_dev qlnxr_dev_t;
+
+
+struct qlnxr_pd {
+        struct ib_pd ibpd;
+        u32 pd_id;
+        struct qlnxr_ucontext *uctx;
+};
+
+struct qlnxr_ucontext {
+        struct ib_ucontext ibucontext;
+        struct qlnxr_dev *dev;
+        struct qlnxr_pd *pd;
+        u64 dpi_addr;
+        u64 dpi_phys_addr;
+        u32 dpi_size;
+        u16 dpi;
+
+        struct list_head mm_head;
+        struct mutex mm_list_lock;
+};
+
+
+
+struct qlnxr_dev_attr {
+        struct ib_device_attr ib_attr;
+};
+
+struct qlnxr_dma_mem {
+        void *va;
+        dma_addr_t pa;
+        u32 size;
+};
+
+struct qlnxr_pbl {
+        struct list_head list_entry;
+        void *va;
+        dma_addr_t pa;
+};
+
+struct qlnxr_queue_info {
+        void *va;
+        dma_addr_t dma;
+        u32 size;
+        u16 len;
+        u16 entry_size;         /* Size of an element in the queue */
+        u16 id;                 /* qid, where to ring the doorbell. */
+        u16 head, tail;
+        bool created;
+};
+
+struct qlnxr_eq {
+        struct qlnxr_queue_info q;
+        u32 vector;
+        int cq_cnt;
+        struct qlnxr_dev *dev;
+        char irq_name[32];
+};
+
+struct qlnxr_mq {
+        struct qlnxr_queue_info sq;
+        struct qlnxr_queue_info cq;
+        bool rearm_cq;
+};
+
+struct phy_info {
+        u16 auto_speeds_supported;
+        u16 fixed_speeds_supported;
+        u16 phy_type;
+        u16 interface_type;
+};
+
+union db_prod64 {
+	struct rdma_pwm_val32_data data;
+        u64 raw;
+};
+
+enum qlnxr_cq_type {
+        QLNXR_CQ_TYPE_GSI,
+        QLNXR_CQ_TYPE_KERNEL,
+        QLNXR_CQ_TYPE_USER
+};
+
+struct qlnxr_pbl_info {
+        u32 num_pbls;
+        u32 num_pbes;
+        u32 pbl_size;
+        u32 pbe_size;
+        bool two_layered;
+};
+
+struct qlnxr_userq {
+        struct ib_umem *umem;
+        struct qlnxr_pbl_info pbl_info;
+        struct qlnxr_pbl *pbl_tbl;
+        u64 buf_addr;
+        size_t buf_len;
+};
+
+struct qlnxr_cq {
+        struct ib_cq		ibcq; /* must be first */
+
+        enum qlnxr_cq_type	cq_type;
+        uint32_t		sig;
+        uint16_t		icid;
+
+        /* relevant to cqs created from kernel space only (ULPs) */
+        spinlock_t		cq_lock;
+        uint8_t			arm_flags;
+        struct ecore_chain	pbl;
+
+        void __iomem		*db_addr; /* db address for cons update*/
+        union db_prod64		db;
+
+        uint8_t			pbl_toggle;
+        union rdma_cqe		*latest_cqe;
+        union rdma_cqe		*toggle_cqe;
+
+        /* TODO: remove since it is redundant with 32 bit chains */
+        uint32_t		cq_cons;
+
+        /* relevant to cqs created from user space only (applications) */
+        struct qlnxr_userq	q;
+
+        /* destroy-IRQ handler race prevention */
+        uint8_t			destroyed;
+        uint16_t		cnq_notif;
+};
+
+
+struct qlnxr_ah {
+        struct ib_ah		ibah;
+        struct ib_ah_attr	attr;
+};
+
+union db_prod32 {
+	struct rdma_pwm_val16_data data;
+        u32 raw;
+};
+
+struct qlnxr_qp_hwq_info {
+        /* WQE Elements*/
+        struct ecore_chain      pbl;
+        u64                     p_phys_addr_tbl;
+        u32                     max_sges;
+
+        /* WQE */
+        u16                     prod;     /* WQE prod index for SW ring */
+        u16                     cons;     /* WQE cons index for SW ring */
+        u16                     wqe_cons;
+        u16                     gsi_cons; /* filled in by GSI implementation */
+        u16                     max_wr;
+
+        /* DB */
+        void __iomem            *db;      /* Doorbell address */
+        union db_prod32         db_data;  /* Doorbell data */
+
+        /* Required for iwarp_only */
+        void __iomem            *iwarp_db2;      /* Doorbell address */
+        union db_prod32         iwarp_db2_data;  /* Doorbell data */
+};
+
+#define QLNXR_INC_SW_IDX(p_info, index)                          \
+        do {                                                    \
+                p_info->index = (p_info->index + 1) &           \
+                        ecore_chain_get_capacity(p_info->pbl)     \
+        } while (0)
+
+struct qlnxr_srq_hwq_info {
+        u32 max_sges;
+        u32 max_wr;
+        struct ecore_chain pbl;
+        u64 p_phys_addr_tbl;
+        u32 wqe_prod;     /* WQE prod index in HW ring */
+        u32 sge_prod;     /* SGE prod index in HW ring */
+        u32 wr_prod_cnt; /* wr producer count */
+        u32 wr_cons_cnt; /* wr consumer count */
+        u32 num_elems;
+
+        u32 *virt_prod_pair_addr; /* producer pair virtual address */
+        dma_addr_t phy_prod_pair_addr; /* producer pair physical address */
+};
+
+struct qlnxr_srq {
+        struct ib_srq ibsrq;
+        struct qlnxr_dev *dev;
+        /* relevant to cqs created from user space only (applications) */
+        struct qlnxr_userq       usrq;
+        struct qlnxr_srq_hwq_info hw_srq;
+        struct ib_umem *prod_umem;
+        u16 srq_id;
+        /* lock to protect srq recv post */
+        spinlock_t lock;
+};
+
+enum qlnxr_qp_err_bitmap {
+        QLNXR_QP_ERR_SQ_FULL     = 1 << 0,
+        QLNXR_QP_ERR_RQ_FULL     = 1 << 1,
+        QLNXR_QP_ERR_BAD_SR      = 1 << 2,
+        QLNXR_QP_ERR_BAD_RR      = 1 << 3,
+        QLNXR_QP_ERR_SQ_PBL_FULL = 1 << 4,
+        QLNXR_QP_ERR_RQ_PBL_FULL = 1 << 5,
+};
+
+struct mr_info {
+        struct qlnxr_pbl *pbl_table;
+        struct qlnxr_pbl_info pbl_info;
+        struct list_head free_pbl_list;
+        struct list_head inuse_pbl_list;
+        u32 completed;
+        u32 completed_handled;
+};
+
+#if __FreeBSD_version < 1102000
+#define DEFINE_IB_FAST_REG
+#else
+#define DEFINE_ALLOC_MR
+#endif
+
+#ifdef DEFINE_IB_FAST_REG
+struct qlnxr_fast_reg_page_list {
+        struct ib_fast_reg_page_list ibfrpl;
+        struct qlnxr_dev *dev;
+        struct mr_info info;
+};
+#endif
+struct qlnxr_qp {
+        struct ib_qp ibqp;              /* must be first */
+        struct qlnxr_dev *dev;
+        struct qlnxr_iw_ep *ep;
+        struct qlnxr_qp_hwq_info sq;
+        struct qlnxr_qp_hwq_info rq;
+
+        u32 max_inline_data;
+
+#if __FreeBSD_version >= 1100000
+        spinlock_t q_lock ____cacheline_aligned;
+#else
+	spinlock_t q_lock;
+#endif
+
+        struct qlnxr_cq *sq_cq;
+        struct qlnxr_cq *rq_cq;
+        struct qlnxr_srq *srq;
+        enum ecore_roce_qp_state state;   /*  QP state */
+        u32 id;
+        struct qlnxr_pd *pd;
+        enum ib_qp_type qp_type;
+        struct ecore_rdma_qp *ecore_qp;
+        u32 qp_id;
+        u16 icid;
+        u16 mtu;
+        int sgid_idx;
+        u32 rq_psn;
+        u32 sq_psn;
+        u32 qkey;
+        u32 dest_qp_num;
+        u32 sig;                /* unique siganture to identify valid QP */
+
+        /* relevant to qps created from kernel space only (ULPs) */
+        u8 prev_wqe_size;
+        u16 wqe_cons;
+        u32 err_bitmap;
+        bool signaled;
+        /* SQ shadow */
+        struct {
+                u64 wr_id;
+                enum ib_wc_opcode opcode;
+                u32 bytes_len;
+                u8 wqe_size;
+                bool  signaled;
+                dma_addr_t icrc_mapping;
+                u32 *icrc;
+#ifdef DEFINE_IB_FAST_REG
+                struct qlnxr_fast_reg_page_list *frmr;
+#endif
+                struct qlnxr_mr *mr;
+        } *wqe_wr_id;
+
+        /* RQ shadow */
+        struct {
+                u64 wr_id;
+                struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
+                uint8_t wqe_size;
+
+                /* for GSI only */
+                u8 smac[ETH_ALEN];
+                u16 vlan_id;
+                int rc;
+        } *rqe_wr_id;
+
+        /* relevant to qps created from user space only (applications) */
+        struct qlnxr_userq usq;
+        struct qlnxr_userq urq;
+        atomic_t refcnt;
+	bool destroyed;
+};
+
+enum qlnxr_mr_type {
+        QLNXR_MR_USER,
+        QLNXR_MR_KERNEL,
+        QLNXR_MR_DMA,
+        QLNXR_MR_FRMR
+};
+
+
+struct qlnxr_mr {
+        struct ib_mr    ibmr;
+        struct ib_umem  *umem;
+
+        struct ecore_rdma_register_tid_in_params hw_mr;
+        enum qlnxr_mr_type type;
+
+        struct qlnxr_dev *dev;
+        struct mr_info info;
+
+        u64 *pages;
+        u32 npages;
+
+	u64 *iova_start; /* valid only for kernel_mr */
+};
+
+
+struct qlnxr_mm {
+        struct {
+                u64 phy_addr;
+                unsigned long len;
+        } key;
+        struct list_head entry;
+};
+
+struct qlnxr_iw_listener {
+        struct qlnxr_dev *dev;
+        struct iw_cm_id *cm_id;
+        int backlog;
+        void *ecore_handle;
+};
+
+struct qlnxr_iw_ep {
+        struct qlnxr_dev *dev;
+        struct iw_cm_id *cm_id;
+        struct qlnxr_qp *qp;
+        void *ecore_context;
+	u8 during_connect;
+};
+
+static inline void
+qlnxr_inc_sw_cons(struct qlnxr_qp_hwq_info *info)
+{
+        info->cons = (info->cons + 1) % info->max_wr;
+        info->wqe_cons++;
+}
+
+static inline void
+qlnxr_inc_sw_prod(struct qlnxr_qp_hwq_info *info)
+{
+        info->prod = (info->prod + 1) % info->max_wr;
+}
+
+static inline struct qlnxr_dev *
+get_qlnxr_dev(struct ib_device *ibdev)
+{
+        return container_of(ibdev, struct qlnxr_dev, ibdev);
+}
+
+static inline struct qlnxr_ucontext *
+get_qlnxr_ucontext(struct ib_ucontext *ibucontext)
+{
+        return container_of(ibucontext, struct qlnxr_ucontext, ibucontext);
+}
+
+static inline struct qlnxr_pd *
+get_qlnxr_pd(struct ib_pd *ibpd)
+{
+        return container_of(ibpd, struct qlnxr_pd, ibpd);
+}
+
+static inline struct qlnxr_cq *
+get_qlnxr_cq(struct ib_cq *ibcq)
+{
+        return container_of(ibcq, struct qlnxr_cq, ibcq);
+}
+
+static inline struct qlnxr_qp *
+get_qlnxr_qp(struct ib_qp *ibqp)
+{
+        return container_of(ibqp, struct qlnxr_qp, ibqp);
+}
+
+static inline struct qlnxr_mr *
+get_qlnxr_mr(struct ib_mr *ibmr)
+{
+        return container_of(ibmr, struct qlnxr_mr, ibmr);
+}
+
+static inline struct qlnxr_ah *
+get_qlnxr_ah(struct ib_ah *ibah)
+{
+        return container_of(ibah, struct qlnxr_ah, ibah);
+}
+
+static inline struct qlnxr_srq *
+get_qlnxr_srq(struct ib_srq *ibsrq)
+{
+        return container_of(ibsrq, struct qlnxr_srq, ibsrq);
+}
+
+static inline bool qlnxr_qp_has_srq(struct qlnxr_qp *qp)
+{
+        return !!qp->srq;
+}
+
+static inline bool qlnxr_qp_has_sq(struct qlnxr_qp *qp)
+{
+        if (qp->qp_type == IB_QPT_GSI)
+                return 0;
+
+        return 1;
+}
+
+static inline bool qlnxr_qp_has_rq(struct qlnxr_qp *qp)
+{
+        if (qp->qp_type == IB_QPT_GSI || qlnxr_qp_has_srq(qp))
+                return 0;
+
+        return 1;
+}
+
+
+#ifdef DEFINE_IB_FAST_REG
+static inline struct qlnxr_fast_reg_page_list *get_qlnxr_frmr_list(
+        struct ib_fast_reg_page_list *ifrpl)
+{
+        return container_of(ifrpl, struct qlnxr_fast_reg_page_list, ibfrpl);
+}
+#endif
+
+#define SET_FIELD2(value, name, flag)                          \
+        do {                                                   \
+                (value) |= ((flag) << (name ## _SHIFT));       \
+        } while (0)
+
+#define QLNXR_RESP_IMM	(RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
+                         RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
+#define QLNXR_RESP_RDMA	(RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
+                         RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
+#define QLNXR_RESP_INV  (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
+                         RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
+
+#define QLNXR_RESP_RDMA_IMM (QLNXR_RESP_IMM | QLNXR_RESP_RDMA)
+
+static inline int
+qlnxr_get_dmac(struct qlnxr_dev *dev, struct ib_ah_attr *ah_attr, u8 *mac_addr)
+{
+#ifdef DEFINE_NO_IP_BASED_GIDS
+        u8 *guid = &ah_attr->grh.dgid.raw[8]; /* GID's 64 MSBs are the GUID */
+#endif
+        union ib_gid zero_sgid = { { 0 } };
+        struct in6_addr in6;
+
+        if (!memcmp(&ah_attr->grh.dgid, &zero_sgid, sizeof(union ib_gid))) {
+                memset(mac_addr, 0x00, ETH_ALEN);
+                return -EINVAL;
+        }
+
+        memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6));
+
+#ifdef DEFINE_NO_IP_BASED_GIDS
+        /* get the MAC address from the GUID i.e. EUI-64 to MAC address */
+        mac_addr[0] = guid[0] ^ 2; /* toggle the local/universal bit to local */
+        mac_addr[1] = guid[1];
+        mac_addr[2] = guid[2];
+        mac_addr[3] = guid[5];
+        mac_addr[4] = guid[6];
+        mac_addr[5] = guid[7];
+#else
+        memcpy(mac_addr, ah_attr->dmac, ETH_ALEN);
+#endif
+        return 0;
+}
+
+extern int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address,
+                uint8_t *new_mac_address);
+
+
+#define QLNXR_ROCE_PKEY_MAX 1
+#define QLNXR_ROCE_PKEY_TABLE_LEN 1
+#define QLNXR_ROCE_PKEY_DEFAULT 0xffff
+
+#if __FreeBSD_version < 1100000
+#define DEFINE_IB_AH_ATTR_WITH_DMAC     (0)
+#define DEFINE_IB_UMEM_WITH_CHUNK	(1)
+#else
+#define DEFINE_IB_AH_ATTR_WITH_DMAC     (1)
+#endif
+
+#define QLNX_IS_IWARP(rdev)	IS_IWARP(ECORE_LEADING_HWFN(rdev->cdev))
+#define QLNX_IS_ROCE(rdev)	IS_ROCE(ECORE_LEADING_HWFN(rdev->cdev))
+
+#define MAX_RXMIT_CONNS		16
+
+#endif /* #ifndef __QLNX_DEF_H_ */
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_os.c b/sys/dev/qlnx/qlnxr/qlnxr_os.c
new file mode 100644
index 000000000000..a9e426e1ab18
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_os.c
@@ -0,0 +1,1366 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/*
+ * File: qlnxr_os.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qlnxr_def.h"
+
+SYSCTL_NODE(_dev, OID_AUTO, qnxr, CTLFLAG_RW, 0, "Qlogic RDMA module");
+
+uint32_t delayed_ack = 0;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, delayed_ack, CTLFLAG_RW, &delayed_ack, 1,
+	"iWARP: Delayed Ack: 0 - Disabled 1 - Enabled. Default: Disabled");
+
+uint32_t timestamp = 1;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, timestamp, CTLFLAG_RW, &timestamp, 1,
+	"iWARP: Timestamp: 0 - Disabled 1 - Enabled. Default:Enabled");
+
+uint32_t rcv_wnd_size = 0;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, rcv_wnd_size, CTLFLAG_RW, &rcv_wnd_size, 1,
+	"iWARP: Receive Window Size in K. Default 1M");
+
+uint32_t crc_needed = 1;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, crc_needed, CTLFLAG_RW, &crc_needed, 1,
+	"iWARP: CRC needed 0 - Disabled 1 - Enabled. Default:Enabled");
+
+uint32_t peer2peer = 1;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, peer2peer, CTLFLAG_RW, &peer2peer, 1,
+	"iWARP: Support peer2peer ULPs 0 - Disabled 1 - Enabled. Default:Enabled");
+
+uint32_t mpa_enhanced = 1;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, mpa_enhanced, CTLFLAG_RW, &mpa_enhanced, 1,
+	"iWARP: MPA Enhanced mode. Default:1");
+
+uint32_t rtr_type = 7;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, rtr_type, CTLFLAG_RW, &rtr_type, 1,
+	"iWARP: RDMAP opcode to use for the RTR message: BITMAP 1: RDMA_SEND 2: RDMA_WRITE 4: RDMA_READ. Default: 7");
+
+
+#define QNXR_WQ_MULTIPLIER_MIN  (1)
+#define QNXR_WQ_MULTIPLIER_MAX  (7)
+#define QNXR_WQ_MULTIPLIER_DFT  (3)
+
+uint32_t wq_multiplier= QNXR_WQ_MULTIPLIER_DFT;
+SYSCTL_UINT(_dev_qnxr, OID_AUTO, wq_multiplier, CTLFLAG_RW, &wq_multiplier, 1,
+	" When creating a WQ the actual number of WQE created will"
+	" be multiplied by this number (default is 3).");
+static ssize_t
+show_rev(struct device *device, struct device_attribute *attr,
+	char *buf)
+{
+        struct qlnxr_dev *dev = dev_get_drvdata(device);
+
+        return sprintf(buf, "0x%x\n", dev->cdev->vendor_id);
+}
+
+static ssize_t
+show_hca_type(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct qlnxr_dev *dev = dev_get_drvdata(device);
+        return sprintf(buf, "QLogic0x%x\n", dev->cdev->device_id);
+}
+
+static ssize_t
+show_fw_ver(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct qlnxr_dev *dev = dev_get_drvdata(device);
+	uint32_t fw_ver = (uint32_t) dev->attr.fw_ver;
+
+	return sprintf(buf, "%d.%d.%d\n",
+		       (fw_ver >> 24) & 0xff, (fw_ver >> 16) & 0xff,
+		       (fw_ver >> 8) & 0xff);
+}
+static ssize_t
+show_board(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct qlnxr_dev *dev = dev_get_drvdata(device);
+	return sprintf(buf, "%x\n", dev->cdev->device_id);
+}
+
+static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
+static DEVICE_ATTR(hca_type, S_IRUGO, show_hca_type, NULL);
+static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
+static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
+
+static struct device_attribute *qlnxr_class_attributes[] = {
+	&dev_attr_hw_rev,
+	&dev_attr_hca_type,
+	&dev_attr_fw_ver,
+	&dev_attr_board_id
+};
+
+static void
+qlnxr_ib_dispatch_event(qlnxr_dev_t *dev, uint8_t port_num,
+	enum ib_event_type type)
+{
+        struct ib_event ibev;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+        ibev.device = &dev->ibdev;
+        ibev.element.port_num = port_num;
+        ibev.event = type;
+
+        ib_dispatch_event(&ibev);
+
+	QL_DPRINT12(dev->ha, "exit\n");
+}
+
+static int
+__qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
+{
+	qlnxr_iw_destroy_listen(cm_id);
+
+	return (0);
+}
+
+static int
+qlnxr_register_device(qlnxr_dev_t *dev)
+{
+	struct ib_device *ibdev;
+	struct iw_cm_verbs *iwcm;
+	int ret;
+
+	QL_DPRINT12(dev->ha, "enter\n");
+
+	ibdev = &dev->ibdev;
+
+	strlcpy(ibdev->name, "qlnxr%d", IB_DEVICE_NAME_MAX);
+
+	memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid));
+	memcpy(&ibdev->node_guid, dev->ha->primary_mac, ETHER_ADDR_LEN);
+
+	memcpy(ibdev->node_desc, QLNXR_NODE_DESC, sizeof(QLNXR_NODE_DESC));
+
+	ibdev->owner = THIS_MODULE;
+	ibdev->uverbs_abi_ver = 7;
+	ibdev->local_dma_lkey = 0;
+
+	ibdev->uverbs_cmd_mask =
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+		(1ull << IB_USER_VERBS_CMD_REG_MR) |
+		(1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+		(1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+		(1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+		(1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+		(1ull << IB_USER_VERBS_CMD_POLL_CQ) |
+		(1ull << IB_USER_VERBS_CMD_POST_SEND) |
+		(1ull << IB_USER_VERBS_CMD_POST_RECV);
+
+        if (QLNX_IS_IWARP(dev)) {
+                ibdev->node_type = RDMA_NODE_RNIC;
+                ibdev->query_gid = qlnxr_iw_query_gid;
+        } else {
+                ibdev->node_type = RDMA_NODE_IB_CA;
+                ibdev->query_gid = qlnxr_query_gid;
+                ibdev->uverbs_cmd_mask |=
+			(1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
+			(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
+			(1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
+			(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
+			(1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
+                ibdev->create_srq = qlnxr_create_srq;
+                ibdev->destroy_srq = qlnxr_destroy_srq;
+                ibdev->modify_srq = qlnxr_modify_srq;
+                ibdev->query_srq = qlnxr_query_srq;
+                ibdev->post_srq_recv = qlnxr_post_srq_recv;
+        }
+
+	ibdev->phys_port_cnt = 1;
+	ibdev->num_comp_vectors = dev->num_cnq;
+
+        /* mandatory verbs. */
+        ibdev->query_device = qlnxr_query_device;
+        ibdev->query_port = qlnxr_query_port;
+        ibdev->modify_port = qlnxr_modify_port;
+        
+	ibdev->alloc_ucontext = qlnxr_alloc_ucontext;
+	ibdev->dealloc_ucontext = qlnxr_dealloc_ucontext;
+        /* mandatory to support user space verbs consumer. */
+        ibdev->mmap = qlnxr_mmap;
+
+        ibdev->alloc_pd = qlnxr_alloc_pd;
+        ibdev->dealloc_pd = qlnxr_dealloc_pd;
+
+        ibdev->create_cq = qlnxr_create_cq;
+        ibdev->destroy_cq = qlnxr_destroy_cq;
+        ibdev->resize_cq = qlnxr_resize_cq;
+        ibdev->req_notify_cq = qlnxr_arm_cq;
+
+        ibdev->create_qp = qlnxr_create_qp;
+        ibdev->modify_qp = qlnxr_modify_qp;
+        ibdev->query_qp = qlnxr_query_qp;
+        ibdev->destroy_qp = qlnxr_destroy_qp;
+
+        ibdev->query_pkey = qlnxr_query_pkey;
+        ibdev->create_ah = qlnxr_create_ah;
+        ibdev->destroy_ah = qlnxr_destroy_ah;
+        ibdev->query_ah = qlnxr_query_ah;
+        ibdev->modify_ah = qlnxr_modify_ah;
+        ibdev->get_dma_mr = qlnxr_get_dma_mr;
+        ibdev->dereg_mr = qlnxr_dereg_mr;
+        ibdev->reg_user_mr = qlnxr_reg_user_mr;
+        
+#if __FreeBSD_version >= 1102000
+	ibdev->alloc_mr = qlnxr_alloc_mr;
+	ibdev->map_mr_sg = qlnxr_map_mr_sg;
+	ibdev->get_port_immutable = qlnxr_get_port_immutable;
+#else
+        ibdev->reg_phys_mr = qlnxr_reg_kernel_mr;
+        ibdev->alloc_fast_reg_mr = qlnxr_alloc_frmr;
+        ibdev->alloc_fast_reg_page_list = qlnxr_alloc_frmr_page_list;
+        ibdev->free_fast_reg_page_list = qlnxr_free_frmr_page_list;
+#endif /* #if __FreeBSD_version >= 1102000 */
+	
+        ibdev->poll_cq = qlnxr_poll_cq;
+        ibdev->post_send = qlnxr_post_send;
+        ibdev->post_recv = qlnxr_post_recv;
+	ibdev->process_mad = qlnxr_process_mad;
+
+
+
+        ibdev->dma_device = &dev->pdev->dev;
+
+	ibdev->get_link_layer = qlnxr_link_layer;
+        
+	if (QLNX_IS_IWARP(dev)) {
+                iwcm = kmalloc(sizeof(*iwcm), GFP_KERNEL);
+	
+		device_printf(dev->ha->pci_dev, "device is IWARP\n");
+		if (iwcm == NULL)
+			return (-ENOMEM);
+
+                ibdev->iwcm = iwcm;
+
+                iwcm->connect = qlnxr_iw_connect;
+                iwcm->accept = qlnxr_iw_accept;
+                iwcm->reject = qlnxr_iw_reject;
+
+#if (__FreeBSD_version >= 1004000) && (__FreeBSD_version < 1102000)
+
+                iwcm->create_listen_ep = qlnxr_iw_create_listen;
+                iwcm->destroy_listen_ep = qlnxr_iw_destroy_listen;
+#else
+                iwcm->create_listen = qlnxr_iw_create_listen;
+                iwcm->destroy_listen = __qlnxr_iw_destroy_listen;
+#endif
+                iwcm->add_ref = qlnxr_iw_qp_add_ref;
+                iwcm->rem_ref = qlnxr_iw_qp_rem_ref;
+                iwcm->get_qp = qlnxr_iw_get_qp;
+        }
+
+        ret = ib_register_device(ibdev, NULL);
+	if (ret) {
+		kfree(iwcm);
+	}
+
+	QL_DPRINT12(dev->ha, "exit\n");
+        return ret;
+}
+
+#define HILO_U64(hi, lo)                ((((u64)(hi)) << 32) + (lo))
+
+static void
+qlnxr_intr(void *handle)
+{
+        struct qlnxr_cnq *cnq = handle;
+        struct qlnxr_cq *cq;
+        struct regpair *cq_handle;
+        u16 hw_comp_cons, sw_comp_cons;
+	qlnx_host_t *ha;
+
+	ha = cnq->dev->ha;
+
+	QL_DPRINT12(ha, "enter cnq = %p\n", handle);
+
+        ecore_sb_ack(cnq->sb, IGU_INT_DISABLE, 0 /*do not update*/);
+
+        ecore_sb_update_sb_idx(cnq->sb);
+
+        hw_comp_cons = le16_to_cpu(*cnq->hw_cons_ptr);
+        sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl);
+
+        rmb();
+
+	QL_DPRINT12(ha, "enter cnq = %p hw_comp_cons = 0x%x sw_comp_cons = 0x%x\n",
+		handle, hw_comp_cons, sw_comp_cons);
+
+        while (sw_comp_cons != hw_comp_cons) {
+                cq_handle = (struct regpair *)ecore_chain_consume(&cnq->pbl);
+                cq = (struct qlnxr_cq *)(uintptr_t)HILO_U64(cq_handle->hi,
+                                cq_handle->lo);
+
+                if (cq == NULL) {
+			QL_DPRINT11(ha, "cq == NULL\n");
+                        break;
+                }
+
+                if (cq->sig != QLNXR_CQ_MAGIC_NUMBER) {
+			QL_DPRINT11(ha,
+				"cq->sig = 0x%x QLNXR_CQ_MAGIC_NUMBER = 0x%x\n",
+				cq->sig, QLNXR_CQ_MAGIC_NUMBER);
+                        break;
+                }
+                cq->arm_flags = 0;
+
+                if (!cq->destroyed && cq->ibcq.comp_handler) {
+			QL_DPRINT11(ha, "calling comp_handler = %p "
+				"ibcq = %p cq_context = 0x%x\n",
+				&cq->ibcq, cq->ibcq.cq_context);
+
+                        (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+                }
+		cq->cnq_notif++;
+
+                sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl);
+
+                cnq->n_comp++;
+        }
+
+        ecore_rdma_cnq_prod_update(cnq->dev->rdma_ctx, cnq->index, sw_comp_cons);
+
+        ecore_sb_ack(cnq->sb, IGU_INT_ENABLE, 1 /*update*/);
+
+	QL_DPRINT12(ha, "exit cnq = %p\n", handle);
+        return;
+}
+
+static void
+qlnxr_release_irqs(struct qlnxr_dev *dev)
+{
+	int i;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+        for (i = 0; i < dev->num_cnq; i++) {
+                if (dev->cnq_array[i].irq_handle)
+                        (void)bus_teardown_intr(dev->ha->pci_dev,
+				dev->cnq_array[i].irq,
+                                dev->cnq_array[i].irq_handle);
+
+                if (dev->cnq_array[i].irq)
+                        (void) bus_release_resource(dev->ha->pci_dev,
+				SYS_RES_IRQ,
+                                dev->cnq_array[i].irq_rid,
+				dev->cnq_array[i].irq);
+	}
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static int
+qlnxr_setup_irqs(struct qlnxr_dev *dev)
+{
+	int start_irq_rid;
+	int i;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	start_irq_rid = dev->sb_start + 2;
+
+	QL_DPRINT12(ha, "enter start_irq_rid = %d num_rss = %d\n",
+		start_irq_rid, dev->ha->num_rss);
+
+
+        for (i = 0; i < dev->num_cnq; i++) {
+
+		dev->cnq_array[i].irq_rid = start_irq_rid + i;
+	
+		dev->cnq_array[i].irq = bus_alloc_resource_any(dev->ha->pci_dev,
+						SYS_RES_IRQ,
+						&dev->cnq_array[i].irq_rid,
+						(RF_ACTIVE | RF_SHAREABLE));
+
+		if (dev->cnq_array[i].irq == NULL) {
+
+			QL_DPRINT11(ha,
+				"bus_alloc_resource_any failed irq_rid = %d\n",
+				dev->cnq_array[i].irq_rid);
+
+			goto qlnxr_setup_irqs_err;
+		}
+			
+                if (bus_setup_intr(dev->ha->pci_dev,
+                                dev->cnq_array[i].irq,
+                                (INTR_TYPE_NET | INTR_MPSAFE),
+                                NULL, qlnxr_intr, &dev->cnq_array[i],
+				&dev->cnq_array[i].irq_handle)) {
+
+			QL_DPRINT11(ha, "bus_setup_intr failed\n");
+			goto qlnxr_setup_irqs_err;
+                }
+		QL_DPRINT12(ha, "irq_rid = %d irq = %p irq_handle = %p\n",
+			dev->cnq_array[i].irq_rid, dev->cnq_array[i].irq,
+			dev->cnq_array[i].irq_handle);
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return (0);
+
+qlnxr_setup_irqs_err:
+	qlnxr_release_irqs(dev);
+
+	QL_DPRINT12(ha, "exit -1\n");
+	return (-1);
+}
+
+static void
+qlnxr_free_resources(struct qlnxr_dev *dev)
+{
+        int i;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter dev->num_cnq = %d\n", dev->num_cnq);
+
+	if (QLNX_IS_IWARP(dev)) {
+		if (dev->iwarp_wq != NULL)
+			destroy_workqueue(dev->iwarp_wq);
+	}
+
+        for (i = 0; i < dev->num_cnq; i++) {
+                qlnx_free_mem_sb(dev->ha, &dev->sb_array[i]);
+                ecore_chain_free(&dev->ha->cdev, &dev->cnq_array[i].pbl);
+        }
+
+	bzero(dev->cnq_array, (sizeof(struct qlnxr_cnq) * QLNXR_MAX_MSIX));
+	bzero(dev->sb_array, (sizeof(struct ecore_sb_info) * QLNXR_MAX_MSIX));
+	bzero(dev->sgid_tbl, (sizeof(union ib_gid) * QLNXR_MAX_SGID));
+
+	if (mtx_initialized(&dev->idr_lock))
+		mtx_destroy(&dev->idr_lock);
+
+	if (mtx_initialized(&dev->sgid_lock))
+		mtx_destroy(&dev->sgid_lock);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+
+static int
+qlnxr_alloc_resources(struct qlnxr_dev *dev)
+{
+	uint16_t n_entries;
+	int i, rc;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+        QL_DPRINT12(ha, "enter\n");
+
+        bzero(dev->sgid_tbl, (sizeof (union ib_gid) * QLNXR_MAX_SGID));
+
+        mtx_init(&dev->idr_lock, "idr_lock", NULL, MTX_DEF);
+        mtx_init(&dev->sgid_lock, "sgid_lock", NULL, MTX_DEF);
+
+        idr_init(&dev->qpidr);
+
+        bzero(dev->sb_array, (sizeof (struct ecore_sb_info) * QLNXR_MAX_MSIX));
+        bzero(dev->cnq_array, (sizeof (struct qlnxr_cnq) * QLNXR_MAX_MSIX));
+
+        dev->sb_start = ecore_rdma_get_sb_id(dev->rdma_ctx, 0);
+
+        QL_DPRINT12(ha, "dev->sb_start = 0x%x\n", dev->sb_start);
+
+        /* Allocate CNQ PBLs */
+
+        n_entries = min_t(u32, ECORE_RDMA_MAX_CNQ_SIZE, QLNXR_ROCE_MAX_CNQ_SIZE);
+
+        for (i = 0; i < dev->num_cnq; i++) {
+                rc = qlnx_alloc_mem_sb(dev->ha, &dev->sb_array[i],
+                                       dev->sb_start + i);
+                if (rc)
+                        goto qlnxr_alloc_resources_exit;
+
+                rc = ecore_chain_alloc(&dev->ha->cdev,
+                                ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+                                ECORE_CHAIN_MODE_PBL,
+                                ECORE_CHAIN_CNT_TYPE_U16,
+                                n_entries,
+                                sizeof(struct regpair *),
+                                &dev->cnq_array[i].pbl,
+                                NULL);
+
+                /* configure cnq, except name since ibdev.name is still NULL */
+                dev->cnq_array[i].dev = dev;
+                dev->cnq_array[i].sb = &dev->sb_array[i];
+                dev->cnq_array[i].hw_cons_ptr =
+                        &(dev->sb_array[i].sb_virt->pi_array[ECORE_ROCE_PROTOCOL_INDEX]);
+                dev->cnq_array[i].index = i;
+                sprintf(dev->cnq_array[i].name, "qlnxr%d@pci:%d",
+                        i, (dev->ha->pci_func));
+
+        }
+
+	QL_DPRINT12(ha, "exit\n");
+        return 0;
+
+qlnxr_alloc_resources_exit:
+
+	qlnxr_free_resources(dev);
+
+	QL_DPRINT12(ha, "exit -ENOMEM\n");
+        return -ENOMEM;
+}
+
+void
+qlnxr_affiliated_event(void *context, u8 e_code, void *fw_handle)
+{
+#define EVENT_TYPE_NOT_DEFINED  0
+#define EVENT_TYPE_CQ           1
+#define EVENT_TYPE_QP           2
+#define EVENT_TYPE_GENERAL      3
+
+        struct qlnxr_dev *dev = (struct qlnxr_dev *)context;
+        struct regpair *async_handle = (struct regpair *)fw_handle;
+        u64 roceHandle64 = ((u64)async_handle->hi << 32) + async_handle->lo;
+        struct qlnxr_cq *cq =  (struct qlnxr_cq *)(uintptr_t)roceHandle64;
+        struct qlnxr_qp *qp =  (struct qlnxr_qp *)(uintptr_t)roceHandle64;
+        u8 event_type = EVENT_TYPE_NOT_DEFINED;
+        struct ib_event event;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter context = %p e_code = 0x%x fw_handle = %p\n",
+		context, e_code, fw_handle);
+
+        if (QLNX_IS_IWARP(dev)) {
+		switch (e_code) {
+
+		case ECORE_IWARP_EVENT_CQ_OVERFLOW:
+			event.event = IB_EVENT_CQ_ERR;
+			event_type = EVENT_TYPE_CQ;
+			break;
+
+		default:
+			QL_DPRINT12(ha,
+				"unsupported event %d on handle=%llx\n",
+				e_code, roceHandle64);
+			break;
+		}
+        } else {
+		switch (e_code) {
+
+		case ROCE_ASYNC_EVENT_CQ_OVERFLOW_ERR:
+			event.event = IB_EVENT_CQ_ERR;
+			event_type = EVENT_TYPE_CQ;
+			break;
+
+		case ROCE_ASYNC_EVENT_SQ_DRAINED:
+			event.event = IB_EVENT_SQ_DRAINED;
+			event_type = EVENT_TYPE_QP;
+			break;
+
+		case ROCE_ASYNC_EVENT_QP_CATASTROPHIC_ERR:
+			event.event = IB_EVENT_QP_FATAL;
+			event_type = EVENT_TYPE_QP;
+			break;
+
+		case ROCE_ASYNC_EVENT_LOCAL_INVALID_REQUEST_ERR:
+			event.event = IB_EVENT_QP_REQ_ERR;
+			event_type = EVENT_TYPE_QP;
+			break;
+
+		case ROCE_ASYNC_EVENT_LOCAL_ACCESS_ERR:
+			event.event = IB_EVENT_QP_ACCESS_ERR;
+			event_type = EVENT_TYPE_QP;
+			break;
+
+		/* NOTE the following are not implemented in FW
+		 *      ROCE_ASYNC_EVENT_CQ_ERR
+		 *      ROCE_ASYNC_EVENT_COMM_EST
+		 */
+		/* TODO associate the following events -
+		 *      ROCE_ASYNC_EVENT_SRQ_LIMIT
+		 *      ROCE_ASYNC_EVENT_LAST_WQE_REACHED
+		 *      ROCE_ASYNC_EVENT_LOCAL_CATASTROPHIC_ERR (un-affiliated)
+		 */
+		default:
+			QL_DPRINT12(ha,
+				"unsupported event 0x%x on fw_handle = %p\n",
+				e_code, fw_handle);
+			break;
+		}
+	}
+
+        switch (event_type) {
+
+        case EVENT_TYPE_CQ:
+                if (cq && cq->sig == QLNXR_CQ_MAGIC_NUMBER) {
+                        struct ib_cq *ibcq = &cq->ibcq;
+
+                        if (ibcq->event_handler) {
+                                event.device     = ibcq->device;
+                                event.element.cq = ibcq;
+                                ibcq->event_handler(&event, ibcq->cq_context);
+                        }
+                } else {
+			QL_DPRINT11(ha,
+				"CQ event with invalid CQ pointer"
+				" Handle = %llx\n", roceHandle64);
+                }
+		QL_DPRINT12(ha,
+			"CQ event 0x%x on handle = %p\n", e_code, cq);
+                break;
+
+        case EVENT_TYPE_QP:
+                if (qp && qp->sig == QLNXR_QP_MAGIC_NUMBER) {
+                        struct ib_qp *ibqp = &qp->ibqp;
+
+                        if (ibqp->event_handler) {
+                                event.device     = ibqp->device;
+                                event.element.qp = ibqp;
+                                ibqp->event_handler(&event, ibqp->qp_context);
+                        }
+                } else {
+			QL_DPRINT11(ha,
+				"QP event 0x%x with invalid QP pointer"
+				" qp handle = %p\n",
+				e_code, roceHandle64);
+                }
+		QL_DPRINT12(ha, "QP event 0x%x on qp handle = %p\n",
+			e_code, qp);
+                break;
+
+        case EVENT_TYPE_GENERAL:
+                break;
+
+        default:
+                break;
+
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+void
+qlnxr_unaffiliated_event(void *context, u8 e_code)
+{
+        struct qlnxr_dev *dev = (struct qlnxr_dev *)context;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter/exit \n");
+	return;
+}
+
+
+static int
+qlnxr_set_device_attr(struct qlnxr_dev *dev)
+{
+	struct ecore_rdma_device *ecore_attr;
+	struct qlnxr_device_attr *attr;
+	u32 page_size;
+
+	ecore_attr = ecore_rdma_query_device(dev->rdma_ctx);
+
+	page_size = ~dev->attr.page_size_caps + 1;
+	if(page_size > PAGE_SIZE) {
+		QL_DPRINT12(dev->ha, "Kernel page size : %ld is smaller than"
+		    " minimum page size : %ld required by qlnxr\n",
+		    PAGE_SIZE, page_size);
+		return -ENODEV;
+	}
+	attr = &dev->attr;
+        attr->vendor_id = ecore_attr->vendor_id;
+        attr->vendor_part_id = ecore_attr->vendor_part_id;
+
+        QL_DPRINT12(dev->ha, "in qlnxr_set_device_attr, vendor : %x device : %x\n",
+		attr->vendor_id, attr->vendor_part_id);
+
+	attr->hw_ver = ecore_attr->hw_ver;
+        attr->fw_ver = ecore_attr->fw_ver;
+        attr->node_guid = ecore_attr->node_guid;
+        attr->sys_image_guid = ecore_attr->sys_image_guid;
+        attr->max_cnq = ecore_attr->max_cnq;
+        attr->max_sge = ecore_attr->max_sge;
+        attr->max_inline = ecore_attr->max_inline;
+        attr->max_sqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_SQE);
+        attr->max_rqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_RQE);
+        attr->max_qp_resp_rd_atomic_resc = ecore_attr->max_qp_resp_rd_atomic_resc;
+        attr->max_qp_req_rd_atomic_resc = ecore_attr->max_qp_req_rd_atomic_resc;
+        attr->max_dev_resp_rd_atomic_resc =
+            ecore_attr->max_dev_resp_rd_atomic_resc;
+        attr->max_cq = ecore_attr->max_cq;
+        attr->max_qp = ecore_attr->max_qp;
+        attr->max_mr = ecore_attr->max_mr;
+	attr->max_mr_size = ecore_attr->max_mr_size;
+        attr->max_cqe = min_t(u64, ecore_attr->max_cqe, QLNXR_MAX_CQES);
+        attr->max_mw = ecore_attr->max_mw;
+        attr->max_fmr = ecore_attr->max_fmr;
+        attr->max_mr_mw_fmr_pbl = ecore_attr->max_mr_mw_fmr_pbl;
+        attr->max_mr_mw_fmr_size = ecore_attr->max_mr_mw_fmr_size;
+        attr->max_pd = ecore_attr->max_pd;
+        attr->max_ah = ecore_attr->max_ah;
+        attr->max_pkey = ecore_attr->max_pkey;
+        attr->max_srq = ecore_attr->max_srq;
+        attr->max_srq_wr = ecore_attr->max_srq_wr;
+        //attr->dev_caps = ecore_attr->dev_caps;
+        attr->page_size_caps = ecore_attr->page_size_caps;
+        attr->dev_ack_delay = ecore_attr->dev_ack_delay;
+        attr->reserved_lkey = ecore_attr->reserved_lkey;
+        attr->bad_pkey_counter = ecore_attr->bad_pkey_counter;
+        attr->max_stats_queues = ecore_attr->max_stats_queues;
+
+        return 0;
+}
+
+
+static int
+qlnxr_init_hw(struct qlnxr_dev *dev)
+{
+        struct ecore_rdma_events events;
+        struct ecore_rdma_add_user_out_params out_params;
+        struct ecore_rdma_cnq_params *cur_pbl;
+        struct ecore_rdma_start_in_params *in_params;
+        dma_addr_t p_phys_table;
+        u32 page_cnt;
+        int rc = 0;
+        int i;
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+        in_params = kzalloc(sizeof(*in_params), GFP_KERNEL);
+        if (!in_params) {
+                rc = -ENOMEM;
+                goto out;
+        }
+
+	bzero(&out_params, sizeof(struct ecore_rdma_add_user_out_params));
+	bzero(&events, sizeof(struct ecore_rdma_events));
+
+        in_params->desired_cnq = dev->num_cnq;
+
+        for (i = 0; i < dev->num_cnq; i++) {
+                cur_pbl = &in_params->cnq_pbl_list[i];
+
+                page_cnt = ecore_chain_get_page_cnt(&dev->cnq_array[i].pbl);
+                cur_pbl->num_pbl_pages = page_cnt;
+
+                p_phys_table = ecore_chain_get_pbl_phys(&dev->cnq_array[i].pbl);
+                cur_pbl->pbl_ptr = (u64)p_phys_table;
+        }
+
+        events.affiliated_event = qlnxr_affiliated_event;
+        events.unaffiliated_event = qlnxr_unaffiliated_event;
+        events.context = dev;
+
+        in_params->events = &events;
+        in_params->roce.cq_mode = ECORE_RDMA_CQ_MODE_32_BITS;
+        in_params->max_mtu = dev->ha->max_frame_size;
+
+
+	if (QLNX_IS_IWARP(dev)) {
+	        if (delayed_ack)
+        	        in_params->iwarp.flags |= ECORE_IWARP_DA_EN;
+
+	        if (timestamp)
+        	        in_params->iwarp.flags |= ECORE_IWARP_TS_EN;
+ 
+	        in_params->iwarp.rcv_wnd_size = rcv_wnd_size*1024;
+	        in_params->iwarp.crc_needed = crc_needed;
+	        in_params->iwarp.ooo_num_rx_bufs =
+        	        (MAX_RXMIT_CONNS * in_params->iwarp.rcv_wnd_size) /
+	                in_params->max_mtu;
+
+	        in_params->iwarp.mpa_peer2peer = peer2peer;
+	        in_params->iwarp.mpa_rev =
+			mpa_enhanced ? ECORE_MPA_REV2 : ECORE_MPA_REV1;
+	        in_params->iwarp.mpa_rtr = rtr_type;
+	}
+
+        memcpy(&in_params->mac_addr[0], dev->ha->primary_mac, ETH_ALEN);
+
+        rc = ecore_rdma_start(dev->rdma_ctx, in_params);
+        if (rc)
+                goto out;
+
+        rc = ecore_rdma_add_user(dev->rdma_ctx, &out_params);
+        if (rc)
+                goto out;
+
+        dev->db_addr = (void *)(uintptr_t)out_params.dpi_addr;
+        dev->db_phys_addr = out_params.dpi_phys_addr;
+        dev->db_size = out_params.dpi_size;
+        dev->dpi = out_params.dpi;
+
+	qlnxr_set_device_attr(dev);
+
+	QL_DPRINT12(ha,
+		"cdev->doorbells = %p, db_phys_addr = %p db_size = 0x%x\n",
+		(void *)ha->cdev.doorbells,
+		(void *)ha->cdev.db_phys_addr, ha->cdev.db_size);
+
+	QL_DPRINT12(ha,
+		"db_addr = %p db_phys_addr = %p db_size = 0x%x dpi = 0x%x\n",
+		(void *)dev->db_addr, (void *)dev->db_phys_addr,
+		dev->db_size, dev->dpi);
+out:
+        kfree(in_params);
+
+	QL_DPRINT12(ha, "exit\n");
+        return rc;
+}
+
+static void
+qlnxr_build_sgid_mac(union ib_gid *sgid, unsigned char *mac_addr,
+	bool is_vlan, u16 vlan_id)
+{
+	sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL);
+	sgid->raw[8] = mac_addr[0] ^ 2;
+	sgid->raw[9] = mac_addr[1];
+	sgid->raw[10] = mac_addr[2];
+	if (is_vlan) {
+		sgid->raw[11] = vlan_id >> 8;
+		sgid->raw[12] = vlan_id & 0xff;
+	} else {
+		sgid->raw[11] = 0xff;
+		sgid->raw[12] = 0xfe;
+	}
+	sgid->raw[13] = mac_addr[3];
+	sgid->raw[14] = mac_addr[4];
+	sgid->raw[15] = mac_addr[5];
+}
+static bool
+qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid);
+
+static void
+qlnxr_add_ip_based_gid(struct qlnxr_dev *dev, struct ifnet *ifp)
+{
+	struct ifaddr *ifa;
+	union ib_gid gid;
+
+	CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr && ifa->ifa_addr->sa_family == AF_INET) {
+
+			QL_DPRINT12(dev->ha, "IP address : %x\n", ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr);
+			ipv6_addr_set_v4mapped(
+				((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr,
+				(struct in6_addr *)&gid);
+			QL_DPRINT12(dev->ha, "gid generated : %llx\n", gid);
+
+			qlnxr_add_sgid(dev, &gid);
+		}
+	}
+	for (int i = 0; i < 16; i++) {
+		QL_DPRINT12(dev->ha, "gid generated : %x\n", gid.raw[i]);
+	}
+}
+
+static bool
+qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid)
+{
+	union ib_gid zero_sgid = { { 0 } };
+	int i;
+	//unsigned long flags;
+	mtx_lock(&dev->sgid_lock);
+	for (i = 0; i < QLNXR_MAX_SGID; i++) {
+		if (!memcmp(&dev->sgid_tbl[i], &zero_sgid,
+				sizeof(union ib_gid))) {
+			/* found free entry */
+			memcpy(&dev->sgid_tbl[i], new_sgid,
+				sizeof(union ib_gid));
+			QL_DPRINT12(dev->ha, "copying sgid : %llx\n",
+					*new_sgid);
+			mtx_unlock(&dev->sgid_lock);
+			//TODO ib_dispatch event here?
+			return true;
+		} else if (!memcmp(&dev->sgid_tbl[i], new_sgid,
+				sizeof(union ib_gid))) {
+			/* entry already present, no addition required */
+			mtx_unlock(&dev->sgid_lock);
+			QL_DPRINT12(dev->ha, "sgid present : %llx\n",
+					*new_sgid);
+			return false;
+		}
+	}	
+	if (i == QLNXR_MAX_SGID) {
+		QL_DPRINT12(dev->ha, "didn't find an empty entry in sgid_tbl\n");
+	}
+	mtx_unlock(&dev->sgid_lock);
+	return false;
+}
+
+static bool qlnxr_del_sgid(struct qlnxr_dev *dev, union ib_gid *gid)
+{
+	int found = false;
+	int i;
+	//unsigned long flags;
+
+	QL_DPRINT12(dev->ha, "removing gid %llx %llx\n",
+			gid->global.interface_id,
+			gid->global.subnet_prefix);
+	mtx_lock(&dev->sgid_lock);
+	/* first is the default sgid which cannot be deleted */
+	for (i = 1; i < QLNXR_MAX_SGID; i++) {
+		if (!memcmp(&dev->sgid_tbl[i], gid, sizeof(union ib_gid))) {
+			/* found matching entry */
+			memset(&dev->sgid_tbl[i], 0, sizeof(union ib_gid));
+			found = true;
+			break;
+		}
+	}
+	mtx_unlock(&dev->sgid_lock);
+
+	return found;
+}
+
+#if __FreeBSD_version < 1100000
+
+static inline int
+is_vlan_dev(struct ifnet *ifp)
+{
+	return (ifp->if_type == IFT_L2VLAN);
+}
+ 
+static inline uint16_t
+vlan_dev_vlan_id(struct ifnet *ifp)
+{
+	uint16_t vtag;
+
+	if (VLAN_TAG(ifp, &vtag) == 0)
+		return (vtag);
+
+	return (0);
+}
+
+#endif /* #if __FreeBSD_version < 1100000 */
+
+static void
+qlnxr_add_sgids(struct qlnxr_dev *dev)
+{
+	qlnx_host_t *ha = dev->ha;
+	u16 vlan_id;
+	bool is_vlan;
+	union ib_gid vgid;
+
+	qlnxr_add_ip_based_gid(dev, ha->ifp);
+	/* MAC/VLAN base GIDs */
+	is_vlan = is_vlan_dev(ha->ifp);
+       	vlan_id = (is_vlan) ? vlan_dev_vlan_id(ha->ifp) : 0;
+	qlnxr_build_sgid_mac(&vgid, ha->primary_mac, is_vlan, vlan_id);
+	qlnxr_add_sgid(dev, &vgid);
+}
+
+static int
+qlnxr_add_default_sgid(struct qlnxr_dev *dev)
+{
+	/* GID Index 0 - Invariant manufacturer-assigned EUI-64 */
+	union ib_gid *sgid = &dev->sgid_tbl[0];
+	struct ecore_rdma_device        *qattr;
+	qlnx_host_t *ha;
+	ha = dev->ha;
+
+	qattr =	ecore_rdma_query_device(dev->rdma_ctx);
+	if(sgid == NULL)
+		QL_DPRINT12(ha, "sgid = NULL?\n");
+
+	sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL);
+	QL_DPRINT12(ha, "node_guid = %llx", dev->attr.node_guid);
+	memcpy(&sgid->raw[8], &qattr->node_guid,
+		sizeof(qattr->node_guid));
+	//memcpy(&sgid->raw[8], &dev->attr.node_guid,
+	//	sizeof(dev->attr.node_guid));
+	QL_DPRINT12(ha, "DEFAULT sgid=[%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x]\n",
+                   sgid->raw[0], sgid->raw[1], sgid->raw[2], sgid->raw[3], sgid->raw[4], sgid->raw[5],
+                   sgid->raw[6], sgid->raw[7], sgid->raw[8], sgid->raw[9], sgid->raw[10], sgid->raw[11],
+                   sgid->raw[12], sgid->raw[13], sgid->raw[14], sgid->raw[15]);
+	return 0;
+}
+
+static int qlnxr_addr_event (struct qlnxr_dev *dev,
+				unsigned long event,
+				struct ifnet *ifp,
+				union ib_gid *gid)
+{
+	bool is_vlan = false;
+	union ib_gid vgid;
+	u16 vlan_id = 0xffff;
+
+	QL_DPRINT12(dev->ha, "Link event occured\n");
+	is_vlan = is_vlan_dev(dev->ha->ifp);
+	vlan_id = (is_vlan) ? vlan_dev_vlan_id(dev->ha->ifp) : 0;
+
+	switch (event) {
+	case NETDEV_UP :
+		qlnxr_add_sgid(dev, gid);
+		if (is_vlan) {
+			qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id);
+			qlnxr_add_sgid(dev, &vgid);
+		}
+		break;
+	case NETDEV_DOWN :
+		qlnxr_del_sgid(dev, gid);
+		if (is_vlan) {
+			qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id);
+			qlnxr_del_sgid(dev, &vgid);
+		}
+		break;
+	default :
+		break;
+	}
+	return 1;
+}
+
+static int qlnxr_inetaddr_event(struct notifier_block *notifier,
+				unsigned long event, void *ptr)
+{
+	struct ifaddr *ifa = ptr;
+	union ib_gid gid;
+	struct qlnxr_dev *dev = container_of(notifier, struct qlnxr_dev, nb_inet);
+	qlnx_host_t *ha = dev->ha;
+
+	ipv6_addr_set_v4mapped(
+			((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr,
+			(struct in6_addr *)&gid);
+	return qlnxr_addr_event(dev, event, ha->ifp, &gid);
+}
+
+static int
+qlnxr_register_inet(struct qlnxr_dev *dev)
+{
+	int ret;
+	dev->nb_inet.notifier_call = qlnxr_inetaddr_event;
+	ret = register_inetaddr_notifier(&dev->nb_inet);
+	if (ret) {
+		QL_DPRINT12(dev->ha, "Failed to register inetaddr\n");
+		return ret;
+	}
+	/* TODO : add for CONFIG_IPV6) */	
+	return 0;	
+}
+
+static int
+qlnxr_build_sgid_tbl(struct qlnxr_dev *dev)
+{
+	qlnxr_add_default_sgid(dev);
+	qlnxr_add_sgids(dev);
+	return 0;
+}
+
+static struct qlnx_rdma_if qlnxr_drv;
+
+static void *
+qlnxr_add(void *eth_dev)
+{
+	struct qlnxr_dev *dev;
+	int ret;
+	//device_t pci_dev;
+	qlnx_host_t *ha;
+
+	ha = eth_dev;
+
+	QL_DPRINT12(ha, "enter [ha = %p]\n", ha);
+
+	dev = (struct qlnxr_dev *)ib_alloc_device(sizeof(struct qlnxr_dev));
+
+	if (dev == NULL)
+		return (NULL);
+
+	dev->ha = eth_dev;
+	dev->cdev = &ha->cdev;
+	/* Added to extend Application support */
+	dev->pdev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
+
+        dev->pdev->dev = *(dev->ha->pci_dev);
+        dev->pdev->device = pci_get_device(dev->ha->pci_dev);
+        dev->pdev->vendor = pci_get_vendor(dev->ha->pci_dev);
+
+	dev->rdma_ctx = &ha->cdev.hwfns[0];
+	dev->wq_multiplier = wq_multiplier;
+	dev->num_cnq = QLNX_NUM_CNQ;
+
+	QL_DPRINT12(ha,
+		"ha = %p dev = %p ha->cdev = %p\n",
+		ha, dev, &ha->cdev);
+	QL_DPRINT12(ha,
+		"dev->cdev = %p dev->rdma_ctx = %p\n",
+		dev->cdev, dev->rdma_ctx);
+
+	ret = qlnxr_alloc_resources(dev);
+
+	if (ret)
+		goto qlnxr_add_err;
+
+	ret = qlnxr_setup_irqs(dev);
+
+	if (ret) {
+		qlnxr_free_resources(dev);
+		goto qlnxr_add_err;
+	}
+
+	ret = qlnxr_init_hw(dev);
+
+	if (ret) {
+		qlnxr_release_irqs(dev);
+		qlnxr_free_resources(dev);
+		goto qlnxr_add_err;
+	}
+
+	qlnxr_register_device(dev);
+	for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) {
+		if (device_create_file(&dev->ibdev.dev, qlnxr_class_attributes[i]))
+			goto sysfs_err;
+	}
+	qlnxr_build_sgid_tbl(dev);
+	//ret = qlnxr_register_inet(dev);
+	QL_DPRINT12(ha, "exit\n");
+	if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) {
+		QL_DPRINT12(ha, "dispatching IB_PORT_ACITVE event\n");
+		qlnxr_ib_dispatch_event(dev, QLNXR_PORT,
+			IB_EVENT_PORT_ACTIVE);
+	}
+
+	return (dev);
+sysfs_err:
+	for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) {
+		device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]);
+	}
+	ib_unregister_device(&dev->ibdev);
+
+qlnxr_add_err:
+	ib_dealloc_device(&dev->ibdev);
+
+	QL_DPRINT12(ha, "exit failed\n");
+	return (NULL);
+}
+
+static void
+qlnxr_remove_sysfiles(struct qlnxr_dev *dev)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i)
+		device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]);
+}
+
+static int
+qlnxr_remove(void *eth_dev, void *qlnx_rdma_dev)
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t *ha;
+
+	dev = qlnx_rdma_dev;
+	ha = eth_dev;
+
+	if ((ha == NULL) || (dev == NULL))
+		return (0);
+
+	QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_dev = %p pd_count = %d\n",
+		ha, qlnx_rdma_dev, dev->pd_count);
+
+	qlnxr_ib_dispatch_event(dev, QLNXR_PORT,
+		IB_EVENT_PORT_ERR);
+
+	if (QLNX_IS_IWARP(dev)) {
+		if (dev->pd_count)
+			return (EBUSY);
+	}
+
+	ib_unregister_device(&dev->ibdev);
+	
+	if (QLNX_IS_ROCE(dev)) {
+		if (dev->pd_count)
+			return (EBUSY);
+	}
+
+	ecore_rdma_remove_user(dev->rdma_ctx, dev->dpi);
+	ecore_rdma_stop(dev->rdma_ctx);
+
+	qlnxr_release_irqs(dev);
+
+	qlnxr_free_resources(dev);
+
+	qlnxr_remove_sysfiles(dev);
+	ib_dealloc_device(&dev->ibdev);
+
+	QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_dev = %p\n", ha, qlnx_rdma_dev);
+	return (0);
+}
+
+int
+qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address,
+	uint8_t *new_mac_address)
+{
+        struct ecore_hwfn *p_hwfn = rdma_ctx;
+        struct qlnx_host *ha;
+        int ret = 0;
+
+        ha = (struct qlnx_host *)(p_hwfn->p_dev);
+        QL_DPRINT2(ha, "enter rdma_ctx (%p)\n", rdma_ctx);
+
+        if (old_mac_address)
+                ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, old_mac_address);
+
+        if (new_mac_address)
+                ret = ecore_llh_add_mac_filter(p_hwfn->p_dev, 0, new_mac_address);
+
+        QL_DPRINT2(ha, "exit rdma_ctx (%p)\n", rdma_ctx);
+        return (ret);
+}
+
+static void
+qlnxr_mac_address_change(struct qlnxr_dev *dev)
+{
+	qlnx_host_t *ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter/exit\n");
+
+	return;
+}
+
+static void
+qlnxr_notify(void *eth_dev, void *qlnx_rdma_dev, enum qlnx_rdma_event event)
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t *ha;
+
+	dev = qlnx_rdma_dev;
+
+	if (dev == NULL)
+		return;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter (%p, %d)\n", qlnx_rdma_dev, event);
+
+        switch (event) {
+
+        case QLNX_ETHDEV_UP:
+		if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state))
+			qlnxr_ib_dispatch_event(dev, QLNXR_PORT,
+				IB_EVENT_PORT_ACTIVE);
+                break;
+
+        case QLNX_ETHDEV_CHANGE_ADDR:
+                qlnxr_mac_address_change(dev);
+                break;
+
+        case QLNX_ETHDEV_DOWN:
+		if (test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state))
+			qlnxr_ib_dispatch_event(dev, QLNXR_PORT,
+				IB_EVENT_PORT_ERR);
+                break;
+        }
+
+	QL_DPRINT12(ha, "exit (%p, %d)\n", qlnx_rdma_dev, event);
+	return;
+}
+
+static int
+qlnxr_mod_load(void)
+{
+	int ret;
+
+
+	qlnxr_drv.add = qlnxr_add;
+	qlnxr_drv.remove = qlnxr_remove;
+	qlnxr_drv.notify = qlnxr_notify;
+
+	ret = qlnx_rdma_register_if(&qlnxr_drv);
+
+	return (0);
+}
+
+static int
+qlnxr_mod_unload(void)
+{
+	int ret;
+
+	ret = qlnx_rdma_deregister_if(&qlnxr_drv);
+	return (ret);
+}
+
+static int
+qlnxr_event_handler(module_t mod, int event, void *arg)
+{
+
+	int ret = 0;
+
+	switch (event) {
+
+	case MOD_LOAD:
+		ret = qlnxr_mod_load();
+		break;
+
+	case MOD_UNLOAD:
+		ret = qlnxr_mod_unload();
+		break;
+
+	default:
+		break;
+	}
+
+        return (ret);
+}
+
+static moduledata_t qlnxr_mod_info = {
+	.name = "qlnxr",
+	.evhand = qlnxr_event_handler,
+};
+
+MODULE_VERSION(qlnxr, 1);
+MODULE_DEPEND(qlnxr, if_qlnxe, 1, 1, 1);
+MODULE_DEPEND(qlnxr, ibcore, 1, 1, 1);
+
+#if __FreeBSD_version >= 1100000
+MODULE_DEPEND(qlnxr, linuxkpi, 1, 1, 1);
+#endif /* #if __FreeBSD_version >= 1100000 */
+
+DECLARE_MODULE(qlnxr, qlnxr_mod_info, SI_SUB_LAST, SI_ORDER_ANY);
+
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_roce.h b/sys/dev/qlnx/qlnxr/qlnxr_roce.h
new file mode 100644
index 000000000000..9a39cb5d18db
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_roce.h
@@ -0,0 +1,675 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef __QLNXR_ROCE_H__
+#define __QLNXR_ROCE_H__ 
+
+
+/*
+ * roce completion notification queue element
+ */
+struct roce_cnqe {
+	struct regpair cq_handle;
+};
+
+
+struct roce_cqe_responder {
+	struct regpair srq_wr_id;
+	struct regpair qp_handle;
+	__le32 imm_data_or_inv_r_Key;
+	__le32 length;
+	__le32 reserved0;
+	__le16 rq_cons;
+	u8 flags;
+#define ROCE_CQE_RESPONDER_TOGGLE_BIT_MASK  0x1
+#define ROCE_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0
+#define ROCE_CQE_RESPONDER_TYPE_MASK        0x3
+#define ROCE_CQE_RESPONDER_TYPE_SHIFT       1
+#define ROCE_CQE_RESPONDER_INV_FLG_MASK     0x1
+#define ROCE_CQE_RESPONDER_INV_FLG_SHIFT    3
+#define ROCE_CQE_RESPONDER_IMM_FLG_MASK     0x1
+#define ROCE_CQE_RESPONDER_IMM_FLG_SHIFT    4
+#define ROCE_CQE_RESPONDER_RDMA_FLG_MASK    0x1
+#define ROCE_CQE_RESPONDER_RDMA_FLG_SHIFT   5
+#define ROCE_CQE_RESPONDER_RESERVED2_MASK   0x3
+#define ROCE_CQE_RESPONDER_RESERVED2_SHIFT  6
+	u8 status;
+};
+
+struct roce_cqe_requester {
+	__le16 sq_cons;
+	__le16 reserved0;
+	__le32 reserved1;
+	struct regpair qp_handle;
+	struct regpair reserved2;
+	__le32 reserved3;
+	__le16 reserved4;
+	u8 flags;
+#define ROCE_CQE_REQUESTER_TOGGLE_BIT_MASK  0x1
+#define ROCE_CQE_REQUESTER_TOGGLE_BIT_SHIFT 0
+#define ROCE_CQE_REQUESTER_TYPE_MASK        0x3
+#define ROCE_CQE_REQUESTER_TYPE_SHIFT       1
+#define ROCE_CQE_REQUESTER_RESERVED5_MASK   0x1F
+#define ROCE_CQE_REQUESTER_RESERVED5_SHIFT  3
+	u8 status;
+};
+
+struct roce_cqe_common {
+	struct regpair reserved0;
+	struct regpair qp_handle;
+	__le16 reserved1[7];
+	u8 flags;
+#define ROCE_CQE_COMMON_TOGGLE_BIT_MASK  0x1
+#define ROCE_CQE_COMMON_TOGGLE_BIT_SHIFT 0
+#define ROCE_CQE_COMMON_TYPE_MASK        0x3
+#define ROCE_CQE_COMMON_TYPE_SHIFT       1
+#define ROCE_CQE_COMMON_RESERVED2_MASK   0x1F
+#define ROCE_CQE_COMMON_RESERVED2_SHIFT  3
+	u8 status;
+};
+
+/*
+ * roce completion queue element
+ */
+union roce_cqe {
+	struct roce_cqe_responder resp;
+	struct roce_cqe_requester req;
+	struct roce_cqe_common cmn;
+};
+
+
+
+
+/*
+ * CQE requester status enumeration
+ */
+enum roce_cqe_requester_status_enum {
+	ROCE_CQE_REQ_STS_OK,
+	ROCE_CQE_REQ_STS_BAD_RESPONSE_ERR,
+	ROCE_CQE_REQ_STS_LOCAL_LENGTH_ERR,
+	ROCE_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR,
+	ROCE_CQE_REQ_STS_LOCAL_PROTECTION_ERR,
+	ROCE_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR,
+	ROCE_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR,
+	ROCE_CQE_REQ_STS_REMOTE_ACCESS_ERR,
+	ROCE_CQE_REQ_STS_REMOTE_OPERATION_ERR,
+	ROCE_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR,
+	ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR,
+	ROCE_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR,
+	MAX_ROCE_CQE_REQUESTER_STATUS_ENUM
+};
+
+
+
+/*
+ * CQE responder status enumeration
+ */
+enum roce_cqe_responder_status_enum {
+	ROCE_CQE_RESP_STS_OK,
+	ROCE_CQE_RESP_STS_LOCAL_ACCESS_ERR,
+	ROCE_CQE_RESP_STS_LOCAL_LENGTH_ERR,
+	ROCE_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR,
+	ROCE_CQE_RESP_STS_LOCAL_PROTECTION_ERR,
+	ROCE_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR,
+	ROCE_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR,
+	ROCE_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR,
+	MAX_ROCE_CQE_RESPONDER_STATUS_ENUM
+};
+
+
+/*
+ * CQE type enumeration
+ */
+enum roce_cqe_type {
+	ROCE_CQE_TYPE_REQUESTER,
+	ROCE_CQE_TYPE_RESPONDER_RQ,
+	ROCE_CQE_TYPE_RESPONDER_SRQ,
+	ROCE_CQE_TYPE_INVALID,
+	MAX_ROCE_CQE_TYPE
+};
+
+
+/*
+ * memory window type enumeration
+ */
+enum roce_mw_type {
+	ROCE_MW_TYPE_1,
+	ROCE_MW_TYPE_2A,
+	MAX_ROCE_MW_TYPE
+};
+
+
+struct roce_rq_sge {
+	struct regpair addr;
+	__le32 length;
+	__le32 flags;
+#define ROCE_RQ_SGE_L_KEY_MASK      0x3FFFFFF
+#define ROCE_RQ_SGE_L_KEY_SHIFT     0
+#define ROCE_RQ_SGE_NUM_SGES_MASK   0x7
+#define ROCE_RQ_SGE_NUM_SGES_SHIFT  26
+#define ROCE_RQ_SGE_RESERVED0_MASK  0x7
+#define ROCE_RQ_SGE_RESERVED0_SHIFT 29
+};
+
+
+struct roce_sq_atomic_wqe {
+	struct regpair remote_va;
+	__le32 xrc_srq;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_ATOMIC_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_ATOMIC_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_ATOMIC_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_ATOMIC_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_ATOMIC_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_ATOMIC_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_ATOMIC_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_ATOMIC_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_ATOMIC_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_ATOMIC_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_ATOMIC_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_ATOMIC_WQE_RESERVED0_SHIFT     5
+	u8 reserved1;
+	u8 prev_wqe_size;
+	struct regpair swap_data;
+	__le32 r_key;
+	__le32 reserved2;
+	struct regpair cmp_data;
+	struct regpair reserved3;
+};
+
+
+/*
+ * First element (16 bytes) of atomic wqe
+ */
+struct roce_sq_atomic_wqe_1st {
+	struct regpair remote_va;
+	__le32 xrc_srq;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_ATOMIC_WQE_1ST_COMP_FLG_MASK       0x1
+#define ROCE_SQ_ATOMIC_WQE_1ST_COMP_FLG_SHIFT      0
+#define ROCE_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_ATOMIC_WQE_1ST_SE_FLG_MASK         0x1
+#define ROCE_SQ_ATOMIC_WQE_1ST_SE_FLG_SHIFT        3
+#define ROCE_SQ_ATOMIC_WQE_1ST_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_ATOMIC_WQE_1ST_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_ATOMIC_WQE_1ST_RESERVED0_MASK      0x7
+#define ROCE_SQ_ATOMIC_WQE_1ST_RESERVED0_SHIFT     5
+	u8 reserved1;
+	u8 prev_wqe_size;
+};
+
+
+/*
+ * Second element (16 bytes) of atomic wqe
+ */
+struct roce_sq_atomic_wqe_2nd {
+	struct regpair swap_data;
+	__le32 r_key;
+	__le32 reserved2;
+};
+
+
+/*
+ * Third element (16 bytes) of atomic wqe
+ */
+struct roce_sq_atomic_wqe_3rd {
+	struct regpair cmp_data;
+	struct regpair reserved3;
+};
+
+
+struct roce_sq_bind_wqe {
+	struct regpair addr;
+	__le32 l_key;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_BIND_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_BIND_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_BIND_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_BIND_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_BIND_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_BIND_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_BIND_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_BIND_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_BIND_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_BIND_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_BIND_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_BIND_WQE_RESERVED0_SHIFT     5
+	u8 access_ctrl;
+#define ROCE_SQ_BIND_WQE_REMOTE_READ_MASK    0x1
+#define ROCE_SQ_BIND_WQE_REMOTE_READ_SHIFT   0
+#define ROCE_SQ_BIND_WQE_REMOTE_WRITE_MASK   0x1
+#define ROCE_SQ_BIND_WQE_REMOTE_WRITE_SHIFT  1
+#define ROCE_SQ_BIND_WQE_ENABLE_ATOMIC_MASK  0x1
+#define ROCE_SQ_BIND_WQE_ENABLE_ATOMIC_SHIFT 2
+#define ROCE_SQ_BIND_WQE_LOCAL_READ_MASK     0x1
+#define ROCE_SQ_BIND_WQE_LOCAL_READ_SHIFT    3
+#define ROCE_SQ_BIND_WQE_LOCAL_WRITE_MASK    0x1
+#define ROCE_SQ_BIND_WQE_LOCAL_WRITE_SHIFT   4
+#define ROCE_SQ_BIND_WQE_RESERVED1_MASK      0x7
+#define ROCE_SQ_BIND_WQE_RESERVED1_SHIFT     5
+	u8 prev_wqe_size;
+	u8 bind_ctrl;
+#define ROCE_SQ_BIND_WQE_ZERO_BASED_MASK     0x1
+#define ROCE_SQ_BIND_WQE_ZERO_BASED_SHIFT    0
+#define ROCE_SQ_BIND_WQE_MW_TYPE_MASK        0x1
+#define ROCE_SQ_BIND_WQE_MW_TYPE_SHIFT       1
+#define ROCE_SQ_BIND_WQE_RESERVED2_MASK      0x3F
+#define ROCE_SQ_BIND_WQE_RESERVED2_SHIFT     2
+	u8 reserved3[2];
+	u8 length_hi;
+	__le32 length_lo;
+	__le32 parent_l_key;
+	__le32 reserved6;
+};
+
+
+/*
+ * First element (16 bytes) of bind wqe
+ */
+struct roce_sq_bind_wqe_1st {
+	struct regpair addr;
+	__le32 l_key;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_BIND_WQE_1ST_COMP_FLG_MASK       0x1
+#define ROCE_SQ_BIND_WQE_1ST_COMP_FLG_SHIFT      0
+#define ROCE_SQ_BIND_WQE_1ST_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_BIND_WQE_1ST_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_BIND_WQE_1ST_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_BIND_WQE_1ST_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_BIND_WQE_1ST_SE_FLG_MASK         0x1
+#define ROCE_SQ_BIND_WQE_1ST_SE_FLG_SHIFT        3
+#define ROCE_SQ_BIND_WQE_1ST_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_BIND_WQE_1ST_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_BIND_WQE_1ST_RESERVED0_MASK      0x7
+#define ROCE_SQ_BIND_WQE_1ST_RESERVED0_SHIFT     5
+	u8 access_ctrl;
+#define ROCE_SQ_BIND_WQE_1ST_REMOTE_READ_MASK    0x1
+#define ROCE_SQ_BIND_WQE_1ST_REMOTE_READ_SHIFT   0
+#define ROCE_SQ_BIND_WQE_1ST_REMOTE_WRITE_MASK   0x1
+#define ROCE_SQ_BIND_WQE_1ST_REMOTE_WRITE_SHIFT  1
+#define ROCE_SQ_BIND_WQE_1ST_ENABLE_ATOMIC_MASK  0x1
+#define ROCE_SQ_BIND_WQE_1ST_ENABLE_ATOMIC_SHIFT 2
+#define ROCE_SQ_BIND_WQE_1ST_LOCAL_READ_MASK     0x1
+#define ROCE_SQ_BIND_WQE_1ST_LOCAL_READ_SHIFT    3
+#define ROCE_SQ_BIND_WQE_1ST_LOCAL_WRITE_MASK    0x1
+#define ROCE_SQ_BIND_WQE_1ST_LOCAL_WRITE_SHIFT   4
+#define ROCE_SQ_BIND_WQE_1ST_RESERVED1_MASK      0x7
+#define ROCE_SQ_BIND_WQE_1ST_RESERVED1_SHIFT     5
+	u8 prev_wqe_size;
+};
+
+
+/*
+ * Second element (16 bytes) of bind wqe
+ */
+struct roce_sq_bind_wqe_2nd {
+	u8 bind_ctrl;
+#define ROCE_SQ_BIND_WQE_2ND_ZERO_BASED_MASK  0x1
+#define ROCE_SQ_BIND_WQE_2ND_ZERO_BASED_SHIFT 0
+#define ROCE_SQ_BIND_WQE_2ND_MW_TYPE_MASK     0x1
+#define ROCE_SQ_BIND_WQE_2ND_MW_TYPE_SHIFT    1
+#define ROCE_SQ_BIND_WQE_2ND_RESERVED2_MASK   0x3F
+#define ROCE_SQ_BIND_WQE_2ND_RESERVED2_SHIFT  2
+	u8 reserved3[2];
+	u8 length_hi;
+	__le32 length_lo;
+	__le32 parent_l_key;
+	__le32 reserved6;
+};
+
+
+/*
+ * Structure with only the SQ WQE common fields. Size is of one SQ element (16B)
+ */
+struct roce_sq_common_wqe {
+	__le32 reserved1[3];
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_COMMON_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_COMMON_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_COMMON_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_COMMON_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_COMMON_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_COMMON_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_COMMON_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_COMMON_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_COMMON_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_COMMON_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_COMMON_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_COMMON_WQE_RESERVED0_SHIFT     5
+	u8 reserved2;
+	u8 prev_wqe_size;
+};
+
+
+struct roce_sq_fmr_wqe {
+	struct regpair addr;
+	__le32 l_key;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_FMR_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_FMR_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_FMR_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_FMR_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_FMR_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_FMR_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_FMR_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_FMR_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_FMR_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_FMR_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_FMR_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_FMR_WQE_RESERVED0_SHIFT     5
+	u8 access_ctrl;
+#define ROCE_SQ_FMR_WQE_REMOTE_READ_MASK    0x1
+#define ROCE_SQ_FMR_WQE_REMOTE_READ_SHIFT   0
+#define ROCE_SQ_FMR_WQE_REMOTE_WRITE_MASK   0x1
+#define ROCE_SQ_FMR_WQE_REMOTE_WRITE_SHIFT  1
+#define ROCE_SQ_FMR_WQE_ENABLE_ATOMIC_MASK  0x1
+#define ROCE_SQ_FMR_WQE_ENABLE_ATOMIC_SHIFT 2
+#define ROCE_SQ_FMR_WQE_LOCAL_READ_MASK     0x1
+#define ROCE_SQ_FMR_WQE_LOCAL_READ_SHIFT    3
+#define ROCE_SQ_FMR_WQE_LOCAL_WRITE_MASK    0x1
+#define ROCE_SQ_FMR_WQE_LOCAL_WRITE_SHIFT   4
+#define ROCE_SQ_FMR_WQE_RESERVED1_MASK      0x7
+#define ROCE_SQ_FMR_WQE_RESERVED1_SHIFT     5
+	u8 prev_wqe_size;
+	u8 fmr_ctrl;
+#define ROCE_SQ_FMR_WQE_PAGE_SIZE_LOG_MASK  0x1F
+#define ROCE_SQ_FMR_WQE_PAGE_SIZE_LOG_SHIFT 0
+#define ROCE_SQ_FMR_WQE_ZERO_BASED_MASK     0x1
+#define ROCE_SQ_FMR_WQE_ZERO_BASED_SHIFT    5
+#define ROCE_SQ_FMR_WQE_BIND_EN_MASK        0x1
+#define ROCE_SQ_FMR_WQE_BIND_EN_SHIFT       6
+#define ROCE_SQ_FMR_WQE_RESERVED2_MASK      0x1
+#define ROCE_SQ_FMR_WQE_RESERVED2_SHIFT     7
+	u8 reserved3[2];
+	u8 length_hi;
+	__le32 length_lo;
+	struct regpair pbl_addr;
+};
+
+
+/*
+ * First element (16 bytes) of fmr wqe
+ */
+struct roce_sq_fmr_wqe_1st {
+	struct regpair addr;
+	__le32 l_key;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_FMR_WQE_1ST_COMP_FLG_MASK       0x1
+#define ROCE_SQ_FMR_WQE_1ST_COMP_FLG_SHIFT      0
+#define ROCE_SQ_FMR_WQE_1ST_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_FMR_WQE_1ST_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_FMR_WQE_1ST_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_FMR_WQE_1ST_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_FMR_WQE_1ST_SE_FLG_MASK         0x1
+#define ROCE_SQ_FMR_WQE_1ST_SE_FLG_SHIFT        3
+#define ROCE_SQ_FMR_WQE_1ST_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_FMR_WQE_1ST_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_FMR_WQE_1ST_RESERVED0_MASK      0x7
+#define ROCE_SQ_FMR_WQE_1ST_RESERVED0_SHIFT     5
+	u8 access_ctrl;
+#define ROCE_SQ_FMR_WQE_1ST_REMOTE_READ_MASK    0x1
+#define ROCE_SQ_FMR_WQE_1ST_REMOTE_READ_SHIFT   0
+#define ROCE_SQ_FMR_WQE_1ST_REMOTE_WRITE_MASK   0x1
+#define ROCE_SQ_FMR_WQE_1ST_REMOTE_WRITE_SHIFT  1
+#define ROCE_SQ_FMR_WQE_1ST_ENABLE_ATOMIC_MASK  0x1
+#define ROCE_SQ_FMR_WQE_1ST_ENABLE_ATOMIC_SHIFT 2
+#define ROCE_SQ_FMR_WQE_1ST_LOCAL_READ_MASK     0x1
+#define ROCE_SQ_FMR_WQE_1ST_LOCAL_READ_SHIFT    3
+#define ROCE_SQ_FMR_WQE_1ST_LOCAL_WRITE_MASK    0x1
+#define ROCE_SQ_FMR_WQE_1ST_LOCAL_WRITE_SHIFT   4
+#define ROCE_SQ_FMR_WQE_1ST_RESERVED1_MASK      0x7
+#define ROCE_SQ_FMR_WQE_1ST_RESERVED1_SHIFT     5
+	u8 prev_wqe_size;
+};
+
+
+/*
+ * Second element (16 bytes) of fmr wqe
+ */
+struct roce_sq_fmr_wqe_2nd {
+	u8 fmr_ctrl;
+#define ROCE_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_MASK  0x1F
+#define ROCE_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_SHIFT 0
+#define ROCE_SQ_FMR_WQE_2ND_ZERO_BASED_MASK     0x1
+#define ROCE_SQ_FMR_WQE_2ND_ZERO_BASED_SHIFT    5
+#define ROCE_SQ_FMR_WQE_2ND_BIND_EN_MASK        0x1
+#define ROCE_SQ_FMR_WQE_2ND_BIND_EN_SHIFT       6
+#define ROCE_SQ_FMR_WQE_2ND_RESERVED2_MASK      0x1
+#define ROCE_SQ_FMR_WQE_2ND_RESERVED2_SHIFT     7
+	u8 reserved3[2];
+	u8 length_hi;
+	__le32 length_lo;
+	struct regpair pbl_addr;
+};
+
+
+struct roce_sq_local_inv_wqe {
+	struct regpair reserved;
+	__le32 inv_l_key;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_LOCAL_INV_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_LOCAL_INV_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_LOCAL_INV_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_LOCAL_INV_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_LOCAL_INV_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_LOCAL_INV_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_LOCAL_INV_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_LOCAL_INV_WQE_RESERVED0_SHIFT     5
+	u8 reserved1;
+	u8 prev_wqe_size;
+};
+
+
+struct roce_sq_rdma_wqe {
+	__le32 imm_data;
+	__le32 length;
+	__le32 xrc_srq;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_RDMA_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_RDMA_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_RDMA_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_RDMA_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_RDMA_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_RDMA_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_RDMA_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_RDMA_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_RDMA_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_RDMA_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_RDMA_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_RDMA_WQE_RESERVED0_SHIFT     5
+	u8 wqe_size;
+	u8 prev_wqe_size;
+	struct regpair remote_va;
+	__le32 r_key;
+	__le32 reserved1;
+};
+
+
+/*
+ * First element (16 bytes) of rdma wqe
+ */
+struct roce_sq_rdma_wqe_1st {
+	__le32 imm_data;
+	__le32 length;
+	__le32 xrc_srq;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_RDMA_WQE_1ST_COMP_FLG_MASK       0x1
+#define ROCE_SQ_RDMA_WQE_1ST_COMP_FLG_SHIFT      0
+#define ROCE_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_RDMA_WQE_1ST_SE_FLG_MASK         0x1
+#define ROCE_SQ_RDMA_WQE_1ST_SE_FLG_SHIFT        3
+#define ROCE_SQ_RDMA_WQE_1ST_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_RDMA_WQE_1ST_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_RDMA_WQE_1ST_RESERVED0_MASK      0x7
+#define ROCE_SQ_RDMA_WQE_1ST_RESERVED0_SHIFT     5
+	u8 wqe_size;
+	u8 prev_wqe_size;
+};
+
+
+/*
+ * Second element (16 bytes) of rdma wqe
+ */
+struct roce_sq_rdma_wqe_2nd {
+	struct regpair remote_va;
+	__le32 r_key;
+	__le32 reserved1;
+};
+
+
+/*
+ * SQ WQE req type enumeration
+ */
+enum roce_sq_req_type {
+	ROCE_SQ_REQ_TYPE_SEND,
+	ROCE_SQ_REQ_TYPE_SEND_WITH_IMM,
+	ROCE_SQ_REQ_TYPE_SEND_WITH_INVALIDATE,
+	ROCE_SQ_REQ_TYPE_RDMA_WR,
+	ROCE_SQ_REQ_TYPE_RDMA_WR_WITH_IMM,
+	ROCE_SQ_REQ_TYPE_RDMA_RD,
+	ROCE_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP,
+	ROCE_SQ_REQ_TYPE_ATOMIC_ADD,
+	ROCE_SQ_REQ_TYPE_LOCAL_INVALIDATE,
+	ROCE_SQ_REQ_TYPE_FAST_MR,
+	ROCE_SQ_REQ_TYPE_BIND,
+	ROCE_SQ_REQ_TYPE_INVALID,
+	MAX_ROCE_SQ_REQ_TYPE
+};
+
+
+struct roce_sq_send_wqe {
+	__le32 inv_key_or_imm_data;
+	__le32 length;
+	__le32 xrc_srq;
+	u8 req_type;
+	u8 flags;
+#define ROCE_SQ_SEND_WQE_COMP_FLG_MASK       0x1
+#define ROCE_SQ_SEND_WQE_COMP_FLG_SHIFT      0
+#define ROCE_SQ_SEND_WQE_RD_FENCE_FLG_MASK   0x1
+#define ROCE_SQ_SEND_WQE_RD_FENCE_FLG_SHIFT  1
+#define ROCE_SQ_SEND_WQE_INV_FENCE_FLG_MASK  0x1
+#define ROCE_SQ_SEND_WQE_INV_FENCE_FLG_SHIFT 2
+#define ROCE_SQ_SEND_WQE_SE_FLG_MASK         0x1
+#define ROCE_SQ_SEND_WQE_SE_FLG_SHIFT        3
+#define ROCE_SQ_SEND_WQE_INLINE_FLG_MASK     0x1
+#define ROCE_SQ_SEND_WQE_INLINE_FLG_SHIFT    4
+#define ROCE_SQ_SEND_WQE_RESERVED0_MASK      0x7
+#define ROCE_SQ_SEND_WQE_RESERVED0_SHIFT     5
+	u8 wqe_size;
+	u8 prev_wqe_size;
+};
+
+
+struct roce_sq_sge {
+	__le32 length;
+	struct regpair addr;
+	__le32 l_key;
+};
+
+
+struct roce_srq_prod {
+	__le16 prod;
+};
+
+
+struct roce_srq_sge {
+	struct regpair addr;
+	__le32 length;
+	__le32 l_key;
+	struct regpair wr_id;
+	u8 flags;
+#define ROCE_SRQ_SGE_NUM_SGES_MASK   0x3
+#define ROCE_SRQ_SGE_NUM_SGES_SHIFT  0
+#define ROCE_SRQ_SGE_RESERVED0_MASK  0x3F
+#define ROCE_SRQ_SGE_RESERVED0_SHIFT 2
+	u8 reserved1;
+	__le16 reserved2;
+	__le32 reserved3;
+};
+
+
+/*
+ * RoCE doorbell data for SQ and RQ
+ */
+struct roce_pwm_val16_data {
+	__le16 icid;
+	__le16 prod_val;
+};
+
+
+union roce_pwm_val16_data_union {
+	struct roce_pwm_val16_data as_struct;
+	__le32 as_dword;
+};
+
+
+/*
+ * RoCE doorbell data for CQ
+ */
+struct roce_pwm_val32_data {
+	__le16 icid;
+	u8 agg_flags;
+	u8 params;
+#define ROCE_PWM_VAL32_DATA_AGG_CMD_MASK    0x3
+#define ROCE_PWM_VAL32_DATA_AGG_CMD_SHIFT   0
+#define ROCE_PWM_VAL32_DATA_BYPASS_EN_MASK  0x1
+#define ROCE_PWM_VAL32_DATA_BYPASS_EN_SHIFT 2
+#define ROCE_PWM_VAL32_DATA_RESERVED_MASK   0x1F
+#define ROCE_PWM_VAL32_DATA_RESERVED_SHIFT  3
+	__le32 cq_cons_val;
+};
+
+
+union roce_pwm_val32_data_union {
+	struct roce_pwm_val32_data as_struct;
+	struct regpair as_repair;
+};
+
+#endif /* __QLNXR_ROCE_H__ */
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_user.h b/sys/dev/qlnx/qlnxr/qlnxr_user.h
new file mode 100644
index 000000000000..ac6755188468
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_user.h
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef __QLNXR_USER_H__
+#define __QLNXR_USER_H__
+
+#define QLNXR_ABI_VERSION		(7)
+#define QLNXR_BE_ROCE_ABI_VERSION	(1)
+
+/* user kernel communication data structures. */
+
+struct qlnxr_alloc_ucontext_resp {
+	u64 db_pa;
+	u32 db_size;
+
+	uint32_t max_send_wr;
+	uint32_t max_recv_wr;
+	uint32_t max_srq_wr;
+	uint32_t sges_per_send_wr;
+	uint32_t sges_per_recv_wr;
+	uint32_t sges_per_srq_wr;
+	int max_cqes;
+	uint8_t dpm_enabled;
+	uint8_t wids_enabled;
+	uint16_t wid_count;
+};
+
+struct qlnxr_alloc_pd_ureq {
+	u64 rsvd1;
+};
+
+struct qlnxr_alloc_pd_uresp {
+	u32 pd_id;
+};
+
+struct qlnxr_create_cq_ureq {
+	uint64_t addr;		/* user space virtual address of CQ buffer */
+	size_t len;		/* size of CQ buffer */
+};
+
+struct qlnxr_create_cq_uresp {
+	u32 db_offset;
+	u16 icid;
+};
+
+struct qlnxr_create_qp_ureq {
+	u32 qp_handle_hi;
+	u32 qp_handle_lo;
+
+	/* SQ */
+	uint64_t sq_addr;	/* user space virtual address of SQ buffer */
+	size_t sq_len;		/* length of SQ buffer */
+
+	/* RQ */
+	uint64_t rq_addr;	/* user space virtual address of RQ buffer */
+	size_t rq_len;		/* length of RQ buffer */
+};
+
+struct qlnxr_create_qp_uresp {
+	u32 qp_id;
+	int atomic_supported;
+
+	/* SQ*/
+	u32 sq_db_offset;
+	u16 sq_icid;
+	
+	/* RQ */
+	u32 rq_db_offset;
+	u16 rq_icid;
+
+	u32 rq_db2_offset;
+};
+
+struct qlnxr_create_srq_ureq {
+	/* user space virtual address of producer pair */
+	uint64_t prod_pair_addr;
+	uint64_t srq_addr;	/* user space virtual address of SQ buffer */
+	size_t srq_len;		/* length of SQ buffer */
+};
+
+struct qlnxr_create_srq_uresp {
+	u16 srq_id;
+};
+
+#endif	/* #ifndef __QLNXR_USER_H__ */
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_verbs.c b/sys/dev/qlnx/qlnxr/qlnxr_verbs.c
new file mode 100644
index 000000000000..7f551647101b
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_verbs.c
@@ -0,0 +1,7306 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/*
+ * File: qlnxr_verbs.c
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "qlnxr_def.h"
+#include "rdma_common.h"
+#include "qlnxr_roce.h"
+#include "qlnxr_cm.h"
+
+#define upper_32_bits(x) (uint32_t)(x >> 32)
+#define lower_32_bits(x) (uint32_t)(x)
+#define HILO_U64(hi, lo)		((((u64)(hi)) << 32) + (lo))
+
+#define TYPEPTR_ADDR_SET(type_ptr, field, vaddr)			\
+	do {								\
+		(type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
+		(type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
+	} while (0)
+
+
+#define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
+	do {							\
+		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
+		(sge)->length = cpu_to_le32(vlength);		\
+		(sge)->flags = cpu_to_le32(vflags);		\
+	} while (0)
+
+#define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
+	do {							\
+		TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id);		\
+		(hdr)->num_sges = num_sge;			\
+	} while (0)
+
+#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
+	do {							\
+		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
+		(sge)->length = cpu_to_le32(vlength);		\
+		(sge)->l_key = cpu_to_le32(vlkey);		\
+	} while (0)
+
+#define NIPQUAD(addr) \
+	((unsigned char *)&addr)[0], \
+	((unsigned char *)&addr)[1], \
+	((unsigned char *)&addr)[2], \
+	((unsigned char *)&addr)[3]
+
+struct ib_srq *qlnxr_create_srq(struct ib_pd *,
+		struct ib_srq_init_attr *,
+		struct ib_udata *);
+
+int qlnxr_destroy_srq(struct ib_srq *);
+
+int qlnxr_modify_srq(struct ib_srq *,
+		struct ib_srq_attr *,
+		enum ib_srq_attr_mask,
+		struct ib_udata *);
+static int
+qlnxr_check_srq_params(struct ib_pd *ibpd,
+	struct qlnxr_dev *dev,
+	struct ib_srq_init_attr *attrs);
+
+static int
+qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
+	struct qlnxr_srq *srq,
+	struct qlnxr_create_srq_ureq *ureq,
+	int access, int dmasync);
+
+static int
+qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
+	struct qlnxr_dev *dev,
+	struct ib_srq_init_attr *init_attr);
+
+extern enum _ecore_status_t
+ecore_rdma_modify_srq(void *rdma_cxt,
+	struct ecore_rdma_modify_srq_in_params *in_params);
+
+extern enum _ecore_status_t
+ecore_rdma_destroy_srq(void *rdma_cxt,
+	struct ecore_rdma_destroy_srq_in_params *in_params);
+
+extern enum _ecore_status_t
+ecore_rdma_create_srq(void *rdma_cxt,
+	struct ecore_rdma_create_srq_in_params *in_params,
+	struct ecore_rdma_create_srq_out_params *out_params);
+
+
+static int
+qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_srq *srq,
+	struct ib_udata *udata);
+
+static void
+qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
+
+static void
+qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
+
+
+static u32
+qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
+
+int
+qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
+	union ib_gid *sgid)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(sgid->raw, 0, sizeof(sgid->raw));
+
+	memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return 0;
+}
+
+int
+qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
+	union ib_gid *sgid)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+	QL_DPRINT12(ha, "enter index: %d\n", index);
+#if 0
+	int ret = 0;
+	/* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
+	//if (!rdma_cap_roce_gid_table(ibdev, port)) {
+	if (!(rdma_protocol_roce(ibdev, port) &&
+		ibdev->add_gid && ibdev->del_gid)) {
+		QL_DPRINT11(ha, "acquire gid failed\n");
+		return -ENODEV;
+	}
+
+	ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
+	if (ret == -EAGAIN) {
+		memcpy(sgid, &zgid, sizeof(*sgid));
+		return 0;
+	}
+#endif
+	if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
+		QL_DPRINT12(ha, "invalid gid index %d\n", index);
+		memset(sgid, 0, sizeof(*sgid));
+		return -EINVAL;
+	}
+	memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
+
+	QL_DPRINT12(ha, "exit : %p\n", sgid);
+
+	return 0;
+}
+
+struct ib_srq *
+qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
+	struct ib_udata *udata)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+	struct ecore_rdma_destroy_srq_in_params destroy_in_params;
+	struct ecore_rdma_create_srq_out_params out_params;
+	struct ecore_rdma_create_srq_in_params in_params;
+	u64 pbl_base_addr, phy_prod_pair_addr;
+	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
+	struct ib_ucontext *ib_ctx = NULL;
+	struct qlnxr_srq_hwq_info *hw_srq;
+	struct qlnxr_ucontext *ctx = NULL;
+	struct qlnxr_create_srq_ureq ureq;
+	u32 page_cnt, page_size;
+	struct qlnxr_srq *srq;
+	int ret = 0;
+
+	dev = get_qlnxr_dev((ibpd->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
+
+	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
+	if (!srq) {
+		QL_DPRINT11(ha, "cannot allocate memory for srq\n");
+		return NULL; //@@@ : TODO what to return here?
+	}
+
+	srq->dev = dev;
+	hw_srq = &srq->hw_srq;
+	spin_lock_init(&srq->lock);
+	memset(&in_params, 0, sizeof(in_params));
+
+	if (udata && ibpd->uobject && ibpd->uobject->context) {
+		ib_ctx = ibpd->uobject->context;
+		ctx = get_qlnxr_ucontext(ib_ctx);
+
+		memset(&ureq, 0, sizeof(ureq));
+		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
+			udata->inlen))) {
+			QL_DPRINT11(ha, "problem"
+				" copying data from user space\n");
+			goto err0;
+		}
+
+		ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
+		if (ret)
+			goto err0;
+
+		page_cnt = srq->usrq.pbl_info.num_pbes;
+		pbl_base_addr = srq->usrq.pbl_tbl->pa;
+		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
+		// @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
+		// page_size = BIT(srq->usrq.umem->page_shift);
+		// else
+		page_size = srq->usrq.umem->page_size;
+	} else {
+		struct ecore_chain *pbl;
+		ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
+		if (ret)
+			goto err0;
+		pbl = &hw_srq->pbl;
+
+		page_cnt = ecore_chain_get_page_cnt(pbl);
+		pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
+		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
+		page_size = pbl->elem_per_page << 4;
+	}
+
+	in_params.pd_id = pd->pd_id;
+	in_params.pbl_base_addr = pbl_base_addr;
+	in_params.prod_pair_addr = phy_prod_pair_addr;
+	in_params.num_pages = page_cnt;
+	in_params.page_size = page_size;
+
+	ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
+	if (ret)
+		goto err1;
+
+	srq->srq_id = out_params.srq_id;
+
+	if (udata) {
+		ret = qlnxr_copy_srq_uresp(dev, srq, udata);
+		if (ret)
+			goto err2;
+	}
+
+	QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
+	return &srq->ibsrq;
+err2:
+	memset(&in_params, 0, sizeof(in_params));
+	destroy_in_params.srq_id = srq->srq_id;
+	ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
+
+err1:
+	if (udata)
+		qlnxr_free_srq_user_params(srq);
+	else
+		qlnxr_free_srq_kernel_params(srq);
+
+err0:
+	kfree(srq);	
+	return ERR_PTR(-EFAULT);
+}
+
+int
+qlnxr_destroy_srq(struct ib_srq *ibsrq)
+{
+	struct qlnxr_dev	*dev;
+	struct qlnxr_srq	*srq;
+	qlnx_host_t		*ha;
+	struct ecore_rdma_destroy_srq_in_params in_params;
+
+	srq = get_qlnxr_srq(ibsrq);
+	dev = srq->dev;
+	ha = dev->ha;
+
+	memset(&in_params, 0, sizeof(in_params));
+	in_params.srq_id = srq->srq_id;
+
+	ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
+
+	if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
+		qlnxr_free_srq_user_params(srq);
+	else
+		qlnxr_free_srq_kernel_params(srq);
+
+	QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
+	kfree(srq);
+	return 0;
+}
+
+int
+qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+	enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
+{
+	struct qlnxr_dev	*dev;
+	struct qlnxr_srq	*srq;
+	qlnx_host_t		*ha;
+	struct ecore_rdma_modify_srq_in_params in_params;
+	int ret = 0;
+
+	srq = get_qlnxr_srq(ibsrq);
+	dev = srq->dev;
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+	if (attr_mask & IB_SRQ_MAX_WR) {
+		QL_DPRINT12(ha, "invalid attribute mask=0x%x"
+			" specified for %p\n", attr_mask, srq);
+		return -EINVAL;
+	}
+
+	if (attr_mask & IB_SRQ_LIMIT) {
+		if (attr->srq_limit >= srq->hw_srq.max_wr) {
+			QL_DPRINT12(ha, "invalid srq_limit=0x%x"
+				" (max_srq_limit = 0x%x)\n",
+			       attr->srq_limit, srq->hw_srq.max_wr);
+			return -EINVAL;	
+		}
+		memset(&in_params, 0, sizeof(in_params));
+		in_params.srq_id = srq->srq_id;
+		in_params.wqe_limit = attr->srq_limit;
+		ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
+		if (ret)
+			return ret;
+	}
+
+	QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
+	return 0;
+}
+
+int
+qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
+{
+	struct qlnxr_dev	*dev;
+	struct qlnxr_srq	*srq;
+	qlnx_host_t		*ha;
+	struct ecore_rdma_device *qattr;
+	srq = get_qlnxr_srq(ibsrq);
+	dev = srq->dev;
+	ha = dev->ha;
+	//qattr = &dev->attr;
+	qattr = ecore_rdma_query_device(dev->rdma_ctx);
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!dev->rdma_ctx) {
+		QL_DPRINT12(ha, "called with invalid params"
+			" rdma_ctx is NULL\n");
+		return -EINVAL;
+	}
+
+	srq_attr->srq_limit = qattr->max_srq;
+	srq_attr->max_wr = qattr->max_srq_wr;
+	srq_attr->max_sge = qattr->max_sge;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+/* Increment srq wr producer by one */
+static
+void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
+{
+	info->wr_prod_cnt++;
+}
+
+/* Increment srq wr consumer by one */
+static 
+void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
+{
+        info->wr_cons_cnt++;
+}
+
+/* get_port_immutable verb is not available in FreeBSD */
+#if 0
+int
+qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
+	struct ib_port_immutable *immutable)
+{
+	struct qlnxr_dev                *dev;
+	qlnx_host_t                     *ha;
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "entered but not implemented!!!\n");
+}
+#endif
+
+int
+qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+	struct ib_recv_wr **bad_wr)
+{
+	struct qlnxr_dev	*dev;
+	struct qlnxr_srq	*srq;
+	qlnx_host_t		*ha;
+	struct qlnxr_srq_hwq_info *hw_srq;
+	struct ecore_chain *pbl;
+	unsigned long flags;
+	int status = 0;
+	u32 num_sge, offset;
+
+	srq = get_qlnxr_srq(ibsrq);
+	dev = srq->dev;
+	ha = dev->ha;
+	hw_srq = &srq->hw_srq;
+
+	QL_DPRINT12(ha, "enter\n");
+	spin_lock_irqsave(&srq->lock, flags);
+
+	pbl = &srq->hw_srq.pbl;
+	while (wr) {
+		struct rdma_srq_wqe_header *hdr;
+		int i;
+
+		if (!qlnxr_srq_elem_left(hw_srq) ||
+		    wr->num_sge > srq->hw_srq.max_sges) {
+			QL_DPRINT11(ha, "WR cannot be posted"
+			    " (%d, %d) || (%d > %d)\n",
+			    hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
+			    wr->num_sge, srq->hw_srq.max_sges);
+			status = -ENOMEM;
+			*bad_wr = wr;
+			break;
+		}
+
+		hdr = ecore_chain_produce(pbl);
+		num_sge = wr->num_sge;
+		/* Set number of sge and WR id in header */
+		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
+
+                /* PBL is maintained in case of WR granularity.
+                 * So increment WR producer in case we post a WR.
+                 */
+		qlnxr_inc_srq_wr_prod(hw_srq);
+		hw_srq->wqe_prod++;
+		hw_srq->sge_prod++;
+
+		QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
+			wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
+
+		for (i = 0; i < wr->num_sge; i++) {
+			struct rdma_srq_sge *srq_sge = 
+			    ecore_chain_produce(pbl);
+			/* Set SGE length, lkey and address */
+			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
+				wr->sg_list[i].length, wr->sg_list[i].lkey);
+
+			QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
+				i, srq_sge->length, srq_sge->l_key,
+				srq_sge->addr.hi, srq_sge->addr.lo);
+			hw_srq->sge_prod++;
+		}
+		wmb();
+		/*
+		 * SRQ prod is 8 bytes. Need to update SGE prod in index
+		 * in first 4 bytes and need to update WQE prod in next
+		 * 4 bytes.
+		 */
+		*(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
+		offset = offsetof(struct rdma_srq_producers, wqe_prod);
+		*((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
+			hw_srq->wqe_prod;
+		/* Flush prod after updating it */
+		wmb();
+		wr = wr->next;
+	}	
+
+	QL_DPRINT12(ha, "Elements in SRQ: %d\n",
+		ecore_chain_get_elem_left(pbl));
+
+	spin_unlock_irqrestore(&srq->lock, flags);	
+	QL_DPRINT12(ha, "exit\n");
+	return status;
+}
+
+int
+#if __FreeBSD_version < 1102000
+qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
+#else
+qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
+	struct ib_udata *udata)
+#endif /* #if __FreeBSD_version < 1102000 */
+
+{
+	struct qlnxr_dev		*dev;
+	struct ecore_rdma_device	*qattr;
+	qlnx_host_t			*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+#if __FreeBSD_version > 1102000
+	if (udata->inlen || udata->outlen)
+		return -EINVAL;
+#endif /* #if __FreeBSD_version > 1102000 */
+
+	if (dev->rdma_ctx == NULL) {
+		return -EINVAL;
+	}
+
+	qattr = ecore_rdma_query_device(dev->rdma_ctx);
+
+	memset(attr, 0, sizeof *attr);
+
+	attr->fw_ver = qattr->fw_ver;
+	attr->sys_image_guid = qattr->sys_image_guid;
+	attr->max_mr_size = qattr->max_mr_size;
+	attr->page_size_cap = qattr->page_size_caps;
+	attr->vendor_id = qattr->vendor_id;
+	attr->vendor_part_id = qattr->vendor_part_id;
+	attr->hw_ver = qattr->hw_ver;
+	attr->max_qp = qattr->max_qp;
+	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
+					IB_DEVICE_RC_RNR_NAK_GEN |
+					IB_DEVICE_LOCAL_DMA_LKEY |
+					IB_DEVICE_MEM_MGT_EXTENSIONS;
+
+	attr->max_sge = qattr->max_sge;
+	attr->max_sge_rd = qattr->max_sge;
+	attr->max_cq = qattr->max_cq;
+	attr->max_cqe = qattr->max_cqe;
+	attr->max_mr = qattr->max_mr;
+	attr->max_mw = qattr->max_mw;
+	attr->max_pd = qattr->max_pd;
+	attr->atomic_cap = dev->atomic_cap;
+	attr->max_fmr = qattr->max_fmr;
+	attr->max_map_per_fmr = 16; /* TBD: FMR */
+
+	/* There is an implicit assumption in some of the ib_xxx apps that the
+	 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
+	 * communication the qp_rd_atom is passed to the other side and used as
+	 * init_rd_atom without check device capabilities for init_rd_atom.
+	 * for this reason, we set the qp_rd_atom to be the minimum between the
+	 * two...There is an additional assumption in mlx4 driver that the
+	 * values are power of two, fls is performed on the value - 1, which
+	 * in fact gives a larger power of two for values which are not a power
+	 * of two. This should be fixed in mlx4 driver, but until then ->
+	 * we provide a value that is a power of two in our code.
+	 */
+	attr->max_qp_init_rd_atom =
+		1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
+	attr->max_qp_rd_atom =
+		min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
+		    attr->max_qp_init_rd_atom);
+
+	attr->max_srq = qattr->max_srq;
+	attr->max_srq_sge = qattr->max_srq_sge;
+	attr->max_srq_wr = qattr->max_srq_wr;
+
+	/* TODO: R&D to more properly configure the following */
+	attr->local_ca_ack_delay = qattr->dev_ack_delay;
+	attr->max_fast_reg_page_list_len = qattr->max_mr/8;
+	attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
+	attr->max_ah = qattr->max_ah;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+static inline void
+get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
+{
+	switch (speed) {
+	case 1000:
+		*ib_speed = IB_SPEED_SDR;
+		*ib_width = IB_WIDTH_1X;
+		break;
+	case 10000:
+		*ib_speed = IB_SPEED_QDR;
+		*ib_width = IB_WIDTH_1X;
+		break;
+
+	case 20000:
+		*ib_speed = IB_SPEED_DDR;
+		*ib_width = IB_WIDTH_4X;
+		break;
+
+	case 25000:
+		*ib_speed = IB_SPEED_EDR;
+		*ib_width = IB_WIDTH_1X;
+		break;
+
+	case 40000:
+		*ib_speed = IB_SPEED_QDR;
+		*ib_width = IB_WIDTH_4X;
+		break;
+
+	case 50000:
+		*ib_speed = IB_SPEED_QDR;
+		*ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
+		break;
+
+	case 100000:
+		*ib_speed = IB_SPEED_EDR;
+		*ib_width = IB_WIDTH_4X;
+		break;
+
+	default:
+		/* Unsupported */
+		*ib_speed = IB_SPEED_SDR;
+		*ib_width = IB_WIDTH_1X;
+	}
+	return;
+}
+
+int
+qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
+	struct ib_port_attr *attr)
+{
+	struct qlnxr_dev	*dev;
+	struct ecore_rdma_port	*rdma_port;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (port > 1) {
+		QL_DPRINT12(ha, "port [%d] > 1 \n", port);
+		return -EINVAL;
+	}
+
+	if (dev->rdma_ctx == NULL) {
+		QL_DPRINT12(ha, "rdma_ctx == NULL\n");
+		return -EINVAL;
+	}
+
+	rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
+	memset(attr, 0, sizeof *attr);
+
+	if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
+		attr->state = IB_PORT_ACTIVE;
+		attr->phys_state = 5;
+	} else {
+		attr->state = IB_PORT_DOWN;
+		attr->phys_state = 3;
+	}
+
+	attr->max_mtu = IB_MTU_4096;
+	attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
+	attr->lid = 0;
+	attr->lmc = 0;
+	attr->sm_lid = 0;
+	attr->sm_sl = 0;
+	attr->port_cap_flags = 0;
+
+	if (QLNX_IS_IWARP(dev)) {
+		attr->gid_tbl_len = 1;
+		attr->pkey_tbl_len = 1;
+	} else {
+		attr->gid_tbl_len = QLNXR_MAX_SGID;
+		attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
+	}
+
+	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
+	attr->qkey_viol_cntr = 0;
+
+	get_link_speed_and_width(rdma_port->link_speed,
+				 &attr->active_speed, &attr->active_width);
+
+	attr->max_msg_sz = rdma_port->max_msg_size;
+	attr->max_vl_num = 4; /* TODO -> figure this one out... */
+
+	QL_DPRINT12(ha, "state = %d phys_state = %d "
+		" link_speed = %d active_speed = %d active_width = %d"
+		" attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
+		" max_msg_sz = 0x%x max_vl_num = 0x%x \n",
+		attr->state, attr->phys_state,
+		rdma_port->link_speed, attr->active_speed,
+		attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
+		attr->max_msg_sz, attr->max_vl_num);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+int
+qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
+	struct ib_port_modify *props)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (port > 1) {
+		QL_DPRINT12(ha, "port (%d) > 1\n", port);
+		return -EINVAL;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+enum rdma_link_layer
+qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
+
+        return IB_LINK_LAYER_ETHERNET;
+}
+
+struct ib_pd *
+qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
+	struct ib_udata *udata)
+{
+	struct qlnxr_pd		*pd = NULL;
+	u16			pd_id;
+	int			rc;
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "ibdev = %p context = %p"
+		" udata = %p enter\n", ibdev, context, udata);
+
+	if (dev->rdma_ctx == NULL) {
+		QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
+		rc = -1;
+		goto err;
+	}
+
+	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
+	if (!pd) {
+		rc = -ENOMEM;
+		QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
+		goto err;
+	}
+
+	rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
+	if (rc)	{
+		QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
+		goto err;
+	}
+
+	pd->pd_id = pd_id;
+
+	if (udata && context) {
+
+		rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
+		if (rc) {
+			QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
+			ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
+			goto err;
+		}
+
+		pd->uctx = get_qlnxr_ucontext(context);
+		pd->uctx->pd = pd;
+	}
+
+	atomic_add_rel_32(&dev->pd_count, 1);
+	QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
+		pd, pd_id, dev->pd_count);
+
+	return &pd->ibpd;
+
+err:
+	kfree(pd);
+	QL_DPRINT12(ha, "exit -1\n");
+	return ERR_PTR(rc);
+}
+
+int
+qlnxr_dealloc_pd(struct ib_pd *ibpd)
+{
+	struct qlnxr_pd		*pd;
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	pd = get_qlnxr_pd(ibpd);
+	dev = get_qlnxr_dev((ibpd->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (pd == NULL) {
+		QL_DPRINT11(ha, "pd = NULL\n");
+	} else {
+		ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
+		kfree(pd);
+		atomic_subtract_rel_32(&dev->pd_count, 1);
+		QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
+			pd, pd->pd_id, dev->pd_count);
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+#define ROCE_WQE_ELEM_SIZE	sizeof(struct rdma_sq_sge)
+#define	RDMA_MAX_SGE_PER_SRQ	(4) /* Should be part of HSI */
+/* Should be part of HSI */
+#define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
+#define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
+
+static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
+static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
+
+int
+qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
+{
+	struct qlnxr_dev	*dev;
+	qlnx_host_t		*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter index = 0x%x\n", index);
+
+	if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 
+		return -EINVAL;
+
+	*pkey = QLNXR_ROCE_PKEY_DEFAULT;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+
+static inline bool
+qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
+       u16 *vlan_id)
+{
+	bool ret = false;
+
+	QL_DPRINT12(ha, "enter \n");
+
+	*vlan_id = 0;
+
+#if __FreeBSD_version >= 1100000
+	u16 tmp_vlan_id;
+
+#if __FreeBSD_version >= 1102000
+	union ib_gid *dgid;
+
+	dgid = &attr->ah_attr.grh.dgid;
+	tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
+
+	if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
+		*vlan_id = tmp_vlan_id;
+		ret = true;
+	}
+#else
+	tmp_vlan_id = attr->vlan_id;
+
+	if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
+		*vlan_id = tmp_vlan_id;
+		ret = true;
+	}
+
+#endif /* #if __FreeBSD_version > 1102000 */
+
+#else
+	ret = true;
+
+#endif /* #if __FreeBSD_version >= 1100000 */
+
+	QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
+
+	return (ret);
+}
+
+static inline void
+get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+	int attr_mask,
+	struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ecore_rdma_modify_qp_in_params *qp_params)
+{
+	int		i;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memcpy(&qp_params->sgid.bytes[0],
+	       &dev->sgid_tbl[qp->sgid_idx].raw[0],
+	       sizeof(qp_params->sgid.bytes));
+	memcpy(&qp_params->dgid.bytes[0],
+	       &attr->ah_attr.grh.dgid.raw[0],
+	       sizeof(qp_params->dgid));
+
+	qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
+
+	for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
+		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
+		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+
+
+static int
+qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
+{
+	struct qlnxr_mm	*mm;
+	qlnx_host_t	*ha;
+
+	ha = uctx->dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
+	if (mm == NULL) {
+		QL_DPRINT11(ha, "mm = NULL\n");
+		return -ENOMEM;
+	}
+
+	mm->key.phy_addr = phy_addr;
+
+	/* This function might be called with a length which is not a multiple
+	 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
+	 * forces this granularity by increasing the requested size if needed.
+	 * When qedr_mmap is called, it will search the list with the updated
+	 * length as a key. To prevent search failures, the length is rounded up
+	 * in advance to PAGE_SIZE.
+	 */
+	mm->key.len = roundup(len, PAGE_SIZE);
+	INIT_LIST_HEAD(&mm->entry);
+
+	mutex_lock(&uctx->mm_list_lock);
+	list_add(&mm->entry, &uctx->mm_head);
+	mutex_unlock(&uctx->mm_list_lock);
+
+	QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
+		(unsigned long long)mm->key.phy_addr,
+		(unsigned long)mm->key.len, uctx);
+
+	return 0;
+}
+
+static bool
+qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
+{
+	bool		found = false;
+	struct qlnxr_mm	*mm;
+	qlnx_host_t	*ha;
+
+	ha = uctx->dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	mutex_lock(&uctx->mm_list_lock);
+	list_for_each_entry(mm, &uctx->mm_head, entry) {
+		if (len != mm->key.len || phy_addr != mm->key.phy_addr)
+			continue;
+
+		found = true;
+		break;
+	}
+	mutex_unlock(&uctx->mm_list_lock);
+
+	QL_DPRINT12(ha,
+		"searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
+		mm->key.phy_addr, mm->key.len, uctx, found);
+
+	return found;
+}
+
+struct
+ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
+                struct ib_udata *udata)
+{
+        int rc;
+        struct qlnxr_ucontext *ctx;
+        struct qlnxr_alloc_ucontext_resp uresp;
+        struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
+        qlnx_host_t *ha = dev->ha;
+        struct ecore_rdma_add_user_out_params oparams;
+
+        if (!udata) {
+                return ERR_PTR(-EFAULT);
+        }
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
+	if (rc) {
+		QL_DPRINT12(ha,
+			"Failed to allocate a DPI for a new RoCE application "
+			",rc = %d. To overcome this, consider to increase "
+			"the number of DPIs, increase the doorbell BAR size "
+			"or just close unnecessary RoCE applications. In "
+			"order to increase the number of DPIs consult the "
+			"README\n", rc);
+		goto err;
+	}
+
+	ctx->dpi = oparams.dpi;
+	ctx->dpi_addr = oparams.dpi_addr;
+	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
+	ctx->dpi_size = oparams.dpi_size;
+	INIT_LIST_HEAD(&ctx->mm_head);
+	mutex_init(&ctx->mm_list_lock);
+
+	memset(&uresp, 0, sizeof(uresp));
+	uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
+				< udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
+	uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
+				< udata->outlen ? 1 : 0; //TODO: figure this out
+	uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
+				< udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 
+        uresp.db_pa = ctx->dpi_phys_addr;
+        uresp.db_size = ctx->dpi_size;
+        uresp.max_send_wr = dev->attr.max_sqe;
+        uresp.max_recv_wr = dev->attr.max_rqe;
+        uresp.max_srq_wr = dev->attr.max_srq_wr;
+        uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
+        uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
+        uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
+        uresp.max_cqes = QLNXR_MAX_CQES;
+	
+	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	if (rc)
+		goto err;
+
+	ctx->dev = dev;
+
+	rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
+	if (rc)
+		goto err;
+	QL_DPRINT12(ha, "Allocated user context %p\n",
+		&ctx->ibucontext);
+	
+	return &ctx->ibucontext;
+err:
+	kfree(ctx);
+	return ERR_PTR(rc);
+}
+
+int
+qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
+{
+        struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
+        struct qlnxr_dev *dev = uctx->dev;
+        qlnx_host_t *ha = dev->ha;
+        struct qlnxr_mm *mm, *tmp;
+        int status = 0;
+
+        QL_DPRINT12(ha, "Deallocating user context %p\n",
+                        uctx);
+
+        if (dev) {
+                ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
+        }
+
+        list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
+                QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
+                                " ctx=%p\n",
+                                mm->key.phy_addr, mm->key.len, uctx);
+                list_del(&mm->entry);
+                kfree(mm);
+        }
+        kfree(uctx);
+        return status;
+}
+
+int
+qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+	struct qlnxr_ucontext	*ucontext = get_qlnxr_ucontext(context);
+	struct qlnxr_dev	*dev = get_qlnxr_dev((context->device));
+	unsigned long		vm_page = vma->vm_pgoff << PAGE_SHIFT;
+	u64 			unmapped_db;
+	unsigned long 		len = (vma->vm_end - vma->vm_start);
+	int 			rc = 0;
+	bool 			found;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+#if __FreeBSD_version > 1102000
+	unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
+#else
+	unmapped_db = dev->db_phys_addr;
+#endif /* #if __FreeBSD_version > 1102000 */
+
+	QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
+		" vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
+		vm_page, vma->vm_pgoff, unmapped_db,
+		dev->db_size, len);
+
+	if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
+		QL_DPRINT11(ha, "Vma_start not page aligned "
+			"vm_start = %ld vma_end = %ld\n", vma->vm_start,
+			vma->vm_end);
+		return -EINVAL;
+	}
+
+	found = qlnxr_search_mmap(ucontext, vm_page, len);
+	if (!found) {
+		QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
+			vma->vm_pgoff);
+		return -EINVAL;
+	}
+
+	QL_DPRINT12(ha, "Mapping doorbell bar\n");
+
+#if __FreeBSD_version > 1102000
+
+	if ((vm_page < unmapped_db) ||
+		((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
+		QL_DPRINT11(ha, "failed pages are outside of dpi;"
+			"page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
+			vm_page, unmapped_db, ucontext->dpi_size);
+		return -EINVAL;
+	}
+
+	if (vma->vm_flags & VM_READ) {
+		QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
+		return -EINVAL;
+	}
+
+	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+	rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
+			vma->vm_page_prot);
+
+#else
+
+	if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
+		dev->db_size))) {
+
+		QL_DPRINT12(ha, "Mapping doorbell bar\n");
+
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
+		rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+					    PAGE_SIZE, vma->vm_page_prot);
+	} else {
+		QL_DPRINT12(ha, "Mapping chains\n");
+		rc = io_remap_pfn_range(vma, vma->vm_start,
+					 vma->vm_pgoff, len, vma->vm_page_prot);
+	}
+
+#endif /* #if __FreeBSD_version > 1102000 */
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+struct ib_mr *
+qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
+{
+	struct qlnxr_mr		*mr;
+	struct qlnxr_dev	*dev = get_qlnxr_dev((ibpd->device));
+	struct qlnxr_pd		*pd = get_qlnxr_pd(ibpd);
+	int			rc;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (acc & IB_ACCESS_MW_BIND) {
+		QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
+	}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		rc = -ENOMEM;
+		QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
+		goto err0;
+	}
+
+	mr->type = QLNXR_MR_DMA;
+
+	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
+	if (rc) {
+		QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
+		goto err1;
+	}
+
+	/* index only, 18 bit long, lkey = itid << 8 | key */
+	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
+	mr->hw_mr.pd = pd->pd_id;
+	mr->hw_mr.local_read = 1;
+	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
+	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
+	mr->hw_mr.dma_mr = true;
+
+	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
+	if (rc) {
+		QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
+		goto err2;
+	}
+
+	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+
+	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
+		mr->hw_mr.remote_atomic) {
+		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+	}
+
+	QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
+
+	return &mr->ibmr;
+
+err2:
+	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
+err1:
+	kfree(mr);
+err0:
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+
+	return ERR_PTR(rc);
+}
+
+static void
+qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
+	struct qlnxr_pbl *pbl)
+{
+	int		i;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	for (i = 0; i < pbl_info->num_pbls; i++) {
+		if (!pbl[i].va)
+			continue;
+		qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
+			pbl_info->pbl_size);
+	}
+	kfree(pbl);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+#define MIN_FW_PBL_PAGE_SIZE (4*1024)
+#define MAX_FW_PBL_PAGE_SIZE (64*1024)
+
+#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
+#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
+#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
+
+static struct qlnxr_pbl *
+qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
+	struct qlnxr_pbl_info *pbl_info, gfp_t flags)
+{
+	void			*va;
+	dma_addr_t		pa;
+	dma_addr_t		*pbl_main_tbl;
+	struct qlnxr_pbl	*pbl_table;
+	int			i, rc = 0;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
+
+	if (!pbl_table) {
+		QL_DPRINT12(ha, "pbl_table = NULL\n");
+		return NULL;
+	}
+
+	for (i = 0; i < pbl_info->num_pbls; i++) {
+		va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
+		if (!va) {
+			QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
+			rc = -ENOMEM;
+			goto err;
+		}
+		memset(va, 0, pbl_info->pbl_size);
+		pbl_table[i].va = va;
+		pbl_table[i].pa = pa;
+	}
+
+	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
+	 * the first one with physical pointers to all of the rest
+	 */
+	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
+	for (i = 0; i < pbl_info->num_pbls - 1; i++)
+		pbl_main_tbl[i] = pbl_table[i + 1].pa;
+
+	QL_DPRINT12(ha, "exit\n");
+	return pbl_table;
+
+err:
+	qlnxr_free_pbl(dev, pbl_info, pbl_table);
+
+	QL_DPRINT12(ha, "exit with error\n");
+	return NULL;
+}
+
+static int
+qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
+	struct qlnxr_pbl_info *pbl_info,
+	u32 num_pbes,
+	int two_layer_capable)
+{
+	u32		pbl_capacity;
+	u32		pbl_size;
+	u32		num_pbls;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
+		if (num_pbes > MAX_PBES_TWO_LAYER) {
+			QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
+				num_pbes);
+			return -EINVAL;
+		}
+
+		/* calculate required pbl page size */
+		pbl_size = MIN_FW_PBL_PAGE_SIZE;
+		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
+			NUM_PBES_ON_PAGE(pbl_size);
+
+		while (pbl_capacity < num_pbes) {
+			pbl_size *= 2;
+			pbl_capacity = pbl_size / sizeof(u64);
+			pbl_capacity = pbl_capacity * pbl_capacity;
+		}
+
+		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
+		num_pbls++; /* One for the layer0 ( points to the pbls) */
+		pbl_info->two_layered = true;
+	} else {
+		/* One layered PBL */
+		num_pbls = 1;
+		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
+				roundup_pow_of_two((num_pbes * sizeof(u64))));
+		pbl_info->two_layered = false;
+	}
+
+	pbl_info->num_pbls = num_pbls;
+	pbl_info->pbl_size = pbl_size;
+	pbl_info->num_pbes = num_pbes;
+
+	QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
+		pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
+
+	return 0;
+}
+
+#define upper_32_bits(x) (uint32_t)(x >> 32)
+#define lower_32_bits(x) (uint32_t)(x)
+
+static void
+qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
+	struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
+{
+	struct regpair		*pbe;
+	struct qlnxr_pbl	*pbl_tbl;
+	struct scatterlist	*sg;
+	int			shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
+	qlnx_host_t		*ha;
+
+#ifdef DEFINE_IB_UMEM_WITH_CHUNK
+        int                     i;
+        struct                  ib_umem_chunk *chunk = NULL;
+#else
+        int                     entry;
+#endif
+
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!pbl_info) {
+		QL_DPRINT11(ha, "PBL_INFO not initialized\n");
+		return;
+	}
+
+	if (!pbl_info->num_pbes) {
+		QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
+		return;
+	}
+
+	/* If we have a two layered pbl, the first pbl points to the rest
+	 * of the pbls and the first entry lays on the second pbl in the table
+	 */
+	if (pbl_info->two_layered)
+		pbl_tbl = &pbl[1];
+	else
+		pbl_tbl = pbl;
+
+	pbe = (struct regpair *)pbl_tbl->va;
+	if (!pbe) {
+		QL_DPRINT12(ha, "pbe is NULL\n");
+		return;
+	}
+
+	pbe_cnt = 0;
+
+	shift = ilog2(umem->page_size);
+
+#ifndef DEFINE_IB_UMEM_WITH_CHUNK
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+
+#else
+	list_for_each_entry(chunk, &umem->chunk_list, list) {
+		/* get all the dma regions from the chunk. */
+		for (i = 0; i < chunk->nmap; i++) {
+			sg = &chunk->page_list[i];
+#endif
+			pages = sg_dma_len(sg) >> shift;
+			for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
+				/* store the page address in pbe */
+				pbe->lo =
+				    cpu_to_le32(sg_dma_address(sg) +
+						(umem->page_size * pg_cnt));
+				pbe->hi =
+				    cpu_to_le32(upper_32_bits
+						((sg_dma_address(sg) +
+						  umem->page_size * pg_cnt)));
+
+				QL_DPRINT12(ha,
+					"Populate pbl table:"
+					" pbe->addr=0x%x:0x%x "
+					" pbe_cnt = %d total_num_pbes=%d"
+					" pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
+					total_num_pbes, pbe);
+
+				pbe_cnt ++;
+				total_num_pbes ++;
+				pbe++;
+
+				if (total_num_pbes == pbl_info->num_pbes)
+					return;
+
+				/* if the given pbl is full storing the pbes,
+				 * move to next pbl.
+				 */
+				if (pbe_cnt ==
+					(pbl_info->pbl_size / sizeof(u64))) {
+					pbl_tbl++;
+					pbe = (struct regpair *)pbl_tbl->va;
+					pbe_cnt = 0;
+				}
+			}
+#ifdef DEFINE_IB_UMEM_WITH_CHUNK
+		}
+#endif
+	}
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static void
+free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
+{
+	struct qlnxr_pbl *pbl, *tmp;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (info->pbl_table)
+		list_add_tail(&info->pbl_table->list_entry,
+			      &info->free_pbl_list);
+
+	if (!list_empty(&info->inuse_pbl_list))
+		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
+
+	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
+		list_del(&pbl->list_entry);
+		qlnxr_free_pbl(dev, &info->pbl_info, pbl);
+	}
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+static int
+qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
+	size_t page_list_len, bool two_layered)
+{
+	int			rc;
+	struct qlnxr_pbl	*tmp;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	INIT_LIST_HEAD(&info->free_pbl_list);
+	INIT_LIST_HEAD(&info->inuse_pbl_list);
+
+	rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
+				  page_list_len, two_layered);
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
+		goto done;
+	}
+
+	info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
+
+	if (!info->pbl_table) {
+		rc = -ENOMEM;
+		QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
+		goto done;
+	}
+
+	QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
+
+	/* in usual case we use 2 PBLs, so we add one to free
+	 * list and allocating another one
+	 */
+	tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
+
+	if (!tmp) {
+		QL_DPRINT11(ha, "Extra PBL is not allocated\n");
+		goto done; /* it's OK if second allocation fails, so rc = 0*/
+	}
+
+	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
+
+	QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
+
+done:
+	if (rc)
+		free_mr_info(dev, info);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+
+	return rc;
+}
+
+
+struct ib_mr *
+#if __FreeBSD_version >= 1102000
+qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
+	u64 usr_addr, int acc, struct ib_udata *udata)
+#else
+qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
+	u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
+#endif /* #if __FreeBSD_version >= 1102000 */
+{
+	int		rc = -ENOMEM;
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
+	struct qlnxr_mr *mr;
+	struct qlnxr_pd *pd;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	pd = get_qlnxr_pd(ibpd);
+
+	QL_DPRINT12(ha, "qedr_register user mr pd = %d"
+		" start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
+		pd->pd_id, start, len, usr_addr, acc);
+
+	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
+		QL_DPRINT11(ha,
+			"(acc & IB_ACCESS_REMOTE_WRITE &&"
+			" !(acc & IB_ACCESS_LOCAL_WRITE))\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
+		return ERR_PTR(rc);
+	}
+
+	mr->type = QLNXR_MR_USER;
+
+	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
+	if (IS_ERR(mr->umem)) {
+		rc = -EFAULT;
+		QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
+		goto err0;
+	}
+
+	rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
+	if (rc) {
+		QL_DPRINT11(ha,
+			"qlnxr_init_mr_info failed [%d]\n", rc);
+		goto err1;
+	}
+
+	qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
+			   &mr->info.pbl_info);
+
+	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
+
+	if (rc) {
+		QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
+		goto err1;
+	}
+
+	/* index only, 18 bit long, lkey = itid << 8 | key */
+	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
+	mr->hw_mr.key = 0;
+	mr->hw_mr.pd = pd->pd_id;
+	mr->hw_mr.local_read = 1;
+	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
+	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
+	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
+	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
+	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
+	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
+	mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
+
+#if __FreeBSD_version >= 1102000
+	mr->hw_mr.fbo = ib_umem_offset(mr->umem);
+#else
+	mr->hw_mr.fbo = mr->umem->offset;
+#endif
+	mr->hw_mr.length = len;
+	mr->hw_mr.vaddr = usr_addr;
+	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
+	mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
+	mr->hw_mr.dma_mr = false;
+
+	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
+	if (rc) {
+		QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
+		goto err2;
+	}
+
+	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
+		mr->hw_mr.remote_atomic)
+		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+
+	QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
+
+	return (&mr->ibmr);
+
+err2:
+	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
+err1:
+	qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
+err0:
+	kfree(mr);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return (ERR_PTR(rc));
+}
+
+int
+qlnxr_dereg_mr(struct ib_mr *ib_mr)
+{
+	struct qlnxr_mr	*mr = get_qlnxr_mr(ib_mr);
+	struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
+	int		rc = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
+		qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
+
+	/* it could be user registered memory. */
+	if (mr->umem)
+		ib_umem_release(mr->umem);
+
+	kfree(mr->pages);
+
+	kfree(mr);
+
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+}
+
+static int
+qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_cq *cq, struct ib_udata *udata)
+{
+	struct qlnxr_create_cq_uresp	uresp;
+	int				rc;
+	qlnx_host_t			*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(&uresp, 0, sizeof(uresp));
+
+	uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
+	uresp.icid = cq->icid;
+
+	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+
+	if (rc) {
+		QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
+			cq->icid, rc);
+	}
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+static void
+consume_cqe(struct qlnxr_cq *cq)
+{
+
+	if (cq->latest_cqe == cq->toggle_cqe)
+		cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
+
+	cq->latest_cqe = ecore_chain_consume(&cq->pbl);
+}
+
+static inline int
+qlnxr_align_cq_entries(int entries)
+{
+	u64 size, aligned_size;
+
+	/* We allocate an extra entry that we don't report to the FW.
+	 * Why?
+	 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
+	 * (for performance purposes). Allocating an extra entry and telling
+	 * the FW we have less prevents overwriting the first entry in case of
+	 * a wrap i.e. when the FW writes the last entry and the application
+	 * hasn't read the first one.
+	 */
+	size = (entries + 1) * QLNXR_CQE_SIZE;
+
+	/* We align to PAGE_SIZE.
+	 * Why?
+	 * Since the CQ is going to be mapped and the mapping is anyhow in whole
+	 * kernel pages we benefit from the possibly extra CQEs.
+	 */
+	aligned_size = ALIGN(size, PAGE_SIZE);
+
+	/* note: for CQs created in user space the result of this function
+	 * should match the size mapped in user space
+	 */
+	return (aligned_size / QLNXR_CQE_SIZE);
+}
+
+static inline int
+qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
+	struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
+	int access, int dmasync, int alloc_and_init)
+{
+	int		page_cnt;
+	int		rc;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	q->buf_addr = buf_addr;
+	q->buf_len = buf_len;
+
+	QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
+	      " dmasync : %x\n", q->buf_addr, q->buf_len,
+		access, dmasync);	
+
+	q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
+
+	if (IS_ERR(q->umem)) {
+		QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
+		return PTR_ERR(q->umem);
+	}
+
+	page_cnt = ib_umem_page_count(q->umem);
+	rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
+				  0 /* SQ and RQ don't support dual layer pbl.
+				     * CQ may, but this is yet uncoded.
+				     */);
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
+		goto err;
+	}
+
+	if (alloc_and_init) {
+		q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
+
+		if (!q->pbl_tbl) {
+			QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
+			rc = -ENOMEM;
+			goto err;
+		}
+
+		qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
+	} else {
+		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
+
+		if (!q->pbl_tbl) {
+			QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
+			rc = -ENOMEM;
+			goto err;
+		}
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+
+err:
+	ib_umem_release(q->umem);
+	q->umem = NULL;
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+#if __FreeBSD_version >= 1102000
+
+struct ib_cq *
+qlnxr_create_cq(struct ib_device *ibdev,
+	const struct ib_cq_init_attr *attr,
+	struct ib_ucontext *ib_ctx,
+	struct ib_udata *udata)
+
+#else 
+
+#if __FreeBSD_version >= 1100000
+
+struct ib_cq *
+qlnxr_create_cq(struct ib_device *ibdev,
+	struct ib_cq_init_attr *attr,
+	struct ib_ucontext *ib_ctx,
+	struct ib_udata *udata)
+
+#else
+
+struct ib_cq *
+qlnxr_create_cq(struct ib_device *ibdev,
+	int entries,
+	int vector,
+	struct ib_ucontext *ib_ctx,
+	struct ib_udata *udata)
+#endif /* #if __FreeBSD_version >= 1100000 */
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+{
+	struct qlnxr_ucontext			*ctx;
+	struct ecore_rdma_destroy_cq_out_params destroy_oparams;
+	struct ecore_rdma_destroy_cq_in_params	destroy_iparams;
+	struct qlnxr_dev			*dev;
+	struct ecore_rdma_create_cq_in_params	params;
+	struct qlnxr_create_cq_ureq		ureq;
+
+#if __FreeBSD_version >= 1100000
+	int					vector = attr->comp_vector;
+	int					entries = attr->cqe;
+#endif
+	struct qlnxr_cq				*cq;
+	int					chain_entries, rc, page_cnt;
+	u64					pbl_ptr;
+	u16					icid;
+	qlnx_host_t				*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "called from %s. entries = %d, "
+		"vector = %d\n",
+		(udata ? "User Lib" : "Kernel"), entries, vector);
+
+        memset(&params, 0, sizeof(struct ecore_rdma_create_cq_in_params));
+        memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
+        memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
+
+	if (entries > QLNXR_MAX_CQES) {
+		QL_DPRINT11(ha,
+			"the number of entries %d is too high. "
+			"Must be equal or below %d.\n",
+			entries, QLNXR_MAX_CQES);
+		return ERR_PTR(-EINVAL);
+	}
+	chain_entries = qlnxr_align_cq_entries(entries);
+	chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
+
+	cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
+
+	if (!cq)
+		return ERR_PTR(-ENOMEM);
+
+	if (udata) {
+		memset(&ureq, 0, sizeof(ureq));
+
+		if (ib_copy_from_udata(&ureq, udata,
+			min(sizeof(ureq), udata->inlen))) {
+			QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
+			goto err0;
+		}
+
+		if (!ureq.len) {
+			QL_DPRINT11(ha, "ureq.len == 0\n");
+			goto err0;
+		}
+
+		cq->cq_type = QLNXR_CQ_TYPE_USER;
+
+		qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
+				     IB_ACCESS_LOCAL_WRITE, 1, 1);
+
+		pbl_ptr = cq->q.pbl_tbl->pa;
+		page_cnt = cq->q.pbl_info.num_pbes;
+		cq->ibcq.cqe = chain_entries;
+	} else {
+		cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
+
+                rc = ecore_chain_alloc(&dev->ha->cdev,
+                           ECORE_CHAIN_USE_TO_CONSUME,
+                           ECORE_CHAIN_MODE_PBL,
+                           ECORE_CHAIN_CNT_TYPE_U32,
+                           chain_entries,
+                           sizeof(union roce_cqe),
+                           &cq->pbl, NULL);
+
+		if (rc)
+			goto err1;
+
+		page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
+		pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
+		cq->ibcq.cqe = cq->pbl.capacity;
+	}
+
+        params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
+        params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
+        params.cnq_id = vector;
+        params.cq_size = chain_entries - 1;
+        params.pbl_num_pages = page_cnt;
+        params.pbl_ptr = pbl_ptr;
+        params.pbl_two_level = 0;
+
+	if (ib_ctx != NULL) {
+		ctx = get_qlnxr_ucontext(ib_ctx);
+        	params.dpi = ctx->dpi;
+	} else {
+        	params.dpi = dev->dpi;
+	}
+
+	rc = ecore_rdma_create_cq(dev->rdma_ctx, &params, &icid);
+	if (rc)
+		goto err2;
+
+	cq->icid = icid;
+	cq->sig = QLNXR_CQ_MAGIC_NUMBER;
+	spin_lock_init(&cq->cq_lock);
+
+	if (ib_ctx) {
+		rc = qlnxr_copy_cq_uresp(dev, cq, udata);
+		if (rc)
+			goto err3;
+	} else {
+		/* Generate doorbell address.
+		 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
+		 * TODO: consider moving to device scope as it is a function of
+		 *       the device.
+		 * TODO: add ifdef if plan to support 16 bit.
+		 */
+		cq->db_addr = dev->db_addr +
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
+		cq->db.data.icid = cq->icid;
+		cq->db.data.params = DB_AGG_CMD_SET <<
+				     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
+
+		/* point to the very last element, passing it we will toggle */
+		cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
+		cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
+
+		/* must be different from pbl_toggle */
+		cq->latest_cqe = NULL;
+		consume_cqe(cq);
+		cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
+	}
+
+	QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
+		" number of entries = 0x%x\n",
+		cq->icid, cq, params.cq_size);
+	QL_DPRINT12(ha,"cq_addr = %p\n", cq);
+	return &cq->ibcq;
+
+err3:
+	destroy_iparams.icid = cq->icid;
+	ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
+err2:
+	if (udata)
+		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
+	else
+		ecore_chain_free(&dev->ha->cdev, &cq->pbl);
+err1:
+	if (udata)
+		ib_umem_release(cq->q.umem);
+err0:
+	kfree(cq);
+
+	QL_DPRINT12(ha, "exit error\n");
+
+	return ERR_PTR(-EINVAL);
+}
+
+int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
+{
+	int			status = 0;
+	struct qlnxr_dev	*dev = get_qlnxr_dev((ibcq->device));
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter/exit\n");
+
+	return status;
+}
+
+int
+qlnxr_destroy_cq(struct ib_cq *ibcq)
+{
+	struct qlnxr_dev			*dev = get_qlnxr_dev((ibcq->device));
+	struct ecore_rdma_destroy_cq_out_params oparams;
+	struct ecore_rdma_destroy_cq_in_params	iparams;
+	struct qlnxr_cq				*cq = get_qlnxr_cq(ibcq);
+	int					rc = 0;
+	qlnx_host_t				*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
+
+	cq->destroyed = 1;
+
+	/* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
+	 * that all completions with notification are dealt with. The rest
+	 * of the completions are not interesting
+	 */
+
+	/* GSIs CQs are handled by driver, so they don't exist in the FW */
+
+	if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
+
+		iparams.icid = cq->icid;
+
+		rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
+
+		if (rc) {
+			QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
+				cq->icid);
+			return rc;
+		}
+
+		QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
+		ecore_chain_free(&dev->ha->cdev, &cq->pbl);
+	}
+
+	if (ibcq->uobject && ibcq->uobject->context) {
+		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
+		ib_umem_release(cq->q.umem);
+	}
+
+	cq->sig = ~cq->sig;
+
+	kfree(cq);
+
+	QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
+
+	return rc;
+}
+
+static int
+qlnxr_check_qp_attrs(struct ib_pd *ibpd,
+	struct qlnxr_dev *dev,
+	struct ib_qp_init_attr *attrs,
+	struct ib_udata *udata)
+{
+	struct ecore_rdma_device	*qattr;
+	qlnx_host_t			*ha;
+
+	qattr = ecore_rdma_query_device(dev->rdma_ctx);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
+	QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
+	QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
+
+#if __FreeBSD_version < 1102000
+	QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
+#endif
+
+	QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
+	QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
+	QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
+	QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
+	QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
+	QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
+		attrs->cap.max_inline_data);
+
+#if __FreeBSD_version < 1102000
+	QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
+		attrs->cap.qpg_tss_mask_sz);
+#endif
+
+	QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
+	QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
+	QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
+	QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
+	QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
+	QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
+		(void *)qattr->sys_image_guid);
+	QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
+	QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
+	QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
+	QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
+	QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
+	QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
+	QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
+		qattr->max_qp_resp_rd_atomic_resc);
+	QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
+		qattr->max_qp_req_rd_atomic_resc);
+	QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
+		qattr->max_dev_resp_rd_atomic_resc);
+	QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
+	QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
+	QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
+	QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
+	QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
+	QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
+	QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
+	QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
+	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
+		qattr->max_mr_mw_fmr_pbl);
+	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
+		(void *)qattr->max_mr_mw_fmr_size);
+	QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
+	QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
+	QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
+	QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
+	QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
+		qattr->max_stats_queues);
+	//QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
+	QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
+		(void *)qattr->page_size_caps);
+	QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
+	QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
+	QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
+		qattr->bad_pkey_counter);
+
+	if ((attrs->qp_type == IB_QPT_GSI) && udata) {
+		QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
+		return -EINVAL;
+	}
+
+	if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
+		QL_DPRINT12(ha, "called from user without context\n");
+		return -EINVAL;
+	}
+
+	/* QP0... attrs->qp_type == IB_QPT_GSI */
+	if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
+		QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 
+			   attrs->qp_type);
+		return -EINVAL;
+	}
+	if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
+		QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
+		return -EINVAL;
+	}
+	/* Skip the check for QP1 to support CM size of 128 */
+	if (attrs->cap.max_send_wr > qattr->max_wqe) {
+		QL_DPRINT12(ha, "cannot create a SQ with %d elements "
+			" (max_send_wr=0x%x)\n",
+			attrs->cap.max_send_wr, qattr->max_wqe);
+		return -EINVAL;
+	}
+	if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
+		QL_DPRINT12(ha, "cannot create a RQ with %d elements"
+			" (max_recv_wr=0x%x)\n",
+			attrs->cap.max_recv_wr, qattr->max_wqe);
+		return -EINVAL;
+	}
+	if (attrs->cap.max_inline_data > qattr->max_inline) {
+		QL_DPRINT12(ha,
+			"unsupported inline data size=0x%x "
+			"requested (max_inline=0x%x)\n",
+			attrs->cap.max_inline_data, qattr->max_inline);
+		return -EINVAL;
+	}
+	if (attrs->cap.max_send_sge > qattr->max_sge) {
+		QL_DPRINT12(ha,
+			"unsupported send_sge=0x%x "
+			"requested (max_send_sge=0x%x)\n",
+			attrs->cap.max_send_sge, qattr->max_sge);
+		return -EINVAL;
+	}
+	if (attrs->cap.max_recv_sge > qattr->max_sge) {
+		QL_DPRINT12(ha,
+			"unsupported recv_sge=0x%x requested "
+			" (max_recv_sge=0x%x)\n",
+			attrs->cap.max_recv_sge, qattr->max_sge);
+		return -EINVAL;
+	}
+	/* unprivileged user space cannot create special QP */
+	if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
+		QL_DPRINT12(ha,
+			"userspace can't create special QPs of type=0x%x\n",
+			attrs->qp_type);
+		return -EINVAL;
+	}
+	/* allow creating only one GSI type of QP */
+	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
+		QL_DPRINT12(ha,
+			"create qp: GSI special QPs already created.\n");
+		return -EINVAL;
+	}
+
+	/* verify consumer QPs are not trying to use GSI QP's CQ */
+	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
+		struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
+		struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
+
+		if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
+		    (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
+			QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
+			return -EINVAL;
+		}
+	}
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+static int
+qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_srq *srq,
+	struct ib_udata *udata)
+{
+	struct qlnxr_create_srq_uresp	uresp;
+	qlnx_host_t			*ha;
+	int				rc;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(&uresp, 0, sizeof(uresp));
+
+	uresp.srq_id = srq->srq_id;
+
+	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+static void
+qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_create_qp_uresp *uresp,
+	struct qlnxr_qp *qp)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	/* Return if QP is associated with SRQ instead of RQ */
+	QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
+
+	if (qp->srq)
+		return;
+
+	/* iWARP requires two doorbells per RQ. */
+	if (QLNX_IS_IWARP(dev)) {
+
+		uresp->rq_db_offset =
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
+		uresp->rq_db2_offset =
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
+
+		QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
+			"uresp->rq_db2_offset = 0x%x\n",
+			uresp->rq_db_offset, uresp->rq_db2_offset);
+	} else {
+		uresp->rq_db_offset =
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
+	}
+	uresp->rq_icid = qp->icid;
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static void
+qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_create_qp_uresp *uresp,
+	struct qlnxr_qp *qp)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
+
+	/* iWARP uses the same cid for rq and sq*/
+	if (QLNX_IS_IWARP(dev)) {
+		uresp->sq_icid = qp->icid;
+		QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
+	} else
+		uresp->sq_icid = qp->icid + 1;
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static int
+qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ib_udata *udata)
+{
+	int				rc;
+	struct qlnxr_create_qp_uresp	uresp;
+	qlnx_host_t			*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
+
+	memset(&uresp, 0, sizeof(uresp));
+	qlnxr_copy_sq_uresp(dev, &uresp, qp);
+	qlnxr_copy_rq_uresp(dev, &uresp, qp);
+
+	uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
+	uresp.qp_id = qp->qp_id;
+
+	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+
+static void
+qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_pd *pd,
+	struct ib_qp_init_attr *attrs)
+{
+	qlnx_host_t			*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	spin_lock_init(&qp->q_lock);
+
+	atomic_set(&qp->refcnt, 1);
+	qp->pd = pd;
+	qp->sig = QLNXR_QP_MAGIC_NUMBER;
+	qp->qp_type = attrs->qp_type;
+	qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
+	qp->sq.max_sges = attrs->cap.max_send_sge;
+	qp->state = ECORE_ROCE_QP_STATE_RESET;
+	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
+	qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
+	qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
+	qp->dev = dev;
+
+	if (!attrs->srq) {
+		/* QP is associated with RQ instead of SRQ */
+		qp->rq.max_sges = attrs->cap.max_recv_sge;
+		QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
+			qp->rq.max_sges, qp->rq_cq->icid);
+	} else {
+		qp->srq = get_qlnxr_srq(attrs->srq);
+	}
+
+	QL_DPRINT12(ha,
+		"QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
+		" state = %d, signaled = %d, use_srq=%d\n",
+		pd->pd_id, qp->qp_type, qp->max_inline_data,
+		qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
+	QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
+		qp->sq.max_sges, qp->sq_cq->icid);
+	return;
+}
+
+static int
+qlnxr_check_srq_params(struct ib_pd *ibpd,
+	struct qlnxr_dev *dev,
+	struct ib_srq_init_attr *attrs)
+{
+	struct ecore_rdma_device *qattr;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+	qattr = ecore_rdma_query_device(dev->rdma_ctx);
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (attrs->attr.max_wr > qattr->max_srq_wqe) {
+		QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
+			" requested (max_srq_wr=0x%x)\n",
+			attrs->attr.max_wr, qattr->max_srq_wr);
+		return -EINVAL;
+	}
+
+	if (attrs->attr.max_sge > qattr->max_sge) {
+		QL_DPRINT12(ha,
+			"unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
+			attrs->attr.max_sge, qattr->max_sge);
+		return -EINVAL;
+	}
+
+	if (attrs->attr.srq_limit > attrs->attr.max_wr) {
+		QL_DPRINT12(ha,
+		       "unsupported srq_limit=0x%x requested"
+			" (max_srq_limit=0x%x)\n",
+			attrs->attr.srq_limit, attrs->attr.srq_limit);
+		return -EINVAL;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+
+static void
+qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
+{
+	struct qlnxr_dev	*dev = srq->dev;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
+	ib_umem_release(srq->usrq.umem);
+	ib_umem_release(srq->prod_umem);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static void
+qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
+{
+	struct qlnxr_srq_hwq_info *hw_srq  = &srq->hw_srq;
+	struct qlnxr_dev	*dev = srq->dev;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	ecore_chain_free(dev->cdev, &hw_srq->pbl);
+
+	qlnx_dma_free_coherent(&dev->cdev,
+		hw_srq->virt_prod_pair_addr,
+		hw_srq->phy_prod_pair_addr,
+		sizeof(struct rdma_srq_producers));
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+static int
+qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
+	struct qlnxr_srq *srq,
+	struct qlnxr_create_srq_ureq *ureq,
+	int access, int dmasync)
+{
+#ifdef DEFINE_IB_UMEM_WITH_CHUNK
+	struct ib_umem_chunk	*chunk;
+#endif
+	struct scatterlist	*sg;
+	int			rc;
+	struct qlnxr_dev	*dev = srq->dev;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
+				  ureq->srq_len, access, dmasync, 1);
+	if (rc)
+		return rc;
+
+	srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
+				     sizeof(struct rdma_srq_producers),
+				     access, dmasync);
+	if (IS_ERR(srq->prod_umem)) {
+
+		qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
+		ib_umem_release(srq->usrq.umem);
+
+		QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
+			PTR_ERR(srq->prod_umem));
+
+		return PTR_ERR(srq->prod_umem);
+	}
+
+#ifdef DEFINE_IB_UMEM_WITH_CHUNK
+	chunk = container_of((&srq->prod_umem->chunk_list)->next,
+			     typeof(*chunk), list);
+	sg = &chunk->page_list[0];
+#else
+	sg = srq->prod_umem->sg_head.sgl;
+#endif
+	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+
+static int
+qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
+	struct qlnxr_dev *dev,
+	struct ib_srq_init_attr *init_attr)
+{
+	struct qlnxr_srq_hwq_info	*hw_srq  = &srq->hw_srq;
+	dma_addr_t			phy_prod_pair_addr;
+	u32				num_elems, max_wr;
+	void				*va;
+	int				rc;
+	qlnx_host_t			*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	va = qlnx_dma_alloc_coherent(&dev->cdev,
+			&phy_prod_pair_addr,
+			sizeof(struct rdma_srq_producers));
+	if (!va) {
+		QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
+		return -ENOMEM;
+	}
+
+	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
+	hw_srq->virt_prod_pair_addr = va;
+
+	max_wr = init_attr->attr.max_wr;
+
+	num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
+
+        rc = ecore_chain_alloc(dev->cdev,
+                   ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+                   ECORE_CHAIN_MODE_PBL,
+                   ECORE_CHAIN_CNT_TYPE_U32,
+                   num_elems,
+                   ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
+                   &hw_srq->pbl, NULL);
+
+	if (rc) {
+		QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
+		goto err0;
+	}
+
+	hw_srq->max_wr = max_wr;
+	hw_srq->num_elems = num_elems;
+	hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+
+err0:
+	qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
+		sizeof(struct rdma_srq_producers));
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+static inline void
+qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
+	struct qlnxr_pd *pd,
+	struct qlnxr_qp *qp,
+	struct ib_qp_init_attr *attrs,
+	bool fmr_and_reserved_lkey,
+	struct ecore_rdma_create_qp_in_params *params)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	/* QP handle to be written in an async event */
+	params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
+	params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
+
+	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
+	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
+	params->pd = pd->pd_id;
+	params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
+	params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
+	params->stats_queue = 0;
+
+	params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
+
+	if (qp->srq) {
+		/* QP is associated with SRQ instead of RQ */
+		params->srq_id = qp->srq->srq_id;
+		params->use_srq = true;
+		QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
+			params->srq_id, params->use_srq);
+		return;
+	}
+
+	params->srq_id = 0;
+	params->use_srq = false;
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+
+static inline void
+qlnxr_qp_user_print( struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp)
+{
+	QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
+		"rq_addr=0x%llx, rq_len=%zd\n",
+		qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
+		qp->urq.buf_len);
+	return;
+}
+
+static int
+qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
+{
+	u32		newid;
+	int		rc;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!QLNX_IS_IWARP(dev))
+		return 0;
+
+	do {
+		if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
+			QL_DPRINT11(ha, "idr_pre_get failed\n");
+			return -ENOMEM;
+		}
+
+		mtx_lock(&dev->idr_lock);
+
+		rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
+
+		mtx_unlock(&dev->idr_lock);
+
+	} while (rc == -EAGAIN);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+
+	return rc;
+}
+
+static void
+qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!QLNX_IS_IWARP(dev))
+		return;
+
+	mtx_lock(&dev->idr_lock);
+	idr_remove(&dev->qpidr, id);
+	mtx_unlock(&dev->idr_lock);
+
+	QL_DPRINT12(ha, "exit \n");
+
+	return;
+}
+
+static inline void
+qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ecore_rdma_create_qp_out_params *out_params)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
+	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
+
+	qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
+			   &qp->usq.pbl_info);
+
+	if (qp->srq) {
+		QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
+		return;
+	}
+
+	qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
+	qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
+
+	qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
+			   &qp->urq.pbl_info);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static int
+qlnxr_create_user_qp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ib_pd *ibpd,
+	struct ib_udata *udata,
+	struct ib_qp_init_attr *attrs)
+{
+	struct ecore_rdma_destroy_qp_out_params d_out_params;
+	struct ecore_rdma_create_qp_in_params in_params;
+	struct ecore_rdma_create_qp_out_params out_params;
+	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
+	struct ib_ucontext *ib_ctx = NULL;
+	struct qlnxr_ucontext *ctx = NULL;
+	struct qlnxr_create_qp_ureq ureq;
+	int alloc_and_init = QLNX_IS_ROCE(dev);
+	int rc = -EINVAL;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	ib_ctx = ibpd->uobject->context;
+	ctx = get_qlnxr_ucontext(ib_ctx);
+
+	memset(&ureq, 0, sizeof(ureq));
+	rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
+
+	if (rc) {
+		QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
+		return rc;
+	}
+
+	/* SQ - read access only (0), dma sync not required (0) */
+	rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
+				  ureq.sq_len, 0, 0,
+				  alloc_and_init);
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
+		return rc;
+	}
+
+	if (!qp->srq) {
+		/* RQ - read access only (0), dma sync not required (0) */
+		rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
+					  ureq.rq_len, 0, 0,
+					  alloc_and_init);
+
+		if (rc) {
+			QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
+			return rc;
+		}
+	}
+
+	memset(&in_params, 0, sizeof(in_params));
+	qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
+	in_params.qp_handle_lo = ureq.qp_handle_lo;
+	in_params.qp_handle_hi = ureq.qp_handle_hi;
+	in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
+	in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
+
+	if (!qp->srq) {
+		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
+		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
+	}
+
+	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
+
+	if (!qp->ecore_qp) {
+		rc = -ENOMEM;
+		QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
+		goto err1;
+	}
+
+	if (QLNX_IS_IWARP(dev))
+		qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
+
+	qp->qp_id = out_params.qp_id;
+	qp->icid = out_params.icid;
+
+	rc = qlnxr_copy_qp_uresp(dev, qp, udata);
+
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
+		goto err;
+	}
+
+	qlnxr_qp_user_print(dev, qp);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+err:
+	rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
+
+	if (rc)
+		QL_DPRINT12(ha, "fatal fault\n");
+
+err1:
+	qlnxr_cleanup_user(dev, qp);
+
+	QL_DPRINT12(ha, "exit[%d]\n", rc);
+	return rc;
+}
+
+static void
+qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
+
+	qp->sq.db = dev->db_addr +
+		DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
+	qp->sq.db_data.data.icid = qp->icid + 1;
+
+	if (!qp->srq) {
+		qp->rq.db = dev->db_addr +
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
+		qp->rq.db_data.data.icid = qp->icid;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static void
+qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp)
+
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
+
+	qp->sq.db = dev->db_addr +
+		DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
+	qp->sq.db_data.data.icid = qp->icid;
+
+	if (!qp->srq) {
+		qp->rq.db = dev->db_addr +
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
+		qp->rq.db_data.data.icid = qp->icid;
+
+		qp->rq.iwarp_db2 = dev->db_addr +
+			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
+		qp->rq.iwarp_db2_data.data.icid = qp->icid;
+		qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
+	}
+
+	QL_DPRINT12(ha,
+		"qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
+		"\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
+		"\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
+		" qp->rq.iwarp_db2.data.prod_val =0x%x\n",
+		qp->sq.db, qp->sq.db_data.data.icid,
+		qp->rq.db, qp->rq.db_data.data.icid,
+		qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
+		qp->rq.iwarp_db2_data.data.value);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static int
+qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ecore_rdma_create_qp_in_params *in_params,
+	u32 n_sq_elems,
+	u32 n_rq_elems)
+{
+	struct ecore_rdma_create_qp_out_params out_params;
+	int		rc;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+        rc = ecore_chain_alloc(
+                dev->cdev,
+                ECORE_CHAIN_USE_TO_PRODUCE,
+                ECORE_CHAIN_MODE_PBL,
+                ECORE_CHAIN_CNT_TYPE_U32,
+                n_sq_elems,
+                QLNXR_SQE_ELEMENT_SIZE,
+                &qp->sq.pbl,
+                NULL);
+
+	if (rc) {
+		QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
+		return rc;
+	}
+
+	in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
+	in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
+
+	if (!qp->srq) {
+
+                rc = ecore_chain_alloc(
+                        dev->cdev,
+                        ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+                        ECORE_CHAIN_MODE_PBL,
+                        ECORE_CHAIN_CNT_TYPE_U32,
+                        n_rq_elems,
+                        QLNXR_RQE_ELEMENT_SIZE,
+                        &qp->rq.pbl,
+                        NULL);
+
+		if (rc) {
+			QL_DPRINT11(ha,
+				"ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
+			return rc;
+		}
+
+		in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
+		in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
+	}
+
+	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
+
+	if (!qp->ecore_qp) {
+		QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
+		return -EINVAL;
+	}
+
+	qp->qp_id = out_params.qp_id;
+	qp->icid = out_params.icid;
+
+	qlnxr_set_roce_db_info(dev, qp);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+static int
+qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ecore_rdma_create_qp_in_params *in_params,
+	u32 n_sq_elems,
+	u32 n_rq_elems)
+{
+	struct ecore_rdma_destroy_qp_out_params d_out_params;
+	struct ecore_rdma_create_qp_out_params out_params;
+	struct ecore_chain_ext_pbl ext_pbl;
+	int rc;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
+						     QLNXR_SQE_ELEMENT_SIZE,
+						     ECORE_CHAIN_MODE_PBL);
+	in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
+						     QLNXR_RQE_ELEMENT_SIZE,
+						     ECORE_CHAIN_MODE_PBL);
+
+	QL_DPRINT12(ha, "n_sq_elems = 0x%x"
+		" n_rq_elems = 0x%x in_params\n"
+		"\t\t\tqp_handle_lo\t\t= 0x%08x\n"
+		"\t\t\tqp_handle_hi\t\t= 0x%08x\n"
+		"\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
+		"\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
+		"\t\t\tuse_srq\t\t\t= 0x%x\n"
+		"\t\t\tsignal_all\t\t= 0x%x\n"
+		"\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
+		"\t\t\tpd\t\t\t= 0x%x\n"
+		"\t\t\tdpi\t\t\t= 0x%x\n"
+		"\t\t\tsq_cq_id\t\t\t= 0x%x\n"
+		"\t\t\tsq_num_pages\t\t= 0x%x\n"
+		"\t\t\tsq_pbl_ptr\t\t= %p\n"
+		"\t\t\tmax_sq_sges\t\t= 0x%x\n"
+		"\t\t\trq_cq_id\t\t\t= 0x%x\n"
+		"\t\t\trq_num_pages\t\t= 0x%x\n"
+		"\t\t\trq_pbl_ptr\t\t= %p\n"
+		"\t\t\tsrq_id\t\t\t= 0x%x\n"
+		"\t\t\tstats_queue\t\t= 0x%x\n",
+		n_sq_elems, n_rq_elems,
+		in_params->qp_handle_lo,
+		in_params->qp_handle_hi,
+		in_params->qp_handle_async_lo,
+		in_params->qp_handle_async_hi,
+		in_params->use_srq,
+		in_params->signal_all,
+		in_params->fmr_and_reserved_lkey,
+		in_params->pd,
+		in_params->dpi,
+		in_params->sq_cq_id,
+		in_params->sq_num_pages,
+		(void *)in_params->sq_pbl_ptr,
+		in_params->max_sq_sges,
+		in_params->rq_cq_id,
+		in_params->rq_num_pages,
+		(void *)in_params->rq_pbl_ptr,
+		in_params->srq_id,
+		in_params->stats_queue );
+
+	memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
+	memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
+
+	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
+
+	if (!qp->ecore_qp) {
+		QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
+		return -EINVAL;
+	}
+
+	/* Now we allocate the chain */
+	ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
+	ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
+
+	QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
+		"ext_pbl.p_pbl_phys = %p\n",
+		ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
+		
+        rc = ecore_chain_alloc(
+                dev->cdev,
+                ECORE_CHAIN_USE_TO_PRODUCE,
+                ECORE_CHAIN_MODE_PBL,
+                ECORE_CHAIN_CNT_TYPE_U32,
+                n_sq_elems,
+                QLNXR_SQE_ELEMENT_SIZE,
+                &qp->sq.pbl,
+                &ext_pbl);
+
+	if (rc) {
+		QL_DPRINT11(ha,
+			"ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
+		goto err;
+	}
+
+	ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
+	ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
+
+	QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
+		"ext_pbl.p_pbl_phys = %p\n",
+		ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
+
+	if (!qp->srq) {
+
+                rc = ecore_chain_alloc(
+                        dev->cdev,
+                        ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
+                        ECORE_CHAIN_MODE_PBL,
+                        ECORE_CHAIN_CNT_TYPE_U32,
+                        n_rq_elems,
+                        QLNXR_RQE_ELEMENT_SIZE,
+                        &qp->rq.pbl,
+                        &ext_pbl);
+
+		if (rc) {
+			QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
+				" failed rc = %d\n", rc);
+			goto err;
+		}
+	}
+
+	QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
+		out_params.qp_id, out_params.icid);
+
+	qp->qp_id = out_params.qp_id;
+	qp->icid = out_params.icid;
+
+	qlnxr_set_iwarp_db_info(dev, qp);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+
+err:
+	ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
+
+	QL_DPRINT12(ha, "exit rc = %d\n", rc);
+	return rc;
+}
+
+static int
+qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct ib_pd *ibpd,
+	struct ib_qp_init_attr *attrs)
+{
+	struct ecore_rdma_create_qp_in_params in_params;
+	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
+	int rc = -EINVAL;
+	u32 n_rq_elems;
+	u32 n_sq_elems;
+	u32 n_sq_entries;
+	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(&in_params, 0, sizeof(in_params));
+
+	/* A single work request may take up to MAX_SQ_WQE_SIZE elements in
+	 * the ring. The ring should allow at least a single WR, even if the
+	 * user requested none, due to allocation issues.
+	 * We should add an extra WR since the prod and cons indices of
+	 * wqe_wr_id are managed in such a way that the WQ is considered full
+	 * when (prod+1)%max_wr==cons. We currently don't do that because we
+	 * double the number of entries due an iSER issue that pushes far more
+	 * WRs than indicated. If we decline its ib_post_send() then we get
+	 * error prints in the dmesg we'd like to avoid.
+	 */
+	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
+			      qattr->max_wqe);
+
+	qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
+			GFP_KERNEL);
+	if (!qp->wqe_wr_id) {
+		QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
+		return -ENOMEM;
+	}
+
+	/* QP handle to be written in CQE */
+	in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
+	in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
+
+	/* A single work request may take up to MAX_RQ_WQE_SIZE elements in
+	 * the ring. There ring should allow at least a single WR, even if the
+	 * user requested none, due to allocation issues.
+	 */
+	qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
+
+	/* Allocate driver internal RQ array */
+	if (!qp->srq) {
+		qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
+					GFP_KERNEL);
+		if (!qp->rqe_wr_id) {
+			QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
+			kfree(qp->wqe_wr_id);
+			return -ENOMEM;
+		}
+	}
+
+	//qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
+
+        in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
+        in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
+
+        in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
+        in_params.fmr_and_reserved_lkey = true;
+        in_params.pd = pd->pd_id;
+        in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
+        in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
+        in_params.stats_queue = 0;
+
+        in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
+
+        if (qp->srq) {
+                /* QP is associated with SRQ instead of RQ */
+                in_params.srq_id = qp->srq->srq_id;
+                in_params.use_srq = true;
+                QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
+                        in_params.srq_id, in_params.use_srq);
+        } else {
+        	in_params.srq_id = 0;
+		in_params.use_srq = false;
+	}
+
+	n_sq_entries = attrs->cap.max_send_wr;
+	n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
+	n_sq_entries = max_t(u32, n_sq_entries, 1);
+	n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
+
+	n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
+
+	if (QLNX_IS_ROCE(dev)) {
+		rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
+						n_sq_elems, n_rq_elems);
+	} else {
+		rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
+						 n_sq_elems, n_rq_elems);
+	}
+
+	if (rc)
+		qlnxr_cleanup_kernel(dev, qp);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+struct ib_qp *
+qlnxr_create_qp(struct ib_pd *ibpd,
+		struct ib_qp_init_attr *attrs,
+		struct ib_udata *udata)
+{
+	struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
+	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
+	struct qlnxr_qp *qp;
+	int rc = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
+		return ERR_PTR(rc);
+	}
+
+	QL_DPRINT12(ha, "called from %s, event_handle=%p,"
+		" eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
+		(udata ? "user library" : "kernel"),
+		attrs->event_handler, pd,
+		get_qlnxr_cq(attrs->send_cq),
+		get_qlnxr_cq(attrs->send_cq)->icid,
+		get_qlnxr_cq(attrs->recv_cq),
+		get_qlnxr_cq(attrs->recv_cq)->icid);
+
+	qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
+
+	if (!qp) {
+		QL_DPRINT11(ha, "kzalloc(qp) failed\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	qlnxr_set_common_qp_params(dev, qp, pd, attrs);
+
+	if (attrs->qp_type == IB_QPT_GSI) {
+		QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
+		return qlnxr_create_gsi_qp(dev, attrs, qp);
+	}
+
+	if (udata) {
+		rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
+
+		if (rc) {
+			QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
+			goto err;
+		}
+	} else {
+		rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
+
+		if (rc) {
+			QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
+			goto err;
+		}
+	}
+
+	qp->ibqp.qp_num = qp->qp_id;
+
+	rc = qlnxr_idr_add(dev, qp, qp->qp_id);
+
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
+		goto err;
+	}
+
+	QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
+
+	return &qp->ibqp;
+err:
+	kfree(qp);
+
+	QL_DPRINT12(ha, "failed exit\n");
+	return ERR_PTR(-EFAULT);
+}
+
+
+static enum ib_qp_state
+qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
+{
+	enum ib_qp_state state = IB_QPS_ERR;
+
+	switch (qp_state) {
+	case ECORE_ROCE_QP_STATE_RESET:
+		state = IB_QPS_RESET;
+		break;
+
+	case ECORE_ROCE_QP_STATE_INIT:
+		state = IB_QPS_INIT;
+		break;
+
+	case ECORE_ROCE_QP_STATE_RTR:
+		state = IB_QPS_RTR;
+		break;
+
+	case ECORE_ROCE_QP_STATE_RTS:
+		state = IB_QPS_RTS;
+		break;
+
+	case ECORE_ROCE_QP_STATE_SQD:
+		state = IB_QPS_SQD;
+		break;
+
+	case ECORE_ROCE_QP_STATE_ERR:
+		state = IB_QPS_ERR;
+		break;
+
+	case ECORE_ROCE_QP_STATE_SQE:
+		state = IB_QPS_SQE;
+		break;
+	}
+	return state;
+}
+
+static enum ecore_roce_qp_state
+qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
+{
+	enum ecore_roce_qp_state ecore_qp_state;
+
+	ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
+
+	switch (qp_state) {
+	case IB_QPS_RESET:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_RESET;
+		break;
+
+	case IB_QPS_INIT:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_INIT;
+		break;
+
+	case IB_QPS_RTR:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_RTR;
+		break;
+
+	case IB_QPS_RTS:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_RTS;
+		break;
+
+	case IB_QPS_SQD:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_SQD;
+		break;
+
+	case IB_QPS_ERR:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_ERR;
+		break;
+
+	default:
+		ecore_qp_state =  ECORE_ROCE_QP_STATE_ERR;
+		break;
+	}
+
+	return (ecore_qp_state);
+}
+
+static void
+qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
+{
+	ecore_chain_reset(&qph->pbl);
+	qph->prod = qph->cons = 0;
+	qph->wqe_cons = 0;
+	qph->db_data.data.value = cpu_to_le16(0);
+
+	return;
+}
+
+static int
+qlnxr_update_qp_state(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	enum ecore_roce_qp_state new_state)
+{
+	int		status = 0;
+	uint32_t	reg_addr;
+	struct ecore_dev *cdev;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+	cdev = &ha->cdev;
+
+	QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
+		qp, new_state, qp->state);
+
+	if (new_state == qp->state) {
+		return 0;
+	}
+
+	switch (qp->state) {
+	case ECORE_ROCE_QP_STATE_RESET:
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_INIT:
+			qp->prev_wqe_size = 0;
+			qlnxr_reset_qp_hwq_info(&qp->sq);
+			if (!(qp->srq))
+				qlnxr_reset_qp_hwq_info(&qp->rq);
+			break;
+		default:
+			status = -EINVAL;
+			break;
+		};
+		break;
+	case ECORE_ROCE_QP_STATE_INIT:
+		/* INIT->XXX */
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_RTR:
+		/* Update doorbell (in case post_recv was done before move to RTR) */
+			if (qp->srq)
+				break;
+			wmb();
+			//writel(qp->rq.db_data.raw, qp->rq.db);
+			//if (QLNX_IS_IWARP(dev))
+			//	writel(qp->rq.iwarp_db2_data.raw,
+			//	       qp->rq.iwarp_db2);
+
+			reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
+					(uint8_t *)cdev->doorbells);
+
+			bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
+			bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
+
+			if (QLNX_IS_IWARP(dev)) {
+				reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
+					(uint8_t *)cdev->doorbells);
+				bus_write_4(ha->pci_dbells, reg_addr,\
+					qp->rq.iwarp_db2_data.raw);
+				bus_barrier(ha->pci_dbells,  0, 0,\
+					BUS_SPACE_BARRIER_READ);
+			}
+
+			
+			mmiowb();
+			break;
+		case ECORE_ROCE_QP_STATE_ERR:
+			/* TBD:flush qps... */
+			break;
+		default:
+			/* invalid state change. */
+			status = -EINVAL;
+			break;
+		};
+		break;
+	case ECORE_ROCE_QP_STATE_RTR:
+		/* RTR->XXX */
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_RTS:
+			break;
+		case ECORE_ROCE_QP_STATE_ERR:
+			break;
+		default:
+			/* invalid state change. */
+			status = -EINVAL;
+			break;
+		};
+		break;
+	case ECORE_ROCE_QP_STATE_RTS:
+		/* RTS->XXX */
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_SQD:
+			break;
+		case ECORE_ROCE_QP_STATE_ERR:
+			break;
+		default:
+			/* invalid state change. */
+			status = -EINVAL;
+			break;
+		};
+		break;
+	case ECORE_ROCE_QP_STATE_SQD:
+		/* SQD->XXX */
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_RTS:
+		case ECORE_ROCE_QP_STATE_ERR:
+			break;
+		default:
+			/* invalid state change. */
+			status = -EINVAL;
+			break;
+		};
+		break;
+	case ECORE_ROCE_QP_STATE_ERR:
+		/* ERR->XXX */
+		switch (new_state) {
+		case ECORE_ROCE_QP_STATE_RESET:
+			if ((qp->rq.prod != qp->rq.cons) ||
+			    (qp->sq.prod != qp->sq.cons)) {
+				QL_DPRINT11(ha,
+					"Error->Reset with rq/sq "
+					"not empty rq.prod=0x%x rq.cons=0x%x"
+					" sq.prod=0x%x sq.cons=0x%x\n",
+					qp->rq.prod, qp->rq.cons,
+					qp->sq.prod, qp->sq.cons);
+				status = -EINVAL;
+			}
+			break;
+		default:
+			status = -EINVAL;
+			break;
+		};
+		break;
+	default:
+		status = -EINVAL;
+		break;
+	};
+
+	QL_DPRINT12(ha, "exit\n");
+	return status;
+}
+
+int
+qlnxr_modify_qp(struct ib_qp	*ibqp,
+	struct ib_qp_attr	*attr,
+	int			attr_mask,
+	struct ib_udata		*udata)
+{
+	int rc = 0;
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
+	struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
+	enum ib_qp_state old_qp_state, new_qp_state;
+	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha,
+		"enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
+		qp, attr_mask, attr->qp_state, udata);
+
+	old_qp_state = qlnxr_get_ibqp_state(qp->state);
+	if (attr_mask & IB_QP_STATE)
+		new_qp_state = attr->qp_state;
+	else
+		new_qp_state = old_qp_state;
+
+	if (QLNX_IS_ROCE(dev)) {
+#if __FreeBSD_version >= 1100000
+		if (!ib_modify_qp_is_ok(old_qp_state,
+					new_qp_state,
+					ibqp->qp_type,
+					attr_mask,
+					IB_LINK_LAYER_ETHERNET)) {
+			QL_DPRINT12(ha,
+				"invalid attribute mask=0x%x"
+				" specified for qpn=0x%x of type=0x%x \n"
+				" old_qp_state=0x%x, new_qp_state=0x%x\n",
+				attr_mask, qp->qp_id, ibqp->qp_type,
+				old_qp_state, new_qp_state);
+			rc = -EINVAL;
+			goto err;
+		}
+#else
+		if (!ib_modify_qp_is_ok(old_qp_state,
+					new_qp_state,
+					ibqp->qp_type,
+					attr_mask )) {
+			QL_DPRINT12(ha,
+				"invalid attribute mask=0x%x"
+				" specified for qpn=0x%x of type=0x%x \n"
+				" old_qp_state=0x%x, new_qp_state=0x%x\n",
+				attr_mask, qp->qp_id, ibqp->qp_type,
+				old_qp_state, new_qp_state);
+			rc = -EINVAL;
+			goto err;
+		}
+
+#endif /* #if __FreeBSD_version >= 1100000 */
+	}
+	/* translate the masks... */
+	if (attr_mask & IB_QP_STATE) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
+		qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
+	}
+
+	// TBD consider changing ecore to be a flag as well...
+	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
+		qp_params.sqd_async = true;
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_PKEY,
+			  1);
+		if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
+			rc = -EINVAL;
+			goto err;
+		}
+
+		qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
+	}
+
+	if (attr_mask & IB_QP_QKEY) {
+		qp->qkey = attr->qkey;
+	}
+
+	/* tbd consider splitting in ecore.. */
+	if (attr_mask & IB_QP_ACCESS_FLAGS) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
+		qp_params.incoming_rdma_read_en =
+			attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
+		qp_params.incoming_rdma_write_en =
+			attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
+		qp_params.incoming_atomic_en =
+			attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
+	}
+
+	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
+		if (attr_mask & IB_QP_PATH_MTU) {
+			if (attr->path_mtu < IB_MTU_256 ||
+			    attr->path_mtu > IB_MTU_4096) {
+
+				QL_DPRINT12(ha,
+					"Only MTU sizes of 256, 512, 1024,"
+					" 2048 and 4096 are supported "
+					" attr->path_mtu = [%d]\n",
+					attr->path_mtu);
+
+				rc = -EINVAL;
+				goto err;
+			}
+			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
+				      ib_mtu_enum_to_int(
+						iboe_get_mtu(dev->ha->ifp->if_mtu)));
+		}
+
+		if (qp->mtu == 0) {
+			qp->mtu = ib_mtu_enum_to_int(
+					iboe_get_mtu(dev->ha->ifp->if_mtu));
+			QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
+				qp->mtu);
+		}
+
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
+			  1);
+
+		qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
+		qp_params.flow_label = attr->ah_attr.grh.flow_label;
+		qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
+
+		qp->sgid_idx = attr->ah_attr.grh.sgid_index;
+
+		get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
+
+		rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
+		if (rc)
+			return rc;
+
+		qp_params.use_local_mac = true;
+		memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
+
+		QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
+		       qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
+		       qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
+		QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
+		       qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
+		       qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
+		QL_DPRINT12(ha,
+			"remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
+			qp_params.remote_mac_addr[0],
+			qp_params.remote_mac_addr[1],
+			qp_params.remote_mac_addr[2],
+			qp_params.remote_mac_addr[3],
+			qp_params.remote_mac_addr[4],
+			qp_params.remote_mac_addr[5]);
+
+		qp_params.mtu = qp->mtu;
+	}
+
+	if (qp_params.mtu == 0) {
+		/* stay with current MTU */
+		if (qp->mtu) {
+			qp_params.mtu = qp->mtu;
+		} else {
+			qp_params.mtu = ib_mtu_enum_to_int(
+						iboe_get_mtu(dev->ha->ifp->if_mtu));
+		}
+	}
+
+	if (attr_mask & IB_QP_TIMEOUT) {
+		SET_FIELD(qp_params.modify_flags, \
+			ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
+
+		qp_params.ack_timeout = attr->timeout;
+		if (attr->timeout) {
+			u32 temp;
+
+			/* 12.7.34 LOCAL ACK TIMEOUT
+			 * Value representing the transport (ACK) timeout for
+			 * use by the remote, expressed as (4.096 μS*2Local ACK
+			 * Timeout)
+			 */
+			/* We use 1UL since the temporal value may be  overflow
+			 * 32 bits
+			 */
+			temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
+			qp_params.ack_timeout = temp; /* FW requires [msec] */
+		}
+		else
+			qp_params.ack_timeout = 0; /* infinite */
+	}
+	if (attr_mask & IB_QP_RETRY_CNT) {
+		SET_FIELD(qp_params.modify_flags,\
+			 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
+		qp_params.retry_cnt = attr->retry_cnt;
+	}
+
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
+			  1);
+		qp_params.rnr_retry_cnt = attr->rnr_retry;
+	}
+
+	if (attr_mask & IB_QP_RQ_PSN) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
+			  1);
+		qp_params.rq_psn = attr->rq_psn;
+		qp->rq_psn = attr->rq_psn;
+	}
+
+	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+		if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
+			rc = -EINVAL;
+			QL_DPRINT12(ha,
+				"unsupported  max_rd_atomic=%d, supported=%d\n",
+				attr->max_rd_atomic,
+				qattr->max_qp_req_rd_atomic_resc);
+			goto err;
+		}
+
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
+			  1);
+		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
+			  1);
+		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
+	}
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
+			  1);
+		qp_params.sq_psn = attr->sq_psn;
+		qp->sq_psn = attr->sq_psn;
+	}
+
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+		if (attr->max_dest_rd_atomic >
+		    qattr->max_qp_resp_rd_atomic_resc) {
+			QL_DPRINT12(ha,
+				"unsupported max_dest_rd_atomic=%d, "
+				"supported=%d\n",
+				attr->max_dest_rd_atomic,
+				qattr->max_qp_resp_rd_atomic_resc);
+
+			rc = -EINVAL;
+			goto err;
+		}
+
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
+			  1);
+		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
+	}
+
+ 	if (attr_mask & IB_QP_DEST_QPN) {
+		SET_FIELD(qp_params.modify_flags,
+			  ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
+			  1);
+
+		qp_params.dest_qp = attr->dest_qp_num;
+		qp->dest_qp_num = attr->dest_qp_num;
+	}
+
+	/*
+	 * Update the QP state before the actual ramrod to prevent a race with
+	 * fast path. Modifying the QP state to error will cause the device to
+	 * flush the CQEs and while polling the flushed CQEs will considered as
+	 * a potential issue if the QP isn't in error state.
+	 */
+	if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
+		(!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
+		qp->state = ECORE_ROCE_QP_STATE_ERR;
+
+	if (qp->qp_type != IB_QPT_GSI)
+		rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
+
+	if (attr_mask & IB_QP_STATE) {
+		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
+			rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
+		qp->state = qp_params.new_state;
+	}
+
+err:
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+}
+
+static int
+qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
+{
+	int ib_qp_acc_flags = 0;
+
+	if (params->incoming_rdma_write_en)
+		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
+	if (params->incoming_rdma_read_en)
+		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
+	if (params->incoming_atomic_en)
+		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
+	if (true) /* FIXME -> local write ?? */
+		ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
+
+	return ib_qp_acc_flags;
+}
+
+static enum ib_mtu
+qlnxr_mtu_int_to_enum(u16 mtu)
+{
+	enum ib_mtu ib_mtu_size;
+
+	switch (mtu) {
+	case 256:
+		ib_mtu_size = IB_MTU_256;
+		break;
+
+	case 512:
+		ib_mtu_size = IB_MTU_512;
+		break;
+
+	case 1024:
+		ib_mtu_size = IB_MTU_1024;
+		break;
+
+	case 2048:
+		ib_mtu_size = IB_MTU_2048;
+		break;
+
+	case 4096:
+		ib_mtu_size = IB_MTU_4096;
+		break;
+
+	default:
+		ib_mtu_size = IB_MTU_1024;
+		break;
+	}
+	return (ib_mtu_size);
+}
+
+int
+qlnxr_query_qp(struct ib_qp *ibqp,
+	struct ib_qp_attr *qp_attr,
+	int attr_mask,
+	struct ib_qp_init_attr *qp_init_attr)
+{
+	int rc = 0;
+	struct ecore_rdma_query_qp_out_params params;
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	struct qlnxr_dev *dev = qp->dev;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(&params, 0, sizeof(params));
+
+	rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, &params);
+	if (rc)
+		goto err;
+
+	memset(qp_attr, 0, sizeof(*qp_attr));
+	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
+
+	qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
+	qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
+
+	/* In some cases in iWARP qelr will ask for the state only */
+	if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
+		QL_DPRINT11(ha, "only state requested\n");
+		return 0;
+	}
+
+	qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
+	qp_attr->path_mig_state = IB_MIG_MIGRATED;
+	qp_attr->rq_psn = params.rq_psn;
+	qp_attr->sq_psn = params.sq_psn;
+	qp_attr->dest_qp_num = params.dest_qp;
+
+	qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(&params);
+
+	QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
+		"path_mtu = %d qp_access_flags = 0x%x\n",
+		qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
+		qp_attr->qp_access_flags);
+
+	qp_attr->cap.max_send_wr = qp->sq.max_wr;
+	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
+	qp_attr->cap.max_send_sge = qp->sq.max_sges;
+	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
+	qp_attr->cap.max_inline_data = qp->max_inline_data;
+	qp_init_attr->cap = qp_attr->cap;
+
+	memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
+	       sizeof(qp_attr->ah_attr.grh.dgid.raw));
+
+	qp_attr->ah_attr.grh.flow_label = params.flow_label;
+	qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
+	qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
+	qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
+
+	qp_attr->ah_attr.ah_flags = IB_AH_GRH;
+	qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
+	qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
+	qp_attr->timeout = params.timeout;
+	qp_attr->rnr_retry = params.rnr_retry;
+	qp_attr->retry_cnt = params.retry_cnt;
+	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
+	qp_attr->pkey_index = params.pkey_index;
+	qp_attr->port_num = 1; /* FIXME -> check this */
+	qp_attr->ah_attr.src_path_bits = 0;
+	qp_attr->ah_attr.static_rate = 0;
+	qp_attr->alt_pkey_index = 0;
+	qp_attr->alt_port_num = 0;
+	qp_attr->alt_timeout = 0;
+	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
+
+	qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
+	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
+	qp_attr->max_rd_atomic = params.max_rd_atomic;
+	qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
+
+	QL_DPRINT12(ha, "max_inline_data=%d\n",
+		qp_attr->cap.max_inline_data);
+
+err:
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+}
+
+
+static void
+qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (qp->usq.umem)
+		ib_umem_release(qp->usq.umem);
+
+	qp->usq.umem = NULL;
+
+	if (qp->urq.umem)
+		ib_umem_release(qp->urq.umem);
+
+	qp->urq.umem = NULL;
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+static void
+qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (qlnxr_qp_has_sq(qp)) {
+		QL_DPRINT12(ha, "freeing SQ\n");
+		ha->qlnxr_debug = 1;
+//		ecore_chain_free(dev->cdev, &qp->sq.pbl);
+		ha->qlnxr_debug = 0;
+		kfree(qp->wqe_wr_id);
+	}
+
+	if (qlnxr_qp_has_rq(qp)) {
+		QL_DPRINT12(ha, "freeing RQ\n");
+		ha->qlnxr_debug = 1;
+	//	ecore_chain_free(dev->cdev, &qp->rq.pbl);
+		ha->qlnxr_debug = 0;
+		kfree(qp->rqe_wr_id);
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+int
+qlnxr_free_qp_resources(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp)
+{
+	int		rc = 0;
+	qlnx_host_t	*ha;
+	struct ecore_rdma_destroy_qp_out_params d_out_params;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+#if 0
+	if (qp->qp_type != IB_QPT_GSI) {
+		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
+				&d_out_params);
+		if (rc)
+			return rc;
+	}
+
+	if (qp->ibqp.uobject && qp->ibqp.uobject->context)
+		qlnxr_cleanup_user(dev, qp);
+	else
+		qlnxr_cleanup_kernel(dev, qp);
+#endif
+
+	if (qp->ibqp.uobject && qp->ibqp.uobject->context)
+		qlnxr_cleanup_user(dev, qp);
+	else
+		qlnxr_cleanup_kernel(dev, qp);
+
+	if (qp->qp_type != IB_QPT_GSI) {
+		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
+				&d_out_params);
+		if (rc)
+			return rc;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+int
+qlnxr_destroy_qp(struct ib_qp *ibqp)
+{
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	struct qlnxr_dev *dev = qp->dev;
+	int rc = 0;
+	struct ib_qp_attr attr;
+	int attr_mask = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
+
+	qp->destroyed = 1;
+
+	if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
+				  ECORE_ROCE_QP_STATE_ERR |
+				  ECORE_ROCE_QP_STATE_INIT))) {
+
+		attr.qp_state = IB_QPS_ERR;
+		attr_mask |= IB_QP_STATE;
+
+		/* change the QP state to ERROR */
+		qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
+	}
+
+	if (qp->qp_type == IB_QPT_GSI)
+		qlnxr_destroy_gsi_qp(dev);
+
+	qp->sig = ~qp->sig;
+
+	qlnxr_free_qp_resources(dev, qp);
+
+	if (atomic_dec_and_test(&qp->refcnt)) {
+		/* TODO: only for iWARP? */
+		qlnxr_idr_remove(dev, qp->qp_id);
+		kfree(qp);
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+}
+
+static inline int
+qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
+{
+	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
+}
+
+static int
+sge_data_len(struct ib_sge *sg_list, int num_sge)
+{
+	int i, len = 0;
+	for (i = 0; i < num_sge; i++)
+		len += sg_list[i].length;
+	return len;
+}
+
+static void
+swap_wqe_data64(u64 *p)
+{
+	int i;
+
+	for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
+		*p = cpu_to_be64(cpu_to_le64(*p));
+}
+
+
+static u32
+qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
+	struct qlnxr_qp		*qp,
+	u8			*wqe_size,
+	struct ib_send_wr	*wr,
+	struct ib_send_wr	**bad_wr,
+	u8			*bits,
+	u8			bit)
+{
+	int i, seg_siz;
+	char *seg_prt, *wqe;
+	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter[%d]\n", data_size);
+
+	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
+		QL_DPRINT12(ha,
+			"Too much inline data in WR:[%d, %d]\n",
+			data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
+		*bad_wr = wr;
+		return 0;
+	}
+
+	if (!data_size)
+		return data_size;
+
+	/* set the bit */
+	*bits |= bit;
+
+	seg_prt = wqe = NULL;
+	seg_siz = 0;
+
+	/* copy data inline */
+	for (i = 0; i < wr->num_sge; i++) {
+		u32 len = wr->sg_list[i].length;
+		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
+
+		while (len > 0) {
+			u32 cur;
+
+			/* new segment required */
+			if (!seg_siz) {
+				wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
+				seg_prt = wqe;
+				seg_siz = sizeof(struct rdma_sq_common_wqe);
+				(*wqe_size)++;
+			}
+
+			/* calculate currently allowed length */
+			cur = MIN(len, seg_siz);
+
+			memcpy(seg_prt, src, cur);
+
+			/* update segment variables */
+			seg_prt += cur;
+			seg_siz -= cur;
+			/* update sge variables */
+			src += cur;
+			len -= cur;
+
+			/* swap fully-completed segments */
+			if (!seg_siz)
+				swap_wqe_data64((u64 *)wqe);
+		}
+	}
+
+	/* swap last not completed segment */
+	if (seg_siz)
+		swap_wqe_data64((u64 *)wqe);
+
+	QL_DPRINT12(ha, "exit\n");
+	return data_size;
+}
+
+static u32
+qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
+	u8 *wqe_size, struct ib_send_wr *wr)
+{
+	int i;
+	u32 data_size = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
+ 
+	for (i = 0; i < wr->num_sge; i++) {
+		struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
+
+		TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
+		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
+		sge->length = cpu_to_le32(wr->sg_list[i].length);
+		data_size += wr->sg_list[i].length;
+	}
+
+	if (wqe_size)
+		*wqe_size += wr->num_sge;
+
+	QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
+	return data_size;
+}
+
+static u32
+qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct rdma_sq_rdma_wqe_1st *rwqe,
+	struct rdma_sq_rdma_wqe_2nd *rwqe2,
+	struct ib_send_wr *wr,
+	struct ib_send_wr **bad_wr)
+{
+	qlnx_host_t	*ha;
+	u32             ret = 0;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
+	TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
+
+	if (wr->send_flags & IB_SEND_INLINE) {
+		u8 flags = 0;
+		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
+		return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
+				wr, bad_wr, &rwqe->flags, flags);
+	}
+
+	ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
+
+	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
+
+	return (ret);
+}
+
+static u32
+qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct rdma_sq_send_wqe *swqe,
+	struct rdma_sq_send_wqe *swqe2,
+	struct ib_send_wr *wr,
+	struct ib_send_wr **bad_wr)
+{
+	qlnx_host_t	*ha;
+	u32             ret = 0;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	memset(swqe2, 0, sizeof(*swqe2));
+
+	if (wr->send_flags & IB_SEND_INLINE) {
+		u8 flags = 0;
+		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
+		return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
+				wr, bad_wr, &swqe->flags, flags);
+	}
+
+	ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
+
+	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
+
+	return (ret);
+}
+
+static void
+qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
+{
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	int work = info->completed - info->completed_handled - 1;
+
+	QL_DPRINT12(ha, "enter [%d]\n", work);
+ 
+	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
+		struct qlnxr_pbl *pbl;
+
+		/* Free all the page list that are possible to be freed
+		 * (all the ones that were invalidated), under the assumption
+		 * that if an FMR was completed successfully that means that
+		 * if there was an invalidate operation before it also ended
+		 */
+		pbl = list_first_entry(&info->inuse_pbl_list,
+				       struct qlnxr_pbl,
+				       list_entry);
+		list_del(&pbl->list_entry);
+		list_add_tail(&pbl->list_entry, &info->free_pbl_list);
+		info->completed_handled++;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+#if __FreeBSD_version >= 1102000
+
+static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
+		struct rdma_sq_fmr_wqe_1st *fwqe1,
+		struct ib_reg_wr *wr)
+{
+	struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
+	struct rdma_sq_fmr_wqe_2nd *fwqe2;
+
+	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
+	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
+	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
+	fwqe1->l_key = wr->key;
+
+	fwqe2->access_ctrl = 0;
+
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
+		!!(wr->access & IB_ACCESS_REMOTE_READ));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
+		!!(wr->access & IB_ACCESS_REMOTE_WRITE));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
+		!!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
+		!!(wr->access & IB_ACCESS_LOCAL_WRITE));
+	fwqe2->fmr_ctrl = 0;
+
+	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
+		ilog2(mr->ibmr.page_size) - 12);
+
+	fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
+	fwqe2->length_lo = mr->ibmr.length;
+	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
+	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
+
+	qp->wqe_wr_id[qp->sq.prod].mr = mr;
+
+	return 0;
+}
+
+#else
+
+static void
+build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr,
+	struct mr_info *info)
+{
+	int i;
+	u64 buf_addr = 0;
+	int num_pbes, total_num_pbes = 0;
+	struct regpair *pbe;
+	struct qlnxr_pbl *pbl_tbl = info->pbl_table;
+	struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	pbe = (struct regpair *)pbl_tbl->va;
+	num_pbes = 0;
+
+	for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
+		buf_addr = wr->wr.fast_reg.page_list->page_list[i];
+		pbe->lo = cpu_to_le32((u32)buf_addr);
+		pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
+
+		num_pbes += 1;
+		pbe++;
+		total_num_pbes++;
+
+		if (total_num_pbes == pbl_info->num_pbes)
+			return;
+
+		/* if the given pbl is full storing the pbes,
+		 * move to next pbl.
+		 */
+		if (num_pbes ==
+		    (pbl_info->pbl_size / sizeof(u64))) {
+			pbl_tbl++;
+			pbe = (struct regpair *)pbl_tbl->va;
+			num_pbes = 0;
+		}
+	}
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+static int
+qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
+{
+	int rc = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (info->completed == 0) {
+		//DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
+		/* first fmr */
+		return 0;
+	}
+
+	qlnx_handle_completed_mrs(dev, info);
+
+	list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
+
+	if (list_empty(&info->free_pbl_list)) {
+		info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
+							  GFP_ATOMIC);
+	} else {
+		info->pbl_table = list_first_entry(&info->free_pbl_list,
+					struct qlnxr_pbl,
+					list_entry);
+		list_del(&info->pbl_table->list_entry);
+	}
+
+	if (!info->pbl_table)
+		rc = -ENOMEM;
+
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+}
+
+static inline int
+qlnxr_prepare_fmr(struct qlnxr_qp *qp,
+	struct rdma_sq_fmr_wqe_1st *fwqe1,
+	struct ib_send_wr *wr)
+{
+	struct qlnxr_dev *dev = qp->dev;
+	u64 fbo;
+	struct qlnxr_fast_reg_page_list *frmr_list =
+		get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
+	struct rdma_sq_fmr_wqe *fwqe2 =
+		(struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
+	int rc = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (wr->wr.fast_reg.page_list_len == 0)
+		BUG();
+
+	rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
+	if (rc)
+		return rc;
+
+	fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
+	fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
+	fwqe1->l_key = wr->wr.fast_reg.rkey;
+
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
+		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
+		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
+		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
+	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
+		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
+
+	fwqe2->fmr_ctrl = 0;
+
+	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
+		   ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
+	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
+
+	fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
+	fwqe2->length_lo = wr->wr.fast_reg.length;
+	fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
+	fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
+
+	/* produce another wqe for fwqe3 */
+	ecore_chain_produce(&qp->sq.pbl);
+
+	fbo = wr->wr.fast_reg.iova_start -
+	    (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
+
+	QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
+		" length = %x pbl_addr %x:%x\n",
+		wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
+		fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
+		fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
+
+	build_frmr_pbes(dev, wr, &frmr_list->info);
+
+	qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+static enum ib_wc_opcode
+qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
+{
+	switch (opcode) {
+	case IB_WR_RDMA_WRITE:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		return IB_WC_RDMA_WRITE;
+	case IB_WR_SEND_WITH_IMM:
+	case IB_WR_SEND:
+	case IB_WR_SEND_WITH_INV:
+		return IB_WC_SEND;
+	case IB_WR_RDMA_READ:
+		return IB_WC_RDMA_READ;
+	case IB_WR_ATOMIC_CMP_AND_SWP:
+		return IB_WC_COMP_SWAP;
+	case IB_WR_ATOMIC_FETCH_AND_ADD:
+		return IB_WC_FETCH_ADD;
+
+#if __FreeBSD_version >= 1102000
+	case IB_WR_REG_MR:
+		return IB_WC_REG_MR;
+#else
+	case IB_WR_FAST_REG_MR:
+		return IB_WC_FAST_REG_MR;
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+	case IB_WR_LOCAL_INV:
+		return IB_WC_LOCAL_INV;
+	default:
+		return IB_WC_SEND;
+	}
+}
+static inline bool
+qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr)
+{
+	int wq_is_full, err_wr, pbl_is_full;
+	struct qlnxr_dev *dev = qp->dev;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
+ 
+	/* prevent SQ overflow and/or processing of a bad WR */
+	err_wr = wr->num_sge > qp->sq.max_sges;
+	wq_is_full = qlnxr_wq_is_full(&qp->sq);
+	pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
+		      QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
+	if (wq_is_full || err_wr || pbl_is_full) {
+		if (wq_is_full &&
+		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
+
+			qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
+
+			QL_DPRINT12(ha,
+				"error: WQ is full. Post send on QP failed"
+				" (this error appears only once) "
+				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
+				qp, wr, qp->err_bitmap);
+		}
+
+		if (err_wr &&
+		    !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
+
+			qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
+
+			QL_DPRINT12(ha,
+				"error: WQ is bad. Post send on QP failed"
+				" (this error appears only once) "
+				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
+				qp, wr, qp->err_bitmap);
+		}
+
+		if (pbl_is_full &&
+		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
+
+			qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
+
+			QL_DPRINT12(ha,
+				"error: WQ PBL is full. Post send on QP failed"
+				" (this error appears only once) "
+				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
+				qp, wr, qp->err_bitmap);
+		}
+		return false;
+	}
+	QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
+	return true;
+}
+
+int
+qlnxr_post_send(struct ib_qp *ibqp,
+	struct ib_send_wr *wr,
+	struct ib_send_wr **bad_wr)
+{
+	struct qlnxr_dev	*dev = get_qlnxr_dev(ibqp->device);
+	struct qlnxr_qp		*qp = get_qlnxr_qp(ibqp);
+	unsigned long 		flags;
+	int 			status = 0, rc = 0;
+	bool			comp;
+	qlnx_host_t		*ha;
+	uint32_t		reg_addr;
+ 
+	*bad_wr = NULL;
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
+		ibqp, wr, bad_wr);
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	if (qp->qp_type == IB_QPT_GSI)
+		return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
+	    (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
+	    (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
+		spin_unlock_irqrestore(&qp->q_lock, flags);
+		*bad_wr = wr;
+		QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
+			qp->icid, qp->state);
+		return -EINVAL;
+	}
+
+	if (!wr) {
+		QL_DPRINT11(ha, "Got an empty post send???\n");
+	}
+
+	while (wr) {
+		struct rdma_sq_common_wqe	*wqe;
+		struct rdma_sq_send_wqe		*swqe;
+		struct rdma_sq_send_wqe		*swqe2;
+		struct rdma_sq_rdma_wqe_1st	*rwqe;
+		struct rdma_sq_rdma_wqe_2nd	*rwqe2;
+		struct rdma_sq_local_inv_wqe	*iwqe;
+		struct rdma_sq_atomic_wqe	*awqe1;
+		struct rdma_sq_atomic_wqe	*awqe2;
+		struct rdma_sq_atomic_wqe	*awqe3;
+		struct rdma_sq_fmr_wqe_1st	*fwqe1;
+
+		if (!qlnxr_can_post_send(qp, wr)) {
+			status = -ENOMEM;
+			*bad_wr = wr;
+			break;
+		}
+
+		wqe = ecore_chain_produce(&qp->sq.pbl);
+
+		qp->wqe_wr_id[qp->sq.prod].signaled =
+			!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
+
+		/* common fields */
+		wqe->flags = 0;
+		wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
+				RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
+
+		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
+			!!(wr->send_flags & IB_SEND_SOLICITED));
+
+		comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
+				(qp->signaled);
+
+		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
+		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,  \
+			!!(wr->send_flags & IB_SEND_FENCE));
+
+		wqe->prev_wqe_size = qp->prev_wqe_size;
+
+		qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
+
+
+		switch (wr->opcode) {
+
+		case IB_WR_SEND_WITH_IMM:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
+			swqe = (struct rdma_sq_send_wqe *)wqe;
+			swqe->wqe_size = 2;
+			swqe2 = (struct rdma_sq_send_wqe *)
+					ecore_chain_produce(&qp->sq.pbl);
+			swqe->inv_key_or_imm_data =
+				cpu_to_le32(wr->ex.imm_data);
+			swqe->length = cpu_to_le32(
+						qlnxr_prepare_sq_send_data(dev,
+							qp, swqe, swqe2, wr,
+							bad_wr));
+
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
+			qp->prev_wqe_size = swqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
+
+			QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
+				swqe->length, wr->ex.imm_data);
+
+			break;
+
+		case IB_WR_SEND:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
+			swqe = (struct rdma_sq_send_wqe *)wqe;
+
+			swqe->wqe_size = 2;
+			swqe2 = (struct rdma_sq_send_wqe *)
+					ecore_chain_produce(&qp->sq.pbl);
+			swqe->length = cpu_to_le32(
+						qlnxr_prepare_sq_send_data(dev,
+							qp, swqe, swqe2, wr,
+							bad_wr));
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
+			qp->prev_wqe_size = swqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
+
+			QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
+				swqe->length);
+
+			break;
+
+		case IB_WR_SEND_WITH_INV:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
+			swqe = (struct rdma_sq_send_wqe *)wqe;
+			swqe2 = (struct rdma_sq_send_wqe *)
+					ecore_chain_produce(&qp->sq.pbl);
+			swqe->wqe_size = 2;
+			swqe->inv_key_or_imm_data =
+				cpu_to_le32(wr->ex.invalidate_rkey);
+			swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
+						qp, swqe, swqe2, wr, bad_wr));
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
+			qp->prev_wqe_size = swqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
+
+			QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
+				swqe->length);
+			break;
+
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
+			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+			rwqe->wqe_size = 2;
+			rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
+			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+					ecore_chain_produce(&qp->sq.pbl);
+			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
+						qp, rwqe, rwqe2, wr, bad_wr));
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+			qp->prev_wqe_size = rwqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+
+			QL_DPRINT12(ha,
+				"RDMA WRITE w/ IMM length = %d imm data=%x\n",
+				rwqe->length, rwqe->imm_data);
+
+			break;
+
+		case IB_WR_RDMA_WRITE:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
+			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+			rwqe->wqe_size = 2;
+			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+					ecore_chain_produce(&qp->sq.pbl);
+			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
+						qp, rwqe, rwqe2, wr, bad_wr));
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+			qp->prev_wqe_size = rwqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+
+			QL_DPRINT12(ha,
+				"RDMA WRITE w/o IMM length = %d\n",
+				rwqe->length);
+
+			break;
+
+		case IB_WR_RDMA_READ_WITH_INV:
+
+			QL_DPRINT12(ha,
+				"RDMA READ WITH INVALIDATE not supported\n");
+
+			*bad_wr = wr;
+			rc = -EINVAL;
+
+			break;
+
+		case IB_WR_RDMA_READ:
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
+			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
+
+			rwqe->wqe_size = 2;
+			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
+					ecore_chain_produce(&qp->sq.pbl);
+			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
+						qp, rwqe, rwqe2, wr, bad_wr));
+
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
+			qp->prev_wqe_size = rwqe->wqe_size;
+			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
+
+			QL_DPRINT12(ha, "RDMA READ length = %d\n",
+				rwqe->length);
+
+			break;
+
+		case IB_WR_ATOMIC_CMP_AND_SWP:
+		case IB_WR_ATOMIC_FETCH_AND_ADD:
+
+			QL_DPRINT12(ha,
+				"ATOMIC operation = %s\n",
+				((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
+					"IB_WR_ATOMIC_CMP_AND_SWP" : 
+					"IB_WR_ATOMIC_FETCH_AND_ADD"));
+
+			awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
+			awqe1->prev_wqe_size = 4;
+
+			awqe2 = (struct rdma_sq_atomic_wqe *)
+					ecore_chain_produce(&qp->sq.pbl);
+
+			TYPEPTR_ADDR_SET(awqe2, remote_va, \
+				atomic_wr(wr)->remote_addr);
+
+			awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
+
+			awqe3 = (struct rdma_sq_atomic_wqe *)
+					ecore_chain_produce(&qp->sq.pbl);
+
+			if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
+				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
+				TYPEPTR_ADDR_SET(awqe3, swap_data,
+						 atomic_wr(wr)->compare_add);
+			} else {
+				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
+				TYPEPTR_ADDR_SET(awqe3, swap_data,
+						 atomic_wr(wr)->swap);
+				TYPEPTR_ADDR_SET(awqe3, cmp_data,
+						 atomic_wr(wr)->compare_add);
+			}
+
+			qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
+
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
+			qp->prev_wqe_size = awqe1->prev_wqe_size;
+
+			break;
+
+		case IB_WR_LOCAL_INV:
+
+			QL_DPRINT12(ha,
+				"INVALIDATE length (IB_WR_LOCAL_INV)\n");
+
+			iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
+			iwqe->prev_wqe_size = 1;
+
+			iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
+			iwqe->inv_l_key = wr->ex.invalidate_rkey;
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
+			qp->prev_wqe_size = iwqe->prev_wqe_size;
+
+			break;
+
+#if __FreeBSD_version >= 1102000
+
+		case IB_WR_REG_MR:
+
+			QL_DPRINT12(ha, "IB_WR_REG_MR\n");
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
+			fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
+			fwqe1->wqe_size = 2;
+
+			rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
+			if (rc) {
+				QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
+				*bad_wr = wr;
+				break;
+			}
+
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
+			qp->prev_wqe_size = fwqe1->wqe_size;
+
+			break;
+#else
+		case IB_WR_FAST_REG_MR:
+
+			QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
+
+			wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
+			fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
+			fwqe1->prev_wqe_size = 3;
+
+			rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
+
+			if (rc) {
+				QL_DPRINT12(ha,
+					"FAST_MR (IB_WR_FAST_REG_MR) failed"
+					" rc = %d\n", rc);
+				*bad_wr = wr;
+				break;
+			}
+
+			qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
+			qp->prev_wqe_size = fwqe1->prev_wqe_size;
+
+			break;
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+		default:
+
+			QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
+
+			rc = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
+
+		if (*bad_wr) {
+			/*
+			 * restore prod to its position before this WR was processed
+			 */
+			ecore_chain_set_prod(&qp->sq.pbl,
+			     le16_to_cpu(qp->sq.db_data.data.value),
+			     wqe);
+			/* restore prev_wqe_size */
+			qp->prev_wqe_size = wqe->prev_wqe_size;
+			status = rc;
+
+			QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
+			break; /* out of the loop */
+		}
+
+		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
+
+		qlnxr_inc_sw_prod(&qp->sq);
+
+		qp->sq.db_data.data.value++;
+
+		wr = wr->next;
+	}
+
+	/* Trigger doorbell
+	 * If there was a failure in the first WR then it will be triggered in
+	 * vane. However this is not harmful (as long as the producer value is
+	 * unchanged). For performance reasons we avoid checking for this
+	 * redundant doorbell.
+	 */
+	wmb();
+	//writel(qp->sq.db_data.raw, qp->sq.db);
+
+	reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
+        bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
+        bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
+
+	mmiowb();
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
+		ibqp, wr, bad_wr);
+
+	return status;
+}
+
+static u32
+qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
+{
+	u32 used;
+
+	/* Calculate number of elements used based on producer
+	 * count and consumer count and subtract it from max
+	 * work request supported so that we get elements left.
+	 */
+	used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
+
+	return hw_srq->max_wr - used;
+}
+
+
+int
+qlnxr_post_recv(struct ib_qp *ibqp,
+	struct ib_recv_wr *wr,
+	struct ib_recv_wr **bad_wr)
+{
+ 	struct qlnxr_qp		*qp = get_qlnxr_qp(ibqp);
+	struct qlnxr_dev	*dev = qp->dev;
+	unsigned long		flags;
+	int			status = 0;
+	qlnx_host_t		*ha;
+	uint32_t		reg_addr;
+
+	ha = dev->ha;
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (qp->qp_type == IB_QPT_GSI) {
+		QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
+		return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
+	}
+
+	if (qp->srq) {
+		QL_DPRINT11(ha, "qp->srq [%p]"
+			" QP is associated with SRQ, cannot post RQ buffers\n",
+			qp->srq);
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&qp->q_lock, flags);
+
+	if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
+		spin_unlock_irqrestore(&qp->q_lock, flags);
+		*bad_wr = wr;
+
+		QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
+
+		return -EINVAL;
+	}
+
+	while (wr) {
+		int i;
+
+		if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
+			QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
+			(wr->num_sge > qp->rq.max_sges)) {
+			status = -ENOMEM;
+			*bad_wr = wr;
+			break;
+		}
+		for (i = 0; i < wr->num_sge; i++) {
+			u32 flags = 0;
+			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
+
+			/* first one must include the number of SGE in the list */
+			if (!i)
+				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
+
+			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
+
+			RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
+				wr->sg_list[i].length, flags);
+		}
+		/* Special case of no sges. FW requires between 1-4 sges...
+		 * in this case we need to post 1 sge with length zero. this is
+		 * because rdma write with immediate consumes an RQ. */
+		if (!wr->num_sge) {
+			u32 flags = 0;
+			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
+
+			/* first one must include the number of SGE in the list */
+			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
+			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
+
+			//RQ_SGE_SET(rqe, 0, 0, flags);
+			rqe->addr.hi = 0;
+			rqe->addr.lo = 0;
+
+			rqe->length = 0;
+			rqe->flags = cpu_to_le32(flags);
+
+			i = 1;
+		}
+
+		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
+		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
+
+		qlnxr_inc_sw_prod(&qp->rq);
+
+		wmb();
+
+		qp->rq.db_data.data.value++;
+
+	//	writel(qp->rq.db_data.raw, qp->rq.db);
+		mmiowb();
+	//	if (QLNX_IS_IWARP(dev)) {
+	//		writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
+	//		mmiowb(); /* for second doorbell */
+	//	}
+
+		reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
+				(uint8_t *)ha->cdev.doorbells);
+
+		bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
+		bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
+
+		if (QLNX_IS_IWARP(dev)) {
+			reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
+						(uint8_t *)ha->cdev.doorbells);
+			bus_write_4(ha->pci_dbells, reg_addr, \
+				qp->rq.iwarp_db2_data.raw);
+			bus_barrier(ha->pci_dbells,  0, 0, \
+				BUS_SPACE_BARRIER_READ);
+		}
+
+		wr = wr->next;
+	}
+
+	spin_unlock_irqrestore(&qp->q_lock, flags);
+
+	QL_DPRINT12(ha, "exit status = 0x%x\n", status);
+
+	return status;
+}
+
+/* In fmr we need to increase the number of fmr completed counter for the fmr
+ * algorithm determining whether we can free a pbl or not.
+ * we need to perform this whether the work request was signaled or not. for
+ * this purpose we call this function from the condition that checks if a wr
+ * should be skipped, to make sure we don't miss it ( possibly this fmr
+ * operation was not signalted)
+ */
+static inline void
+qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
+{
+#if __FreeBSD_version >= 1102000
+
+	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
+		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
+#else
+	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
+		qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+}
+
+static int
+process_req(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	int num_entries,
+	struct ib_wc *wc,
+	u16 hw_cons,
+	enum ib_wc_status status,
+	int force)
+{
+	u16		cnt = 0;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	while (num_entries && qp->sq.wqe_cons != hw_cons) {
+		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
+			qlnxr_chk_if_fmr(qp);
+			/* skip WC */
+			goto next_cqe;
+		}
+
+		/* fill WC */
+		wc->status = status;
+		wc->vendor_err = 0;
+		wc->wc_flags = 0;
+		wc->src_qp = qp->id;
+		wc->qp = &qp->ibqp;
+
+		// common section
+		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
+		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
+
+		switch (wc->opcode) {
+
+		case IB_WC_RDMA_WRITE:
+
+			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
+
+			QL_DPRINT12(ha,
+				"opcode = IB_WC_RDMA_WRITE bytes = %d\n",
+				qp->wqe_wr_id[qp->sq.cons].bytes_len);
+			break;
+
+		case IB_WC_COMP_SWAP:
+		case IB_WC_FETCH_ADD:
+			wc->byte_len = 8;
+			break;
+
+#if __FreeBSD_version >= 1102000
+		case IB_WC_REG_MR:
+			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
+			break;
+#else
+		case IB_WC_FAST_REG_MR:
+			qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
+			break;
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+		case IB_WC_RDMA_READ:
+		case IB_WC_SEND:
+
+			QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
+			break;
+		default:
+			;//DP_ERR("TBD ERROR");
+		}
+
+		num_entries--;
+		wc++;
+		cnt++;
+next_cqe:
+		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
+			ecore_chain_consume(&qp->sq.pbl);
+		qlnxr_inc_sw_cons(&qp->sq);
+	}
+
+	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
+	return cnt;
+}
+
+static int
+qlnxr_poll_cq_req(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	int num_entries,
+	struct ib_wc *wc,
+	struct rdma_cqe_requester *req)
+{
+	int		cnt = 0;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
+ 
+	switch (req->status) {
+
+	case RDMA_CQE_REQ_STS_OK:
+
+		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
+			IB_WC_SUCCESS, 0);
+		break;
+
+	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
+
+		if (qp->state != ECORE_ROCE_QP_STATE_ERR)
+		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
+				  IB_WC_WR_FLUSH_ERR, 1);
+		break;
+
+	default: /* other errors case */
+
+		/* process all WQE before the cosumer */
+		qp->state = ECORE_ROCE_QP_STATE_ERR;
+		cnt = process_req(dev, qp, cq, num_entries, wc,
+				req->sq_cons - 1, IB_WC_SUCCESS, 0);
+		wc += cnt;
+		/* if we have extra WC fill it with actual error info */
+
+		if (cnt < num_entries) {
+			enum ib_wc_status wc_status;
+
+			switch (req->status) {
+			case 	RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
+				wc_status = IB_WC_BAD_RESP_ERR;
+				break;
+			case 	RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
+				wc_status = IB_WC_LOC_LEN_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
+				wc_status = IB_WC_LOC_QP_OP_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
+				wc_status = IB_WC_LOC_PROT_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
+				wc_status = IB_WC_MW_BIND_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
+				wc_status = IB_WC_REM_INV_REQ_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
+				wc_status = IB_WC_REM_ACCESS_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
+				wc_status = IB_WC_REM_OP_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
+				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
+				break;
+			case    RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
+				wc_status = IB_WC_RETRY_EXC_ERR;
+				break;
+			default:
+				wc_status = IB_WC_GENERAL_ERR;
+			}
+
+			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
+					wc_status, 1 /* force use of WC */);
+		}
+	}
+
+	QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
+	return cnt;
+}
+
+static void
+__process_resp_one(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	struct ib_wc *wc,
+	struct rdma_cqe_responder *resp,
+	u64 wr_id)
+{
+	enum ib_wc_status	wc_status = IB_WC_SUCCESS;
+#if __FreeBSD_version < 1102000
+	u8			flags;
+#endif
+	qlnx_host_t		*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
+		qp, resp->status);
+ 
+	wc->opcode = IB_WC_RECV;
+	wc->wc_flags = 0;
+
+	switch (resp->status) {
+
+	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
+		wc_status = IB_WC_LOC_ACCESS_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
+		wc_status = IB_WC_LOC_LEN_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
+		wc_status = IB_WC_LOC_QP_OP_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
+		wc_status = IB_WC_LOC_PROT_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
+		wc_status = IB_WC_MW_BIND_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
+		wc_status = IB_WC_REM_INV_RD_REQ_ERR;
+		break;
+
+	case RDMA_CQE_RESP_STS_OK:
+
+#if __FreeBSD_version >= 1102000
+		if (resp->flags & QLNXR_RESP_IMM) {
+			wc->ex.imm_data =
+				le32_to_cpu(resp->imm_data_or_inv_r_Key);
+			wc->wc_flags |= IB_WC_WITH_IMM;
+
+			if (resp->flags & QLNXR_RESP_RDMA)
+				wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+
+			if (resp->flags & QLNXR_RESP_INV) {
+				QL_DPRINT11(ha,
+					"Invalid flags QLNXR_RESP_INV [0x%x]"
+					"qp = %p qp->id = 0x%x cq = %p"
+					" cq->icid = 0x%x\n",
+					resp->flags, qp, qp->id, cq, cq->icid );
+			}
+		} else if (resp->flags & QLNXR_RESP_INV) {
+			wc->ex.imm_data =
+				le32_to_cpu(resp->imm_data_or_inv_r_Key);
+			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+
+			if (resp->flags & QLNXR_RESP_RDMA) {
+				QL_DPRINT11(ha,
+					"Invalid flags QLNXR_RESP_RDMA [0x%x]"
+					"qp = %p qp->id = 0x%x cq = %p"
+					" cq->icid = 0x%x\n",
+					resp->flags, qp, qp->id, cq, cq->icid );
+			}
+		} else if (resp->flags & QLNXR_RESP_RDMA) {
+			QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
+				"qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
+				resp->flags, qp, qp->id, cq, cq->icid );
+		}
+#else
+		wc_status = IB_WC_SUCCESS;
+		wc->byte_len = le32_to_cpu(resp->length);
+
+		flags = resp->flags & QLNXR_RESP_RDMA_IMM;
+
+		switch (flags) {
+
+		case QLNXR_RESP_RDMA_IMM:
+			/* update opcode */
+			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+			/* fall to set imm data */
+		case QLNXR_RESP_IMM:
+			wc->ex.imm_data =
+				le32_to_cpu(resp->imm_data_or_inv_r_Key);
+			wc->wc_flags |= IB_WC_WITH_IMM;
+			break;
+		case QLNXR_RESP_RDMA:
+			QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
+				"qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
+				resp->flags, qp, qp->id, cq, cq->icid );
+			break;
+		default:
+			/* valid configuration, but nothing todo here */
+			;
+		}
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+		break;
+	default:
+		wc_status = IB_WC_GENERAL_ERR;
+	}
+
+	/* fill WC */
+	wc->status = wc_status;
+	wc->vendor_err = 0;
+	wc->src_qp = qp->id;
+	wc->qp = &qp->ibqp;
+	wc->wr_id = wr_id;
+
+	QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
+
+	return;
+}
+
+static int
+process_resp_one_srq(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	struct ib_wc *wc,
+	struct rdma_cqe_responder *resp)
+{
+	struct qlnxr_srq	*srq = qp->srq;
+	u64			wr_id;
+	qlnx_host_t		*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
+
+	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
+		wc->status = IB_WC_WR_FLUSH_ERR;
+		wc->vendor_err = 0;
+		wc->wr_id = wr_id;
+		wc->byte_len = 0;
+		wc->src_qp = qp->id;
+		wc->qp = &qp->ibqp;
+		wc->wr_id = wr_id;
+	} else {
+		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
+	}
+
+	/* PBL is maintained in case of WR granularity.
+	 * So increment WR consumer after consuming WR
+	 */
+	srq->hw_srq.wr_cons_cnt++;
+
+	QL_DPRINT12(ha, "exit\n");
+	return 1;
+}
+
+static int
+process_resp_one(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	struct ib_wc *wc,
+	struct rdma_cqe_responder *resp)
+{
+	qlnx_host_t	*ha = dev->ha;
+	u64		wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
+
+	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
+		ecore_chain_consume(&qp->rq.pbl);
+	qlnxr_inc_sw_cons(&qp->rq);
+
+	QL_DPRINT12(ha, "exit\n");
+	return 1;
+}
+
+static int
+process_resp_flush(struct qlnxr_qp *qp,
+	int num_entries,
+	struct ib_wc *wc,
+	u16 hw_cons)
+{
+	u16		cnt = 0;
+	qlnx_host_t	*ha = qp->dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	while (num_entries && qp->rq.wqe_cons != hw_cons) {
+		/* fill WC */
+		wc->status = IB_WC_WR_FLUSH_ERR;
+		wc->vendor_err = 0;
+		wc->wc_flags = 0;
+		wc->src_qp = qp->id;
+		wc->byte_len = 0;
+		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
+		wc->qp = &qp->ibqp;
+		num_entries--;
+		wc++;
+		cnt++;
+		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
+			ecore_chain_consume(&qp->rq.pbl);
+		qlnxr_inc_sw_cons(&qp->rq);
+	}
+
+	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
+	return cnt;
+}
+
+static void
+try_consume_resp_cqe(struct qlnxr_cq *cq,
+	struct qlnxr_qp *qp,
+	struct rdma_cqe_responder *resp,
+	int *update)
+{
+	if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
+		consume_cqe(cq);
+		*update |= 1;
+	}
+}
+
+static int
+qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	int num_entries,
+	struct ib_wc *wc,
+	struct rdma_cqe_responder *resp,
+	int *update)
+{
+	int		cnt;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
+	consume_cqe(cq);
+	*update |= 1;
+
+	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
+	return cnt;
+}
+
+static int
+qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
+	struct qlnxr_qp *qp,
+	struct qlnxr_cq *cq,
+	int num_entries,
+	struct ib_wc *wc,
+	struct rdma_cqe_responder *resp,
+	int *update)
+{
+	int		cnt;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
+		cnt = process_resp_flush(qp, num_entries, wc,
+				resp->rq_cons);
+		try_consume_resp_cqe(cq, qp, resp, update);
+	} else {
+		cnt = process_resp_one(dev, qp, cq, wc, resp);
+		consume_cqe(cq);
+		*update |= 1;
+	}
+
+	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
+	return cnt;
+}
+
+static void
+try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
+	struct rdma_cqe_requester *req, int *update)
+{
+	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
+		consume_cqe(cq);
+		*update |= 1;
+	}
+}
+
+static void
+doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
+{
+	uint64_t	reg_addr;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	wmb();
+	cq->db.data.agg_flags = flags;
+	cq->db.data.value = cpu_to_le32(cons);
+
+	reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
+				(uint8_t *)(ha->cdev.doorbells));
+
+	bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
+	bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
+
+	QL_DPRINT12(ha, "exit\n");
+	return;
+
+//#ifdef __LP64__
+//	writeq(cq->db.raw, cq->db_addr);
+//#else
+	/* Note that since the FW allows 64 bit write only, in 32bit systems
+	 * the value of db_addr must be low enough. This is currently not
+	 * enforced.
+	 */
+//	writel(cq->db.raw & 0xffffffff, cq->db_addr);
+//	mmiowb();
+//#endif
+}
+
+
+static int
+is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
+{
+	struct rdma_cqe_requester *resp_cqe = &cqe->req;
+	return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
+			cq->pbl_toggle;
+}
+
+int
+qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct qlnxr_cq	*cq = get_qlnxr_cq(ibcq);
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
+	int		done = 0;
+	union rdma_cqe	*cqe = cq->latest_cqe;
+	int 		update = 0;
+	u32		old_cons, new_cons;
+	unsigned long	flags;
+	qlnx_host_t	*ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+ 
+	if (cq->destroyed) {
+		QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
+			cq, cq->icid);
+		return 0;
+	}
+
+	if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
+		return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+
+	old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
+
+	while (num_entries && is_valid_cqe(cq, cqe)) {
+		int cnt = 0;
+		struct qlnxr_qp *qp;
+		struct rdma_cqe_requester *resp_cqe;
+		enum rdma_cqe_type cqe_type;
+
+		/* prevent speculative reads of any field of CQE */
+		rmb();
+
+		resp_cqe = &cqe->req;
+		qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
+						resp_cqe->qp_handle.lo);
+
+		if (!qp) {
+			QL_DPRINT11(ha, "qp = NULL\n");
+			break;
+		}
+
+		wc->qp = &qp->ibqp;
+
+		cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
+
+		switch (cqe_type) {
+		case RDMA_CQE_TYPE_REQUESTER:
+			cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
+					wc, &cqe->req);
+			try_consume_req_cqe(cq, qp, &cqe->req, &update);
+			break;
+		case RDMA_CQE_TYPE_RESPONDER_RQ:
+			cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
+					wc, &cqe->resp, &update);
+			break;
+		case RDMA_CQE_TYPE_RESPONDER_SRQ:
+			cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
+					wc, &cqe->resp, &update);
+			break;
+		case RDMA_CQE_TYPE_INVALID:
+		default:
+			QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
+			break;
+		}
+		num_entries -= cnt;
+		wc += cnt;
+		done += cnt;
+
+		cqe = cq->latest_cqe;
+	}
+	new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
+
+	cq->cq_cons += new_cons - old_cons;
+
+	if (update) {
+		/* doorbell notifies abount latest VALID entry,
+		 * but chain already point to the next INVALID one
+		 */
+		doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
+		QL_DPRINT12(ha, "cq = %p cons = 0x%x "
+			"arm_flags = 0x%x db.icid = 0x%x\n", cq,
+			(cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
+	}
+
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+	QL_DPRINT12(ha, "exit\n");
+ 
+	return done;
+}
+
+
+int
+qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+        struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
+        unsigned long sflags;
+        struct qlnxr_dev *dev;
+	qlnx_host_t	*ha;
+
+	dev = get_qlnxr_dev((ibcq->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
+		"cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
+		flags, cq, cq->cq_cons, cq->cq_type);
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	if (cq->destroyed) {
+		QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
+			cq, cq->icid);
+		return -EINVAL;
+	}
+
+        if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
+                return 0;
+        }
+
+        spin_lock_irqsave(&cq->cq_lock, sflags);
+
+        cq->arm_flags = 0;
+
+        if (flags & IB_CQ_SOLICITED) {
+                cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
+        }
+        if (flags & IB_CQ_NEXT_COMP) {
+                cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
+        }
+
+        doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
+
+        spin_unlock_irqrestore(&cq->cq_lock, sflags);
+
+	QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
+        return 0;
+}
+
+
+static struct qlnxr_mr *
+__qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
+{
+	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
+	struct qlnxr_mr *mr;
+	int		rc = -ENOMEM;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+ 
+	QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
+		" pd_id = %d max_page_list_len = %d\n",
+		ibpd, pd, pd->pd_id, max_page_list_len);
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
+		return ERR_PTR(rc);
+	}
+
+	mr->dev = dev;
+	mr->type = QLNXR_MR_FRMR;
+
+	rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
+				  1 /* allow dual layer pbl */);
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
+		goto err0;
+	}
+
+	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
+	if (rc) {
+		QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
+		goto err0;
+	}
+
+	/* index only, 18 bit long, lkey = itid << 8 | key */
+	mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
+	mr->hw_mr.key = 0;
+	mr->hw_mr.pd = pd->pd_id;
+	mr->hw_mr.local_read = 1;
+	mr->hw_mr.local_write = 0;
+	mr->hw_mr.remote_read = 0;
+	mr->hw_mr.remote_write = 0;
+	mr->hw_mr.remote_atomic = 0;
+	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
+	mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
+	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
+	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
+	mr->hw_mr.fbo = 0;
+	mr->hw_mr.length = 0;
+	mr->hw_mr.vaddr = 0;
+	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
+	mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
+	mr->hw_mr.dma_mr = false;
+
+	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
+	if (rc) {
+		QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
+		goto err1;
+	}
+
+	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+	mr->ibmr.rkey = mr->ibmr.lkey;
+
+	QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
+		mr, mr->ibmr.lkey);
+
+	return mr;
+
+err1:
+	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
+err0:
+	kfree(mr);
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return ERR_PTR(rc);
+}
+
+#if __FreeBSD_version >= 1102000
+
+struct ib_mr *
+qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
+{
+	struct qlnxr_dev *dev;
+	struct qlnxr_mr *mr;
+	qlnx_host_t     *ha;
+
+	dev = get_qlnxr_dev(ibpd->device);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (mr_type != IB_MR_TYPE_MEM_REG)
+		return ERR_PTR(-EINVAL);
+
+	mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
+
+	if (IS_ERR(mr))
+		return ERR_PTR(-EINVAL);
+
+	QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
+
+	return &mr->ibmr;
+}
+
+static int
+qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
+{
+	struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
+	struct qlnxr_pbl *pbl_table;
+	struct regpair *pbe;
+	struct qlnxr_dev *dev;
+	qlnx_host_t     *ha;
+	u32 pbes_in_page;
+
+	dev = mr->dev;
+	ha = dev->ha;
+
+	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
+		QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
+		return -ENOMEM;
+	}
+
+	QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
+		((void *)addr));
+
+	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
+	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
+	pbe = (struct regpair *)pbl_table->va;
+	pbe +=  mr->npages % pbes_in_page;
+	pbe->lo = cpu_to_le32((u32)addr);
+	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
+
+	mr->npages++;
+
+	QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
+		((void *)addr));
+	return 0;
+}
+
+int
+qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+	int sg_nents, unsigned int *sg_offset)
+{
+	int             ret;
+	struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
+	qlnx_host_t     *ha;
+
+	if (mr == NULL)
+		return (-1);
+
+	if (mr->dev == NULL)
+		return (-1);
+
+	ha = mr->dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	mr->npages = 0;
+	qlnx_handle_completed_mrs(mr->dev, &mr->info);
+
+	ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
+
+	QL_DPRINT12(ha, "exit ret = %d\n", ret);
+
+	return (ret);
+}
+
+#else
+
+struct ib_mr *
+qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
+{
+	struct qlnxr_dev *dev;
+	struct qlnxr_mr *mr;
+	qlnx_host_t	*ha;
+	struct ib_mr *ibmr = NULL;
+
+	dev = get_qlnxr_dev((ibpd->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
+
+	if (IS_ERR(mr)) {
+		ibmr = ERR_PTR(-EINVAL);
+	} else {
+		ibmr = &mr->ibmr;
+	}
+
+	QL_DPRINT12(ha, "exit %p\n", ibmr);
+	return (ibmr);
+}
+
+void
+qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
+{
+	struct qlnxr_fast_reg_page_list *frmr_list;
+
+	frmr_list = get_qlnxr_frmr_list(page_list);
+ 
+	free_mr_info(frmr_list->dev, &frmr_list->info);
+
+	kfree(frmr_list->ibfrpl.page_list);
+	kfree(frmr_list);
+
+	return;
+}
+
+struct ib_fast_reg_page_list *
+qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
+{
+	struct qlnxr_fast_reg_page_list *frmr_list = NULL;
+	struct qlnxr_dev		*dev;
+	int				size = page_list_len * sizeof(u64);
+	int				rc = -ENOMEM;
+	qlnx_host_t			*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
+	if (!frmr_list) {
+		QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
+		goto err;
+	}
+
+	frmr_list->dev = dev;
+	frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
+	if (!frmr_list->ibfrpl.page_list) {
+		QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
+		goto err0;
+	}
+
+	rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
+			  1 /* allow dual layer pbl */);
+	if (rc)
+		goto err1;
+
+	QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
+
+	return &frmr_list->ibfrpl;
+
+err1:
+	kfree(frmr_list->ibfrpl.page_list);
+err0:
+	kfree(frmr_list);
+err:
+	QL_DPRINT12(ha, "exit with error\n");
+
+	return ERR_PTR(rc);
+}
+
+static int
+qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
+	int buf_cnt, uint64_t *total_size)
+{
+	u64 size = 0;
+
+	*total_size = 0;
+
+	if (!buf_cnt || buf_list == NULL) {
+		QL_DPRINT11(ha,
+			"failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
+		return (-1);
+	}
+
+	size = buf_list->size;
+
+	if (!size) {
+		QL_DPRINT11(ha,
+			"failed buf_list = %p buf_cnt = %d"
+			" buf_list->size = 0\n", buf_list, buf_cnt);
+		return (-1);
+	}
+
+	while (buf_cnt) {
+
+		*total_size += buf_list->size;
+
+		if (buf_list->size != size) {
+			QL_DPRINT11(ha,
+				"failed buf_list = %p buf_cnt = %d"
+				" all buffers should have same size\n",
+				buf_list, buf_cnt);
+			return (-1);
+		}
+
+		buf_list++;
+		buf_cnt--;
+	}
+	return (0);
+}
+
+static size_t
+qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
+	int buf_cnt)
+{
+	int	i;
+	size_t	num_pages = 0;
+	u64	size;
+
+	for (i = 0; i < buf_cnt; i++) {
+
+		size = 0;
+		while (size < buf_list->size) {
+			size += PAGE_SIZE;
+			num_pages++;
+		}
+		buf_list++;
+	}
+	return (num_pages);
+}
+
+static void
+qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
+	struct ib_phys_buf *buf_list, int buf_cnt,
+	struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
+{
+	struct regpair		*pbe;
+	struct qlnxr_pbl	*pbl_tbl;
+	int			pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
+	qlnx_host_t		*ha;
+        int                     i;
+	u64			pbe_addr;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!pbl_info) {
+		QL_DPRINT11(ha, "PBL_INFO not initialized\n");
+		return;
+	}
+
+	if (!pbl_info->num_pbes) {
+		QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
+		return;
+	}
+
+	/* If we have a two layered pbl, the first pbl points to the rest
+	 * of the pbls and the first entry lays on the second pbl in the table
+	 */
+	if (pbl_info->two_layered)
+		pbl_tbl = &pbl[1];
+	else
+		pbl_tbl = pbl;
+
+	pbe = (struct regpair *)pbl_tbl->va;
+	if (!pbe) {
+		QL_DPRINT12(ha, "pbe is NULL\n");
+		return;
+	}
+
+	pbe_cnt = 0;
+
+	for (i = 0; i < buf_cnt; i++) {
+
+		pages = buf_list->size >> PAGE_SHIFT;
+
+		for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
+			/* store the page address in pbe */
+
+			pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
+
+			pbe->lo = cpu_to_le32((u32)pbe_addr);
+			pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
+
+			QL_DPRINT12(ha, "Populate pbl table:"
+				" pbe->addr=0x%x:0x%x "
+				" pbe_cnt = %d total_num_pbes=%d"
+				" pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
+				total_num_pbes, pbe);
+
+			pbe_cnt ++;
+			total_num_pbes ++;
+			pbe++;
+
+			if (total_num_pbes == pbl_info->num_pbes)
+				return;
+
+			/* if the given pbl is full storing the pbes,
+			 * move to next pbl.  */
+
+			if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
+				pbl_tbl++;
+				pbe = (struct regpair *)pbl_tbl->va;
+				pbe_cnt = 0;
+			}
+		}
+		buf_list++;
+	}
+	QL_DPRINT12(ha, "exit\n");
+	return;
+}
+
+struct ib_mr *
+qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
+	struct ib_phys_buf *buf_list,
+	int buf_cnt, int acc, u64 *iova_start)
+{
+	int		rc = -ENOMEM;
+	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
+	struct qlnxr_mr *mr;
+	struct qlnxr_pd *pd;
+	qlnx_host_t	*ha;
+	size_t		num_pages = 0;
+	uint64_t	length;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	pd = get_qlnxr_pd(ibpd);
+
+	QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
+		" iova_start = %p, acc = %d\n",
+		pd->pd_id, buf_list, buf_cnt, iova_start, acc);
+
+	//if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
+	//	QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
+	//		" !(acc & IB_ACCESS_LOCAL_WRITE))\n");
+	//	return ERR_PTR(-EINVAL);
+	//}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
+		return ERR_PTR(rc);
+	}
+
+	mr->type = QLNXR_MR_KERNEL;
+	mr->iova_start = iova_start;
+
+	rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
+	if (rc)
+		goto err0;
+
+	num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
+	if (!num_pages)
+		goto err0;
+
+	rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
+	if (rc) {
+		QL_DPRINT11(ha,
+			"qlnxr_init_mr_info failed [%d]\n", rc);
+		goto err1;
+	}
+
+	qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
+		   &mr->info.pbl_info);
+
+	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
+
+	if (rc) {
+		QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
+		goto err1;
+	}
+
+	/* index only, 18 bit long, lkey = itid << 8 | key */
+	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
+	mr->hw_mr.key = 0;
+	mr->hw_mr.pd = pd->pd_id;
+	mr->hw_mr.local_read = 1;
+	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
+	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
+	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
+	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
+	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
+	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
+	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
+	mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
+
+	mr->hw_mr.fbo = 0;
+
+	mr->hw_mr.length = length;
+	mr->hw_mr.vaddr = (uint64_t)iova_start;
+	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
+	mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
+	mr->hw_mr.dma_mr = false;
+
+	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
+	if (rc) {
+		QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
+		goto err2;
+	}
+
+	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
+		mr->hw_mr.remote_atomic)
+		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
+
+	QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
+
+	return (&mr->ibmr);
+
+err2:
+	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
+err1:
+	qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
+err0:
+	kfree(mr);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return (ERR_PTR(rc));
+}
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+struct ib_ah *
+#if __FreeBSD_version >= 1102000
+qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
+	struct ib_udata *udata)
+#else
+qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
+#endif /* #if __FreeBSD_version >= 1102000 */
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t	*ha;
+	struct qlnxr_ah *ah;
+
+	dev = get_qlnxr_dev((ibpd->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "in create_ah\n");
+
+	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
+	if (!ah) {
+		QL_DPRINT12(ha, "no address handle can be allocated\n");
+		return ERR_PTR(-ENOMEM);
+	}
+	
+	ah->attr = *attr;	
+ 
+	return &ah->ibah;
+}
+
+int
+qlnxr_destroy_ah(struct ib_ah *ibah)
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t     *ha;
+	struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
+	
+	dev = get_qlnxr_dev((ibah->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "in destroy_ah\n");
+
+	kfree(ah);
+	return 0;
+}
+
+int
+qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t     *ha;
+
+	dev = get_qlnxr_dev((ibah->device));
+	ha = dev->ha;
+	QL_DPRINT12(ha, "Query AH not supported\n");
+	return -EINVAL;
+}
+
+int
+qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t     *ha;
+
+	dev = get_qlnxr_dev((ibah->device));
+	ha = dev->ha;
+	QL_DPRINT12(ha, "Modify AH not supported\n");
+	return -ENOSYS;
+}
+
+#if __FreeBSD_version >= 1102000
+int
+qlnxr_process_mad(struct ib_device *ibdev,
+		int process_mad_flags,
+		u8 port_num,
+		const struct ib_wc *in_wc,
+		const struct ib_grh *in_grh,
+		const struct ib_mad_hdr *mad_hdr,
+		size_t in_mad_size,
+		struct ib_mad_hdr *out_mad,
+		size_t *out_mad_size,
+		u16 *out_mad_pkey_index)
+
+#else
+
+int
+qlnxr_process_mad(struct ib_device *ibdev,
+                        int process_mad_flags,
+                        u8 port_num,
+                        struct ib_wc *in_wc,
+                        struct ib_grh *in_grh,
+                        struct ib_mad *in_mad,
+                        struct ib_mad *out_mad)
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+{
+	struct qlnxr_dev *dev;
+	qlnx_host_t	*ha;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+	QL_DPRINT12(ha, "process mad not supported\n");
+
+	return -ENOSYS;
+//	QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
+//               in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
+//               in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
+//               in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
+//               in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
+
+//	return IB_MAD_RESULT_SUCCESS;	
+}
+
+
+#if __FreeBSD_version >= 1102000
+int
+qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+	struct ib_port_immutable *immutable)
+{
+	struct qlnxr_dev        *dev;
+	qlnx_host_t             *ha;
+	struct ib_port_attr     attr;
+	int                     err;
+
+	dev = get_qlnxr_dev(ibdev);
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	err = qlnxr_query_port(ibdev, port_num, &attr);
+	if (err)
+		return err;
+
+	if (QLNX_IS_IWARP(dev)) {
+		immutable->pkey_tbl_len = 1;
+		immutable->gid_tbl_len = 1;
+		immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
+		immutable->max_mad_size = 0;
+	} else {
+		immutable->pkey_tbl_len = attr.pkey_tbl_len;
+		immutable->gid_tbl_len = attr.gid_tbl_len;
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+		immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+}
+#endif /* #if __FreeBSD_version > 1102000 */
+
+
+/***** iWARP related functions *************/
+
+
+static void
+qlnxr_iw_mpa_request(void *context,
+	struct ecore_iwarp_cm_event_params *params)
+{
+	struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
+	struct qlnxr_dev *dev = listener->dev;
+	struct qlnxr_iw_ep *ep;
+	struct iw_cm_event event;
+	struct sockaddr_in *laddr;
+	struct sockaddr_in *raddr;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
+		QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
+			params->cm_info->ip_version);
+		return;
+	}
+ 
+	ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
+
+	if (!ep) {
+		QL_DPRINT11(ha, "kzalloc{ep) failed\n");
+		return;
+	}
+
+	ep->dev = dev;
+	ep->ecore_context = params->ep_context;
+
+	memset(&event, 0, sizeof(event));
+
+	event.event = IW_CM_EVENT_CONNECT_REQUEST;
+	event.status = params->status;
+
+	laddr = (struct sockaddr_in *)&event.local_addr;
+	raddr = (struct sockaddr_in *)&event.remote_addr;
+
+	laddr->sin_family = AF_INET;
+	raddr->sin_family = AF_INET;
+
+	laddr->sin_port = htons(params->cm_info->local_port);
+	raddr->sin_port = htons(params->cm_info->remote_port);
+
+	laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
+	raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
+
+	event.provider_data = (void *)ep;
+	event.private_data = (void *)params->cm_info->private_data;
+	event.private_data_len = (u8)params->cm_info->private_data_len;
+
+#if __FreeBSD_version >= 1100000
+	event.ord = params->cm_info->ord;
+	event.ird = params->cm_info->ird;
+#endif /* #if __FreeBSD_version >= 1100000 */
+
+	listener->cm_id->event_handler(listener->cm_id, &event);
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+static void
+qlnxr_iw_issue_event(void *context,
+	 struct ecore_iwarp_cm_event_params *params,
+	 enum iw_cm_event_type event_type,
+	 char *str)
+{
+	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+	struct qlnxr_dev *dev = ep->dev;
+	struct iw_cm_event event;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	memset(&event, 0, sizeof(event));
+	event.status = params->status;
+	event.event = event_type;
+
+	if (params->cm_info != NULL) {
+#if __FreeBSD_version >= 1100000
+		event.ird = params->cm_info->ird;
+		event.ord = params->cm_info->ord;
+		QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
+		QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
+#endif /* #if __FreeBSD_version >= 1100000 */
+
+		event.private_data_len = params->cm_info->private_data_len;
+		event.private_data = (void *)params->cm_info->private_data;
+		QL_DPRINT12(ha, "private_data_len=[%d] \n",
+			event.private_data_len);
+	}
+
+	QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
+	QL_DPRINT12(ha, "status=[%d] \n", event.status);
+	
+	if (ep) {
+		if (ep->cm_id)
+			ep->cm_id->event_handler(ep->cm_id, &event);
+		else
+			QL_DPRINT11(ha, "ep->cm_id == NULL \n");
+	} else {
+		QL_DPRINT11(ha, "ep == NULL \n");
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+static void
+qlnxr_iw_close_event(void *context,
+	 struct ecore_iwarp_cm_event_params *params)
+{
+	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+	struct qlnxr_dev *dev = ep->dev;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+ 
+	if (ep->cm_id) {
+		qlnxr_iw_issue_event(context,
+				    params,
+				    IW_CM_EVENT_CLOSE,
+				    "IW_CM_EVENT_EVENT_CLOSE");
+		ep->cm_id->rem_ref(ep->cm_id);
+		ep->cm_id = NULL;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return;
+}
+
+#if __FreeBSD_version >= 1102000
+
+static void
+qlnxr_iw_passive_complete(void *context,
+        struct ecore_iwarp_cm_event_params *params)
+{
+        struct qlnxr_iw_ep      *ep = (struct qlnxr_iw_ep *)context;
+        struct qlnxr_dev        *dev = ep->dev;
+        qlnx_host_t             *ha;
+
+        ha = dev->ha;
+
+        /* We will only reach the following state if MPA_REJECT was called on
+         * passive. In this case there will be no associated QP.
+         */
+        if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
+                QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
+                kfree(ep);
+                return;
+        }
+
+        /* We always issue an established event, however, ofed does not look
+         * at event code for established. So if there was a failure, we follow
+         * with close...
+         */
+        qlnxr_iw_issue_event(context,
+                params,
+                IW_CM_EVENT_ESTABLISHED,
+                "IW_CM_EVENT_ESTABLISHED");
+
+        if (params->status < 0) {
+                qlnxr_iw_close_event(context, params);
+        }
+
+        return;
+}
+
+struct qlnxr_discon_work {
+        struct work_struct work;
+        struct qlnxr_iw_ep *ep;
+        enum ecore_iwarp_event_type event;
+        int status;
+};
+
+static void
+qlnxr_iw_disconnect_worker(struct work_struct *work)
+{
+        struct qlnxr_discon_work *dwork =
+                container_of(work, struct qlnxr_discon_work, work);
+        struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
+        struct qlnxr_iw_ep *ep = dwork->ep;
+        struct qlnxr_dev *dev = ep->dev;
+        struct qlnxr_qp *qp = ep->qp;
+        struct iw_cm_event event;
+
+        if (qp->destroyed) {
+                kfree(dwork);
+                qlnxr_iw_qp_rem_ref(&qp->ibqp);
+                return;
+        }
+
+        memset(&event, 0, sizeof(event));
+        event.status = dwork->status;
+        event.event = IW_CM_EVENT_DISCONNECT;
+
+        /* Success means graceful disconnect was requested. modifying
+         * to SQD is translated to graceful disconnect. O/w reset is sent
+         */
+        if (dwork->status)
+                qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
+        else
+                qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
+
+        kfree(dwork);
+
+        if (ep->cm_id)
+                ep->cm_id->event_handler(ep->cm_id, &event);
+
+        SET_FIELD(qp_params.modify_flags,
+                  ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
+
+        ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
+
+        qlnxr_iw_qp_rem_ref(&qp->ibqp);
+
+        return;
+}
+
+void
+qlnxr_iw_disconnect_event(void *context,
+        struct ecore_iwarp_cm_event_params *params)
+{
+        struct qlnxr_discon_work *work;
+        struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+        struct qlnxr_dev *dev = ep->dev;
+        struct qlnxr_qp *qp = ep->qp;
+
+        work = kzalloc(sizeof(*work), GFP_ATOMIC);
+        if (!work)
+                return;
+
+        qlnxr_iw_qp_add_ref(&qp->ibqp);
+        work->ep = ep;
+        work->event = params->event;
+        work->status = params->status;
+
+        INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
+        queue_work(dev->iwarp_wq, &work->work);
+
+        return;
+}
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+static int
+qlnxr_iw_mpa_reply(void *context,
+	struct ecore_iwarp_cm_event_params *params)
+{
+        struct qlnxr_iw_ep	*ep = (struct qlnxr_iw_ep *)context;
+        struct qlnxr_dev	*dev = ep->dev;
+        struct ecore_iwarp_send_rtr_in rtr_in;
+        int			rc;
+	qlnx_host_t		*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
+        rtr_in.ep_context = params->ep_context;
+
+        rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
+
+	QL_DPRINT12(ha, "exit rc = %d\n", rc);
+        return rc;
+}
+
+
+void
+qlnxr_iw_qp_event(void *context,
+	struct ecore_iwarp_cm_event_params *params,
+	enum ib_event_type ib_event,
+	char *str)
+{
+        struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+        struct qlnxr_dev *dev = ep->dev;
+        struct ib_qp *ibqp = &(ep->qp->ibqp);
+        struct ib_event event;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha,
+		"[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
+		context, params->event, str, ibqp->event_handler);
+
+        if (ibqp->event_handler) {
+                event.event = ib_event;
+                event.device = ibqp->device;
+                event.element.qp = ibqp;
+                ibqp->event_handler(&event, ibqp->qp_context);
+        }
+
+	return;
+}
+
+int
+qlnxr_iw_event_handler(void *context,
+	struct ecore_iwarp_cm_event_params *params)
+{
+	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+	struct qlnxr_dev *dev = ep->dev;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
+		"enter\n", context, params->event);
+ 
+	switch (params->event) {
+
+	/* Passive side request received */
+	case ECORE_IWARP_EVENT_MPA_REQUEST:
+		qlnxr_iw_mpa_request(context, params);
+		break;
+
+        case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
+                qlnxr_iw_mpa_reply(context, params);
+                break;
+
+	/* Passive side established ( ack on mpa response ) */
+	case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
+
+#if __FreeBSD_version >= 1102000
+
+		ep->during_connect = 0;
+		qlnxr_iw_passive_complete(context, params);
+
+#else
+		qlnxr_iw_issue_event(context,
+				    params,
+				    IW_CM_EVENT_ESTABLISHED,
+				    "IW_CM_EVENT_ESTABLISHED");
+#endif /* #if __FreeBSD_version >= 1102000 */
+		break;
+
+	/* Active side reply received */
+	case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
+		ep->during_connect = 0;
+		qlnxr_iw_issue_event(context,
+				    params,
+				    IW_CM_EVENT_CONNECT_REPLY,
+				    "IW_CM_EVENT_CONNECT_REPLY");
+		if (params->status < 0) {
+			struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
+
+			ep->cm_id->rem_ref(ep->cm_id);
+			ep->cm_id = NULL;
+		}
+		break;
+
+	case ECORE_IWARP_EVENT_DISCONNECT:
+
+#if __FreeBSD_version >= 1102000
+		qlnxr_iw_disconnect_event(context, params);
+#else
+		qlnxr_iw_issue_event(context,
+				    params,
+				    IW_CM_EVENT_DISCONNECT,
+				    "IW_CM_EVENT_DISCONNECT");
+		qlnxr_iw_close_event(context, params);
+#endif /* #if __FreeBSD_version >= 1102000 */
+		break;
+
+	case ECORE_IWARP_EVENT_CLOSE:
+		ep->during_connect = 0;
+		qlnxr_iw_close_event(context, params);
+		break;
+
+        case ECORE_IWARP_EVENT_RQ_EMPTY:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "IWARP_EVENT_RQ_EMPTY");
+                break;
+
+        case ECORE_IWARP_EVENT_IRQ_FULL:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "IWARP_EVENT_IRQ_FULL");
+                break;
+
+        case ECORE_IWARP_EVENT_LLP_TIMEOUT:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "IWARP_EVENT_LLP_TIMEOUT");
+                break;
+
+        case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
+                                 "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
+                break;
+
+        case ECORE_IWARP_EVENT_CQ_OVERFLOW:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "QED_IWARP_EVENT_CQ_OVERFLOW");
+                break;
+
+        case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "QED_IWARP_EVENT_QP_CATASTROPHIC");
+                break;
+
+        case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
+                                 "IWARP_EVENT_LOCAL_ACCESS_ERROR");
+                break;
+
+        case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
+                qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
+                                 "IWARP_EVENT_REMOTE_OPERATION_ERROR");
+                break;
+
+        case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
+		QL_DPRINT12(ha, "Got terminate message"
+			" ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
+                break;
+
+	default:
+		QL_DPRINT12(ha,
+			"Unknown event [0x%x] received \n", params->event);
+		break;
+	};
+
+	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
+		"exit\n", context, params->event);
+	return 0;
+}
+
+static int
+qlnxr_addr4_resolve(struct qlnxr_dev *dev,
+			      struct sockaddr_in *src_in,
+			      struct sockaddr_in *dst_in,
+			      u8 *dst_mac)
+{
+	int rc;
+
+#if __FreeBSD_version >= 1100000
+	rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
+			dst_mac, NULL, NULL);
+#else
+	struct llentry *lle;
+
+	rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
+			dst_mac, &lle);
+#endif
+
+	QL_DPRINT12(dev->ha, "rc = %d "
+		"sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
+		"Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
+		dst_in->sin_len, dst_in->sin_family,
+		NIPQUAD((dst_in->sin_addr.s_addr)),
+		dst_mac[0], dst_mac[1], dst_mac[2],
+		dst_mac[3], dst_mac[4], dst_mac[5]);
+
+	return rc;
+}
+
+int
+qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
+{
+	struct qlnxr_dev *dev;
+	struct ecore_iwarp_connect_out out_params;
+	struct ecore_iwarp_connect_in in_params;
+	struct qlnxr_iw_ep *ep;
+	struct qlnxr_qp *qp;
+	struct sockaddr_in *laddr;
+	struct sockaddr_in *raddr;
+	int rc = 0;
+	qlnx_host_t	*ha;
+
+	dev = get_qlnxr_dev((cm_id->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
+		"enter \n", cm_id, conn_param);
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	qp = idr_find(&dev->qpidr, conn_param->qpn);
+
+	laddr = (struct sockaddr_in *)&cm_id->local_addr;
+	raddr = (struct sockaddr_in *)&cm_id->remote_addr;
+
+	QL_DPRINT12(ha,
+		"local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
+		NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
+		NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
+
+	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
+	if (!ep) {
+		QL_DPRINT11(ha, "struct qlnxr_iw_ep "
+			"alloc memory failed\n");
+		return -ENOMEM;
+	}
+
+	ep->dev = dev;
+	ep->qp = qp;
+	cm_id->add_ref(cm_id);
+	ep->cm_id = cm_id;
+
+	memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
+	memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
+
+	in_params.event_cb = qlnxr_iw_event_handler;
+	in_params.cb_context = ep;
+
+	in_params.cm_info.ip_version = ECORE_TCP_IPV4;
+
+	in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
+	in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
+	in_params.cm_info.remote_port = ntohs(raddr->sin_port);
+	in_params.cm_info.local_port = ntohs(laddr->sin_port);
+	in_params.cm_info.vlan = 0;
+	in_params.mss = dev->ha->ifp->if_mtu - 40;
+
+	QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
+		"local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
+		"vlan = %d\n",
+		NIPQUAD((in_params.cm_info.remote_ip[0])),
+		NIPQUAD((in_params.cm_info.local_ip[0])),
+		in_params.cm_info.remote_port, in_params.cm_info.local_port,
+		in_params.cm_info.vlan);
+
+	rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
+
+	if (rc) {
+		QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
+		goto err;
+	}
+
+	QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
+		" private_data_len=%d rq_psn=%d\n",
+		conn_param->ord, conn_param->ird, conn_param->private_data,
+		conn_param->private_data_len, qp->rq_psn);
+
+	in_params.cm_info.ord = conn_param->ord;
+	in_params.cm_info.ird = conn_param->ird;
+	in_params.cm_info.private_data = conn_param->private_data;
+	in_params.cm_info.private_data_len = conn_param->private_data_len;
+	in_params.qp = qp->ecore_qp;
+
+	memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
+
+	rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
+
+	if (rc) {
+		QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
+		goto err;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+
+	return rc;
+
+err:
+	cm_id->rem_ref(cm_id);
+	kfree(ep);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+int
+qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
+{
+	struct qlnxr_dev *dev;
+	struct qlnxr_iw_listener *listener;
+	struct ecore_iwarp_listen_in iparams;
+	struct ecore_iwarp_listen_out oparams;
+	struct sockaddr_in *laddr;
+	qlnx_host_t	*ha;
+	int rc;
+
+	dev = get_qlnxr_dev((cm_id->device));
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+
+	laddr = (struct sockaddr_in *)&cm_id->local_addr;
+
+	listener = kzalloc(sizeof(*listener), GFP_KERNEL);
+
+	if (listener == NULL) {
+		QL_DPRINT11(ha, "listener memory alloc failed\n");
+		return -ENOMEM;
+	}
+
+	listener->dev = dev;
+	cm_id->add_ref(cm_id);
+	listener->cm_id = cm_id;
+	listener->backlog = backlog;
+
+	memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
+	memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
+
+	iparams.cb_context = listener;
+	iparams.event_cb = qlnxr_iw_event_handler;
+	iparams.max_backlog = backlog;
+
+	iparams.ip_version = ECORE_TCP_IPV4;
+
+	iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
+	iparams.port = ntohs(laddr->sin_port);
+	iparams.vlan = 0;
+
+	QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
+		NIPQUAD((laddr->sin_addr.s_addr)),
+		laddr->sin_port, iparams.port);
+
+	rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
+	if (rc) {
+		QL_DPRINT11(ha,
+			"ecore_iwarp_create_listen failed rc = %d\n", rc);
+		goto err;
+	}
+
+	listener->ecore_handle = oparams.handle;
+	cm_id->provider_data = listener;
+
+	QL_DPRINT12(ha, "exit\n");
+	return rc;
+
+err:
+	cm_id->rem_ref(cm_id);
+	kfree(listener);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return rc;
+}
+
+void
+qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
+{
+	struct qlnxr_iw_listener *listener = cm_id->provider_data;
+	struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
+	int rc = 0;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter\n");
+
+	if (listener->ecore_handle)
+		rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
+				listener->ecore_handle);
+
+	cm_id->rem_ref(cm_id);
+
+	QL_DPRINT12(ha, "exit [%d]\n", rc);
+	return;
+}
+
+int
+qlnxr_iw_accept(struct iw_cm_id *cm_id,
+	struct iw_cm_conn_param *conn_param)
+{
+	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
+	struct qlnxr_dev *dev = ep->dev;
+	struct qlnxr_qp *qp;
+	struct ecore_iwarp_accept_in params;
+	int rc;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter  qpid=%d\n", conn_param->qpn);
+
+	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return -EINVAL;
+ 
+	qp = idr_find(&dev->qpidr, conn_param->qpn);
+	if (!qp) {
+		QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
+			conn_param->qpn);
+		return -EINVAL;
+	}
+	ep->qp = qp;
+	qp->ep = ep;
+	cm_id->add_ref(cm_id);
+	ep->cm_id = cm_id;
+
+	params.ep_context = ep->ecore_context;
+	params.cb_context = ep;
+	params.qp = ep->qp->ecore_qp;
+	params.private_data = conn_param->private_data;
+	params.private_data_len = conn_param->private_data_len;
+	params.ird = conn_param->ird;
+	params.ord = conn_param->ord;
+
+	rc = ecore_iwarp_accept(dev->rdma_ctx, &params);
+	if (rc) {
+		QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
+		goto err;
+	}
+
+	QL_DPRINT12(ha, "exit\n");
+	return 0;
+err:
+	cm_id->rem_ref(cm_id);
+	QL_DPRINT12(ha, "exit rc = %d\n", rc);
+	return rc;
+}
+
+int
+qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
+{
+#if __FreeBSD_version >= 1102000
+
+        struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
+        struct qlnxr_dev *dev = ep->dev;
+        struct ecore_iwarp_reject_in params;
+        int rc;
+
+        params.ep_context = ep->ecore_context;
+        params.cb_context = ep;
+        params.private_data = pdata;
+        params.private_data_len = pdata_len;
+        ep->qp = NULL;
+
+        rc = ecore_iwarp_reject(dev->rdma_ctx, &params);
+
+        return rc;
+
+#else
+
+	printf("iWARP reject_cr not implemented\n");
+	return -EINVAL;
+
+#endif /* #if __FreeBSD_version >= 1102000 */
+}
+
+void
+qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
+{
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	qlnx_host_t	*ha;
+
+	ha = qp->dev->ha;
+
+	QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
+ 
+	atomic_inc(&qp->refcnt);
+
+	QL_DPRINT12(ha, "exit \n");
+	return;
+}
+
+void
+qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
+{
+	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
+	qlnx_host_t	*ha;
+
+	ha = qp->dev->ha;
+
+	QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
+
+	if (atomic_dec_and_test(&qp->refcnt)) {
+		qlnxr_idr_remove(qp->dev, qp->qp_id);
+		kfree(qp);
+	}
+
+	QL_DPRINT12(ha, "exit \n");
+	return;
+}
+
+struct ib_qp *
+qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
+{
+	struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
+	struct ib_qp *qp;
+	qlnx_host_t	*ha;
+
+	ha = dev->ha;
+
+	QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
+
+	qp = idr_find(&dev->qpidr, qpn);
+
+	QL_DPRINT12(ha, "exit qp = %p\n", qp);
+
+	return (qp);
+}
diff --git a/sys/dev/qlnx/qlnxr/qlnxr_verbs.h b/sys/dev/qlnx/qlnxr/qlnxr_verbs.h
new file mode 100644
index 000000000000..c202f4457b85
--- /dev/null
+++ b/sys/dev/qlnx/qlnxr/qlnxr_verbs.h
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2018-2019 Cavium, Inc.
+ * All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * $FreeBSD$
+ */
+
+
+#ifndef __QLNXR_VERBS_H__
+#define __QLNXR_VERBS_H__
+
+extern int qlnxr_iw_query_gid(struct ib_device *,
+			uint8_t port,
+			int index,
+			union ib_gid *gid);
+
+extern int qlnxr_query_gid(struct ib_device *,
+			u8 port,
+			int index,
+			union ib_gid *gid);
+
+extern struct ib_srq *qlnxr_create_srq(struct ib_pd *,
+			struct ib_srq_init_attr *,
+			struct ib_udata *);
+
+extern int qlnxr_destroy_srq(struct ib_srq *);
+
+
+extern int qlnxr_modify_srq(struct ib_srq *,
+			struct ib_srq_attr *,
+			enum ib_srq_attr_mask,
+			struct ib_udata *);
+
+extern int qlnxr_query_srq(struct ib_srq *,
+			struct ib_srq_attr *);
+
+extern int qlnxr_post_srq_recv(struct ib_srq *,
+			struct ib_recv_wr *,
+			struct ib_recv_wr **bad_recv_wr);
+
+#if __FreeBSD_version < 1102000
+extern int qlnxr_query_device(struct ib_device *, struct ib_device_attr *);
+#else
+extern int qlnxr_query_device(struct ib_device *, struct ib_device_attr *,
+		struct ib_udata *);
+extern int qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+		struct ib_port_immutable *immutable);
+#endif
+
+extern int qlnxr_query_port(struct ib_device *,
+			u8 port,
+			struct ib_port_attr *props);
+
+extern int qlnxr_modify_port(struct ib_device *,
+			u8 port,
+			int mask,
+			struct ib_port_modify *props);
+
+extern enum rdma_link_layer qlnxr_link_layer(struct ib_device *device,
+			uint8_t port_num);
+
+struct ib_pd *qlnxr_alloc_pd(struct ib_device *,
+			struct ib_ucontext *,
+			struct ib_udata *);
+
+extern int qlnxr_dealloc_pd(struct ib_pd *pd);
+
+#if __FreeBSD_version >= 1102000
+extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
+			const struct ib_cq_init_attr *attr,
+			struct ib_ucontext *ib_ctx,
+			struct ib_udata *udata);
+#else
+#if __FreeBSD_version >= 1100000
+extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
+			struct ib_cq_init_attr *attr,
+			struct ib_ucontext *ib_ctx,
+			struct ib_udata *udata);
+#else
+extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
+			int cqe,
+			int comp_vector,
+			struct ib_ucontext *ib_ctx,
+			struct ib_udata *udata);
+#endif
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+extern int qlnxr_destroy_cq(struct ib_cq *);
+
+extern int qlnxr_resize_cq(struct ib_cq *,
+			int cqe,
+			struct ib_udata *);
+
+extern int qlnxr_poll_cq(struct ib_cq *,
+			int num_entries,
+			struct ib_wc *wc);
+
+
+extern struct ib_qp *qlnxr_create_qp(struct ib_pd *,
+		       struct ib_qp_init_attr *attrs,
+		       struct ib_udata *);
+
+extern int qlnxr_modify_qp(struct ib_qp *,
+			struct ib_qp_attr *attr,
+			int attr_mask,
+			struct ib_udata *udata);
+
+extern int qlnxr_query_qp(struct ib_qp *,
+			struct ib_qp_attr *qp_attr,
+			int qp_attr_mask,
+			struct ib_qp_init_attr *);
+
+extern int qlnxr_destroy_qp(struct ib_qp *);
+
+extern int qlnxr_query_pkey(struct ib_device *,
+			u8 port,
+			u16 index,
+			u16 *pkey);
+
+#if __FreeBSD_version >= 1102000
+extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd,
+			struct ib_ah_attr *attr, struct ib_udata *udata);
+#else
+extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd,
+			struct ib_ah_attr *attr);
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+extern int qlnxr_destroy_ah(struct ib_ah *ibah);
+
+extern int qlnxr_query_ah(struct ib_ah *ibah,
+			struct ib_ah_attr *attr);
+
+extern int qlnxr_modify_ah(struct ib_ah *ibah,
+			struct ib_ah_attr *attr);
+
+#if __FreeBSD_version >= 1102000
+extern int qlnxr_process_mad(struct ib_device *ibdev,
+			int process_mad_flags,
+			u8 port_num,
+			const struct ib_wc *in_wc,
+			const struct ib_grh *in_grh,
+			const struct ib_mad_hdr *mad_hdr,
+			size_t in_mad_size,
+			struct ib_mad_hdr *out_mad,
+			size_t *out_mad_size,
+			u16 *out_mad_pkey_index);
+#else
+extern int qlnxr_process_mad(struct ib_device *ibdev,
+			int process_mad_flags,
+			u8 port_num,
+			struct ib_wc *in_wc,
+			struct ib_grh *in_grh,
+			struct ib_mad *in_mad,
+			struct ib_mad *out_mad);
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+extern int qlnxr_post_send(struct ib_qp *,
+			struct ib_send_wr *,
+			struct ib_send_wr **bad_wr);
+
+extern int qlnxr_post_recv(struct ib_qp *,
+			struct ib_recv_wr *,
+			struct ib_recv_wr **bad_wr);
+
+extern int qlnxr_arm_cq(struct ib_cq *,
+			enum ib_cq_notify_flags flags);
+
+extern struct ib_mr *qlnxr_get_dma_mr(struct ib_pd *,
+			int acc);
+
+#if __FreeBSD_version < 1102000
+extern struct ib_mr *qlnxr_reg_kernel_mr(struct ib_pd *,
+			struct ib_phys_buf *buffer_list,
+			int num_phys_buf,
+			int acc,
+			u64 *iova_start);
+#endif /* #if __FreeBSD_version < 1102000 */
+
+extern int qlnxr_dereg_mr(struct ib_mr *);
+
+#if __FreeBSD_version >= 1102000
+extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *,
+			u64 start,
+			u64 length,
+			u64 virt,
+			int acc,
+			struct ib_udata *);
+#else
+extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *,
+			u64 start,
+			u64 length,
+			u64 virt,
+			int acc,
+			struct ib_udata *,
+			int mr_id);
+#endif /* #if __FreeBSD_version >= 1102000 */
+
+#if __FreeBSD_version >= 1102000
+
+extern struct ib_mr *qlnxr_alloc_mr(struct ib_pd *pd,
+			enum ib_mr_type mr_type, u32 max_num_sg);
+extern int qlnxr_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg,
+			int sg_nents, unsigned int *sg_offset);
+#else
+
+extern struct ib_mr *qlnxr_alloc_frmr(struct ib_pd *pd,
+			int max_page_list_len);
+
+
+extern struct ib_fast_reg_page_list *qlnxr_alloc_frmr_page_list(
+			struct ib_device *ibdev,
+			int page_list_len);
+
+extern void qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list);
+
+#endif /* #if  __FreeBSD_version >= 1102000 */
+
+extern struct ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
+                struct ib_udata *udata);
+
+extern int qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx);
+
+
+extern int qlnxr_mmap(struct ib_ucontext *, struct vm_area_struct *vma);
+
+extern int qlnxr_iw_connect(struct iw_cm_id *cm_id,
+		    struct iw_cm_conn_param *conn_param);
+
+extern int qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog);
+
+void qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id);
+
+extern int qlnxr_iw_accept(struct iw_cm_id *cm_id,
+		      struct iw_cm_conn_param *conn_param);
+
+extern int qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
+
+extern void qlnxr_iw_qp_add_ref(struct ib_qp *qp);
+
+extern void qlnxr_iw_qp_rem_ref(struct ib_qp *qp);
+
+extern struct ib_qp *qlnxr_iw_get_qp(struct ib_device *dev, int qpn);
+
+#endif /* #ifndef __QLNXR_VERBS_H__ */
+