mlx5en: add IPSEC_OFFLOAD support

Right now, only IPv4 transport mode, with aes-gcm ESP, is supported.
Driver also cooperates with NAT-T, and obeys socket policies, which
makes IKEd like StrongSwan working.

Sponsored by:	NVIDIA networking

53 files changed, 11214 insertions, 4328 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 5250b65135e6..6e69c128ee65 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4952,6 +4952,8 @@ dev/mlx5/mlx5_core/mlx5_alloc.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_cmd.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_core/mlx5_crypto.c		optional mlx5 pci	\
+	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_cq.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_diag_cnt.c		optional mlx5 pci	\
@@ -4962,15 +4964,17 @@ dev/mlx5/mlx5_core/mlx5_eq.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_eswitch.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
-dev/mlx5/mlx5_core/mlx5_fc_cmd.c		optional mlx5 pci	\
+dev/mlx5/mlx5_core/mlx5_fs_chains.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_fs_cmd.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_core/mlx5_fs_core.c		optional mlx5 pci	\
+	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_fs_counters.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
-dev/mlx5/mlx5_core/mlx5_fs_tcp.c		optional mlx5 pci	\
+dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
-dev/mlx5/mlx5_core/mlx5_fs_tree.c		optional mlx5 pci	\
+dev/mlx5/mlx5_core/mlx5_fs_tcp.c		optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_fw.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
@@ -5012,8 +5016,18 @@ dev/mlx5/mlx5_core/mlx5_vsc.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
 dev/mlx5/mlx5_core/mlx5_wq.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_lib/mlx5_aso.c			optional mlx5 pci	\
+	compile-with "${OFED_C}"
 dev/mlx5/mlx5_lib/mlx5_gid.c			optional mlx5 pci	\
 	compile-with "${OFED_C}"
+dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c		optional mlx5 pci	\
+	compile-with "${OFED_C}"
+dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c	optional mlx5 pci	\
+	compile-with "${OFED_C}"
+dev/mlx5/mlx5_accel/mlx5_ipsec.c		optional mlx5 pci	\
+	compile-with "${OFED_C}"
+dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c		optional mlx5 pci	\
+	compile-with "${OFED_C}"
 
 dev/mlx5/mlx5_en/mlx5_en_dim.c			optional mlx5en pci inet inet6	\
 	compile-with "${OFED_C}"
diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h
index 02de7f66e260..cc551e65dead 100644
--- a/sys/dev/mlx5/cq.h
+++ b/sys/dev/mlx5/cq.h
@@ -111,6 +111,12 @@ struct mlx5_cq_modify_params {
 	} params;
 };
 
+enum {
+	CQE_STRIDE_64 = 0,
+	CQE_STRIDE_128 = 1,
+	CQE_STRIDE_128_PAD = 2,
+};
+
 static inline int cqe_sz_to_mlx_sz(u8 size)
 {
 	return size == 64 ? CQE_SIZE_64 : CQE_SIZE_128;
diff --git a/sys/dev/mlx5/crypto.h b/sys/dev/mlx5/crypto.h
new file mode 100644
index 000000000000..3b2c4c218ef2
--- /dev/null
+++ b/sys/dev/mlx5/crypto.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef __MLX5_CRYPTO_H__
+#define	__MLX5_CRYPTO_H__
+
+struct mlx5_core_dev;
+
+int mlx5_encryption_key_create(struct mlx5_core_dev *, u32 pdn, u32 key_type,
+    const void *p_key, u32 key_len, u32 *p_obj_id);
+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
+
+#endif					/* __MLX5_CRYPTO_H__ */
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index 4b7cf8686794..50995d4f70a7 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -148,12 +148,14 @@ __mlx5_mask16(typ, fld))
 		tmp;							  \
 		})
 
-#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 8
-#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 8
+#define MLX5_RDMA_RX_NUM_COUNTERS_PRIOS 2
+#define MLX5_RDMA_TX_NUM_COUNTERS_PRIOS 1
+#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16
+#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16
 #define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1
 #define MLX5_BY_PASS_NUM_PRIOS (MLX5_BY_PASS_NUM_REGULAR_PRIOS +\
-                                    MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
-                                    MLX5_BY_PASS_NUM_MULTICAST_PRIOS)
+				MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
+				MLX5_BY_PASS_NUM_MULTICAST_PRIOS)
 
 /* insert a value to a struct */
 #define MLX5_VSC_SET(typ, p, fld, v) do { \
@@ -391,6 +393,8 @@ enum {
 	MLX5_OPCODE_UMR			= 0x25,
 	MLX5_OPCODE_QOS_REMAP		= 0x2a,
 
+	MLX5_OPCODE_ACCESS_ASO          = 0x2d, 
+
 	MLX5_OPCODE_SIGNATURE_CANCELED	= (1 << 15),
 };
 
@@ -567,6 +571,11 @@ struct mlx5_eqe_vport_change {
 	__be32		rsvd1[6];
 };
 
+struct mlx5_eqe_obj_change {
+        u8      rsvd0[2];
+        __be16  obj_type;
+        __be32  obj_id;
+};
 
 #define PORT_MODULE_EVENT_MODULE_STATUS_MASK  0xF
 #define PORT_MODULE_EVENT_ERROR_TYPE_MASK     0xF
@@ -638,6 +647,7 @@ union ev_data {
 	struct mlx5_eqe_dct             dct;
 	struct mlx5_eqe_temp_warning	temp_warning;
 	struct mlx5_eqe_xrq_err		xrq_err;
+	struct mlx5_eqe_obj_change      obj_change;
 } __packed;
 
 struct mlx5_eqe {
@@ -703,7 +713,12 @@ struct mlx5_cqe64 {
 	u8		l4_hdr_type_etc;
 	__be16		vlan_info;
 	__be32		srqn; /* [31:24]: lro_num_seg, [23:0]: srqn */
-	__be32		imm_inval_pkey;
+	union {
+		__be32 immediate;
+		__be32 inval_rkey;
+		__be32 pkey;
+		__be32 ft_metadata;
+	};
 	u8		rsvd40[4];
 	__be32		byte_cnt;
 	__be64		timestamp;
@@ -919,6 +934,7 @@ enum {
 	MLX5_MATCH_OUTER_HEADERS	= 1 << 0,
 	MLX5_MATCH_MISC_PARAMETERS	= 1 << 1,
 	MLX5_MATCH_INNER_HEADERS	= 1 << 2,
+	MLX5_MATCH_MISC_PARAMETERS_2	= 1 << 3,
 
 };
 
@@ -988,12 +1004,19 @@ enum mlx5_cap_type {
 	MLX5_CAP_VECTOR_CALC,
 	MLX5_CAP_QOS,
 	MLX5_CAP_DEBUG,
-	MLX5_CAP_NVME,
-	MLX5_CAP_DMC,
-	MLX5_CAP_DEC,
+	MLX5_CAP_RESERVED_14,
+	MLX5_CAP_DEV_MEM,
+	MLX5_CAP_RESERVED_16,
 	MLX5_CAP_TLS,
+	MLX5_CAP_VDPA_EMULATION = 0x13,
 	MLX5_CAP_DEV_EVENT = 0x14,
+	MLX5_CAP_IPSEC,
+	MLX5_CAP_CRYPTO = 0x1a,
+	MLX5_CAP_DEV_SHAMPO = 0x1d,
+	MLX5_CAP_MACSEC = 0x1f,
 	MLX5_CAP_GENERAL_2 = 0x20,
+	MLX5_CAP_PORT_SELECTION = 0x25,
+	MLX5_CAP_ADV_VIRTUALIZATION = 0x26,
 	/* NUM OF CAP Types */
 	MLX5_CAP_NUM
 };
@@ -1058,6 +1081,9 @@ enum mlx5_mcam_feature_groups {
 #define MLX5_CAP_FLOWTABLE(mdev, cap) \
 	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
 
+#define MLX5_CAP64_FLOWTABLE(mdev, cap) \
+	MLX5_GET64(flow_table_nic_cap, (mdev)->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
+
 #define MLX5_CAP_FLOWTABLE_MAX(mdev, cap) \
 	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_max[MLX5_CAP_FLOW_TABLE], cap)
 
@@ -1067,6 +1093,54 @@ enum mlx5_mcam_feature_groups {
 #define MLX5_CAP_FLOWTABLE_NIC_RX_MAX(mdev, cap) \
 	MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive.cap)
 
+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
+                MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX_MAX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX_MAX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
+        MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
+	MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
 #define MLX5_CAP_ESW_FLOWTABLE(mdev, cap) \
 	MLX5_GET(flow_table_eswitch_cap, \
 		 mdev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
@@ -1093,14 +1167,46 @@ enum mlx5_mcam_feature_groups {
 #define MLX5_CAP_ESW_INGRESS_ACL_MAX(mdev, cap) \
 	MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, flow_table_properties_esw_acl_ingress.cap)
 
+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2(mdev, cap) \
+        MLX5_CAP_ESW_FLOWTABLE(mdev, ft_field_support_2_esw_fdb.cap)
+
+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2_MAX(mdev, cap) \
+        MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, ft_field_support_2_esw_fdb.cap)
+
 #define MLX5_CAP_ESW(mdev, cap) \
 	MLX5_GET(e_switch_cap, \
 		 mdev->hca_caps_cur[MLX5_CAP_ESWITCH], cap)
 
+#define MLX5_CAP64_ESW_FLOWTABLE(mdev, cap) \
+	MLX5_GET64(flow_table_eswitch_cap, \
+		   (mdev)->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
+
 #define MLX5_CAP_ESW_MAX(mdev, cap) \
 	MLX5_GET(e_switch_cap, \
 		 mdev->hca_caps_max[MLX5_CAP_ESWITCH], cap)
 
+#define MLX5_CAP_PORT_SELECTION(mdev, cap) \
+	MLX5_GET(port_selection_cap, \
+		 mdev->hca_caps_cur[MLX5_CAP_PORT_SELECTION], cap)
+
+#define MLX5_CAP_PORT_SELECTION_MAX(mdev, cap) \
+	MLX5_GET(port_selection_cap, \
+		 mdev->hca_caps_max[MLX5_CAP_PORT_SELECTION], cap)
+
+#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \
+	MLX5_GET(adv_virtualization_cap, \
+		 mdev->hca_caps_cur[MLX5_CAP_ADV_VIRTUALIZATION], cap)
+
+#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \
+	MLX5_GET(adv_virtualization_cap, \
+		 mdev->hca_caps_max[MLX5_CAP_ADV_VIRTUALIZATION], cap)
+
+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \
+	MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap)
+
+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION_MAX(mdev, cap) \
+	MLX5_CAP_PORT_SELECTION_MAX(mdev, flow_table_properties_port_selection.cap)
+
 #define MLX5_CAP_ODP(mdev, cap)\
 	MLX5_GET(odp_cap, mdev->hca_caps_cur[MLX5_CAP_ODP], cap)
 
@@ -1169,6 +1275,9 @@ enum mlx5_mcam_feature_groups {
 #define	MLX5_CAP_DEV_EVENT(mdev, cap)\
 	MLX5_ADDR_OF(device_event_cap, (mdev)->hca_caps_cur[MLX5_CAP_DEV_EVENT], cap)
 
+#define	MLX5_CAP_IPSEC(mdev, cap) \
+	MLX5_GET(ipsec_cap, (mdev)->hca_caps_cur[MLX5_CAP_IPSEC], cap)
+
 enum {
 	MLX5_CMD_STAT_OK			= 0x0,
 	MLX5_CMD_STAT_INT_ERR			= 0x1,
diff --git a/sys/dev/mlx5/doorbell.h b/sys/dev/mlx5/doorbell.h
index 11ce9ae6649f..4a32e74623a0 100644
--- a/sys/dev/mlx5/doorbell.h
+++ b/sys/dev/mlx5/doorbell.h
@@ -61,10 +61,12 @@ static inline void mlx5_write64(__be32 val[2], void __iomem *dest,
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(doorbell_lock, flags);
+	if (doorbell_lock)
+		spin_lock_irqsave(doorbell_lock, flags);
 	__raw_writel((__force u32) val[0], dest);
 	__raw_writel((__force u32) val[1], dest + 4);
-	spin_unlock_irqrestore(doorbell_lock, flags);
+	if (doorbell_lock)
+		spin_unlock_irqrestore(doorbell_lock, flags);
 }
 
 #endif
diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
index 6ebe4171a503..db1c9f0bb46c 100644
--- a/sys/dev/mlx5/driver.h
+++ b/sys/dev/mlx5/driver.h
@@ -634,11 +634,13 @@ struct mlx5_priv {
 #endif
 	struct mlx5_pme_stats pme_stats;
 
+	struct mlx5_flow_steering *steering;
 	struct mlx5_eswitch	*eswitch;
 
 	struct mlx5_bfreg_data		bfregs;
 	struct mlx5_uars_page	       *uar;
 	struct mlx5_fc_stats		fc_stats;
+	struct mlx5_ft_pool             *ft_pool;
 };
 
 enum mlx5_device_state {
@@ -728,6 +730,10 @@ struct mlx5_core_dev {
 	struct mlx5_flow_root_namespace *esw_ingress_root_ns;
 	struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
 	struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
+	struct mlx5_flow_root_namespace *nic_tx_root_ns;
+	struct mlx5_flow_root_namespace *rdma_tx_root_ns;
+	struct mlx5_flow_root_namespace *rdma_rx_root_ns;
+
 	u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER];
 	struct mlx5_crspace_regmap *dump_rege;
 	uint32_t *dump_data;
@@ -756,6 +762,7 @@ struct mlx5_core_dev {
 #ifdef CONFIG_MLX5_FPGA
 	struct mlx5_fpga_device	*fpga;
 #endif
+	struct xarray ipsec_sadb;
 };
 
 enum {
diff --git a/sys/dev/mlx5/fs.h b/sys/dev/mlx5/fs.h
index 8107726bc76a..ee0d28b6758a 100644
--- a/sys/dev/mlx5/fs.h
+++ b/sys/dev/mlx5/fs.h
@@ -33,8 +33,33 @@
 #include <dev/mlx5/device.h>
 #include <dev/mlx5/driver.h>
 
+enum mlx5_flow_destination_type {
+        MLX5_FLOW_DESTINATION_TYPE_NONE,
+        MLX5_FLOW_DESTINATION_TYPE_VPORT,
+        MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
+        MLX5_FLOW_DESTINATION_TYPE_TIR,
+        MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER,
+        MLX5_FLOW_DESTINATION_TYPE_UPLINK,
+        MLX5_FLOW_DESTINATION_TYPE_PORT,
+        MLX5_FLOW_DESTINATION_TYPE_COUNTER,
+        MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM,
+        MLX5_FLOW_DESTINATION_TYPE_RANGE,
+        MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE,
+};
+
+enum {
+        MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO  = 1 << 16,
+        MLX5_FLOW_CONTEXT_ACTION_ENCRYPT        = 1 << 17,
+        MLX5_FLOW_CONTEXT_ACTION_DECRYPT        = 1 << 18,
+        MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS    = 1 << 19,
+};
+
 enum {
-	MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO	= 1 << 16,
+        MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT = BIT(0),
+        MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1),
+        MLX5_FLOW_TABLE_TERMINATION = BIT(2),
+        MLX5_FLOW_TABLE_UNMANAGED = BIT(3),
+        MLX5_FLOW_TABLE_OTHER_VPORT = BIT(4),
 };
 
 /*Flow tag*/
@@ -61,54 +86,149 @@ enum {
 
 enum mlx5_flow_namespace_type {
 	MLX5_FLOW_NAMESPACE_BYPASS,
+	MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
+	MLX5_FLOW_NAMESPACE_LAG,
 	MLX5_FLOW_NAMESPACE_OFFLOADS,
+	MLX5_FLOW_NAMESPACE_ETHTOOL,
 	MLX5_FLOW_NAMESPACE_KERNEL,
 	MLX5_FLOW_NAMESPACE_LEFTOVERS,
-	MLX5_FLOW_NAMESPACE_SNIFFER_RX,
-	MLX5_FLOW_NAMESPACE_SNIFFER_TX,
+	MLX5_FLOW_NAMESPACE_ANCHOR,
+	MLX5_FLOW_NAMESPACE_FDB_BYPASS,
 	MLX5_FLOW_NAMESPACE_FDB,
 	MLX5_FLOW_NAMESPACE_ESW_EGRESS,
 	MLX5_FLOW_NAMESPACE_ESW_INGRESS,
+	MLX5_FLOW_NAMESPACE_SNIFFER_RX,
+	MLX5_FLOW_NAMESPACE_SNIFFER_TX,
+	MLX5_FLOW_NAMESPACE_EGRESS,
+	MLX5_FLOW_NAMESPACE_EGRESS_IPSEC,
+	MLX5_FLOW_NAMESPACE_EGRESS_MACSEC,
+	MLX5_FLOW_NAMESPACE_RDMA_RX,
+	MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL,
+	MLX5_FLOW_NAMESPACE_RDMA_TX,
+	MLX5_FLOW_NAMESPACE_PORT_SEL,
+	MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS,
+	MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS,
+	MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC,
+	MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC,
+};
+
+enum {
+	FDB_BYPASS_PATH,
+	FDB_TC_OFFLOAD,
+	FDB_FT_OFFLOAD,
+	FDB_TC_MISS,
+	FDB_BR_OFFLOAD,
+	FDB_SLOW_PATH,
+	FDB_PER_VPORT,
 };
 
 struct mlx5_flow_table;
 struct mlx5_flow_group;
 struct mlx5_flow_rule;
 struct mlx5_flow_namespace;
+struct mlx5_flow_handle;
+
+enum {
+	FLOW_CONTEXT_HAS_TAG = BIT(0),
+};
+
+struct mlx5_flow_context {
+	u32 flags;
+	u32 flow_tag;
+	u32 flow_source;
+};
 
 struct mlx5_flow_spec {
 	u8   match_criteria_enable;
 	u32  match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
 	u32  match_value[MLX5_ST_SZ_DW(fte_match_param)];
+	struct mlx5_flow_context flow_context;
+};
+
+enum {
+	MLX5_FLOW_DEST_VPORT_VHCA_ID      = BIT(0),
+	MLX5_FLOW_DEST_VPORT_REFORMAT_ID  = BIT(1),
+};
+
+enum mlx5_flow_dest_range_field {
+	MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN = 0,
 };
 
 struct mlx5_flow_destination {
-	u32	type;
+	enum mlx5_flow_destination_type type;
 	union {
-		u32			tir_num;
-		struct mlx5_flow_table	*ft;
-		u32			vport_num;
+		u32                     tir_num;
+		u32                     ft_num;
+		struct mlx5_flow_table  *ft;
+		u32                     counter_id;
+		struct {
+			u16             num;
+			u16             vhca_id;
+			struct mlx5_pkt_reformat *pkt_reformat;
+			u8              flags;
+		} vport;
+		struct {
+			struct mlx5_flow_table         *hit_ft;
+			struct mlx5_flow_table         *miss_ft;
+			enum mlx5_flow_dest_range_field field;
+			u32                             min;
+			u32                             max;
+		} range;
+		u32                     sampler_id;
 	};
 };
 
-enum mlx5_flow_act_actions {
-	MLX5_FLOW_ACT_ACTIONS_FLOW_TAG = 1 << 0,
-	MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR = 1 << 1,
-	MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT = 1 << 2,
-	MLX5_FLOW_ACT_ACTIONS_COUNT = 1 << 3,
+struct mlx5_exe_aso {
+	u32 object_id;
+	u8 type;
+	u8 return_reg_id;
+	union {
+		u32 ctrl_data;
+		struct {
+			u8 meter_idx;
+			u8 init_color;
+		} flow_meter;
+	};
+};
+
+enum {
+	FLOW_ACT_NO_APPEND = BIT(0),
+	FLOW_ACT_IGNORE_FLOW_LEVEL = BIT(1),
+};
+
+struct mlx5_fs_vlan {
+	u16 ethtype;
+	u16 vid;
+	u8  prio;
+};
+
+#define MLX5_FS_VLAN_DEPTH      2
+
+enum mlx5_flow_act_crypto_type {
+	MLX5_FLOW_ACT_CRYPTO_TYPE_IPSEC,
+};
+
+enum mlx5_flow_act_crypto_op {
+	MLX5_FLOW_ACT_CRYPTO_OP_ENCRYPT,
+	MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT,
 };
 
-enum MLX5_FLOW_ACT_FLAGS {
-	MLX5_FLOW_ACT_NO_APPEND = 1 << 0,
+struct mlx5_flow_act_crypto_params {
+	u32 obj_id;
+	u8 type; /* see enum mlx5_flow_act_crypto_type */
+	u8 op; /* see enum mlx5_flow_act_crypto_op */
 };
 
 struct mlx5_flow_act {
-	u32 actions; /* See enum mlx5_flow_act_actions */
-	u32 flags;
-	u32 flow_tag;
-	struct mlx5_modify_hdr *modify_hdr;
+	u32 action;
+	struct mlx5_modify_hdr  *modify_hdr;
 	struct mlx5_pkt_reformat *pkt_reformat;
-	struct mlx5_fc *counter;
+	struct mlx5_flow_act_crypto_params crypto;
+	u32 flags;
+	struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH];
+	struct ib_counters *counters;
+	struct mlx5_flow_group *fg;
+	struct mlx5_exe_aso exe_aso;
 };
 
 #define FT_NAME_STR_SZ 20
@@ -136,6 +256,28 @@ static inline bool outer_header_zero(u32 *match_criteria)
 }
 
 struct mlx5_flow_namespace *
+mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
+                                  enum mlx5_flow_namespace_type type,
+                                  int vport);
+
+struct mlx5_flow_table_attr {
+        int prio;
+        int max_fte;
+        u32 level;
+        u32 flags;
+        u16 uid;
+        struct mlx5_flow_table *next_ft;
+
+        struct {
+                int max_num_groups;
+                int num_reserved_entries;
+        } autogroup;
+};
+
+struct mlx5_flow_namespace *
+mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev, int n);
+
+struct mlx5_flow_namespace *
 mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 			enum mlx5_flow_namespace_type type);
 
@@ -145,24 +287,19 @@ mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
  */
 struct mlx5_flow_table *
 mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
-				    int prio,
-				    const char *name,
-				    int num_flow_table_entries,
-				    int max_num_groups,
-				    int num_reserved_entries);
+				    struct mlx5_flow_table_attr *ft_attr);
 
 struct mlx5_flow_table *
 mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
-							 u16 vport,
-							 int prio,
-							 const char *name,
-							 int num_flow_table_entries);
+                             struct mlx5_flow_table_attr *ft_attr, u16 vport);
+
+struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
+                                               struct mlx5_flow_namespace *ns,
+                                               int prio, u32 level);
 
 struct mlx5_flow_table *
 mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
-		       int prio,
-		       const char *name,
-		       int num_flow_table_entries);
+                       struct mlx5_flow_table_attr *ft_attr);
 int mlx5_destroy_flow_table(struct mlx5_flow_table *ft);
 
 /* inbox should be set with the following values:
@@ -175,18 +312,17 @@ struct mlx5_flow_group *
 mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in);
 void mlx5_destroy_flow_group(struct mlx5_flow_group *fg);
 
-/* Single destination per rule.
- * Group ID is implied by the match criteria.
- */
-struct mlx5_flow_rule *
-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
-		   u8 match_criteria_enable,
-		   u32 *match_criteria,
-		   u32 *match_value,
-		   u32 sw_action,
-		   struct mlx5_flow_act *flow_act,
-		   struct mlx5_flow_destination *dest);
-void mlx5_del_flow_rule(struct mlx5_flow_rule **);
+struct mlx5_flow_handle *
+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+                    const struct mlx5_flow_spec *spec,
+                    struct mlx5_flow_act *flow_act,
+                    struct mlx5_flow_destination *dest,
+                    int num_dest);
+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp);
+
+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handler,
+                                 struct mlx5_flow_destination *new_dest,
+                                 struct mlx5_flow_destination *old_dest);
 
 /*The following API is for sniffer*/
 typedef int (*rule_event_fn)(struct mlx5_flow_rule *rule,
@@ -292,4 +428,8 @@ int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter,
                   u64 *packets, u64 *bytes);
 u32 mlx5_fc_id(struct mlx5_fc *counter);
 /******* End of Flow counters API ******/
+
+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft);
+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
 #endif
diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h
index c020d41cd875..1658542fc9c6 100644
--- a/sys/dev/mlx5/mlx5_accel/ipsec.h
+++ b/sys/dev/mlx5/mlx5_accel/ipsec.h
@@ -1,137 +1,277 @@
 /*-
- * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
  *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
  *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
  *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
  */
 
 #ifndef __MLX5_ACCEL_IPSEC_H__
 #define __MLX5_ACCEL_IPSEC_H__
 
-#ifdef CONFIG_MLX5_ACCEL
-
+#include <sys/mbuf.h>
 #include <dev/mlx5/driver.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_lib/aso.h>
+
+#define MLX5E_IPSEC_SADB_RX_BITS 10
+#define MLX5_IPSEC_METADATA_MARKER(ipsec_metadata) ((ipsec_metadata >> 31) & 0x1)
+
+struct mlx5e_priv;
+struct mlx5e_tx_wqe;
+struct mlx5e_ipsec_tx;
+struct mlx5e_ipsec_rx;
+
+struct aes_gcm_keymat {
+	u64   seq_iv;
+
+	u32   salt;
+	u32   icv_len;
 
-enum {
-	MLX5_ACCEL_IPSEC_DEVICE = BIT(1),
-	MLX5_ACCEL_IPSEC_IPV6 = BIT(2),
-	MLX5_ACCEL_IPSEC_ESP = BIT(3),
-	MLX5_ACCEL_IPSEC_LSO = BIT(4),
+	u32   key_len;
+	u32   aes_key[256 / 32];
 };
 
-#define MLX5_IPSEC_SADB_IP_AH       BIT(7)
-#define MLX5_IPSEC_SADB_IP_ESP      BIT(6)
-#define MLX5_IPSEC_SADB_SA_VALID    BIT(5)
-#define MLX5_IPSEC_SADB_SPI_EN      BIT(4)
-#define MLX5_IPSEC_SADB_DIR_SX      BIT(3)
-#define MLX5_IPSEC_SADB_IPV6        BIT(2)
+struct mlx5e_ipsec_priv_bothdir {
+	struct mlx5e_ipsec_sa_entry *priv_in;
+	struct mlx5e_ipsec_sa_entry *priv_out;
+};
+
+struct mlx5e_ipsec_work {
+        struct work_struct work;
+        struct mlx5e_ipsec_sa_entry *sa_entry;
+        void *data;
+};
 
-enum {
-	MLX5_IPSEC_CMD_ADD_SA = 0,
-	MLX5_IPSEC_CMD_DEL_SA = 1,
+struct mlx5e_ipsec_dwork {
+	struct delayed_work dwork;
+	struct mlx5e_ipsec_sa_entry *sa_entry;
+	struct mlx5e_ipsec_priv_bothdir *pb;
 };
 
-enum mlx5_accel_ipsec_enc_mode {
-	MLX5_IPSEC_SADB_MODE_NONE = 0,
-	MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1,
-	MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3,
+struct mlx5e_ipsec_aso {
+        u8 __aligned(64) ctx[MLX5_ST_SZ_BYTES(ipsec_aso)];
+        dma_addr_t dma_addr;
+        struct mlx5_aso *aso;
+        /* Protect ASO WQ access, as it is global to whole IPsec */
+        spinlock_t lock;
 };
 
-#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \
-			      MLX5_ACCEL_IPSEC_DEVICE)
+struct mlx5_replay_esn {
+	u32 replay_window;
+	u32 esn;
+	u32 esn_msb;
+	u8 overlap : 1;
+	u8 trigger : 1;
+};
+
+struct mlx5_accel_esp_xfrm_attrs {
+	u32   spi;
+	struct aes_gcm_keymat aes_gcm;
 
-struct mlx5_accel_ipsec_sa {
-	__be32 cmd;
-	u8 key_enc[32];
-	u8 key_auth[32];
-	__be32 sip[4];
-	__be32 dip[4];
 	union {
-		struct {
-			__be32 reserved;
-			u8 salt_iv[8];
-			__be32 salt;
-		} __packed gcm;
-		struct {
-			u8 salt[16];
-		} __packed cbc;
-	};
-	__be32 spi;
-	__be32 sw_sa_handle;
-	__be16 tfclen;
-	u8 enc_mode;
-	u8 sip_masklen;
-	u8 dip_masklen;
-	u8 flags;
-	u8 reserved[2];
-} __packed;
-
-/**
- * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command
- * @mdev: mlx5 device
- * @cmd: command to execute
- * May be called from atomic context. Returns context pointer, or error
- * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic
- * context, to cleanup the context pointer
- */
-void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev,
-				   struct mlx5_accel_ipsec_sa *cmd);
-
-/**
- * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion
- * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec
- * Sleeps (killable) until command execution is complete.
- * Returns the command result, or -EINTR if killed
- */
-int mlx5_accel_ipsec_sa_cmd_wait(void *context);
+		__be32 a4;
+		__be32 a6[4];
+	} saddr;
+
+	union {
+		__be32 a4;
+		__be32 a6[4];
+	} daddr;
+
+	u8 dir : 2;
+	u8 encap : 1;
+	u8 drop : 1;
+	u8 family;
+	struct mlx5_replay_esn replay_esn;
+	u32 authsize;
+	u32 reqid;
+	u16 sport;
+	u16 dport;
+};
+
+enum mlx5_ipsec_cap {
+	MLX5_IPSEC_CAP_CRYPTO		= 1 << 0,
+	MLX5_IPSEC_CAP_ESN		= 1 << 1,
+	MLX5_IPSEC_CAP_PACKET_OFFLOAD	= 1 << 2,
+	MLX5_IPSEC_CAP_ROCE             = 1 << 3,
+	MLX5_IPSEC_CAP_PRIO             = 1 << 4,
+	MLX5_IPSEC_CAP_TUNNEL           = 1 << 5,
+	MLX5_IPSEC_CAP_ESPINUDP         = 1 << 6,
+};
+
+struct mlx5e_ipsec {
+	struct mlx5_core_dev *mdev;
+	struct workqueue_struct *wq;
+	struct mlx5e_ipsec_tx *tx;
+	struct mlx5e_ipsec_rx *rx_ipv4;
+	struct mlx5e_ipsec_rx *rx_ipv6;
+	struct mlx5e_ipsec_aso *aso;
+	u32 pdn;
+	u32 mkey;
+};
+
+struct mlx5e_ipsec_rule {
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_handle *kspi_rule;
+	struct mlx5_flow_handle *reqid_rule;
+	struct mlx5_modify_hdr *modify_hdr;
+	struct mlx5_pkt_reformat *pkt_reformat;
+	struct mlx5_fc *fc;
+};
+
+struct mlx5e_ipsec_esn_state {
+	u32 esn;
+	u32 esn_msb;
+	u8 overlap: 1;
+};     
 
-u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev);
+struct mlx5e_ipsec_sa_entry {
+	struct secasvar *savp;
+	if_t ifp;
+	struct mlx5e_ipsec *ipsec;
+	struct mlx5_accel_esp_xfrm_attrs attrs;
+	struct mlx5e_ipsec_rule ipsec_rule;
+	struct mlx5e_ipsec_dwork *dwork;
+	struct mlx5e_ipsec_work *work;
+	u32 ipsec_obj_id;
+	u32 enc_key_id;
+	u16 kspi; /* Stack allocated unique SA identifier */
+	struct mlx5e_ipsec_esn_state esn_state;
+};
+
+struct upspec {
+        u16 dport;
+        u16 sport;
+        u8 proto;
+};
+
+struct mlx5_accel_pol_xfrm_attrs {
+        union {
+                __be32 a4;
+                __be32 a6[4];
+        } saddr;
 
-unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);
-int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
-				   unsigned int count);
+        union {
+                __be32 a4;
+                __be32 a6[4];
+        } daddr;
 
-int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);
-void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
+	struct upspec upspec;
 
-#else
+        u8 family;
+        u8 action;
+        u8 dir : 2;
+        u32 reqid;
+        u32 prio;
+};
+
+struct mlx5e_ipsec_pol_entry {
+	struct secpolicy *sp;
+	struct mlx5e_ipsec *ipsec;
+	struct mlx5e_ipsec_rule ipsec_rule;
+	struct mlx5_accel_pol_xfrm_attrs attrs;
+};
 
-#define MLX5_IPSEC_DEV(mdev) false
+/* This function doesn't really belong here, but let's put it here for now */
+void mlx5_object_change_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe);
 
-static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)
+int mlx5e_ipsec_init(struct mlx5e_priv *priv);
+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv);
+
+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec);
+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);
+
+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
+
+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev);
+
+static inline struct mlx5_core_dev *
+mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
-	return 0;
+	return sa_entry->ipsec->mdev;
 }
 
-static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev)
+static inline struct mlx5_core_dev *
+mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
 {
+	return pol_entry->ipsec->mdev;
 }
 
-#endif
+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
+					struct mlx5_accel_esp_xfrm_attrs *attrs,
+					u8 dir);
 
+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);
+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);
+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry);
+struct ipsec_accel_out_tag;
+void mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf *mb, struct mlx5e_tx_wqe *wqe,
+    struct ipsec_accel_out_tag *tag);
+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
+static inline int mlx5e_accel_ipsec_get_metadata(unsigned int id)
+{
+	return MLX5_ETH_WQE_FT_META_IPSEC << 23 |  id;
+}
+static inline void
+mlx5e_accel_ipsec_handle_tx(struct mbuf *mb, struct mlx5e_tx_wqe *wqe)
+{
+	struct ipsec_accel_out_tag *tag;
+
+	tag = (struct ipsec_accel_out_tag *)m_tag_find(mb,
+	    PACKET_TAG_IPSEC_ACCEL_OUT, NULL);
+	if (tag != NULL)
+		mlx5e_accel_ipsec_handle_tx_wqe(mb, wqe, tag);
+}
+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv);
+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv);
+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv);
+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv);
+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb);
+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe);
+static inline int mlx5e_accel_ipsec_flow(struct mlx5_cqe64 *cqe)
+{
+	return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
+}
+
+static inline void mlx5e_accel_ipsec_handle_rx(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+{
+	u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
+
+	if (!MLX5_IPSEC_METADATA_MARKER(ipsec_meta_data)) {
+		struct m_tag *mtag;
+
+		mtag = m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
+		if (mtag != NULL)
+			m_tag_delete(mb, mtag);
+
+		return;
+	}
+
+	mlx5e_accel_ipsec_handle_rx_cqe(mb, cqe);
+}
 #endif	/* __MLX5_ACCEL_IPSEC_H__ */
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
new file mode 100644
index 000000000000..555847717779
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
@@ -0,0 +1,747 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "opt_ipsec.h"
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/key_var.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec.h>
+#include <netipsec/xform.h>
+#include <netipsec/ipsec_offload.h>
+#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+#define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000)
+
+static int mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv);
+
+static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x)
+{
+	return (struct mlx5e_ipsec_sa_entry *)x;
+}
+
+static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x)
+{
+	return (struct mlx5e_ipsec_pol_entry *)x;
+}
+
+static void
+mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry *sa_entry,
+    u64 *packets, u64 *bytes)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+	mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes);
+}
+
+static struct mlx5e_ipsec_sa_entry *
+mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir *pb,
+    struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	return (pb->priv_in == sa_entry ? pb->priv_out : pb->priv_in);
+}
+
+static void
+mlx5e_ipsec_handle_counters(struct work_struct *_work)
+{
+	struct mlx5e_ipsec_dwork *dwork =
+	    container_of(_work, struct mlx5e_ipsec_dwork, dwork.work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry;
+	struct mlx5e_ipsec_sa_entry *other_sa_entry;
+	u64 bytes, bytes1, packets1, packets;
+
+	if (sa_entry->attrs.drop)
+		return;
+	other_sa_entry = mlx5e_ipsec_other_sa_entry(dwork->pb, sa_entry);
+	if (other_sa_entry == NULL || other_sa_entry->attrs.drop)
+		return;
+
+	mlx5e_ipsec_handle_counters_onedir(sa_entry, &packets, &bytes);
+	mlx5e_ipsec_handle_counters_onedir(other_sa_entry, &packets1, &bytes1);
+	packets += packets1;
+	bytes += bytes1;
+	
+#ifdef IPSEC_OFFLOAD
+	ipsec_accel_drv_sa_lifetime_update(sa_entry->savp, sa_entry->ifp,
+	    sa_entry->kspi, bytes, packets);
+#endif
+
+	queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork,
+	    MLX5_IPSEC_RESCHED);
+}
+
+static int
+mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry,
+    struct mlx5e_ipsec_priv_bothdir *pb)
+{
+        struct mlx5e_ipsec_dwork *dwork;
+
+        dwork = kzalloc(sizeof(*dwork), GFP_KERNEL);
+        if (!dwork)
+		return (ENOMEM);
+
+        dwork->sa_entry = sa_entry;
+	dwork->pb = pb;
+        INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters);
+        sa_entry->dwork = dwork;
+        return 0;
+}
+
+static int mlx5_xform_ah_authsize(const struct auth_hash *esph)
+{
+        int alen;
+
+        if (esph == NULL)
+                return 0;
+
+        switch (esph->type) {
+        case CRYPTO_SHA2_256_HMAC:
+        case CRYPTO_SHA2_384_HMAC:
+        case CRYPTO_SHA2_512_HMAC:
+                alen = esph->hashsize / 2;      /* RFC4868 2.3 */
+                break;
+
+        case CRYPTO_POLY1305:
+        case CRYPTO_AES_NIST_GMAC:
+                alen = esph->hashsize;
+                break;
+
+        default:
+                alen = AH_HMAC_HASHLEN;
+                break;
+        }
+
+        return alen;
+}
+
+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
+					struct mlx5_accel_esp_xfrm_attrs *attrs,
+					u8 dir)
+{
+	struct secasvar *savp = sa_entry->savp;
+	const struct auth_hash *esph = savp->tdb_authalgxform;
+	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
+	struct secasindex *saidx = &savp->sah->saidx;
+	struct seckey *key_encap = savp->key_enc;
+	int key_len;
+
+	memset(attrs, 0, sizeof(*attrs));
+
+	/* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
+	key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
+
+	memcpy(aes_gcm->aes_key, key_encap->key_data, key_len);
+	aes_gcm->key_len = key_len;
+
+	/* salt and seq_iv */
+	aes_gcm->seq_iv = 0;
+	memcpy(&aes_gcm->salt, key_encap->key_data + key_len,
+	       sizeof(aes_gcm->salt));
+
+	switch (savp->alg_enc) {
+	case SADB_X_EALG_AESGCM8:
+		attrs->authsize = 8 / 4; /* in dwords */
+		break;
+	case SADB_X_EALG_AESGCM12:
+		attrs->authsize = 12 / 4; /* in dwords */
+		break;
+	case SADB_X_EALG_AESGCM16:
+		attrs->authsize = 16 / 4; /* in dwords */
+		break;
+	default: break;
+	}
+
+	/* iv len */
+	aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense
+
+	attrs->dir = dir;
+	/* spi - host order */
+	attrs->spi = ntohl(savp->spi);
+	attrs->family = saidx->dst.sa.sa_family;
+	attrs->reqid = saidx->reqid;
+
+	if (saidx->src.sa.sa_family == AF_INET) {
+		attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr;
+		attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr;
+	} else {
+		memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16);
+		memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16);
+	}
+
+	if (savp->natt) {
+		attrs->encap = true;
+		attrs->sport = savp->natt->sport;
+		attrs->dport = savp->natt->dport;
+	}
+
+	if (savp->flags & SADB_X_SAFLAGS_ESN) {
+		/* We support replay window with ESN only */
+		attrs->replay_esn.trigger = true;
+		if (sa_entry->esn_state.esn_msb)
+			attrs->replay_esn.esn = sa_entry->esn_state.esn;
+		else
+			/* According to RFC4303, section "3.3.3. Sequence Number Generation",
+			 * the first packet sent using a given SA will contain a sequence
+			 * number of 1.
+			 */
+			attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1);
+		attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;
+		attrs->replay_esn.overlap = sa_entry->esn_state.overlap;
+
+	        if (savp->replay) {
+			switch (savp->replay->wsize) {
+			case 4:
+	                     attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT;
+			     break;
+			case 8:
+	                     attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT;
+			     break;
+			case 16:
+	                     attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT;
+			     break;
+			case 32:
+	                     attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT;
+			     break;
+			default:
+			     /* Do nothing */
+			     break;
+	                }
+		}
+        }
+}
+
+static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
+				     struct secasvar *savp)
+{
+	struct secasindex *saidx = &savp->sah->saidx;
+	struct seckey *key_encp = savp->key_enc;
+	int keylen;
+
+	if (!(mlx5_ipsec_device_caps(mdev) &
+				MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+		mlx5_core_err(mdev, "FULL offload is not supported\n");
+		return (EINVAL);
+	}
+	if (savp->alg_enc == SADB_EALG_NONE) {
+		mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n");
+		return (EINVAL);
+	}
+	if (savp->alg_enc != SADB_X_EALG_AESGCM16) {
+		mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n");
+		return (EINVAL);
+	}
+	if (savp->tdb_compalgxform) {
+		mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n");
+		return (EINVAL);
+	}
+	if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) {
+		mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n");
+		return (EINVAL);
+	}
+	if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) ||
+	    (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) {
+		mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n");
+		return (EINVAL);
+	}
+	if (saidx->proto != IPPROTO_ESP) {
+		mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n");
+		return (EINVAL);
+	}
+	/* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
+	keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
+	if (keylen != 128/8 && keylen != 256 / 8) {
+		mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
+		return (EINVAL);
+	}
+
+	if (saidx->mode != IPSEC_MODE_TRANSPORT) {
+		mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n");
+		return (EINVAL);
+	}
+
+	if (savp->natt) {
+		if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) {
+			mlx5_core_err(mdev, "Encapsulation is not supported\n");
+			return (EINVAL);
+		}
+        }
+
+        if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 &&
+	    savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) {
+		mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize);
+		return (EINVAL);
+	}
+
+	if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) {
+		if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) {
+			mlx5_core_err(mdev, "ESN is not supported\n");
+			return (EINVAL);
+		}
+	} else if (savp->replay != NULL && savp->replay->wsize != 0) {
+		mlx5_core_warn(mdev,
+		    "non-ESN but replay-protect SA offload is not supported\n");
+		return (EINVAL);
+	}
+        return 0;
+}
+
+static int
+mlx5e_if_sa_newkey_onedir(struct ifnet *ifp, void *sav, int dir,
+    u_int drv_spi, struct mlx5e_ipsec_sa_entry **privp,
+    struct mlx5e_ipsec_priv_bothdir *pb)
+{
+	struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
+	struct mlx5e_priv *priv = if_getsoftc(ifp);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_ipsec *ipsec = priv->ipsec;
+	int err;
+
+	if (priv->gone != 0 || ipsec == NULL)
+		return (EOPNOTSUPP);
+
+	err = mlx5e_xfrm_validate_state(mdev, sav);
+	if (err)
+		return err;
+
+	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
+	if (sa_entry == NULL)
+		return (ENOMEM);
+
+	sa_entry->kspi = drv_spi;
+	sa_entry->savp = sav;
+	sa_entry->ifp = ifp;
+	sa_entry->ipsec = ipsec;
+
+	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir);
+
+	err = mlx5e_ipsec_create_dwork(sa_entry, pb);
+	if (err)
+		goto err_xfrm;
+
+	/* create hw context */
+	err = mlx5_ipsec_create_sa_ctx(sa_entry);
+	if (err)
+		goto err_sa_ctx;
+
+	err = mlx5e_accel_ipsec_fs_add_rule(sa_entry);
+	if (err)
+		goto err_fs;
+
+	*privp = sa_entry;
+	if (sa_entry->dwork)
+		queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED);
+
+	err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL);
+	if (err)
+		goto err_xa;
+
+	return 0;
+
+err_xa:
+	if (sa_entry->dwork)
+		cancel_delayed_work_sync(&sa_entry->dwork->dwork);
+	mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+err_fs:
+	mlx5_ipsec_free_sa_ctx(sa_entry);
+err_sa_ctx:
+	kfree(sa_entry->dwork);
+err_xfrm:
+	kfree(sa_entry);
+	mlx5_en_err(ifp, "Device failed to offload this state");
+	return err;
+}
+
+static int
+mlx5e_if_sa_newkey(struct ifnet *ifp, void *sav, u_int dev_spi, void **privp)
+{
+	struct mlx5e_ipsec_priv_bothdir *pb;
+	int error;
+
+	pb = malloc(sizeof(struct mlx5e_ipsec_priv_bothdir), M_DEVBUF,
+	    M_WAITOK | M_ZERO);
+	error = mlx5e_if_sa_newkey_onedir(ifp, sav, IPSEC_DIR_INBOUND,
+	    dev_spi, &pb->priv_in, pb);
+	if (error != 0) {
+		free(pb, M_DEVBUF);
+		return (error);
+	}
+	error = mlx5e_if_sa_newkey_onedir(ifp, sav, IPSEC_DIR_OUTBOUND,
+	    dev_spi, &pb->priv_out, pb);
+	if (error == 0) {
+		*privp = pb;
+	} else {
+		mlx5e_if_sa_deinstall(ifp, dev_spi, pb->priv_in);
+		free(pb, M_DEVBUF);
+	}
+	return (error);
+}
+
+static void
+mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, void *priv)
+{
+	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct mlx5e_ipsec_sa_entry *old;
+
+	old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id);
+	WARN_ON(old != sa_entry);
+
+	mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+	mlx5_ipsec_free_sa_ctx(sa_entry);
+	kfree(sa_entry->dwork);
+	kfree(sa_entry);
+}
+
+static int
+mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv)
+{
+	struct mlx5e_ipsec_priv_bothdir pb, *pbp;
+
+	pbp = priv;
+	pb = *(struct mlx5e_ipsec_priv_bothdir *)priv;
+	pbp->priv_in = pbp->priv_out = NULL;
+
+	if (pb.priv_in->dwork != NULL)
+		cancel_delayed_work_sync(&pb.priv_in->dwork->dwork);
+	if (pb.priv_out->dwork != NULL)
+		cancel_delayed_work_sync(&pb.priv_out->dwork->dwork);
+
+	mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_in);
+	mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_out);
+	free(pbp, M_DEVBUF);
+	return (0);
+}
+
+static void
+mlx5e_if_sa_cnt_one(struct ifnet *ifp, void *sa, uint32_t drv_spi,
+    void *priv, u64 *bytes, u64 *packets)
+{
+	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+	mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes);
+}
+
+static int
+mlx5e_if_sa_cnt(struct ifnet *ifp, void *sa, uint32_t drv_spi,
+    void *priv, struct seclifetime *lt)
+{
+	struct mlx5e_ipsec_priv_bothdir *pb;
+	u64 packets_in, packets_out;
+	u64 bytes_in, bytes_out;
+
+	pb = priv;
+	mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_in,
+	    &bytes_in, &packets_in);
+	mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_out,
+	    &bytes_out, &packets_out);
+	/* TODO: remove this casting once Kostia changes allocation type to be u64 */
+	lt->bytes = bytes_in + bytes_out;
+	lt->allocations = (uint32_t)(packets_in + packets_out);
+	return (0);
+}
+
+static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
+                                      struct secpolicy *sp, struct inpcb *inp)
+{
+	struct secpolicyindex *spidx = &sp->spidx;
+
+	if (!(mlx5_ipsec_device_caps(mdev) &
+				MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+		mlx5_core_err(mdev, "FULL offload is not supported\n");
+		return (EINVAL);
+	}
+
+        if (sp->tcount > 1) {
+		mlx5_core_err(mdev, "Can offload exactly one template, "
+		    "not %d\n", sp->tcount);
+                return (EINVAL);
+        }
+
+        if (sp->policy == IPSEC_POLICY_BYPASS &&
+            !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
+		mlx5_core_err(mdev, "Device does not support policy priority\n");
+		return (EINVAL);
+	}
+
+	if (sp->tcount > 0 && inp != NULL) {
+		mlx5_core_err(mdev, "Not valid input data\n");
+		return (EINVAL);
+	}
+
+	if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) {
+		mlx5_core_err(mdev, "Wrong policy direction\n");
+		return (EINVAL);
+	}
+
+	if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) {
+		mlx5_core_err(mdev, "Device supports transport mode only");
+		return (EINVAL);
+	}
+
+        if (sp->policy != IPSEC_POLICY_DISCARD &&
+            sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) {
+                mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n");
+		return (EINVAL);
+        }
+
+	if (sp->policy == IPSEC_POLICY_BYPASS && !inp) {
+		mlx5_core_err(mdev, "Missing port information for IKE bypass\n");
+		return (EINVAL);
+	}
+
+	if (inp != NULL) {
+		INP_RLOCK(inp);
+		if (inp->inp_socket == NULL || inp->inp_socket->so_proto->
+		    pr_protocol != IPPROTO_UDP) {
+			mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n",
+			    inp->inp_socket == NULL ? -1 :
+			    inp->inp_socket->so_proto->pr_protocol);
+			INP_RUNLOCK(inp);
+			return (EINVAL);
+		}
+		INP_RUNLOCK(inp);
+	}
+
+        /* TODO fill relevant bits */
+	return 0;
+}
+
+static void mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
+					      struct mlx5_accel_pol_xfrm_attrs *attrs,
+					      struct inpcb *inp)
+{
+	struct secpolicy *sp = pol_entry->sp;
+	struct secpolicyindex *spidx = &sp->spidx;
+
+	memset(attrs, 0, sizeof(*attrs));
+
+	if (!inp) {
+		if (spidx->src.sa.sa_family == AF_INET) {
+			attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr;
+			attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr;
+		} else if (spidx->src.sa.sa_family == AF_INET6) {
+			memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16);
+			memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16);
+		} else {
+			KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family));
+		}
+		attrs->family = spidx->src.sa.sa_family;
+		attrs->prio = 0;
+		attrs->action = sp->policy;
+		attrs->reqid = sp->req[0]->saidx.reqid;
+	} else {
+		INP_RLOCK(inp);
+		if ((inp->inp_vflag & INP_IPV4) != 0) {
+			attrs->saddr.a4 = inp->inp_laddr.s_addr;
+			attrs->daddr.a4 = inp->inp_faddr.s_addr;
+			attrs->family = AF_INET;
+		} else if ((inp->inp_vflag & INP_IPV6) != 0) {
+			memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16);
+			memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16);
+			attrs->family = AF_INET6;
+		} else {
+			KASSERT(0, ("unsupported family %d", inp->inp_vflag));
+		}
+		attrs->upspec.dport = inp->inp_fport;
+		attrs->upspec.sport = inp->inp_lport;
+		attrs->upspec.proto = inp->inp_ip_p;
+		INP_RUNLOCK(inp);
+
+		/* Give highest priority for PCB policies */
+		attrs->prio = 1;
+		attrs->action = IPSEC_POLICY_IPSEC;
+	}
+	attrs->dir = spidx->dir;
+}
+
+static int mlx5e_if_spd_install(struct ifnet *ifp, void *sp, void *inp1,
+    void **ifdatap)
+{
+	struct mlx5e_ipsec_pol_entry *pol_entry;
+	struct mlx5e_priv *priv;
+	int err;
+
+	priv = if_getsoftc(ifp);
+	if (priv->gone || !priv->ipsec)
+		return (EOPNOTSUPP);
+
+	err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1);
+	if (err)
+		return err;
+
+	pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL);
+	if (!pol_entry)
+		return (ENOMEM);
+
+	pol_entry->sp = sp;
+	pol_entry->ipsec = priv->ipsec;
+
+	mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, inp1);
+	err = mlx5e_accel_ipsec_fs_add_pol(pol_entry);
+	if (err)
+		goto err_pol;
+	*ifdatap = pol_entry;
+
+	return 0;
+
+err_pol:
+	kfree(pol_entry);
+	mlx5_en_err(ifp, "Device failed to offload this policy");
+	return err;
+}
+
+
+static int mlx5e_if_spd_deinstall(struct ifnet *ifp, void *sp, void *ifdata)
+{
+	struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(ifdata);
+
+	mlx5e_accel_ipsec_fs_del_pol(pol_entry);
+	kfree(pol_entry);
+	return 0;
+}
+
+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
+{
+	struct mlx5e_ipsec *pipsec = priv->ipsec;
+	if (!pipsec)
+		return;
+
+	mlx5e_accel_ipsec_fs_cleanup(pipsec);
+	destroy_workqueue(pipsec->wq);
+	mlx5e_ipsec_aso_cleanup(pipsec);
+	kfree(pipsec);
+	priv->ipsec = NULL;
+}
+
+static int
+mlx5e_if_ipsec_hwassist(if_t ifnet, void *sav __unused,
+    uint32_t drv_spi __unused, void *priv __unused)
+{
+	return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP |
+	    CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP));
+}
+
+static const struct if_ipsec_accel_methods  mlx5e_ipsec_funcs = {
+	.if_sa_newkey = mlx5e_if_sa_newkey,
+	.if_sa_deinstall = mlx5e_if_sa_deinstall,
+	.if_spdadd = mlx5e_if_spd_install,
+	.if_spddel = mlx5e_if_spd_deinstall,
+	.if_sa_cnt = mlx5e_if_sa_cnt,
+	.if_hwassist = mlx5e_if_ipsec_hwassist,
+};
+
+int mlx5e_ipsec_init(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_ipsec *pipsec;
+	if_t ifp = priv->ifp;
+	int ret;
+
+	mlx5_core_info(mdev, "ipsec "
+	    "offload %d log_max_dek %d gen_obj_types %d "
+	    "ipsec_encrypt %d ipsec_decrypt %d "
+	    "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d "
+	    "ipsec_full_offload %d "
+	    "reformat_add_esp_trasport %d reformat_del_esp_trasport %d "
+	    "decap %d "
+	    "ignore_flow_level_tx %d ignore_flow_level_rx %d "
+	    "reformat_natt_tx %d reformat_natt_rx %d "
+	    "ipsec_esn %d\n",
+	    MLX5_CAP_GEN(mdev, ipsec_offload) != 0,
+	    MLX5_CAP_GEN(mdev, log_max_dek) != 0,
+	    (MLX5_CAP_GEN_64(mdev, general_obj_types) &
+		MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0,
+	    MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0,
+	    MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0,
+	    MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0,
+            MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0,
+            MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0,
+            MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+	        reformat_add_esp_transport_over_udp) != 0,
+	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+		reformat_del_esp_transport_over_udp) != 0,
+	    MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0);
+
+	if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+		mlx5_core_dbg(mdev, "Not an IPSec offload device\n");
+		return 0;
+	}
+
+	xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC);
+
+	pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL);
+	if (pipsec == NULL)
+		return (ENOMEM);
+
+	pipsec->mdev = mdev;
+	pipsec->pdn = priv->pdn;
+	pipsec->mkey = priv->mr.key;
+
+	ret = mlx5e_ipsec_aso_init(pipsec);
+	if (ret)
+		goto err_ipsec_aso;
+
+	pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0);
+	if (pipsec->wq == NULL) {
+		ret = ENOMEM;
+		goto err_ipsec_wq;
+	}
+
+	ret = mlx5e_accel_ipsec_fs_init(pipsec);
+	if (ret)
+		goto err_ipsec_alloc;
+
+	if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs);
+	priv->ipsec = pipsec;
+	mlx5_core_dbg(mdev, "IPSec attached to netdevice\n");
+	return 0;
+
+err_ipsec_alloc:
+	destroy_workqueue(pipsec->wq);
+err_ipsec_wq:
+	mlx5e_ipsec_aso_cleanup(pipsec);
+err_ipsec_aso:
+	kfree(pipsec);
+	mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret);
+	return ret;
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
new file mode 100644
index 000000000000..a6a0398f9dca
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
@@ -0,0 +1,2047 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "opt_ipsec.h"
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/key_var.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec.h>
+#include <netipsec/xform.h>
+#include <netipsec/ipsec_offload.h>
+#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <dev/mlx5/mlx5_core/fs_chains.h>
+
+/*
+ * TX tables are organized differently for Ethernet and for RoCE:
+ *
+ *                       +=========+
+ *       Ethernet Tx     | SA KSPI | match
+ * --------------------->|Flowtable|----->+         +
+ *                       |         |\     |        / \
+ *                       +=========+ |    |       /   \         +=========+     +=========+
+ *                              miss |    |      /     \        |  Status |     |         |
+ *                      DROP<--------+    |---->|Encrypt|------>|Flowtable|---->|  TX NS  |
+ *                                        |      \     /        |         |     |         |
+ *                                        |       \   /         +=========+     +=========+
+ *       +=========+      +=========+     |        \ /               |
+ *  RoCE |  Policy | match|SA ReqId |match|         +                |
+ *  Tx   |Flowtable|----->|Flowtable|---->+                          |
+ *  ---->|IP header|      |ReqId+IP |                                |
+ *       |         |      | header  |--------------------------------+
+ *       +=========+      +=========+         miss                   |
+ *            |                                                      |
+ *            |                   miss                               |
+ *            +-------------------------------------------------------
+ *
+ *                                                                                  +=========+
+ *                                                                                  |   RDMA  |
+ *                                                                                  |Flowtable|
+ *                                                                                  |         |
+ * Rx Tables and rules:                                                             +=========+
+ *                                             +                                        /
+ *       +=========+      +=========+         / \         +=========+      +=========+ /match
+ *       |  Policy |      |   SA    |        /   \        |  Status |      |  RoCE   |/
+ *  ---->|Flowtable| match|Flowtable| match /     \       |Flowtable|----->|Flowtable|
+ *       |IP header|----->|IP header|----->|Decrypt|----->|         |      | Roce V2 |
+ *       |         |      |+ESP+SPI |       \     /       |         |      | UDP port|\
+ *       +=========+      +=========+        \   /        +=========+      +=========+ \miss
+ *             |               |              \ /                                       \
+ *             |               |               +                                      +=========+
+ *             |     miss      |          miss                                       | Ethernet|
+ *             +--------------->---------------------------------------------------->|  RX NS  |
+ *                                                                                   |         |
+ *                                                                                   +=========+
+ *
+ */
+
+#define NUM_IPSEC_FTE BIT(15)
+#define IPSEC_TUNNEL_DEFAULT_TTL 0x40
+
+struct mlx5e_ipsec_fc {
+	struct mlx5_fc *cnt;
+	struct mlx5_fc *drop;
+};
+
+struct mlx5e_ipsec_ft {
+	struct mutex mutex; /* Protect changes to this struct */
+	struct mlx5_flow_table *pol;
+	struct mlx5_flow_table *sa_kspi;
+	struct mlx5_flow_table *sa;
+	struct mlx5_flow_table *status;
+	u32 refcnt;
+};
+
+struct mlx5e_ipsec_tx_roce {
+	struct mlx5_flow_group *g;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_namespace *ns;
+};
+
+struct mlx5e_ipsec_miss {
+	struct mlx5_flow_group *group;
+	struct mlx5_flow_handle *rule;
+};
+
+struct mlx5e_ipsec_tx {
+	struct mlx5e_ipsec_ft ft;
+	struct mlx5e_ipsec_miss pol;
+	struct mlx5e_ipsec_miss kspi_miss;
+	struct mlx5e_ipsec_rule status;
+	struct mlx5e_ipsec_rule kspi_bypass_rule; /*rule for IPSEC bypass*/
+	struct mlx5_flow_namespace *ns;
+	struct mlx5e_ipsec_fc *fc;
+	struct mlx5_fs_chains *chains;
+	struct mlx5e_ipsec_tx_roce roce;
+};
+
+struct mlx5e_ipsec_rx_roce {
+	struct mlx5_flow_group *g;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_handle *rule;
+	struct mlx5e_ipsec_miss roce_miss;
+
+	struct mlx5_flow_table *ft_rdma;
+	struct mlx5_flow_namespace *ns_rdma;
+};
+
+struct mlx5e_ipsec_rx {
+	struct mlx5e_ipsec_ft ft;
+	struct mlx5e_ipsec_miss pol;
+	struct mlx5e_ipsec_miss sa;
+	struct mlx5e_ipsec_rule status;
+	struct mlx5_flow_namespace *ns;
+	struct mlx5e_ipsec_fc *fc;
+	struct mlx5_fs_chains *chains;
+	struct mlx5e_ipsec_rx_roce roce;
+};
+
+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
+                                     u16 kspi);
+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec);
+
+static void setup_fte_no_frags(struct mlx5_flow_spec *spec)
+{
+	/* Non fragmented */
+	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0);
+}
+
+static void setup_fte_esp(struct mlx5_flow_spec *spec)
+{
+	/* ESP header */
+	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP);
+}
+
+static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap)
+{
+	/* SPI number */
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
+
+	if (encap) {
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.inner_esp_spi);
+		MLX5_SET(fte_match_param, spec->match_value, misc_parameters.inner_esp_spi, spi);
+	} else {
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi);
+		MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi);
+	}
+}
+
+static struct mlx5_fs_chains *
+ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft,
+		    enum mlx5_flow_namespace_type ns, int base_prio,
+		    int base_level, struct mlx5_flow_table **root_ft)
+{
+	struct mlx5_chains_attr attr = {};
+	struct mlx5_fs_chains *chains;
+	struct mlx5_flow_table *ft;
+	int err;
+
+	attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
+		     MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
+	attr.max_grp_num = 2;
+	attr.default_ft = miss_ft;
+	attr.ns = ns;
+	attr.fs_base_prio = base_prio;
+	attr.fs_base_level = base_level;
+	chains = mlx5_chains_create(mdev, &attr);
+	if (IS_ERR(chains))
+		return chains;
+
+	/* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */
+	ft = mlx5_chains_get_table(chains, 0, 1, 0);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_chains_get;
+	}
+
+	*root_ft = ft;
+	return chains;
+
+err_chains_get:
+	mlx5_chains_destroy(chains);
+	return ERR_PTR(err);
+}
+
+static void ipsec_chains_destroy(struct mlx5_fs_chains *chains)
+{
+	mlx5_chains_put_table(chains, 0, 1, 0);
+	mlx5_chains_destroy(chains);
+}
+
+static struct mlx5_flow_table *
+ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+	return mlx5_chains_get_table(chains, 0, prio + 1, 0);
+}
+
+static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+	mlx5_chains_put_table(chains, 0, prio + 1, 0);
+}
+
+static struct mlx5_flow_table *ipsec_rx_ft_create(struct mlx5_flow_namespace *ns,
+						  int level, int prio,
+						  int max_num_groups)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+
+	ft_attr.max_fte = NUM_IPSEC_FTE;
+	ft_attr.level = level;
+	ft_attr.prio = prio;
+	ft_attr.autogroup.max_num_groups = max_num_groups;
+	ft_attr.autogroup.num_reserved_entries = 1;
+
+	return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+}
+
+static int ipsec_miss_create(struct mlx5_core_dev *mdev,
+			     struct mlx5_flow_table *ft,
+			     struct mlx5e_ipsec_miss *miss,
+			     struct mlx5_flow_destination *dest)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_spec *spec;
+	u32 *flow_group_in;
+	int err = 0;
+
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!flow_group_in || !spec) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	/* Create miss_group */
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1);
+	miss->group = mlx5_create_flow_group(ft, flow_group_in);
+	if (IS_ERR(miss->group)) {
+		err = PTR_ERR(miss->group);
+		mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n",
+			      err);
+		goto out;
+	}
+
+	if (dest)
+		flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
+	else
+		flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
+	/* Create miss rule */
+	miss->rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1);
+	if (IS_ERR(miss->rule)) {
+		mlx5_destroy_flow_group(miss->group);
+		err = PTR_ERR(miss->rule);
+		mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n",
+			      err);
+		goto out;
+	}
+out:
+	kvfree(flow_group_in);
+	kvfree(spec);
+	return err;
+}
+
+static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir,
+                               struct mlx5_flow_act *flow_act)
+{
+        u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+        enum mlx5_flow_namespace_type ns_type;
+        struct mlx5_modify_hdr *modify_hdr;
+
+        MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
+        switch (dir) {
+        case IPSEC_DIR_INBOUND:
+                MLX5_SET(set_action_in, action, field,
+                         MLX5_ACTION_IN_FIELD_METADATA_REG_B);
+                ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
+                break;
+        case IPSEC_DIR_OUTBOUND:
+                MLX5_SET(set_action_in, action, field,
+                         MLX5_ACTION_IN_FIELD_METADATA_REG_C_0);
+                ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        MLX5_SET(set_action_in, action, data, val);
+        MLX5_SET(set_action_in, action, offset, 0);
+        MLX5_SET(set_action_in, action, length, 32);
+
+        modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action);
+        if (IS_ERR(modify_hdr)) {
+                mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n",
+                              PTR_ERR(modify_hdr));
+                return PTR_ERR(modify_hdr);
+        }
+
+        flow_act->modify_hdr = modify_hdr;
+        flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+        return 0;
+}
+
+static int
+setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs,
+			     struct mlx5_pkt_reformat_params *reformat_params)
+{
+	struct udphdr *udphdr;
+	size_t bfflen = 16;
+	char *reformatbf;
+	__be32 spi;
+	void *hdr;
+
+	if (attrs->family == AF_INET)
+		if (attrs->encap)
+			reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4;
+		else
+			reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4;
+	else
+		reformat_params->type =
+			MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6;
+
+	if (attrs->encap)
+		bfflen += sizeof(*udphdr);
+	reformatbf = kzalloc(bfflen, GFP_KERNEL);
+	if (!reformatbf)
+		return -ENOMEM;
+
+	hdr = reformatbf;
+	if (attrs->encap) {
+		udphdr = (struct udphdr *)reformatbf;
+		udphdr->uh_sport = attrs->sport;
+		udphdr->uh_dport = attrs->dport;
+		hdr += sizeof(*udphdr);
+	}
+
+	/* convert to network format */
+	spi = htonl(attrs->spi);
+	memcpy(hdr, &spi, 4);
+
+	reformat_params->param_0 = attrs->authsize;
+	reformat_params->size = bfflen;
+	reformat_params->data = reformatbf;
+
+	return 0;
+}
+
+static int setup_pkt_reformat(struct mlx5_core_dev *mdev,
+			      struct mlx5_accel_esp_xfrm_attrs *attrs,
+			      struct mlx5_flow_act *flow_act)
+{
+	enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
+	struct mlx5_pkt_reformat_params reformat_params = {};
+	struct mlx5_pkt_reformat *pkt_reformat;
+	int ret;
+
+	if (attrs->dir == IPSEC_DIR_INBOUND) {
+		if (attrs->encap)
+			reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP;
+		else
+			reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT;
+		ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
+		goto cmd;
+	}
+
+	ret = setup_pkt_transport_reformat(attrs, &reformat_params);
+	if (ret)
+		return ret;
+cmd:
+	pkt_reformat =
+		mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type);
+	if (reformat_params.data)
+		kfree(reformat_params.data);
+	if (IS_ERR(pkt_reformat))
+		return PTR_ERR(pkt_reformat);
+
+	flow_act->pkt_reformat = pkt_reformat;
+	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+	return 0;
+}
+
+static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr,
+                            __be32 *daddr)
+{
+        spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+        MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
+        MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
+
+        memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+                            outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
+        memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+                            outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
+        MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                         outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
+        MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                         outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+}
+
+static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr,
+                            __be32 *daddr)
+{
+        spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+        MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
+        MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
+
+        memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+                            outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
+        memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+                            outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
+        memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+                            outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
+        memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+                            outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
+}
+
+static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+	struct mlx5_flow_destination dest[2] = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	struct mlx5e_ipsec_rx *rx;
+	struct mlx5_fc *counter;
+	int err;
+
+	rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	if (attrs->family == AF_INET)
+		setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+	else
+		setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+	if (!attrs->encap)
+		setup_fte_esp(spec);
+
+	setup_fte_spi(spec, attrs->spi, attrs->encap);
+	setup_fte_no_frags(spec);
+
+	if (!attrs->drop) {
+		err = setup_modify_header(mdev, sa_entry->kspi | BIT(31), IPSEC_DIR_INBOUND,
+					  &flow_act);
+		if (err)
+			goto err_mod_header;
+	}
+
+	err = setup_pkt_reformat(mdev, attrs, &flow_act);
+	if (err)
+		goto err_pkt_reformat;
+
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_add_cnt;
+	}
+
+	flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
+	flow_act.crypto.op = MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT;
+	flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
+	flow_act.flags |= FLOW_ACT_NO_APPEND;
+
+	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
+		MLX5_FLOW_CONTEXT_ACTION_COUNT;
+
+	if (attrs->drop)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+	else
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[0].ft = rx->ft.status;
+	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest[1].counter_id = mlx5_fc_id(counter);
+
+	rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);
+		goto err_add_flow;
+	}
+
+	kvfree(spec);
+	ipsec_rule->rule = rule;
+	ipsec_rule->fc = counter;
+	ipsec_rule->modify_hdr = flow_act.modify_hdr;
+	ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
+	return 0;
+
+err_add_flow:
+	mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
+	mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
+err_pkt_reformat:
+	if (flow_act.modify_hdr)
+		mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
+err_mod_header:
+	kvfree(spec);
+
+	return err;
+}
+
+static struct mlx5_flow_table *ipsec_tx_ft_create(struct mlx5_flow_namespace *ns,
+						  int level, int prio,
+						  int max_num_groups)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+
+        ft_attr.autogroup.num_reserved_entries = 1;
+        ft_attr.autogroup.max_num_groups = max_num_groups;
+        ft_attr.max_fte = NUM_IPSEC_FTE;
+        ft_attr.level = level;
+        ft_attr.prio = prio;
+
+	return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+}
+
+static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *fte;
+	int err;
+
+	/* create fte */
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_COUNT |
+		MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest.counter_id = mlx5_fc_id(tx->fc->cnt);
+	fte = mlx5_add_flow_rules(tx->ft.status, NULL, &flow_act, &dest, 1);
+	if (IS_ERR_OR_NULL(fte)) {
+		err = PTR_ERR(fte);
+		mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err);
+		goto err_rule;
+	}
+
+	tx->status.rule = fte;
+	return 0;
+
+err_rule:
+	return err;
+}
+
+static void tx_destroy_roce(struct mlx5e_ipsec_tx *tx) {
+	if (!tx->roce.ft)
+		return;
+
+	mlx5_del_flow_rules(&tx->roce.rule);
+	mlx5_destroy_flow_group(tx->roce.g);
+	mlx5_destroy_flow_table(tx->roce.ft);
+	tx->roce.ft = NULL;
+}
+
+/* IPsec TX flow steering */
+static void tx_destroy(struct mlx5e_ipsec_tx *tx)
+{
+	tx_destroy_roce(tx);
+	if (tx->chains) {
+		ipsec_chains_destroy(tx->chains);
+	} else {
+		mlx5_del_flow_rules(&tx->pol.rule);
+		mlx5_destroy_flow_group(tx->pol.group);
+		mlx5_destroy_flow_table(tx->ft.pol);
+	}
+	mlx5_destroy_flow_table(tx->ft.sa);
+	mlx5_del_flow_rules(&tx->kspi_miss.rule);
+	mlx5_destroy_flow_group(tx->kspi_miss.group);
+	mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
+	mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+	mlx5_destroy_flow_table(tx->ft.sa_kspi);
+	mlx5_del_flow_rules(&tx->status.rule);
+	mlx5_destroy_flow_table(tx->ft.status);
+}
+
+static int ipsec_tx_roce_rule_setup(struct mlx5_core_dev *mdev,
+				    struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_destination dst = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *rule;
+	int err = 0;
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+	dst.ft = tx->ft.pol;
+	rule = mlx5_add_flow_rules(tx->roce.ft, NULL, &flow_act, &dst, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "Fail to add TX roce ipsec rule err=%d\n",
+			      err);
+		goto out;
+	}
+	tx->roce.rule = rule;
+
+out:
+	return err;
+}
+
+static int ipsec_tx_create_roce(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_group *g;
+	int ix = 0;
+	int err;
+	u32 *in;
+
+	if (!tx->roce.ns)
+		return -EOPNOTSUPP;
+
+	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	ft_attr.max_fte = 1;
+	ft = mlx5_create_flow_table(tx->roce.ns, &ft_attr);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		mlx5_core_err(mdev, "Fail to create ipsec tx roce ft err=%d\n",
+			      err);
+		goto fail_table;
+	}
+	tx->roce.ft = ft;
+
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += 1;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	g = mlx5_create_flow_group(ft, in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		mlx5_core_err(mdev, "Fail to create ipsec tx roce group err=%d\n",
+			      err);
+		goto fail_group;
+	}
+	tx->roce.g = g;
+
+	err = ipsec_tx_roce_rule_setup(mdev, tx);
+	if (err) {
+		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
+		goto fail_rule;
+	}
+
+	kvfree(in);
+	return 0;
+
+fail_rule:
+	mlx5_destroy_flow_group(tx->roce.g);
+fail_group:
+	mlx5_destroy_flow_table(tx->roce.ft);
+	tx->roce.ft = NULL;
+fail_table:
+	kvfree(in);
+	return err;
+}
+
+/*
+ * Setting a rule in KSPI table for values that should bypass IPSEC.
+ *
+ * mdev - mlx5 core device
+ * tx - IPSEC TX
+ * return - 0 for success errno for failure
+ */
+static int tx_create_kspi_bypass_rules(struct mlx5_core_dev *mdev,
+                                       struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_act flow_act_kspi = {};
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	dest.ft = tx->ft.status;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	flow_act_kspi.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+	setup_fte_reg_a_with_tag(spec, IPSEC_ACCEL_DRV_SPI_BYPASS);
+	rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act_kspi,
+								&dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n",
+                      err);
+		goto err_add_kspi_rule;
+	}
+	tx->kspi_bypass_rule.kspi_rule = rule;
+
+	/* set the rule for packets withoiut ipsec tag. */
+	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	memset(spec, 0, sizeof(*spec));
+	setup_fte_reg_a_no_tag(spec);
+	rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", err);
+		goto err_add_rule;
+	}
+	tx->kspi_bypass_rule.rule = rule;
+
+	kvfree(spec);
+	return 0;
+err_add_rule:
+	mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+err_add_kspi_rule:
+	kvfree(spec);
+	return err;
+}
+
+
+static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_table *ft;
+	int err;
+
+	/*
+	 *  Tx flow is different for ethernet traffic then for RoCE packets
+	 *  For Ethernet packets we start in SA KSPI table that matches KSPI of SA rule
+	 *  to the KSPI in the packet metadata
+	 *  For RoCE traffic we start in Policy table, then move to SA table
+	 *  which matches either reqid of the SA rule to reqid reported by policy table
+	 *  or ip header fields of SA to the packet IP header fields.
+	 *  Tables are ordered by their level so we set kspi
+	 *  with level 0 to have it first one for ethernet traffic.
+	 *  For RoCE the RoCE TX table direct the packets to policy table explicitly
+	 */
+	ft = ipsec_tx_ft_create(tx->ns, 0, 0, 4);
+	if (IS_ERR(ft))
+		return PTR_ERR(ft);
+	tx->ft.sa_kspi = ft;
+
+	ft = ipsec_tx_ft_create(tx->ns, 2, 0, 4);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_reqid_ft;
+	}
+	tx->ft.sa = ft;
+
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+		tx->chains = ipsec_chains_create(
+				mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 1,
+				&tx->ft.pol);
+		if (IS_ERR(tx->chains)) {
+			err = PTR_ERR(tx->chains);
+			goto err_pol_ft;
+		}
+	} else {
+		ft = ipsec_tx_ft_create(tx->ns, 1, 0, 2);
+		if (IS_ERR(ft)) {
+			err = PTR_ERR(ft);
+			goto err_pol_ft;
+		}
+		tx->ft.pol = ft;
+		dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+		dest.ft = tx->ft.sa;
+		err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest);
+		if (err)
+			goto err_pol_miss;
+	}
+
+	ft = ipsec_tx_ft_create(tx->ns, 2, 0, 1);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_status_ft;
+	}
+	tx->ft.status = ft;
+
+	/* set miss rule for kspi table with drop action*/
+	err = ipsec_miss_create(mdev, tx->ft.sa_kspi, &tx->kspi_miss, NULL);
+	if (err)
+		goto err_kspi_miss;
+
+	err = tx_create_kspi_bypass_rules(mdev, tx);
+	if (err)
+		goto err_kspi_rule;
+
+	err = ipsec_counter_rule_tx(mdev, tx);
+	if (err)
+		goto err_status_rule;
+
+	err = ipsec_tx_create_roce(mdev, tx);
+	if (err)
+		goto err_counter_rule;
+
+	return 0;
+
+err_counter_rule:
+	mlx5_del_flow_rules(&tx->status.rule);
+err_status_rule:
+	mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
+	mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+err_kspi_rule:
+	mlx5_destroy_flow_table(tx->ft.status);
+err_status_ft:
+	if (tx->chains) {
+		ipsec_chains_destroy(tx->chains);
+	} else {
+		mlx5_del_flow_rules(&tx->pol.rule);
+		mlx5_destroy_flow_group(tx->pol.group);
+	}
+err_pol_miss:
+	if (!tx->chains)
+		mlx5_destroy_flow_table(tx->ft.pol);
+err_pol_ft:
+	mlx5_del_flow_rules(&tx->kspi_miss.rule);
+	mlx5_destroy_flow_group(tx->kspi_miss.group);
+err_kspi_miss:
+	mlx5_destroy_flow_table(tx->ft.sa);
+err_reqid_ft:
+	mlx5_destroy_flow_table(tx->ft.sa_kspi);
+	return err;
+}
+
+static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
+		  struct mlx5e_ipsec_tx *tx)
+{
+	int err;
+
+	if (tx->ft.refcnt)
+		goto skip;
+
+	err = tx_create(mdev, tx);
+	if (err)
+		return err;
+
+skip:
+	tx->ft.refcnt++;
+	return 0;
+}
+
+static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx)
+{
+	if (--tx->ft.refcnt)
+		return;
+
+	tx_destroy(tx);
+}
+
+static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev,
+					struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+	int err;
+
+	mutex_lock(&tx->ft.mutex);
+	err = tx_get(mdev, ipsec, tx);
+	mutex_unlock(&tx->ft.mutex);
+	if (err)
+		return ERR_PTR(err);
+
+	return tx;
+}
+
+static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev,
+                                                struct mlx5e_ipsec *ipsec,
+                                                u32 prio)
+{
+        struct mlx5e_ipsec_tx *tx = ipsec->tx;
+        struct mlx5_flow_table *ft;
+        int err;
+
+        mutex_lock(&tx->ft.mutex);
+        err = tx_get(mdev, ipsec, tx);
+        if (err)
+            goto err_get;
+
+        ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol;
+        if (IS_ERR(ft)) {
+                err = PTR_ERR(ft);
+                goto err_get_ft;
+        }
+
+        mutex_unlock(&tx->ft.mutex);
+        return ft;
+
+err_get_ft:
+        tx_put(ipsec, tx);
+err_get:
+        mutex_unlock(&tx->ft.mutex);
+        return ERR_PTR(err);
+}
+
+static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio)
+{
+        struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+        mutex_lock(&tx->ft.mutex);
+        if (tx->chains)
+                ipsec_chains_put_table(tx->chains, prio);
+
+        tx_put(ipsec, tx);
+        mutex_unlock(&tx->ft.mutex);
+}
+
+static void tx_ft_put(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+	mutex_lock(&tx->ft.mutex);
+	tx_put(ipsec, tx);
+	mutex_unlock(&tx->ft.mutex);
+}
+
+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
+									 u16 kspi)
+{
+       /* Add IPsec indicator in metadata_reg_a. */
+       spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+       MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                        misc_parameters_2.metadata_reg_a);
+       MLX5_SET(fte_match_param, spec->match_value,
+                misc_parameters_2.metadata_reg_a,
+                MLX5_ETH_WQE_FT_META_IPSEC << 23 |  kspi);
+}
+
+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec)
+{
+       /* Add IPsec indicator in metadata_reg_a. */
+       spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+       MLX5_SET(fte_match_param, spec->match_criteria,
+                misc_parameters_2.metadata_reg_a,
+				MLX5_ETH_WQE_FT_META_IPSEC << 23);
+       MLX5_SET(fte_match_param, spec->match_value,
+                misc_parameters_2.metadata_reg_a,
+                0);
+}
+
+static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid)
+{
+	/* Pass policy check before choosing this SA */
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+			 misc_parameters_2.metadata_reg_c_0);
+	MLX5_SET(fte_match_param, spec->match_value,
+		 misc_parameters_2.metadata_reg_c_0, reqid);
+}
+
+static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec)
+{
+        switch (upspec->proto) {
+        case IPPROTO_UDP:
+                if (upspec->dport) {
+                        MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+                                         spec->match_criteria, udp_dport);
+                        MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+                                 udp_dport, upspec->dport);
+                }
+
+                if (upspec->sport) {
+                        MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+                                         spec->match_criteria, udp_sport);
+                        MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+                                 udp_dport, upspec->sport);
+                }
+                break;
+        case IPPROTO_TCP:
+                if (upspec->dport) {
+                        MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+                                         spec->match_criteria, tcp_dport);
+                        MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+                                 tcp_dport, upspec->dport);
+                }
+
+                if (upspec->sport) {
+                        MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+                                         spec->match_criteria, tcp_sport);
+                        MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+                                 tcp_dport, upspec->sport);
+                }
+                break;
+        default:
+                return;
+        }
+
+        spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol);
+	MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto);
+}
+
+static int tx_add_kspi_rule(struct mlx5e_ipsec_sa_entry *sa_entry,
+							struct mlx5e_ipsec_tx *tx,
+							struct mlx5_flow_act *flow_act,
+							struct mlx5_flow_destination *dest,
+							int num_dest)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	setup_fte_no_frags(spec);
+	setup_fte_reg_a_with_tag(spec, sa_entry->kspi);
+
+	rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, flow_act, dest, num_dest);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "fail to add TX ipsec kspi rule err=%d\n", err);
+		goto err_add_kspi_flow;
+	}
+	ipsec_rule->kspi_rule = rule;
+	kvfree(spec);
+	return 0;
+
+err_add_kspi_flow:
+	kvfree(spec);
+	return err;
+}
+
+static int tx_add_reqid_ip_rules(struct mlx5e_ipsec_sa_entry *sa_entry,
+								struct mlx5e_ipsec_tx *tx,
+								struct mlx5_flow_act *flow_act,
+								struct mlx5_flow_destination *dest,
+								int num_dest)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
+
+	if(attrs->reqid) {
+		setup_fte_no_frags(spec);
+		setup_fte_reg_c0(spec, attrs->reqid);
+		rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
+		if (IS_ERR(rule)) {
+			err = PTR_ERR(rule);
+			mlx5_core_err(mdev, "fail to add TX ipsec reqid rule err=%d\n", err);
+			goto err_add_reqid_rule;
+		}
+		ipsec_rule->reqid_rule = rule;
+		memset(spec, 0, sizeof(*spec));
+	}
+
+	if (attrs->family == AF_INET)
+		setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+	else
+		setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+	setup_fte_no_frags(spec);
+
+	rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "fail to add TX ipsec ip rule err=%d\n", err);
+		goto err_add_ip_rule;
+	}
+	ipsec_rule->rule = rule;
+	kvfree(spec);
+	return 0;
+
+err_add_ip_rule:
+	mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
+err_add_reqid_rule:
+	kvfree(spec);
+	return err;
+}
+
+static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+	struct mlx5_flow_destination dest[2] = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5e_ipsec_tx *tx;
+	struct mlx5_fc *counter;
+	int err;
+
+	tx = tx_ft_get(mdev, ipsec);
+	if (IS_ERR(tx))
+		return PTR_ERR(tx);
+
+	err = setup_pkt_reformat(mdev, attrs, &flow_act);
+	if (err)
+		goto err_pkt_reformat;
+
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_add_cnt;
+	}
+
+	flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
+        flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
+        flow_act.flags |= FLOW_ACT_NO_APPEND;
+        flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
+                           MLX5_FLOW_CONTEXT_ACTION_COUNT;
+
+	if (attrs->drop)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+	else
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+	dest[0].ft = tx->ft.status;
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest[1].counter_id = mlx5_fc_id(counter);
+
+	err = tx_add_kspi_rule(sa_entry, tx, &flow_act, dest, 2);
+	if (err) {
+		goto err_add_kspi_rule;
+	}
+
+	err = tx_add_reqid_ip_rules(sa_entry, tx, &flow_act, dest, 2);
+	if (err) {
+		goto err_add_reqid_ip_rule;
+	}
+
+	ipsec_rule->fc = counter;
+	ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
+	return 0;
+
+err_add_reqid_ip_rule:
+	mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
+err_add_kspi_rule:
+	mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
+	if (flow_act.pkt_reformat)
+		mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
+err_pkt_reformat:
+	tx_ft_put(ipsec);
+	return err;
+}
+
+static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+        struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
+        struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+        struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx;
+        struct mlx5_flow_destination dest[2] = {};
+        struct mlx5_flow_act flow_act = {};
+        struct mlx5_flow_handle *rule;
+        struct mlx5_flow_spec *spec;
+        struct mlx5_flow_table *ft;
+        int err, dstn = 0;
+
+        ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio);
+        if (IS_ERR(ft))
+            return PTR_ERR(ft);
+
+        spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+        if (!spec) {
+            err = -ENOMEM;
+            goto err_alloc;
+        }
+
+        if (attrs->family == AF_INET)
+                setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+        else
+                setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+        setup_fte_no_frags(spec);
+	setup_fte_upper_proto_match(spec, &attrs->upspec);
+
+        switch (attrs->action) {
+        case IPSEC_POLICY_IPSEC:
+                flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+                /*if (!attrs->reqid)
+                    break;*/
+                err = setup_modify_header(mdev, attrs->reqid,
+                                          IPSEC_DIR_OUTBOUND, &flow_act);
+                if (err)
+                        goto err_mod_header;
+                 break;
+        case IPSEC_POLICY_DISCARD:
+                flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
+                                   MLX5_FLOW_CONTEXT_ACTION_COUNT;
+                dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+                dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop);
+                dstn++;
+                break;
+        default:
+                err = -EINVAL;
+                goto err_mod_header;
+        }
+
+        flow_act.flags |= FLOW_ACT_NO_APPEND;
+        dest[dstn].ft = tx->ft.sa;
+        dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+        dstn++;
+        rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
+        if (IS_ERR(rule)) {
+                err = PTR_ERR(rule);
+                mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
+                goto err_action;
+        }
+
+        kvfree(spec);
+        pol_entry->ipsec_rule.rule = rule;
+        pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr;
+        return 0;
+
+err_action:
+        if (flow_act.modify_hdr)
+                mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
+err_mod_header:
+        kvfree(spec);
+err_alloc:
+        tx_ft_put_policy(pol_entry->ipsec, attrs->prio);
+        return err;
+}
+
+static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+        struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
+        struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+	struct mlx5e_ipsec *ipsec = pol_entry->ipsec;
+        struct mlx5_flow_destination dest[2];
+        struct mlx5_flow_act flow_act = {};
+        struct mlx5_flow_handle *rule;
+        struct mlx5_flow_spec *spec;
+        struct mlx5_flow_table *ft;
+        struct mlx5e_ipsec_rx *rx;
+        int err, dstn = 0;
+
+        rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
+        ft = rx->chains ? ipsec_chains_get_table(rx->chains, attrs->prio) : rx->ft.pol;
+        if (IS_ERR(ft))
+                return PTR_ERR(ft);
+
+        spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+        if (!spec) {
+                err = -ENOMEM;
+                goto err_alloc;
+        }
+
+        if (attrs->family == AF_INET)
+                setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+        else
+                setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+        setup_fte_no_frags(spec);
+	setup_fte_upper_proto_match(spec, &attrs->upspec);
+
+        switch (attrs->action) {
+        case IPSEC_POLICY_IPSEC:
+                flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+                break;
+        case IPSEC_POLICY_DISCARD:
+                flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
+                dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+                dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop);
+                dstn++;
+                break;
+        default:
+                err = -EINVAL;
+                goto err_action;
+        }
+
+        flow_act.flags |= FLOW_ACT_NO_APPEND;
+        dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+        dest[dstn].ft = rx->ft.sa;
+        dstn++;
+        rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
+        if (IS_ERR(rule)) {
+                err = PTR_ERR(rule);
+                mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err);
+                goto err_action;
+        }
+
+        kvfree(spec);
+        pol_entry->ipsec_rule.rule = rule;
+        return 0;
+
+err_action:
+        kvfree(spec);
+err_alloc:
+        if (rx->chains)
+                ipsec_chains_put_table(rx->chains, attrs->prio);
+        return err;
+}
+
+static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
+	struct mlx5_core_dev *mdev = ipsec->mdev;
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+	mlx5_fc_destroy(mdev, rx_ipv4->fc->drop);
+	mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
+	kfree(rx_ipv4->fc);
+	mlx5_fc_destroy(mdev, tx->fc->drop);
+	mlx5_fc_destroy(mdev, tx->fc->cnt);
+	kfree(tx->fc);
+}
+
+static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
+	struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6;
+	struct mlx5_core_dev *mdev = ipsec->mdev;
+	struct mlx5e_ipsec_tx *tx = ipsec->tx;
+	struct mlx5e_ipsec_fc *fc;
+	struct mlx5_fc *counter;
+	int err;
+
+	fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
+	if (!fc)
+		return -ENOMEM;
+
+	tx->fc = fc;
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_tx_fc_alloc;
+	}
+
+	fc->cnt = counter;
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_tx_fc_cnt;
+	}
+
+	fc->drop = counter;
+
+	fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
+	if (!fc) {
+		err = -ENOMEM;
+		goto err_tx_fc_drop;
+	}
+
+	/* Both IPv4 and IPv6 point to same flow counters struct. */
+	rx_ipv4->fc = fc;
+	rx_ipv6->fc = fc;
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_rx_fc_alloc;
+	}
+
+	fc->cnt = counter;
+	counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(counter)) {
+		err = PTR_ERR(counter);
+		goto err_rx_fc_cnt;
+	}
+
+	fc->drop = counter;
+	return 0;
+
+err_rx_fc_cnt:
+	mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
+err_rx_fc_alloc:
+	kfree(rx_ipv4->fc);
+err_tx_fc_drop:
+	mlx5_fc_destroy(mdev, tx->fc->drop);
+err_tx_fc_cnt:
+	mlx5_fc_destroy(mdev, tx->fc->cnt);
+err_tx_fc_alloc:
+	kfree(tx->fc);
+	return err;
+}
+
+static int ipsec_status_rule(struct mlx5_core_dev *mdev,
+			     struct mlx5e_ipsec_rx *rx,
+			     struct mlx5_flow_destination *dest)
+{
+	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_modify_hdr *modify_hdr;
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	/* Action to copy 7 bit ipsec_syndrome to regB[24:30] */
+	MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY);
+	MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME);
+	MLX5_SET(copy_action_in, action, src_offset, 0);
+	MLX5_SET(copy_action_in, action, length, 7);
+	MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
+	MLX5_SET(copy_action_in, action, dst_offset, 24);
+
+	modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL,
+					      1, action);
+
+	if (IS_ERR(modify_hdr)) {
+		err = PTR_ERR(modify_hdr);
+		mlx5_core_err(mdev,
+			      "fail to alloc ipsec copy modify_header_id err=%d\n", err);
+		goto out_spec;
+	}
+
+	/* create fte */
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
+		MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+		MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	flow_act.modify_hdr = modify_hdr;
+
+	rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err);
+		goto out;
+	}
+
+	kvfree(spec);
+	rx->status.rule = rule;
+	rx->status.modify_hdr = modify_hdr;
+	return 0;
+
+out:
+	mlx5_modify_header_dealloc(mdev, modify_hdr);
+out_spec:
+	kvfree(spec);
+	return err;
+}
+
+static void ipsec_fs_rx_roce_rules_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
+{
+	if (!rx_roce->ns_rdma)
+		return;
+
+	mlx5_del_flow_rules(&rx_roce->roce_miss.rule);
+	mlx5_del_flow_rules(&rx_roce->rule);
+	mlx5_destroy_flow_group(rx_roce->roce_miss.group);
+	mlx5_destroy_flow_group(rx_roce->g);
+}
+
+static void ipsec_fs_rx_catchall_rules_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx)
+{
+	mutex_lock(&rx->ft.mutex);
+	mlx5_del_flow_rules(&rx->sa.rule);
+	mlx5_destroy_flow_group(rx->sa.group);
+	if (rx->chains == NULL) {
+		mlx5_del_flow_rules(&rx->pol.rule);
+		mlx5_destroy_flow_group(rx->pol.group);
+	}
+	mlx5_del_flow_rules(&rx->status.rule);
+	mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
+	ipsec_fs_rx_roce_rules_destroy(&rx->roce);
+	mutex_unlock(&rx->ft.mutex);
+}
+
+static void ipsec_fs_rx_roce_table_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
+{
+	if (!rx_roce->ns_rdma)
+		return;
+
+	mlx5_destroy_flow_table(rx_roce->ft_rdma);
+	mlx5_destroy_flow_table(rx_roce->ft);
+}
+
+static void ipsec_fs_rx_table_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx)
+{
+	mutex_lock(&rx->ft.mutex);
+	if (rx->chains) {
+		ipsec_chains_destroy(rx->chains);
+	} else {
+		mlx5_del_flow_rules(&rx->pol.rule);
+		mlx5_destroy_flow_table(rx->ft.pol);
+        }
+	mlx5_destroy_flow_table(rx->ft.sa);
+	mlx5_destroy_flow_table(rx->ft.status);
+	ipsec_fs_rx_roce_table_destroy(&rx->roce);
+	mutex_unlock(&rx->ft.mutex);
+}
+
+static void ipsec_roce_setup_udp_dport(struct mlx5_flow_spec *spec, u16 dport)
+{
+	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
+}
+
+static int ipsec_roce_rx_rule_setup(struct mlx5_flow_destination *default_dst,
+				    struct mlx5e_ipsec_rx_roce *roce, struct mlx5_core_dev *mdev)
+{
+	struct mlx5_flow_destination dst = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err = 0;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	ipsec_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
+
+	//flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;//not needed it is added in command
+	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+	dst.ft = roce->ft_rdma;
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "Fail to add RX roce ipsec rule err=%d\n",
+			      err);
+		goto fail_add_rule;
+	}
+
+	roce->rule = rule;
+
+	rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, default_dst, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_err(mdev, "Fail to add RX roce ipsec miss rule err=%d\n",
+			      err);
+		goto fail_add_default_rule;
+	}
+
+	roce->roce_miss.rule = rule;
+
+	kvfree(spec);
+	return 0;
+
+fail_add_default_rule:
+	mlx5_del_flow_rules(&roce->rule);
+fail_add_rule:
+	kvfree(spec);
+	return err;
+}
+
+static int ipsec_roce_rx_rules(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *defdst,
+			       struct mlx5_core_dev *mdev)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	void *outer_headers_c;
+	u32 *in;
+	int err = 0;
+	int ix = 0;
+	u8 *mc;
+
+	if (!rx->roce.ns_rdma)
+		return 0;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
+	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
+	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
+
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += 1;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	g = mlx5_create_flow_group(rx->roce.ft, in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		mlx5_core_err(mdev, "Fail to create ipsec rx roce group at nic err=%d\n", err);
+		goto fail_group;
+	}
+	rx->roce.g = g;
+
+	memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += 1;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	g = mlx5_create_flow_group(rx->roce.ft, in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		mlx5_core_err(mdev, "Fail to create ipsec rx roce miss group at nic err=%d\n",
+			      err);
+		goto fail_mgroup;
+	}
+	rx->roce.roce_miss.group = g;
+
+	err = ipsec_roce_rx_rule_setup(defdst, &rx->roce, mdev);
+	if (err)
+		goto fail_setup_rule;
+
+	kvfree(in);
+	return 0;
+
+fail_setup_rule:
+	mlx5_destroy_flow_group(rx->roce.roce_miss.group);
+fail_mgroup:
+	mlx5_destroy_flow_group(rx->roce.g);
+fail_group:
+	kvfree(in);
+	return err;
+}
+
+static int ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv,
+				      struct mlx5e_ipsec_rx *rx,
+				      struct mlx5_flow_destination *defdst)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5_flow_destination dest[2] = {};
+	int err = 0;
+
+	mutex_lock(&rx->ft.mutex);
+	/* IPsec RoCE RX rules */
+	err = ipsec_roce_rx_rules(rx, defdst, mdev);
+	if (err)
+		goto out;
+
+	/* IPsec Rx IP Status table rule */
+	dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	if (rx->roce.ft)
+		dest[0].ft = rx->roce.ft;
+	else
+		dest[0].ft = priv->fts.vlan.t;
+
+	dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+        dest[1].counter_id = mlx5_fc_id(rx->fc->cnt);
+        err = ipsec_status_rule(mdev, rx, dest);
+        if (err)
+                goto err_roce_rules_destroy;
+
+	if (!rx->chains) {
+		/* IPsec Rx IP policy default miss rule */
+		err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, defdst);
+		if (err)
+			goto err_status_rule_destroy;
+	}
+
+	/* FIXME: This is workaround to current design
+	 * which installs SA on firt packet. So we need to forward this
+	 * packet to the stack. It doesn't work with RoCE and eswitch traffic,
+	 */
+	err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, defdst);
+	if (err)
+		goto err_status_sa_rule_destroy;
+
+	mutex_unlock(&rx->ft.mutex);
+	return 0;
+
+err_status_sa_rule_destroy:
+	if (!rx->chains) {
+		mlx5_del_flow_rules(&rx->pol.rule);
+		mlx5_destroy_flow_group(rx->pol.group);
+	}
+err_status_rule_destroy:
+	mlx5_del_flow_rules(&rx->status.rule);
+	mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
+err_roce_rules_destroy:
+	ipsec_fs_rx_roce_rules_destroy(&rx->roce);
+out:
+	mutex_unlock(&rx->ft.mutex);
+	return err;
+}
+
+static int ipsec_fs_rx_roce_tables_create(struct mlx5e_ipsec_rx *rx,
+					  int rx_init_level, int rdma_init_level)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_table *ft;
+	int err = 0;
+
+	if (!rx->roce.ns_rdma)
+		return 0;
+
+	ft_attr.max_fte = 2;
+	ft_attr.level = rx_init_level;
+	ft = mlx5_create_flow_table(rx->ns, &ft_attr);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		return err;
+	}
+	rx->roce.ft = ft;
+
+	ft_attr.max_fte = 0;
+	ft_attr.level = rdma_init_level;
+	ft = mlx5_create_flow_table(rx->roce.ns_rdma, &ft_attr);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto out;
+	}
+	rx->roce.ft_rdma = ft;
+
+	return 0;
+out:
+	mlx5_destroy_flow_table(rx->roce.ft);
+	rx->roce.ft = NULL;
+	return err;
+}
+
+static int ipsec_fs_rx_table_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx,
+				    int rx_init_level, int rdma_init_level)
+{
+	struct mlx5_flow_namespace *ns = rx->ns;
+	struct mlx5_flow_table *ft;
+	int err = 0;
+
+	mutex_lock(&rx->ft.mutex);
+
+	/* IPsec Rx IP SA table create */
+	ft = ipsec_rx_ft_create(ns, rx_init_level + 1, 0, 1);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto out;
+	}
+	rx->ft.sa = ft;
+
+	/* IPsec Rx IP Status table create */
+	ft = ipsec_rx_ft_create(ns, rx_init_level + 2, 0, 1);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_sa_table_destroy;
+	}
+	rx->ft.status = ft;
+
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+		rx->chains = ipsec_chains_create(mdev, rx->ft.sa,
+				MLX5_FLOW_NAMESPACE_KERNEL, 0,
+				rx_init_level, &rx->ft.pol);
+		if (IS_ERR(rx->chains)) {
+			err = PTR_ERR(rx->chains);
+			goto err_status_table_destroy;
+		}
+	} else {
+		ft = ipsec_rx_ft_create(ns, rx_init_level, 0, 1);
+		if (IS_ERR(ft)) {
+			err = PTR_ERR(ft);
+			goto err_status_table_destroy;
+		}
+		rx->ft.pol = ft;
+	}
+
+	/* IPsec RoCE RX tables create*/
+	err = ipsec_fs_rx_roce_tables_create(rx, rx_init_level + 3,
+					     rdma_init_level);
+	if (err)
+		goto err_pol_table_destroy;
+
+	goto out;
+
+err_pol_table_destroy:
+	mlx5_destroy_flow_table(rx->ft.pol);
+err_status_table_destroy:
+	mlx5_destroy_flow_table(rx->ft.status);
+err_sa_table_destroy:
+	mlx5_destroy_flow_table(rx->ft.sa);
+out:
+	mutex_unlock(&rx->ft.mutex);
+	return err;
+}
+
+#define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA | MLX5_FT_NIC_TX_RDMA_2_NIC_TX)
+
+static void mlx5e_accel_ipsec_fs_init_roce(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5_core_dev *mdev = ipsec->mdev;
+	struct mlx5_flow_namespace *ns;
+
+	if ((MLX5_CAP_GEN_2(ipsec->mdev, flow_table_type_2_type) &
+	      NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) {
+		mlx5_core_dbg(mdev, "Failed to init roce ns, capabilities not supported\n");
+		return;
+	}
+
+	ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
+	if (!ns) {
+		mlx5_core_err(mdev, "Failed to init roce rx ns\n");
+		return;
+	}
+
+	ipsec->rx_ipv4->roce.ns_rdma = ns;
+	ipsec->rx_ipv6->roce.ns_rdma = ns;
+
+	ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
+	if (!ns) {
+		ipsec->rx_ipv4->roce.ns_rdma = NULL;
+		ipsec->rx_ipv6->roce.ns_rdma = NULL;
+		mlx5_core_err(mdev, "Failed to init roce tx ns\n");
+		return;
+	}
+
+	ipsec->tx->roce.ns = ns;
+}
+
+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
+		return tx_add_rule(sa_entry);
+
+	return rx_add_rule(sa_entry);
+}
+
+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+	mlx5_del_flow_rules(&ipsec_rule->rule);
+	mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
+	if (ipsec_rule->reqid_rule)
+		mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
+	mlx5_fc_destroy(mdev, ipsec_rule->fc);
+	mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat);
+	if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) {
+		tx_ft_put(sa_entry->ipsec);
+		return;
+	}
+
+	if (ipsec_rule->modify_hdr)
+		mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
+}
+
+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+	if (pol_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
+		return tx_add_policy(pol_entry);
+
+	return rx_add_policy(pol_entry);
+}
+
+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+	struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+
+	mlx5_del_flow_rules(&ipsec_rule->rule);
+
+	if (pol_entry->attrs.dir == IPSEC_DIR_INBOUND) {
+		struct mlx5e_ipsec_rx *rx;
+
+                rx = (pol_entry->attrs.family == AF_INET)
+                         ? pol_entry->ipsec->rx_ipv4
+                         : pol_entry->ipsec->rx_ipv6;
+                if (rx->chains)
+                        ipsec_chains_put_table(rx->chains,
+                                               pol_entry->attrs.prio);
+                return;
+	}
+
+	if (ipsec_rule->modify_hdr)
+		mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
+
+	tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio);
+}
+
+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv)
+{
+	/* Check if IPsec supported */
+	if (!priv->ipsec)
+		return;
+
+	ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv4);
+	ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
+}
+
+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv)
+{
+	struct mlx5e_ipsec *ipsec = priv->ipsec;
+	struct mlx5_flow_destination dest = {};
+	int err = 0;
+
+	/* Check if IPsec supported */
+	if (!ipsec)
+		return 0;
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft = priv->fts.vlan.t;
+	err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv6, &dest);
+	if (err)
+		goto out;
+
+	err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv4, &dest);
+	if (err)
+		ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
+out:
+	return err;
+}
+
+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_ipsec *ipsec = priv->ipsec;
+
+	/* Check if IPsec supported */
+	if (!ipsec)
+		return;
+
+	ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv6);
+	ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv4);
+}
+
+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv)
+{
+	struct mlx5e_ipsec *ipsec = priv->ipsec;
+	int err = 0;
+
+	/* Check if IPsec supported */
+	if (!ipsec)
+		return 0;
+
+	err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv4, 0, 0);
+	if (err)
+		goto out;
+
+	err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv6, 4, 1);
+	if (err) {
+		ipsec_fs_rx_table_destroy(priv->mdev, ipsec->rx_ipv4);
+		goto out;
+	}
+
+	priv->fts.ipsec_ft = priv->ipsec->rx_ipv4->ft.pol;
+out:
+	return err;
+}
+
+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec)
+{
+	WARN_ON(ipsec->tx->ft.refcnt);
+	mutex_destroy(&ipsec->rx_ipv6->ft.mutex);
+	mutex_destroy(&ipsec->rx_ipv4->ft.mutex);
+	mutex_destroy(&ipsec->tx->ft.mutex);
+	ipsec_fs_destroy_counters(ipsec);
+	kfree(ipsec->rx_ipv6);
+	kfree(ipsec->rx_ipv4);
+	kfree(ipsec->tx);
+}
+
+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5_flow_namespace *tns, *rns;
+	int err = -ENOMEM;
+
+	tns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
+	if (!tns)
+		return -EOPNOTSUPP;
+
+	rns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
+	if (!rns)
+		return -EOPNOTSUPP;
+
+	ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL);
+	if (!ipsec->tx)
+		return -ENOMEM;
+
+	ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL);
+	if (!ipsec->rx_ipv4)
+		goto err_tx;
+
+	ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL);
+	if (!ipsec->rx_ipv6)
+		goto err_rx_ipv4;
+
+	err = ipsec_fs_init_counters(ipsec);
+	if (err)
+		goto err_rx_ipv6;
+
+	ipsec->tx->ns = tns;
+	mutex_init(&ipsec->tx->ft.mutex);
+	ipsec->rx_ipv4->ns = rns;
+	ipsec->rx_ipv6->ns = rns;
+	mutex_init(&ipsec->rx_ipv4->ft.mutex);
+	mutex_init(&ipsec->rx_ipv6->ft.mutex);
+
+	mlx5e_accel_ipsec_fs_init_roce(ipsec);
+
+	return 0;
+
+err_rx_ipv6:
+	kfree(ipsec->rx_ipv6);
+err_rx_ipv4:
+	kfree(ipsec->rx_ipv4);
+err_tx:
+	kfree(ipsec->tx);
+	return err;
+}
+
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+        struct mlx5e_ipsec_sa_entry sa_entry_shadow = {};
+        int err;
+
+        memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry));
+        memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule));
+
+        err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow);
+        if (err)
+                return;
+        mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+        memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry));
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
new file mode 100644
index 000000000000..978e5f25ceaf
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
@@ -0,0 +1,486 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/ipsec.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/crypto.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
+{
+	u32 caps = 0;
+
+	if (!MLX5_CAP_GEN(mdev, ipsec_offload))
+		return 0;
+
+	if (!MLX5_CAP_GEN(mdev, log_max_dek))
+		return 0;
+
+	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
+	    MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+		return 0;
+
+	if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
+	    !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
+		return 0;
+
+	if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) ||
+	    !MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
+		return 0;
+
+        if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload)) {
+                if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+                                              reformat_add_esp_trasport) &&
+                    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+                                              reformat_del_esp_trasport) &&
+                    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
+			caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
+
+                if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) &&
+                    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level))
+			caps |= MLX5_IPSEC_CAP_PRIO;
+
+		if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_transport_over_udp) &&
+		    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_transport_over_udp))
+			caps |= MLX5_IPSEC_CAP_ESPINUDP;
+        }
+
+        if (!caps)
+		return 0;
+
+	if (MLX5_CAP_IPSEC(mdev, ipsec_esn))
+		caps |= MLX5_IPSEC_CAP_ESN;
+
+	return caps;
+}
+EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
+
+static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
+				     struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+	void *aso_ctx;
+
+	aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
+	/* ASO context */
+	MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn);
+	MLX5_SET(ipsec_obj, obj, full_offload, 1);
+	MLX5_SET(ipsec_aso, aso_ctx, valid, 1);
+	/* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used
+	 * in flow steering to perform matching against. Please be
+	 * aware that this register was chosen arbitrary and can't
+	 * be used in other places as long as IPsec packet offload
+	 * active.
+	 */
+	MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
+	if (attrs->replay_esn.trigger) {
+		MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);
+
+		if (attrs->dir == IPSEC_DIR_INBOUND) {
+			MLX5_SET(ipsec_aso, aso_ctx, window_sz,
+				 attrs->replay_esn.replay_window);
+			if (attrs->replay_esn.replay_window != 0)
+				MLX5_SET(ipsec_aso, aso_ctx, mode,
+				    MLX5_IPSEC_ASO_REPLAY_PROTECTION);
+			else
+				MLX5_SET(ipsec_aso, aso_ctx, mode,
+				    MLX5_IPSEC_ASO_MODE);
+		}
+		MLX5_SET(ipsec_aso, aso_ctx, mode_parameter,
+			 attrs->replay_esn.esn);
+	}
+
+	switch (attrs->dir) {
+	case IPSEC_DIR_OUTBOUND:
+		if (attrs->replay_esn.replay_window != 0)
+			MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);
+		else
+			MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_MODE);
+		break;
+	default:
+		break;
+	}
+}
+
+static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+	u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
+	void *obj, *salt_p, *salt_iv_p;
+	int err;
+
+	obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
+
+	/* salt and seq_iv */
+	salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
+	memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
+
+	MLX5_SET(ipsec_obj, obj, icv_length, MLX5_IPSEC_OBJECT_ICV_LEN_16B);
+	salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
+	memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
+
+	/* esn */
+	if (attrs->replay_esn.trigger) {
+		MLX5_SET(ipsec_obj, obj, esn_en, 1);
+		MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+		MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
+	}
+
+	/* enc./dec. key */
+	MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);
+
+	/* general object fields set */
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJ);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+
+	mlx5e_ipsec_packet_setup(obj, sa_entry->ipsec->pdn, attrs);
+
+	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		sa_entry->ipsec_obj_id =
+			MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+	return err;
+}
+
+static void mlx5_destroy_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+		 MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
+
+	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct aes_gcm_keymat *aes_gcm = &sa_entry->attrs.aes_gcm;
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+	int err;
+
+	/* key */
+	err = mlx5_encryption_key_create(mdev, sa_entry->ipsec->pdn,
+					 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC,
+					 aes_gcm->aes_key,
+					 aes_gcm->key_len,
+					 &sa_entry->enc_key_id);
+	if (err) {
+		mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
+		return err;
+	}
+
+	err = mlx5_create_ipsec_obj(sa_entry);
+	if (err) {
+		mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
+		goto err_enc_key;
+	}
+
+	return 0;
+
+err_enc_key:
+	mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
+	return err;
+}
+
+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+	mlx5_destroy_ipsec_obj(sa_entry);
+	mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
+}
+
+static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
+				 struct mlx5_wqe_aso_ctrl_seg *data)
+{
+	if (!data)
+		return;
+
+	ctrl->data_mask_mode = data->data_mask_mode;
+	ctrl->condition_1_0_operand = data->condition_1_0_operand;
+	ctrl->condition_1_0_offset = data->condition_1_0_offset;
+	ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
+	ctrl->condition_0_data = data->condition_0_data;
+	ctrl->condition_0_mask = data->condition_0_mask;
+	ctrl->condition_1_data = data->condition_1_data;
+	ctrl->condition_1_mask = data->condition_1_mask;
+	ctrl->bitwise_data = data->bitwise_data;
+	ctrl->data_mask = data->data_mask;
+}
+
+static int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
+				 struct mlx5_wqe_aso_ctrl_seg *data)
+{
+	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+	struct mlx5e_ipsec_aso *aso = ipsec->aso;
+	struct mlx5_wqe_aso_ctrl_seg *ctrl;
+	struct mlx5_aso_wqe *wqe;
+	unsigned long expires;
+	u8 ds_cnt;
+	int ret;
+
+	spin_lock_bh(&aso->lock);
+	memset(aso->ctx, 0, sizeof(aso->ctx));
+	wqe = mlx5_aso_get_wqe(aso->aso);
+	ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
+	mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
+			   MLX5_ACCESS_ASO_OPC_MOD_IPSEC);
+
+	ctrl = &wqe->aso_ctrl;
+	ctrl->va_l = cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN);
+	ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
+	ctrl->l_key = cpu_to_be32(ipsec->mkey);
+	mlx5e_ipsec_aso_copy(ctrl, data);
+
+	mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
+	expires = jiffies + msecs_to_jiffies(10);
+	do {
+		ret = mlx5_aso_poll_cq(aso->aso, false);
+		if (ret)
+			/* We are in atomic context */
+			udelay(10);
+	} while (ret && time_is_after_jiffies(expires));
+	spin_unlock_bh(&aso->lock);
+
+	return ret;
+}
+
+#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L
+
+static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
+                                 const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+        struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+        u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
+        u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
+        u64 modify_field_select = 0;
+        u64 general_obj_types;
+        void *obj;
+        int err;
+
+        general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+        if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+                return -EINVAL;
+
+        /* general object fields set */
+        MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJ);
+        MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+        MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
+        err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+        if (err) {
+                mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
+                              sa_entry->ipsec_obj_id, err);
+                return err;
+        }
+
+        obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
+        modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
+
+        /* esn */
+        if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
+            !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
+                return -EOPNOTSUPP;
+
+        obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
+        MLX5_SET64(ipsec_obj, obj, modify_field_select,
+                   MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP |
+                           MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
+        MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+        MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
+
+        /* general object fields set */
+        MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJ);
+
+        return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
+				       const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+	int err;
+
+	err = mlx5_modify_ipsec_obj(sa_entry, attrs);
+	if (err)
+		return;
+
+	memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
+}
+
+static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry,
+				   struct mlx5_wqe_aso_ctrl_seg *data)
+{
+	data->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
+	data->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE << 4;
+
+	mlx5e_ipsec_aso_query(sa_entry, data);
+}
+
+#define MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET 0
+
+static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
+					 u32 mode_param)
+{
+	struct mlx5_accel_esp_xfrm_attrs attrs = {};
+	struct mlx5_wqe_aso_ctrl_seg data = {};
+
+	if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
+		sa_entry->esn_state.esn_msb++;
+		sa_entry->esn_state.overlap = 0;
+	} else {
+		sa_entry->esn_state.overlap = 1;
+	}
+
+	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs, sa_entry->attrs.dir);
+
+	mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
+
+	data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
+	data.bitwise_data = cpu_to_be64(BIT_ULL(54));
+	data.data_mask = data.bitwise_data;
+
+	mlx5e_ipsec_aso_update(sa_entry, &data);
+}
+
+static void mlx5e_ipsec_handle_event(struct work_struct *_work)
+{
+	struct mlx5e_ipsec_work *work =
+		container_of(_work, struct mlx5e_ipsec_work, work);
+	struct mlx5e_ipsec_sa_entry *sa_entry = work->data;
+	struct mlx5_accel_esp_xfrm_attrs *attrs;
+	struct mlx5e_ipsec_aso *aso;
+	int ret;
+
+	aso = sa_entry->ipsec->aso;
+	attrs = &sa_entry->attrs;
+
+	/* TODO: Kostia, this event should be locked/protected
+	 * from concurent SA delete.
+	 */
+	ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
+	if (ret)
+		goto unlock;
+
+	if (attrs->replay_esn.trigger &&
+	    !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
+		u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
+
+	        mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
+	}
+
+unlock:
+	kfree(work);
+}
+
+void mlx5_object_change_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
+{
+	struct mlx5e_ipsec_sa_entry *sa_entry;
+	struct mlx5_eqe_obj_change *object;
+	struct mlx5e_ipsec_work *work;
+	u16 type;
+
+	object = &eqe->data.obj_change;
+	type = be16_to_cpu(object->obj_type);
+
+	if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
+		return;
+
+	sa_entry = xa_load(&dev->ipsec_sadb, be32_to_cpu(object->obj_id));
+	if (!sa_entry)
+		return;
+
+	work = kmalloc(sizeof(*work), GFP_ATOMIC);
+	if (!work)
+		return;
+
+	INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
+	work->data = sa_entry;
+
+	queue_work(sa_entry->ipsec->wq, &work->work);
+}
+
+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5_core_dev *mdev = ipsec->mdev;
+	struct mlx5e_ipsec_aso *aso;
+	struct device *pdev;
+	int err;
+
+	aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL);
+	if (!aso)
+		return -ENOMEM;
+
+	pdev = &mdev->pdev->dev;
+	aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+	err = dma_mapping_error(pdev, aso->dma_addr);
+	if (err)
+		goto err_dma;
+
+	aso->aso = mlx5_aso_create(mdev, ipsec->pdn);
+	if (IS_ERR(aso->aso)) {
+		err = PTR_ERR(aso->aso);
+		goto err_aso_create;
+	}
+
+	spin_lock_init(&aso->lock);
+	ipsec->aso = aso;
+	return 0;
+
+err_aso_create:
+	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+err_dma:
+	kfree(aso);
+	return err;
+}
+
+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
+{
+	struct mlx5_core_dev *mdev = ipsec->mdev;
+	struct mlx5e_ipsec_aso *aso;
+	struct device *pdev;
+
+	aso = ipsec->aso;
+	pdev = &mdev->pdev->dev;
+
+	mlx5_aso_destroy(aso->aso);
+	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+	kfree(aso);
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
new file mode 100644
index 000000000000..5ff8e021b196
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
@@ -0,0 +1,76 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec_offload.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+#define MLX5_IPSEC_METADATA_HANDLE(ipsec_metadata) (ipsec_metadata & 0xFFFFFF)
+
+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb)
+{
+	struct mlx5e_priv *priv;
+	struct ipsec_accel_in_tag *tag;
+	struct m_tag *mtag;
+
+	priv = if_getsoftc(ifp);
+	if (priv->ipsec == NULL)
+		return (0);
+
+	mtag = m_tag_get(PACKET_TAG_IPSEC_ACCEL_IN, sizeof(*tag), M_NOWAIT);
+	if (mtag == NULL)
+		return -ENOMEM;
+
+	m_tag_prepend(mb, mtag);
+	return 0;
+}
+
+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+{
+	struct ipsec_accel_in_tag *tag;
+	u32  drv_spi;
+
+	drv_spi = MLX5_IPSEC_METADATA_HANDLE(be32_to_cpu(cqe->ft_metadata));
+	tag = (struct ipsec_accel_in_tag *) m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
+	WARN_ON(tag == NULL);
+	if (tag)
+		tag->drv_spi = drv_spi;
+
+	return 0;
+}
+
+void
+mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf *mb, struct mlx5e_tx_wqe *wqe,
+    struct ipsec_accel_out_tag *tag)
+{
+	wqe->eth.flow_table_metadata = cpu_to_be32(
+	    mlx5e_accel_ipsec_get_metadata(tag->drv_spi));
+}
diff --git a/sys/dev/mlx5/mlx5_core/eswitch.h b/sys/dev/mlx5/mlx5_core/eswitch.h
index ca03da287543..50d06951bf07 100644
--- a/sys/dev/mlx5/mlx5_core/eswitch.h
+++ b/sys/dev/mlx5/mlx5_core/eswitch.h
@@ -29,6 +29,8 @@
 #include <linux/if_ether.h>
 #include <dev/mlx5/device.h>
 
+#define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_flow_table)
+
 #define MLX5_MAX_UC_PER_VPORT(dev) \
 	(1 << MLX5_CAP_GEN(dev, log_max_current_uc_list))
 
@@ -83,15 +85,15 @@ struct l2addr_node {
 struct vport_ingress {
 	struct mlx5_flow_table *acl;
 	struct mlx5_flow_group *drop_grp;
-	struct mlx5_flow_rule  *drop_rule;
+	struct mlx5_flow_handle  *drop_rule;
 };
 
 struct vport_egress {
 	struct mlx5_flow_table *acl;
 	struct mlx5_flow_group *allowed_vlans_grp;
 	struct mlx5_flow_group *drop_grp;
-	struct mlx5_flow_rule  *allowed_vlan;
-	struct mlx5_flow_rule  *drop_rule;
+	struct mlx5_flow_handle  *allowed_vlan;
+	struct mlx5_flow_handle  *drop_rule;
 };
 
 struct mlx5_vport {
diff --git a/sys/dev/mlx5/mlx5_core/fs_chains.h b/sys/dev/mlx5/mlx5_core/fs_chains.h
new file mode 100644
index 000000000000..e703a98981b6
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_chains.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2020 Mellanox Technologies. */
+
+#ifndef __ML5_ESW_CHAINS_H__
+#define __ML5_ESW_CHAINS_H__
+
+#include <dev/mlx5/fs.h>
+
+struct mlx5_fs_chains;
+
+enum mlx5_chains_flags {
+	MLX5_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
+	MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED = BIT(1),
+	MLX5_CHAINS_FT_TUNNEL_SUPPORTED = BIT(2),
+};
+
+struct mlx5_chains_attr {
+	enum mlx5_flow_namespace_type ns;
+	int fs_base_prio;
+	int fs_base_level;
+	u32 flags;
+	u32 max_grp_num;
+	struct mlx5_flow_table *default_ft;
+};
+
+bool
+mlx5_chains_prios_supported(struct mlx5_fs_chains *chains);
+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains);
+bool
+mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains);
+
+struct mlx5_flow_table *
+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+		      u32 level);
+void
+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+		      u32 level);
+
+struct mlx5_flow_table *
+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains);
+
+struct mlx5_flow_table *
+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains);
+void
+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
+				 struct mlx5_flow_table *ft);
+
+int
+mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
+			      u32 *chain_mapping);
+int
+mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains,
+			      u32 chain_mapping);
+
+struct mlx5_fs_chains *
+mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr);
+void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
+
+void
+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
+		       struct mlx5_flow_table *ft);
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains);
+
+#endif /* __ML5_ESW_CHAINS_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/fs_cmd.h b/sys/dev/mlx5/mlx5_core/fs_cmd.h
new file mode 100644
index 000000000000..a2b2d537ac45
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_cmd.h
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _MLX5_FS_CMD_
+#define _MLX5_FS_CMD_
+
+#include "fs_core.h"
+
+struct mlx5_flow_cmds {
+	int (*create_flow_table)(struct mlx5_flow_root_namespace *ns,
+				 struct mlx5_flow_table *ft,
+				 struct mlx5_flow_table_attr *ft_attr,
+				 struct mlx5_flow_table *next_ft);
+	int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns,
+				  struct mlx5_flow_table *ft);
+
+	int (*modify_flow_table)(struct mlx5_flow_root_namespace *ns,
+				 struct mlx5_flow_table *ft,
+				 struct mlx5_flow_table *next_ft);
+
+	int (*create_flow_group)(struct mlx5_flow_root_namespace *ns,
+				 struct mlx5_flow_table *ft,
+				 u32 *in,
+				 struct mlx5_flow_group *fg);
+
+	int (*destroy_flow_group)(struct mlx5_flow_root_namespace *ns,
+				  struct mlx5_flow_table *ft,
+				  struct mlx5_flow_group *fg);
+
+	int (*create_fte)(struct mlx5_flow_root_namespace *ns,
+			  struct mlx5_flow_table *ft,
+			  struct mlx5_flow_group *fg,
+			  struct fs_fte *fte);
+
+	int (*update_fte)(struct mlx5_flow_root_namespace *ns,
+			  struct mlx5_flow_table *ft,
+			  struct mlx5_flow_group *fg,
+			  int modify_mask,
+			  struct fs_fte *fte);
+
+	int (*delete_fte)(struct mlx5_flow_root_namespace *ns,
+			  struct mlx5_flow_table *ft,
+			  struct fs_fte *fte);
+
+	int (*update_root_ft)(struct mlx5_flow_root_namespace *ns,
+			      struct mlx5_flow_table *ft,
+			      u32 underlay_qpn,
+			      bool disconnect);
+
+	int (*packet_reformat_alloc)(struct mlx5_flow_root_namespace *ns,
+				     struct mlx5_pkt_reformat_params *params,
+				     enum mlx5_flow_namespace_type namespace,
+				     struct mlx5_pkt_reformat *pkt_reformat);
+
+	void (*packet_reformat_dealloc)(struct mlx5_flow_root_namespace *ns,
+					struct mlx5_pkt_reformat *pkt_reformat);
+
+	int (*modify_header_alloc)(struct mlx5_flow_root_namespace *ns,
+				   u8 namespace, u8 num_actions,
+				   void *modify_actions,
+				   struct mlx5_modify_hdr *modify_hdr);
+
+	void (*modify_header_dealloc)(struct mlx5_flow_root_namespace *ns,
+				      struct mlx5_modify_hdr *modify_hdr);
+
+	int (*set_peer)(struct mlx5_flow_root_namespace *ns,
+			struct mlx5_flow_root_namespace *peer_ns);
+
+	int (*create_ns)(struct mlx5_flow_root_namespace *ns);
+	int (*destroy_ns)(struct mlx5_flow_root_namespace *ns);
+
+	u32 (*get_capabilities)(struct mlx5_flow_root_namespace *ns,
+				enum fs_flow_table_type ft_type);
+};
+
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id);
+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
+			   enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
+			   u32 *id);
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
+		      u64 *packets, u64 *bytes);
+
+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len);
+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
+			   u32 *out);
+
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type);
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
+
+#endif
diff --git a/sys/dev/mlx5/mlx5_core/fs_core.h b/sys/dev/mlx5/mlx5_core/fs_core.h
index 05757f493469..1d7339e6b7d5 100644
--- a/sys/dev/mlx5/mlx5_core/fs_core.h
+++ b/sys/dev/mlx5/mlx5_core/fs_core.h
@@ -1,14 +1,11 @@
-/*-
- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd.  All rights reserved.
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
  *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
  *
  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -26,306 +23,327 @@
 #ifndef _MLX5_FS_CORE_
 #define _MLX5_FS_CORE_
 
-#include <asm/atomic.h>
-#include <linux/completion.h>
-#include <linux/mutex.h>
 #include <dev/mlx5/fs.h>
 
-enum fs_type {
+#define FDB_TC_MAX_CHAIN 3
+#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
+#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
+
+/* The index of the last real chain (FT) + 1 as chain zero is valid as well */
+#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
+
+#define FDB_TC_MAX_PRIO 16
+#define FDB_TC_LEVELS_PER_PRIO 2
+
+struct mlx5_flow_definer {
+	enum mlx5_flow_namespace_type ns_type;
+	u32 id;
+};
+
+struct mlx5_modify_hdr {
+	enum mlx5_flow_namespace_type ns_type;
+	union {
+		u32 id;
+	};
+};
+
+struct mlx5_pkt_reformat {
+	enum mlx5_flow_namespace_type ns_type;
+	int reformat_type; /* from mlx5_ifc */
+	union {
+		u32 id;
+	};
+};
+
+/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
+ * and those are in parallel to one another when going over them to connect
+ * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
+ * parallel namespace will not automatically connect to the first flow table
+ * found in any prio in any next namespace, but skip the entire containing
+ * TYPE_PRIO_CHAINS prio.
+ *
+ * This is used to implement tc chains, each chain of prios is a different
+ * namespace inside a containing TYPE_PRIO_CHAINS prio.
+ */
+
+enum fs_node_type {
 	FS_TYPE_NAMESPACE,
 	FS_TYPE_PRIO,
+	FS_TYPE_PRIO_CHAINS,
 	FS_TYPE_FLOW_TABLE,
 	FS_TYPE_FLOW_GROUP,
 	FS_TYPE_FLOW_ENTRY,
 	FS_TYPE_FLOW_DEST
 };
 
-enum fs_ft_type {
+/**********************************************************************************************************/
+
+
+#define fs_ft_type fs_flow_table_type
+/************************************************************************************************************/
+enum fs_flow_table_type {
 	FS_FT_NIC_RX          = 0x0,
+	FS_FT_NIC_TX          = 0x1,
 	FS_FT_ESW_EGRESS_ACL  = 0x2,
 	FS_FT_ESW_INGRESS_ACL = 0x3,
 	FS_FT_FDB             = 0X4,
-	FS_FT_SNIFFER_RX      = 0x5,
-	FS_FT_SNIFFER_TX      = 0x6
+	FS_FT_SNIFFER_RX	= 0X5,
+	FS_FT_SNIFFER_TX	= 0X6,
+	FS_FT_RDMA_RX		= 0X7,
+	FS_FT_RDMA_TX		= 0X8,
+	FS_FT_PORT_SEL		= 0X9,
+	FS_FT_MAX_TYPE = FS_FT_PORT_SEL,
+};
+
+enum fs_flow_table_op_mod {
+	FS_FT_OP_MOD_NORMAL,
+	FS_FT_OP_MOD_LAG_DEMUX,
 };
 
 enum fs_fte_status {
 	FS_FTE_STATUS_EXISTING = 1UL << 0,
 };
 
-/* Should always be the first variable in the struct */
-struct fs_base {
-	struct list_head			list;
-	struct fs_base			*parent;
-	enum fs_type			type;
-	struct kref			refcount;
+enum mlx5_flow_steering_mode {
+	MLX5_FLOW_STEERING_MODE_DMFS,
+	MLX5_FLOW_STEERING_MODE_SMFS
+};
+
+enum mlx5_flow_steering_capabilty {
+	MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
+	MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
+	MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2,
+};
+
+struct mlx5_flow_steering {
+	struct mlx5_core_dev *dev;
+	enum   mlx5_flow_steering_mode	mode;
+	struct kmem_cache		*fgs_cache;
+	struct kmem_cache               *ftes_cache;
+	struct mlx5_flow_root_namespace *root_ns;
+	struct mlx5_flow_root_namespace *fdb_root_ns;
+	struct mlx5_flow_namespace	**fdb_sub_ns;
+	struct mlx5_flow_root_namespace **esw_egress_root_ns;
+	struct mlx5_flow_root_namespace **esw_ingress_root_ns;
+	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
+	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
+	struct mlx5_flow_root_namespace	*rdma_rx_root_ns;
+	struct mlx5_flow_root_namespace	*rdma_tx_root_ns;
+	struct mlx5_flow_root_namespace	*egress_root_ns;
+	struct mlx5_flow_root_namespace	*port_sel_root_ns;
+	int esw_egress_acl_vports;
+	int esw_ingress_acl_vports;
+};
+
+struct fs_node {
+	struct list_head	list;
+	struct list_head	children;
+	enum fs_node_type	type;
+	struct fs_node		*parent;
+	struct fs_node		*root;
 	/* lock the node for writing and traversing */
-	struct mutex		lock;
-	struct completion			complete;
-	atomic_t			users_refcount;
-	const char			*name;
+	struct rw_semaphore	lock;
+	refcount_t		refcount;
+	bool			active;
+	void			(*del_hw_func)(struct fs_node *);
+	void			(*del_sw_func)(struct fs_node *);
+	atomic_t		version;
 };
 
 struct mlx5_flow_rule {
-	struct fs_base				base;
+	struct fs_node				node;
+	struct mlx5_flow_table			*ft;
 	struct mlx5_flow_destination		dest_attr;
-	struct list_head			clients_data;
-	/*protect clients lits*/
-	struct mutex			clients_lock;
-};
-
-struct fs_fte {
-	struct fs_base				base;
-	u32					val[MLX5_ST_SZ_DW(fte_match_param)];
-	uint32_t				dests_size;
-	struct list_head			dests;
-	uint32_t				index; /* index in ft */
-	struct mlx5_flow_act			flow_act;
-	u32					sw_action; /* enum mlx5_rule_fwd_action */
-	enum fs_fte_status			status;
+	/* next_ft should be accessed under chain_lock and only of
+	 * destination type is FWD_NEXT_fT.
+	 */
+	struct list_head			next_ft;
+	u32					sw_action;
 };
 
-struct fs_star_rule {
-	struct	mlx5_flow_group	 *fg;
-	struct	fs_fte		*fte;
+struct mlx5_flow_handle {
+	int num_rules;
+	struct mlx5_flow_rule *rule[];
 };
 
+/* Type of children is mlx5_flow_group */
 struct mlx5_flow_table {
-	struct fs_base			base;
-	/* sorted list by start_index */
-	struct list_head		fgs;
+	struct fs_node			node;
+	u32				id;
+	u16				vport;
+	unsigned int			max_fte;
+	unsigned int			level;
+	enum fs_flow_table_type		type;
+	enum fs_flow_table_op_mod	op_mod;
 	struct {
 		bool			active;
-		unsigned int		max_types;
+		unsigned int		required_groups;
 		unsigned int		group_size;
-		unsigned int		num_types;
+		unsigned int		num_groups;
 		unsigned int		max_fte;
 	} autogroup;
-	unsigned int			max_fte;
-	unsigned int			level;
-	uint32_t			id;
-	u16                             vport;
-	enum fs_ft_type			type;
-	struct fs_star_rule		star_rule;
-	unsigned int			shared_refcount;
+	/* Protect fwd_rules */
+	struct mutex			lock;
+	/* FWD rules that point on this flow table */
+	struct list_head		fwd_rules;
+	u32				flags;
+	struct xarray			fgs_xa;
+	enum mlx5_flow_table_miss_action def_miss_action;
+	struct mlx5_flow_namespace	*ns;
 };
 
-enum fs_prio_flags {
-	MLX5_CORE_FS_PRIO_SHARED = 1
+struct mlx5_ft_underlay_qp {
+	struct list_head list;
+	u32 qpn;
+};
+
+#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_e00
+/* Calculate the fte_match_param length and without the reserved length.
+ * Make sure the reserved field is the last.
+ */
+#define MLX5_ST_SZ_DW_MATCH_PARAM					    \
+	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
+	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
+			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
+					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
+			   MLX5_BYTE_OFF(fte_match_param,		     \
+					 MLX5_FTE_MATCH_PARAM_RESERVED)))
+
+/* Type of children is mlx5_flow_rule */
+struct fs_fte {
+	struct fs_node			node;
+	u32				val[MLX5_ST_SZ_DW_MATCH_PARAM];
+	u32				dests_size;
+	u32				fwd_dests;
+	u32				index;
+	struct mlx5_flow_context	flow_context;
+	struct mlx5_flow_act		action;
+	enum fs_fte_status		status;
+	struct mlx5_fc			*counter;
+	int				modify_mask;
 };
 
+/* Type of children is mlx5_flow_table/namespace */
 struct fs_prio {
-	struct fs_base			base;
-	struct list_head		objs; /* each object is a namespace or ft */
-	unsigned int			max_ft;
-	unsigned int			num_ft;
-	unsigned int			max_ns;
+	struct fs_node			node;
+	unsigned int			num_levels;
+	unsigned int			start_level;
 	unsigned int			prio;
-	/*When create shared flow table, this lock should be taken*/
-	struct mutex		shared_lock;
-	u8				flags;
+	unsigned int			num_ft;
 };
 
+/* Type of children is fs_prio */
 struct mlx5_flow_namespace {
 	/* parent == NULL => root ns */
-	struct	fs_base			base;
-	/* sorted by priority number */
-	struct	list_head		prios; /* list of fs_prios */
-	struct  list_head		list_notifiers;
-	struct	rw_semaphore		notifiers_rw_sem;
-	struct  rw_semaphore		dests_rw_sem;
+	struct	fs_node			node;
+	enum mlx5_flow_table_miss_action def_miss_action;
+};
+
+struct mlx5_flow_group_mask {
+	u8	match_criteria_enable;
+	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
+};
+
+/* Type of children is fs_fte */
+struct mlx5_flow_group {
+	struct fs_node			node;
+	struct mlx5_flow_group_mask	mask;
+	u32				start_index;
+	u32				max_ftes;
+	struct ida			fte_allocator;
+	u32				id;
+	struct xarray			ftes_xa;
 };
 
 struct mlx5_flow_root_namespace {
 	struct mlx5_flow_namespace	ns;
-	struct mlx5_flow_table		*ft_level_0;
-	enum   fs_ft_type		table_type;
+	enum   mlx5_flow_steering_mode	mode;
+	enum   fs_flow_table_type	table_type;
 	struct mlx5_core_dev		*dev;
 	struct mlx5_flow_table		*root_ft;
-	/* When chaining flow-tables, this lock should be taken */
-	struct mutex		fs_chain_lock;
+	/* Should be held when chaining flow tables */
+	struct mutex			chain_lock;
+	struct list_head		underlay_qpns;
+	const struct mlx5_flow_cmds	*cmds;
 };
 
-struct mlx5_flow_group {
-	struct fs_base			base;
-	struct list_head		ftes;
-	struct mlx5_core_fs_mask	mask;
-	uint32_t			start_index;
-	uint32_t			max_ftes;
-	uint32_t			num_ftes;
-	uint32_t			id;
-};
+int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
+void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
+void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
+			      struct delayed_work *dwork,
+			      unsigned long delay);
+void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
+				      unsigned long interval);
 
-struct mlx5_flow_handler {
-	struct list_head list;
-	rule_event_fn add_dst_cb;
-	rule_event_fn del_dst_cb;
-	void *client_context;
-	struct mlx5_flow_namespace *ns;
-};
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
 
-struct fs_client_priv_data {
-	struct mlx5_flow_handler *fs_handler;
-	struct list_head list;
-	void   *client_dst_data;
-};
+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
+				 struct mlx5_flow_root_namespace *peer_ns);
 
-struct mlx5_modify_hdr {
-	enum mlx5_flow_namespace_type ns_type;
-	u32 id;
-};
+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
+				 enum mlx5_flow_steering_mode mode);
 
-struct mlx5_pkt_reformat {
-        enum mlx5_flow_namespace_type ns_type;
-        int reformat_type; /* from mlx5_ifc */
-	u32 id;
-};
+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
+void mlx5_fs_core_free(struct mlx5_core_dev *dev);
+int mlx5_fs_core_init(struct mlx5_core_dev *dev);
+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);
 
-void _fs_remove_node(struct kref *kref);
-#define fs_get_obj(v, _base)  {v = container_of((_base), typeof(*v), base); }
-#define fs_get_parent(v, child)  {v = (child)->base.parent ?		     \
-				  container_of((child)->base.parent,	     \
-					       typeof(*v), base) : NULL; }
+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);
 
-#define fs_list_for_each_entry(pos, cond, root)		\
-	list_for_each_entry(pos, root, base.list)	\
-		if (!(cond)) {} else
+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);
 
-#define fs_list_for_each_entry_continue(pos, cond, root)	\
-	list_for_each_entry_continue(pos, root, base.list)	\
-		if (!(cond)) {} else
+struct mlx5_flow_root_namespace *find_root(struct fs_node *node);
 
-#define fs_list_for_each_entry_reverse(pos, cond, root)		\
-	list_for_each_entry_reverse(pos, root, base.list)	\
-		if (!(cond)) {} else
+#define fs_get_obj(v, _node)  {v = container_of((_node), typeof(*v), node); }
 
-#define fs_list_for_each_entry_continue_reverse(pos, cond, root)	\
-	list_for_each_entry_continue_reverse(pos, root, base.list)	\
-		if (!(cond)) {} else
+#define fs_list_for_each_entry(pos, root)		\
+	list_for_each_entry(pos, root, node.list)
 
-#define fs_for_each_ft(pos, prio)			\
-	fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_TABLE, \
-			       &(prio)->objs)
+#define fs_list_for_each_entry_safe(pos, tmp, root)		\
+	list_for_each_entry_safe(pos, tmp, root, node.list)
 
-#define fs_for_each_ft_reverse(pos, prio)			\
-	fs_list_for_each_entry_reverse(pos,			\
-				       (pos)->base.type == FS_TYPE_FLOW_TABLE, \
-				       &(prio)->objs)
+#define fs_for_each_ns_or_ft_reverse(pos, prio)				\
+	list_for_each_entry_reverse(pos, &(prio)->node.children, list)
 
-#define fs_for_each_ns(pos, prio)			\
-	fs_list_for_each_entry(pos,			\
-			       (pos)->base.type == FS_TYPE_NAMESPACE, \
-			       &(prio)->objs)
-
-#define fs_for_each_ns_or_ft_reverse(pos, prio)			\
-	list_for_each_entry_reverse(pos, &(prio)->objs, list)		\
-		if (!((pos)->type == FS_TYPE_NAMESPACE ||		\
-		      (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft(pos, prio)			\
-	list_for_each_entry(pos, &(prio)->objs, list)		\
-		if (!((pos)->type == FS_TYPE_NAMESPACE ||	\
-		      (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft_continue_reverse(pos, prio)		\
-	list_for_each_entry_continue_reverse(pos, &(prio)->objs, list)	\
-		if (!((pos)->type == FS_TYPE_NAMESPACE ||		\
-		      (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft_continue(pos, prio)			\
-	list_for_each_entry_continue(pos, &(prio)->objs, list)		\
-		if (!((pos)->type == FS_TYPE_NAMESPACE ||		\
-		      (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
+#define fs_for_each_ns_or_ft(pos, prio)					\
+	list_for_each_entry(pos, (&(prio)->node.children), list)
 
 #define fs_for_each_prio(pos, ns)			\
-	fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_PRIO, \
-			       &(ns)->prios)
+	fs_list_for_each_entry(pos, &(ns)->node.children)
 
-#define fs_for_each_prio_reverse(pos, ns)			\
-	fs_list_for_each_entry_reverse(pos, (pos)->base.type == FS_TYPE_PRIO, \
-				       &(ns)->prios)
+#define fs_for_each_ns(pos, prio)			\
+	fs_list_for_each_entry(pos, &(prio)->node.children)
 
-#define fs_for_each_prio_continue(pos, ns)			\
-	fs_list_for_each_entry_continue(pos, (pos)->base.type == FS_TYPE_PRIO, \
-				       &(ns)->prios)
+#define fs_for_each_ft(pos, prio)			\
+	fs_list_for_each_entry(pos, &(prio)->node.children)
 
-#define fs_for_each_prio_continue_reverse(pos, ns)			\
-	fs_list_for_each_entry_continue_reverse(pos,			\
-						(pos)->base.type == FS_TYPE_PRIO, \
-						&(ns)->prios)
+#define fs_for_each_ft_safe(pos, tmp, prio)			\
+	fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
 
 #define fs_for_each_fg(pos, ft)			\
-	fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_GROUP, \
-			       &(ft)->fgs)
+	fs_list_for_each_entry(pos, &(ft)->node.children)
 
 #define fs_for_each_fte(pos, fg)			\
-	fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_ENTRY, \
-			       &(fg)->ftes)
+	fs_list_for_each_entry(pos, &(fg)->node.children)
+
 #define fs_for_each_dst(pos, fte)			\
-	fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_DEST, \
-			       &(fte)->dests)
-
-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
-			  u16 vport, enum fs_ft_type type, unsigned int level,
-			  unsigned int log_size, const char *name, unsigned int *table_id);
-
-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_ft_type type, unsigned int table_id);
-
-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
-			  u32 *in,
-			  u16 vport,
-			  enum fs_ft_type type, unsigned int table_id,
-			  unsigned int *group_id);
-
-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_ft_type type, unsigned int table_id,
-			   unsigned int group_id);
-
-
-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
-			u16 vport,
-			enum fs_fte_status *fte_status,
-			u32 *match_val,
-			enum fs_ft_type type, unsigned int table_id,
-			unsigned int index, unsigned int group_id,
-			struct mlx5_flow_act *flow_act,
-			u32 sw_action, int dest_size,
-			struct list_head *dests);  /* mlx5_flow_desination */
-
-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_fte_status *fte_status,
-			   enum fs_ft_type type, unsigned int table_id,
-			   unsigned int index);
-
-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
-			    enum fs_ft_type type,
-			    unsigned int id);
-
-int mlx5_init_fs(struct mlx5_core_dev *dev);
-void mlx5_cleanup_fs(struct mlx5_core_dev *dev);
-void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
-				      unsigned long interval);
+	fs_list_for_each_entry(pos, &(fte)->node.children)
+
+#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) (		\
+	(type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) :		\
+	(type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) :		\
+	(type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) :		\
+	(type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) :		\
+	(type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :		\
+	(type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) :		\
+	(type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) :		\
+	(type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) :		\
+	(type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) :      \
+	(type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) :      \
+	(BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\
+	)
 
-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
-				 enum mlx5_flow_namespace_type namespace,
-				 u8 num_actions,
-				 void *modify_actions,
-				 struct mlx5_modify_hdr *modify_hdr);
-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
-				    struct mlx5_modify_hdr *modify_hdr);
-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
-				   struct mlx5_pkt_reformat_params *params,
-				   enum mlx5_flow_namespace_type namespace,
-				   struct mlx5_pkt_reformat *pkt_reformat);
-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
-				      struct mlx5_pkt_reformat *pkt_reformat);
-int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
-void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
-void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
-			      struct delayed_work *dwork,
-			      unsigned long delay);
 #endif
diff --git a/sys/dev/mlx5/mlx5_core/fs_ft_pool.h b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
new file mode 100644
index 000000000000..b81e70d51bd6
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#ifndef __MLX5_FS_FT_POOL_H__
+#define __MLX5_FS_FT_POOL_H__
+
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <linux/compiler.h>
+
+#define POOL_NEXT_SIZE 0
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev);
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev);
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+			  int desired_size);
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz);
+
+#endif /* __MLX5_FS_FT_POOL_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/fs_tcp.h b/sys/dev/mlx5/mlx5_core/fs_tcp.h
index fa11ad9c4cb5..e2433af53a42 100644
--- a/sys/dev/mlx5/mlx5_core/fs_tcp.h
+++ b/sys/dev/mlx5/mlx5_core/fs_tcp.h
@@ -27,15 +27,15 @@
 #define	__MLX5E_ACCEL_FS_TCP_H__
 
 struct inpcb;
-struct mlx5_flow_rule;
+struct mlx5_flow_handle;
 struct mlx5e_priv;
 
 int	mlx5e_accel_fs_tcp_create(struct mlx5e_priv *);
 void	mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *);
-struct mlx5_flow_rule *
+struct mlx5_flow_handle *
 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *,
     struct inpcb *, uint32_t tirn, uint32_t flow_tag, uint16_t vlan_id);
 #define	MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN 0xFFFF
-void	mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *);
+void	mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *);
 
 #endif					/* __MLX5E_ACCEL_FS_TCP_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
index d46feb4b9e5b..8ce30bc24e50 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
@@ -614,6 +614,9 @@ const char *mlx5_command_str(int command)
 	MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJ);
 	MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJ);
 	MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJ);
+	MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
+	MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
+	MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
 	default: return "unknown command opcode";
 	}
 }
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h
index f0b1dde60323..f63bb2070bcf 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_core.h
+++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h
@@ -163,4 +163,14 @@ enum {
 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
 void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);
 
+/************************************************ TESTTEST********************************************/
+static inline int mlx5_init_fs(struct mlx5_core_dev *dev)
+{
+	return 0;
+}
+
+static inline int mlx5_cleanup_fs(struct mlx5_core_dev *dev)
+{
+	return 0;
+}
 #endif /* __MLX5_CORE_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_crypto.c b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
new file mode 100644
index 000000000000..03804219e0b3
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
@@ -0,0 +1,94 @@
+/*-
+ * Copyright (c) 2019-2021, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "opt_rss.h"
+#include "opt_ratelimit.h"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/crypto.h>
+
+int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn, u32 key_type,
+    const void *p_key, u32 key_len, u32 *p_obj_id)
+{
+	u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
+	u64 general_obj_types;
+	int err;
+
+	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
+		return -EINVAL;
+
+	switch (key_len) {
+	case 128 / 8:
+		memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
+		    encryption_key_object.key[4]), p_key, 128 / 8);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
+			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
+			 key_type);
+		break;
+	case 256 / 8:
+		memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
+		    encryption_key_object.key[0]), p_key, 256 / 8);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
+			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
+		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
+			 key_type);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
+	MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
+
+	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+	if (err == 0)
+		*p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
+
+	/* avoid leaking key on the stack */
+	explicit_bzero(in, sizeof(in));
+
+	return err;
+}
+
+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
+
+	MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
+	MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
+	MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
+
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
index 2d5b53b6482a..29c12e41650e 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
@@ -33,6 +33,7 @@
 #include <dev/mlx5/mlx5_fpga/core.h>
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
 #include <dev/mlx5/mlx5_core/eswitch.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
 
 #ifdef  RSS
 #include <net/rss_config.h>
@@ -165,6 +166,8 @@ static const char *eqe_type_str(u8 type)
 		return "MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT";
 	case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT:
 		return "MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT";
+	case MLX5_EVENT_TYPE_OBJECT_CHANGE:
+		return "MLX5_EVENT_TYPE_OBJECT_CHANGE";
 	default:
 		return "Unrecognized event";
 	}
@@ -370,6 +373,10 @@ static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 			mlx5_temp_warning_event(dev, eqe);
 			break;
 
+		case MLX5_EVENT_TYPE_OBJECT_CHANGE:
+			mlx5_object_change_event(dev, eqe);
+			break;
+
 		default:
 			mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
 				       eqe->type, eq->eqn);
@@ -571,6 +578,10 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
 		    MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT);
 	}
 
+	if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
+		async_event_mask |=
+			(1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
+
 	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
 				 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD);
 	if (err) {
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
index 15f5f0ff0336..a7bff68b68e6 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
@@ -64,7 +64,7 @@ struct esw_uc_addr {
 /* E-Switch MC FDB table hash node */
 struct esw_mc_addr { /* SRIOV only */
 	struct l2addr_node     node;
-	struct mlx5_flow_rule *uplink_rule; /* Forward to uplink rule */
+	struct mlx5_flow_handle *uplink_rule; /* Forward to uplink rule */
 	u32                    refcnt;
 };
 
@@ -73,7 +73,7 @@ struct vport_addr {
 	struct l2addr_node     node;
 	u8                     action;
 	u32                    vport;
-	struct mlx5_flow_rule *flow_rule; /* SRIOV only */
+	struct mlx5_flow_handle *flow_rule; /* SRIOV only */
 };
 
 enum {
@@ -215,59 +215,54 @@ static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
 }
 
 /* E-Switch FDB */
-static struct mlx5_flow_rule *
+static struct mlx5_flow_handle *
 esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
 {
-	int match_header = MLX5_MATCH_OUTER_HEADERS;
 	struct mlx5_flow_destination dest;
-	struct mlx5_flow_rule *flow_rule = NULL;
+	struct mlx5_flow_handle *flow_rule = NULL;
 	struct mlx5_flow_act flow_act = {};
-	u32 *match_v;
-	u32 *match_c;
+	struct mlx5_flow_spec *spec;
 	u8 *dmac_v;
 	u8 *dmac_c;
 
-	match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	if (!match_v || !match_c) {
-		printf("mlx5_core: WARN: ""FDB: Failed to alloc match parameters\n");
+	spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec) {
+		printf("mlx5_core: WARN: ""FDB: Failed to alloc flow spec\n");
 		goto out;
 	}
-	dmac_v = MLX5_ADDR_OF(fte_match_param, match_v,
+	dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 			      outer_headers.dmac_47_16);
-	dmac_c = MLX5_ADDR_OF(fte_match_param, match_c,
+	dmac_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 			      outer_headers.dmac_47_16);
 
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 	ether_addr_copy(dmac_v, mac);
 	/* Match criteria mask */
 	memset(dmac_c, 0xff, 6);
 
 	dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
-	dest.vport_num = vport;
+	dest.vport.num = vport;
 
 	esw_debug(esw->dev,
 		  "\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
 		  dmac_v, dmac_c, vport);
+	flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
 	flow_rule =
-		mlx5_add_flow_rule(esw->fdb_table.fdb,
-				   match_header,
-				   match_c,
-				   match_v,
-				   MLX5_FLOW_RULE_FWD_ACTION_DEST,
-				   &flow_act, &dest);
+		mlx5_add_flow_rules(esw->fdb_table.fdb, spec,
+				   &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(flow_rule)) {
 		printf("mlx5_core: WARN: ""FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n", dmac_v, dmac_c, vport, PTR_ERR(flow_rule));
 		flow_rule = NULL;
 	}
 out:
-	kfree(match_v);
-	kfree(match_c);
+	kfree(spec);
 	return flow_rule;
 }
 
 static int esw_create_fdb_table(struct mlx5_eswitch *esw)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_core_dev *dev = esw->dev;
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *fdb;
@@ -295,7 +290,9 @@ static int esw_create_fdb_table(struct mlx5_eswitch *esw)
 	/* (-2) Since MaorG said so .. */
 	table_size = BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size)) - 2;
 
-	fdb = mlx5_create_flow_table(root_ns, 0, "FDB", table_size);
+	ft_attr.prio = 0;
+	ft_attr.max_fte = table_size;
+	fdb = mlx5_create_flow_table(root_ns, &ft_attr);
 	if (IS_ERR_OR_NULL(fdb)) {
 		err = PTR_ERR(fdb);
 		esw_warn(dev, "Failed to create FDB Table err %d\n", err);
@@ -397,7 +394,7 @@ static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 
 	mlx5_mpfs_del_mac(esw->dev, esw_uc->table_index);
 
-	mlx5_del_flow_rule(&vaddr->flow_rule);
+	mlx5_del_flow_rules(&vaddr->flow_rule);
 
 	l2addr_hash_del(esw_uc);
 	return 0;
@@ -456,12 +453,12 @@ static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 		  vport, mac, vaddr->flow_rule, esw_mc->refcnt,
 		  esw_mc->uplink_rule);
 
-	mlx5_del_flow_rule(&vaddr->flow_rule);
+	mlx5_del_flow_rules(&vaddr->flow_rule);
 
 	if (--esw_mc->refcnt)
 		return 0;
 
-	mlx5_del_flow_rule(&esw_mc->uplink_rule);
+	mlx5_del_flow_rules(&esw_mc->uplink_rule);
 
 	l2addr_hash_del(esw_mc);
 	return 0;
@@ -602,13 +599,13 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw,
 					struct mlx5_vport *vport)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_flow_group *vlan_grp = NULL;
 	struct mlx5_flow_group *drop_grp = NULL;
 	struct mlx5_core_dev *dev = esw->dev;
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *acl;
 	void *match_criteria;
-	char table_name[32];
 	u32 *flow_group_in;
 	int table_size = 2;
 	int err = 0;
@@ -629,8 +626,8 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw,
 	if (!flow_group_in)
 		return;
 
-	snprintf(table_name, 32, "egress_%d", vport->vport);
-	acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
+	ft_attr.max_fte = table_size;
+	acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
 	if (IS_ERR_OR_NULL(acl)) {
 		err = PTR_ERR(acl);
 		esw_warn(dev, "Failed to create E-Switch vport[%d] egress flow Table, err(%d)\n",
@@ -678,8 +675,8 @@ out:
 static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw,
 					   struct mlx5_vport *vport)
 {
-	mlx5_del_flow_rule(&vport->egress.allowed_vlan);
-	mlx5_del_flow_rule(&vport->egress.drop_rule);
+	mlx5_del_flow_rules(&vport->egress.allowed_vlan);
+	mlx5_del_flow_rules(&vport->egress.drop_rule);
 }
 
 static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw,
@@ -703,12 +700,12 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw,
 					 struct mlx5_vport *vport)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_core_dev *dev = esw->dev;
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *acl;
 	struct mlx5_flow_group *g;
 	void *match_criteria;
-	char table_name[32];
 	u32 *flow_group_in;
 	int table_size = 1;
 	int err = 0;
@@ -729,8 +726,8 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw,
 	if (!flow_group_in)
 		return;
 
-	snprintf(table_name, 32, "ingress_%d", vport->vport);
-	acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
+	ft_attr.max_fte = table_size;
+	acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
 	if (IS_ERR_OR_NULL(acl)) {
 		err = PTR_ERR(acl);
 		esw_warn(dev, "Failed to create E-Switch vport[%d] ingress flow Table, err(%d)\n",
@@ -763,7 +760,7 @@ out:
 static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw,
 					    struct mlx5_vport *vport)
 {
-	mlx5_del_flow_rule(&vport->ingress.drop_rule);
+	mlx5_del_flow_rules(&vport->ingress.drop_rule);
 }
 
 static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw,
@@ -785,9 +782,7 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 				    struct mlx5_vport *vport)
 {
 	struct mlx5_flow_act flow_act = {};
-	struct mlx5_flow_destination dest;
-	u32 *match_v;
-	u32 *match_c;
+	struct mlx5_flow_spec *spec;
 	int err = 0;
 
 	if (IS_ERR_OR_NULL(vport->ingress.acl)) {
@@ -806,35 +801,28 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 		  "vport[%d] configure ingress rules, vlan(%d) qos(%d)\n",
 		  vport->vport, vport->vlan, vport->qos);
 
-	match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	if (!match_v || !match_c) {
+	spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec) {
 		err = -ENOMEM;
 		esw_warn(esw->dev, "vport[%d] configure ingress rules failed, err(%d)\n",
 			 vport->vport, err);
 		goto out;
 	}
-	MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
-	MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
-
-	dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
-	dest.vport_num = vport->vport;
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
 
+	flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 	vport->ingress.drop_rule =
-		mlx5_add_flow_rule(vport->ingress.acl,
-				   MLX5_MATCH_OUTER_HEADERS,
-				   match_c,
-				   match_v,
-				   MLX5_FLOW_RULE_FWD_ACTION_DROP,
-				   &flow_act, &dest);
+		mlx5_add_flow_rules(vport->ingress.acl, spec,
+				    &flow_act, NULL, 0);
 	if (IS_ERR_OR_NULL(vport->ingress.drop_rule)) {
 		err = PTR_ERR(vport->ingress.drop_rule);
 		printf("mlx5_core: WARN: ""vport[%d] configure ingress rules, err(%d)\n", vport->vport, err);
 		vport->ingress.drop_rule = NULL;
 	}
 out:
-	kfree(match_v);
-	kfree(match_c);
+	kfree(spec);
 	return err;
 }
 
@@ -842,9 +830,7 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 				   struct mlx5_vport *vport)
 {
 	struct mlx5_flow_act flow_act = {};
-	struct mlx5_flow_destination dest;
-	u32 *match_v;
-	u32 *match_c;
+	struct mlx5_flow_spec *spec;
 	int err = 0;
 
 	if (IS_ERR_OR_NULL(vport->egress.acl)) {
@@ -862,9 +848,8 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 		  "vport[%d] configure egress rules, vlan(%d) qos(%d)\n",
 		  vport->vport, vport->vlan, vport->qos);
 
-	match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	if (!match_v || !match_c) {
+	spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec) {
 		err = -ENOMEM;
 		esw_warn(esw->dev, "vport[%d] configure egress rules failed, err(%d)\n",
 			 vport->vport, err);
@@ -872,21 +857,17 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 	}
 
 	/* Allowed vlan rule */
-	MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
-	MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
-	MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.first_vid);
-	MLX5_SET(fte_match_param, match_v, outer_headers.first_vid, vport->vlan);
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->vlan);
 
-	dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
-	dest.vport_num = vport->vport;
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_ALLOW;
 
 	vport->egress.allowed_vlan =
-		mlx5_add_flow_rule(vport->egress.acl,
-				   MLX5_MATCH_OUTER_HEADERS,
-				   match_c,
-				   match_v,
-				   MLX5_FLOW_RULE_FWD_ACTION_ALLOW,
-				   &flow_act, &dest);
+		mlx5_add_flow_rules(vport->egress.acl, spec,
+				    &flow_act, NULL, 0);
 	if (IS_ERR_OR_NULL(vport->egress.allowed_vlan)) {
 		err = PTR_ERR(vport->egress.allowed_vlan);
 		printf("mlx5_core: WARN: ""vport[%d] configure egress allowed vlan rule failed, err(%d)\n", vport->vport, err);
@@ -894,24 +875,17 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 		goto out;
 	}
 
-	/* Drop others rule (star rule) */
-	memset(match_c, 0, MLX5_ST_SZ_BYTES(fte_match_param));
-	memset(match_v, 0, MLX5_ST_SZ_BYTES(fte_match_param));
+	flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
 	vport->egress.drop_rule =
-		mlx5_add_flow_rule(vport->egress.acl,
-				   0,
-				   match_c,
-				   match_v,
-				   MLX5_FLOW_RULE_FWD_ACTION_DROP,
-				   &flow_act, &dest);
+		mlx5_add_flow_rules(vport->egress.acl, NULL,
+				   &flow_act, NULL, 0);
 	if (IS_ERR_OR_NULL(vport->egress.drop_rule)) {
 		err = PTR_ERR(vport->egress.drop_rule);
 		printf("mlx5_core: WARN: ""vport[%d] configure egress drop rule failed, err(%d)\n", vport->vport, err);
 		vport->egress.drop_rule = NULL;
 	}
 out:
-	kfree(match_v);
-	kfree(match_c);
+	kfree(spec);
 	return err;
 }
 
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
deleted file mode 100644
index f3410249e67f..000000000000
--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*-
- * Copyright (c) 2022 NVIDIA corporation & affiliates.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <dev/mlx5/driver.h>
-#include <dev/mlx5/device.h>
-#include <dev/mlx5/mlx5_ifc.h>
-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
-
-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
-			   enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
-			   u32 *id)
-{
-	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
-	int err;
-
-	MLX5_SET(alloc_flow_counter_in, in, opcode,
-		 MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
-	MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
-
-	err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
-	if (!err)
-		*id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
-	return err;
-}
-
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
-{
-	return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
-}
-
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
-{
-	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
-
-	MLX5_SET(dealloc_flow_counter_in, in, opcode,
-		 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
-	MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
-	return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
-}
-
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
-		      u64 *packets, u64 *bytes)
-{
-	u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
-		MLX5_ST_SZ_BYTES(traffic_counter)] = {};
-	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
-	void *stats;
-	int err = 0;
-
-	MLX5_SET(query_flow_counter_in, in, opcode,
-		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
-	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
-	MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		return err;
-
-	stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
-	*packets = MLX5_GET64(traffic_counter, stats, packets);
-	*bytes = MLX5_GET64(traffic_counter, stats, octets);
-	return 0;
-}
-
-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
-			   u32 *out)
-{
-	int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
-	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
-
-	MLX5_SET(query_flow_counter_in, in, opcode,
-		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
-	MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
-	MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
-}
-
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
deleted file mode 100644
index 3adebb3ca94c..000000000000
--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _MLX5_FC_CMD_
-#define _MLX5_FC_CMD_
-
-#include "fs_core.h"
-
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id);
-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
-			   enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
-			   u32 *id);
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
-		      u64 *packets, u64 *bytes);
-
-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
-			   u32 *out);
-static inline int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
-{
-	return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
-		MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
-}
-
-#endif
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
new file mode 100644
index 000000000000..21c1914fd864
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
@@ -0,0 +1,664 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2020 Mellanox Technologies.
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/fs.h>
+
+#include "mlx5_core.h"
+#include "fs_chains.h"
+#include "fs_ft_pool.h"
+#include "fs_core.h"
+
+#define chains_lock(chains) ((chains)->lock)
+#define chains_xa(chains) ((chains)->chains_xa)
+#define prios_xa(chains) ((chains)->prios_xa)
+#define chains_default_ft(chains) ((chains)->chains_default_ft)
+#define chains_end_ft(chains) ((chains)->chains_end_ft)
+#define FT_TBL_SZ (64 * 1024)
+
+struct mlx5_fs_chains {
+	struct mlx5_core_dev *dev;
+
+	struct xarray chains_xa;
+	struct xarray prios_xa;
+	/* Protects above chains_ht and prios_ht */
+	struct mutex lock;
+
+	struct mlx5_flow_table *chains_default_ft;
+	struct mlx5_flow_table *chains_end_ft;
+
+	enum mlx5_flow_namespace_type ns;
+	u32 group_num;
+	u32 flags;
+	int fs_base_prio;
+	int fs_base_level;
+};
+
+struct fs_chain {
+	u32 chain;
+
+	int ref;
+	int id;
+	uint32_t xa_idx;
+
+	struct mlx5_fs_chains *chains;
+	struct list_head prios_list;
+	struct mlx5_flow_handle *restore_rule;
+	struct mlx5_modify_hdr *miss_modify_hdr;
+};
+
+struct prio_key {
+	u32 chain;
+	u32 prio;
+	u32 level;
+};
+
+struct prio {
+	struct list_head list;
+
+	struct prio_key key;
+	uint32_t xa_idx;
+
+	int ref;
+
+	struct fs_chain *chain;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_table *next_ft;
+	struct mlx5_flow_group *miss_group;
+	struct mlx5_flow_handle *miss_rule;
+};
+
+/*
+static const struct rhashtable_params chain_params = {
+	.head_offset = offsetof(struct fs_chain, node),
+	.key_offset = offsetof(struct fs_chain, chain),
+	.key_len = sizeof_field(struct fs_chain, chain),
+	.automatic_shrinking = true,
+};
+
+static const struct rhashtable_params prio_params = {
+	.head_offset = offsetof(struct prio, node),
+	.key_offset = offsetof(struct prio, key),
+	.key_len = sizeof_field(struct prio, key),
+	.automatic_shrinking = true,
+};
+*/
+
+bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
+{
+	return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
+}
+
+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
+{
+	return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
+}
+
+bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
+{
+	return mlx5_chains_prios_supported(chains) &&
+	       mlx5_chains_ignore_flow_level_supported(chains);
+}
+
+u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
+{
+	if (!mlx5_chains_prios_supported(chains))
+		return 1;
+
+	if (mlx5_chains_ignore_flow_level_supported(chains))
+		return UINT_MAX - 1;
+
+	/* We should get here only for eswitch case */
+	return FDB_TC_MAX_CHAIN;
+}
+
+u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
+{
+	return mlx5_chains_get_chain_range(chains) + 1;
+}
+
+u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
+{
+	if (mlx5_chains_ignore_flow_level_supported(chains))
+		return UINT_MAX;
+
+	if (!chains->dev->priv.eswitch)
+		return 1;
+
+	/* We should get here only for eswitch case */
+	return FDB_TC_MAX_PRIO;
+}
+
+static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
+{
+	if (mlx5_chains_ignore_flow_level_supported(chains))
+		return UINT_MAX;
+
+	/* Same value for FDB and NIC RX tables */
+	return FDB_TC_LEVELS_PER_PRIO;
+}
+
+void
+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
+		       struct mlx5_flow_table *ft)
+{
+	chains_end_ft(chains) = ft;
+}
+
+static struct mlx5_flow_table *
+mlx5_chains_create_table(struct mlx5_fs_chains *chains,
+			 u32 chain, u32 prio, u32 level)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_namespace *ns;
+	struct mlx5_flow_table *ft;
+	int sz;
+
+	if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
+		ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
+				  MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+
+	sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
+	ft_attr.max_fte = sz;
+
+	/* We use chains_default_ft(chains) as the table's next_ft till
+	 * ignore_flow_level is allowed on FT creation and not just for FTEs.
+	 * Instead caller should add an explicit miss rule if needed.
+	 */
+	ft_attr.next_ft = chains_default_ft(chains);
+
+	/* The root table(chain 0, prio 1, level 0) is required to be
+	 * connected to the previous fs_core managed prio.
+	 * We always create it, as a managed table, in order to align with
+	 * fs_core logic.
+	 */
+	if (!mlx5_chains_ignore_flow_level_supported(chains) ||
+	    (chain == 0 && prio == 1 && level == 0)) {
+		ft_attr.level = chains->fs_base_level;
+		ft_attr.prio = chains->fs_base_prio;
+		ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
+			mlx5_get_fdb_sub_ns(chains->dev, chain) :
+			mlx5_get_flow_namespace(chains->dev, chains->ns);
+	} else {
+		ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
+		ft_attr.prio = chains->fs_base_prio;
+		/* Firmware doesn't allow us to create another level 0 table,
+		 * so we create all unmanaged tables as level 1 (base + 1).
+		 *
+		 * To connect them, we use explicit miss rules with
+		 * ignore_flow_level. Caller is responsible to create
+		 * these rules (if needed).
+		 */
+		ft_attr.level = chains->fs_base_level + 1;
+		ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
+	}
+
+	ft_attr.autogroup.num_reserved_entries = 2;
+	ft_attr.autogroup.max_num_groups = chains->group_num;
+	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+	if (IS_ERR(ft)) {
+		mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
+			       (int)PTR_ERR(ft), chain, prio, level, sz);
+		return ft;
+	}
+
+	return ft;
+}
+
+static struct fs_chain *
+mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
+{
+	struct fs_chain *chain_s = NULL;
+	int err;
+
+	chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
+	if (!chain_s)
+		return ERR_PTR(-ENOMEM);
+
+	chain_s->chains = chains;
+	chain_s->chain = chain;
+	INIT_LIST_HEAD(&chain_s->prios_list);
+
+	err = xa_alloc(&chains_xa(chains), &chain_s->xa_idx, chain_s,
+		       xa_limit_32b, GFP_KERNEL);
+	if (err)
+		goto err_insert;
+
+	return chain_s;
+
+err_insert:
+	kvfree(chain_s);
+	return ERR_PTR(err);
+}
+
+static void
+mlx5_chains_destroy_chain(struct fs_chain *chain)
+{
+	struct mlx5_fs_chains *chains = chain->chains;
+
+	xa_erase(&chains_xa(chains), chain->xa_idx);
+	kvfree(chain);
+}
+
+static struct fs_chain *
+mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
+{
+	struct fs_chain *chain_s = NULL;
+	unsigned long idx;
+
+	xa_for_each(&chains_xa(chains), idx, chain_s) {
+		if (chain_s->chain == chain)
+			break;
+	}
+
+	if (!chain_s) {
+		chain_s = mlx5_chains_create_chain(chains, chain);
+		if (IS_ERR(chain_s))
+			return chain_s;
+	}
+
+	chain_s->ref++;
+
+	return chain_s;
+}
+
+static struct mlx5_flow_handle *
+mlx5_chains_add_miss_rule(struct fs_chain *chain,
+			  struct mlx5_flow_table *ft,
+			  struct mlx5_flow_table *next_ft)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_act act = {};
+
+	act.flags  = FLOW_ACT_NO_APPEND;
+	if (mlx5_chains_ignore_flow_level_supported(chain->chains))
+		act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
+
+	act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	dest.type  = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft = next_ft;
+
+	return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
+}
+
+static int
+mlx5_chains_update_prio_prevs(struct prio *prio,
+			      struct mlx5_flow_table *next_ft)
+{
+	struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
+	struct fs_chain *chain = prio->chain;
+	struct prio *pos;
+	int n = 0, err;
+
+	if (prio->key.level)
+		return 0;
+
+	/* Iterate in reverse order until reaching the level 0 rule of
+	 * the previous priority, adding all the miss rules first, so we can
+	 * revert them if any of them fails.
+	 */
+	pos = prio;
+	list_for_each_entry_continue_reverse(pos,
+					     &chain->prios_list,
+					     list) {
+		miss_rules[n] = mlx5_chains_add_miss_rule(chain,
+							  pos->ft,
+							  next_ft);
+		if (IS_ERR(miss_rules[n])) {
+			err = PTR_ERR(miss_rules[n]);
+			goto err_prev_rule;
+		}
+
+		n++;
+		if (!pos->key.level)
+			break;
+	}
+
+	/* Success, delete old miss rules, and update the pointers. */
+	n = 0;
+	pos = prio;
+	list_for_each_entry_continue_reverse(pos,
+					     &chain->prios_list,
+					     list) {
+		mlx5_del_flow_rules(&pos->miss_rule);
+
+		pos->miss_rule = miss_rules[n];
+		pos->next_ft = next_ft;
+
+		n++;
+		if (!pos->key.level)
+			break;
+	}
+
+	return 0;
+
+err_prev_rule:
+	while (--n >= 0)
+		mlx5_del_flow_rules(&miss_rules[n]);
+
+	return err;
+}
+
+static void
+mlx5_chains_put_chain(struct fs_chain *chain)
+{
+	if (--chain->ref == 0)
+		mlx5_chains_destroy_chain(chain);
+}
+
+static struct prio *
+mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
+			u32 chain, u32 prio, u32 level)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_handle *miss_rule;
+	struct mlx5_flow_group *miss_group;
+	struct mlx5_flow_table *next_ft;
+	struct mlx5_flow_table *ft;
+	struct fs_chain *chain_s;
+	struct list_head *pos;
+	struct prio *prio_s;
+	u32 *flow_group_in;
+	int err;
+
+	chain_s = mlx5_chains_get_chain(chains, chain);
+	if (IS_ERR(chain_s))
+		return ERR_CAST(chain_s);
+
+	prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!prio_s || !flow_group_in) {
+		err = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Chain's prio list is sorted by prio and level.
+	 * And all levels of some prio point to the next prio's level 0.
+	 * Example list (prio, level):
+	 * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
+	 * In hardware, we will we have the following pointers:
+	 * (3,0) -> (5,0) -> (7,0) -> Slow path
+	 * (3,1) -> (5,0)
+	 * (5,1) -> (7,0)
+	 * (6,1) -> (7,0)
+	 */
+
+	/* Default miss for each chain: */
+	next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
+		  chains_default_ft(chains) :
+		  chains_end_ft(chains);
+	list_for_each(pos, &chain_s->prios_list) {
+		struct prio *p = list_entry(pos, struct prio, list);
+
+		/* exit on first pos that is larger */
+		if (prio < p->key.prio || (prio == p->key.prio &&
+					   level < p->key.level)) {
+			/* Get next level 0 table */
+			next_ft = p->key.level == 0 ? p->ft : p->next_ft;
+			break;
+		}
+	}
+
+	ft = mlx5_chains_create_table(chains, chain, prio, level);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_create;
+	}
+
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
+		 ft->max_fte - 2);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
+		 ft->max_fte - 1);
+	miss_group = mlx5_create_flow_group(ft, flow_group_in);
+	if (IS_ERR(miss_group)) {
+		err = PTR_ERR(miss_group);
+		goto err_group;
+	}
+
+	/* Add miss rule to next_ft */
+	miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
+	if (IS_ERR(miss_rule)) {
+		err = PTR_ERR(miss_rule);
+		goto err_miss_rule;
+	}
+
+	prio_s->miss_group = miss_group;
+	prio_s->miss_rule = miss_rule;
+	prio_s->next_ft = next_ft;
+	prio_s->chain = chain_s;
+	prio_s->key.chain = chain;
+	prio_s->key.prio = prio;
+	prio_s->key.level = level;
+	prio_s->ft = ft;
+
+	err = xa_alloc(&prios_xa(chains), &prio_s->xa_idx, prio_s,
+		       xa_limit_32b, GFP_KERNEL);
+	if (err)
+		goto err_insert;
+
+	list_add(&prio_s->list, pos->prev);
+
+	/* Table is ready, connect it */
+	err = mlx5_chains_update_prio_prevs(prio_s, ft);
+	if (err)
+		goto err_update;
+
+	kvfree(flow_group_in);
+	return prio_s;
+
+err_update:
+	list_del(&prio_s->list);
+	xa_erase(&prios_xa(chains), prio_s->xa_idx);
+err_insert:
+	mlx5_del_flow_rules(&miss_rule);
+err_miss_rule:
+	mlx5_destroy_flow_group(miss_group);
+err_group:
+	mlx5_destroy_flow_table(ft);
+err_create:
+err_alloc:
+	kvfree(prio_s);
+	kvfree(flow_group_in);
+	mlx5_chains_put_chain(chain_s);
+	return ERR_PTR(err);
+}
+
+static void
+mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
+			 struct prio *prio)
+{
+	struct fs_chain *chain = prio->chain;
+
+	WARN_ON(mlx5_chains_update_prio_prevs(prio,
+					      prio->next_ft));
+
+	list_del(&prio->list);
+	xa_erase(&prios_xa(chains), prio->xa_idx);
+	mlx5_del_flow_rules(&prio->miss_rule);
+	mlx5_destroy_flow_group(prio->miss_group);
+	mlx5_destroy_flow_table(prio->ft);
+	mlx5_chains_put_chain(chain);
+	kvfree(prio);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+		      u32 level)
+{
+	struct mlx5_flow_table *prev_fts;
+	struct prio *prio_s;
+	unsigned long idx;
+	int l = 0;
+
+	if ((chain > mlx5_chains_get_chain_range(chains) &&
+	     chain != mlx5_chains_get_nf_ft_chain(chains)) ||
+	    prio > mlx5_chains_get_prio_range(chains) ||
+	    level > mlx5_chains_get_level_range(chains))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	/* create earlier levels for correct fs_core lookup when
+	 * connecting tables.
+	 */
+	for (l = 0; l < level; l++) {
+		prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
+		if (IS_ERR(prev_fts)) {
+			prio_s = ERR_CAST(prev_fts);
+			goto err_get_prevs;
+		}
+	}
+
+	mutex_lock(&chains_lock(chains));
+	xa_for_each(&prios_xa(chains), idx, prio_s) {
+		if (chain == prio_s->key.chain &&
+		    prio == prio_s->key.prio &&
+		    level == prio_s->key.level)
+			break;
+	}
+	if (!prio_s) {
+		prio_s = mlx5_chains_create_prio(chains, chain,
+						 prio, level);
+		if (IS_ERR(prio_s))
+			goto err_create_prio;
+	}
+
+	++prio_s->ref;
+	mutex_unlock(&chains_lock(chains));
+
+	return prio_s->ft;
+
+err_create_prio:
+	mutex_unlock(&chains_lock(chains));
+err_get_prevs:
+	while (--l >= 0)
+		mlx5_chains_put_table(chains, chain, prio, l);
+	return ERR_CAST(prio_s);
+}
+
+void
+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+		      u32 level)
+{
+	struct prio *prio_s;
+	unsigned long idx;
+
+	mutex_lock(&chains_lock(chains));
+
+	xa_for_each(&prios_xa(chains), idx, prio_s) {
+		if (chain == prio_s->key.chain &&
+		    prio == prio_s->key.prio &&
+		    level == prio_s->key.level)
+			break;
+	}
+	if (!prio_s)
+		goto err_get_prio;
+
+	if (--prio_s->ref == 0)
+		mlx5_chains_destroy_prio(chains, prio_s);
+	mutex_unlock(&chains_lock(chains));
+
+	while (level-- > 0)
+		mlx5_chains_put_table(chains, chain, prio, level);
+
+	return;
+
+err_get_prio:
+	mutex_unlock(&chains_lock(chains));
+	WARN_ONCE(1,
+		  "Couldn't find table: (chain: %d prio: %d level: %d)",
+		  chain, prio, level);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
+{
+	return chains_end_ft(chains);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
+{
+	u32 chain, prio, level;
+	int err;
+
+	if (!mlx5_chains_ignore_flow_level_supported(chains)) {
+		err = -EOPNOTSUPP;
+
+		mlx5_core_warn(chains->dev,
+			       "Couldn't create global flow table, ignore_flow_level not supported.");
+		goto err_ignore;
+	}
+
+	chain = mlx5_chains_get_chain_range(chains),
+	prio = mlx5_chains_get_prio_range(chains);
+	level = mlx5_chains_get_level_range(chains);
+
+	return mlx5_chains_create_table(chains, chain, prio, level);
+
+err_ignore:
+	return ERR_PTR(err);
+}
+
+void
+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
+				 struct mlx5_flow_table *ft)
+{
+	mlx5_destroy_flow_table(ft);
+}
+
+static struct mlx5_fs_chains *
+mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
+{
+	struct mlx5_fs_chains *chains;
+
+	chains = kzalloc(sizeof(*chains), GFP_KERNEL);
+	if (!chains)
+		return ERR_PTR(-ENOMEM);
+
+	chains->dev = dev;
+	chains->flags = attr->flags;
+	chains->ns = attr->ns;
+	chains->group_num = attr->max_grp_num;
+	chains->fs_base_prio = attr->fs_base_prio;
+	chains->fs_base_level = attr->fs_base_level;
+	chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft;
+
+	xa_init(&chains_xa(chains));
+	xa_init(&prios_xa(chains));
+
+	mutex_init(&chains_lock(chains));
+
+	return chains;
+}
+
+static void
+mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
+{
+	mutex_destroy(&chains_lock(chains));
+	xa_destroy(&prios_xa(chains));
+	xa_destroy(&chains_xa(chains));
+
+	kfree(chains);
+}
+
+struct mlx5_fs_chains *
+mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
+{
+	struct mlx5_fs_chains *chains;
+
+	chains = mlx5_chains_init(dev, attr);
+
+	return chains;
+}
+
+void
+mlx5_chains_destroy(struct mlx5_fs_chains *chains)
+{
+	mlx5_chains_cleanup(chains);
+}
+
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains)
+{
+	mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
index 0f827f0e69d3..b3c118fedc9b 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
@@ -1,468 +1,993 @@
-/*-
- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd.  All rights reserved.
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
  *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
  *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 
-#include "opt_rss.h"
-#include "opt_ratelimit.h"
-
-#include <linux/types.h>
-#include <linux/module.h>
-#include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/driver.h>
 #include <dev/mlx5/device.h>
-#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_ifc.h>
+
+#include "fs_core.h"
+#include "fs_cmd.h"
+#include "fs_ft_pool.h"
+#include "mlx5_core.h"
+#include "eswitch.h"
 
-#include <dev/mlx5/mlx5_core/fs_core.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
+static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns,
+					struct mlx5_flow_table *ft,
+					u32 underlay_qpn,
+					bool disconnect)
+{
+	return 0;
+}
 
-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
-			    enum fs_ft_type type,
-			    unsigned int id)
+static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns,
+					   struct mlx5_flow_table *ft,
+					   struct mlx5_flow_table_attr *ft_attr,
+					   struct mlx5_flow_table *next_ft)
 {
-	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0};
+	int max_fte = ft_attr->max_fte;
 
-	if (!dev)
-		return -EINVAL;
+	ft->max_fte = max_fte ? roundup_pow_of_two(max_fte) : 1;
+
+	return 0;
+}
+
+static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
+					    struct mlx5_flow_table *ft)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns,
+					   struct mlx5_flow_table *ft,
+					   struct mlx5_flow_table *next_ft)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns,
+					   struct mlx5_flow_table *ft,
+					   u32 *in,
+					   struct mlx5_flow_group *fg)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
+					    struct mlx5_flow_table *ft,
+					    struct mlx5_flow_group *fg)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns,
+				    struct mlx5_flow_table *ft,
+				    struct mlx5_flow_group *group,
+				    struct fs_fte *fte)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns,
+				    struct mlx5_flow_table *ft,
+				    struct mlx5_flow_group *group,
+				    int modify_mask,
+				    struct fs_fte *fte)
+{
+	return -EOPNOTSUPP;
+}
+
+static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns,
+				    struct mlx5_flow_table *ft,
+				    struct fs_fte *fte)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
+					       struct mlx5_pkt_reformat_params *params,
+					       enum mlx5_flow_namespace_type namespace,
+					       struct mlx5_pkt_reformat *pkt_reformat)
+{
+	return 0;
+}
+
+static void mlx5_cmd_stub_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
+						  struct mlx5_pkt_reformat *pkt_reformat)
+{
+}
+
+static int mlx5_cmd_stub_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
+					     u8 namespace, u8 num_actions,
+					     void *modify_actions,
+					     struct mlx5_modify_hdr *modify_hdr)
+{
+	return 0;
+}
+
+static void mlx5_cmd_stub_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
+						struct mlx5_modify_hdr *modify_hdr)
+{
+}
+
+static int mlx5_cmd_stub_set_peer(struct mlx5_flow_root_namespace *ns,
+				  struct mlx5_flow_root_namespace *peer_ns)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_create_ns(struct mlx5_flow_root_namespace *ns)
+{
+	return 0;
+}
+
+static int mlx5_cmd_stub_destroy_ns(struct mlx5_flow_root_namespace *ns)
+{
+	return 0;
+}
+
+static u32 mlx5_cmd_stub_get_capabilities(struct mlx5_flow_root_namespace *ns,
+					  enum fs_flow_table_type ft_type)
+{
+	return 0;
+}
+
+static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns,
+				   struct mlx5_flow_table *ft, u32 underlay_qpn,
+				   bool disconnect)
+{
+	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+
+	if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
+	    underlay_qpn == 0)
+		return 0;
 
 	MLX5_SET(set_flow_table_root_in, in, opcode,
 		 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
-	MLX5_SET(set_flow_table_root_in, in, table_type, type);
-	MLX5_SET(set_flow_table_root_in, in, table_id, id);
+	MLX5_SET(set_flow_table_root_in, in, table_type, ft->type);
+
+	if (disconnect)
+		MLX5_SET(set_flow_table_root_in, in, op_mod, 1);
+	else
+		MLX5_SET(set_flow_table_root_in, in, table_id, ft->id);
 
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn);
+	MLX5_SET(set_flow_table_root_in, in, vport_number, ft->vport);
+	MLX5_SET(set_flow_table_root_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+	return mlx5_cmd_exec_in(dev, set_flow_table_root, in);
 }
 
-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
-			  u16 vport, enum fs_ft_type type, unsigned int level,
-			  unsigned int log_size, const char *name, unsigned int *table_id)
+static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
+				      struct mlx5_flow_table *ft,
+				      struct mlx5_flow_table_attr *ft_attr,
+				      struct mlx5_flow_table *next_ft)
 {
-	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
+	int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
+	int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+	int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION);
+	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+	unsigned int size;
 	int err;
 
-	if (!dev)
-		return -EINVAL;
+	size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte);
+	if (!size)
+		return -ENOSPC;
 
 	MLX5_SET(create_flow_table_in, in, opcode,
 		 MLX5_CMD_OP_CREATE_FLOW_TABLE);
 
-	MLX5_SET(create_flow_table_in, in, table_type, type);
-	MLX5_SET(create_flow_table_in, in, flow_table_context.level, level);
-	MLX5_SET(create_flow_table_in, in, flow_table_context.log_size,
-		 log_size);
-	if (strstr(name, FS_REFORMAT_KEYWORD) != NULL)
-		MLX5_SET(create_flow_table_in, in,
-			 flow_table_context.reformat_en, 1);
-	if (vport) {
-		MLX5_SET(create_flow_table_in, in, vport_number, vport);
-		MLX5_SET(create_flow_table_in, in, other_vport, 1);
+	MLX5_SET(create_flow_table_in, in, uid, ft_attr->uid);
+	MLX5_SET(create_flow_table_in, in, table_type, ft->type);
+	MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level);
+	MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0);
+	MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
+	MLX5_SET(create_flow_table_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+	MLX5_SET(create_flow_table_in, in, flow_table_context.decap_en,
+		 en_decap);
+	MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en,
+		 en_encap);
+	MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table,
+		 term);
+
+	switch (ft->op_mod) {
+	case FS_FT_OP_MOD_NORMAL:
+		if (next_ft) {
+			MLX5_SET(create_flow_table_in, in,
+				 flow_table_context.table_miss_action,
+				 MLX5_FLOW_TABLE_MISS_ACTION_FWD);
+			MLX5_SET(create_flow_table_in, in,
+				 flow_table_context.table_miss_id, next_ft->id);
+		} else {
+			MLX5_SET(create_flow_table_in, in,
+				 flow_table_context.table_miss_action,
+				 ft->def_miss_action);
+		}
+		break;
+
+	case FS_FT_OP_MOD_LAG_DEMUX:
+		MLX5_SET(create_flow_table_in, in, op_mod, 0x1);
+		if (next_ft)
+			MLX5_SET(create_flow_table_in, in,
+				 flow_table_context.lag_master_next_table_id,
+				 next_ft->id);
+		break;
 	}
 
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (!err)
-		*table_id = MLX5_GET(create_flow_table_out, out, table_id);
+	err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out);
+	if (!err) {
+		ft->id = MLX5_GET(create_flow_table_out, out,
+				  table_id);
+		ft->max_fte = size;
+	} else {
+		mlx5_ft_pool_put_sz(ns->dev, size);
+	}
 
 	return err;
 }
 
-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_ft_type type, unsigned int table_id)
+static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
+				       struct mlx5_flow_table *ft)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0};
-
-	if (!dev)
-		return -EINVAL;
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+	int err;
 
 	MLX5_SET(destroy_flow_table_in, in, opcode,
 		 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
-	MLX5_SET(destroy_flow_table_in, in, table_type, type);
-	MLX5_SET(destroy_flow_table_in, in, table_id, table_id);
-	if (vport) {
-		MLX5_SET(destroy_flow_table_in, in, vport_number, vport);
-		MLX5_SET(destroy_flow_table_in, in, other_vport, 1);
+	MLX5_SET(destroy_flow_table_in, in, table_type, ft->type);
+	MLX5_SET(destroy_flow_table_in, in, table_id, ft->id);
+	MLX5_SET(destroy_flow_table_in, in, vport_number, ft->vport);
+	MLX5_SET(destroy_flow_table_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+	err = mlx5_cmd_exec_in(dev, destroy_flow_table, in);
+	if (!err)
+		mlx5_ft_pool_put_sz(ns->dev, ft->max_fte);
+
+	return err;
+}
+
+static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns,
+				      struct mlx5_flow_table *ft,
+				      struct mlx5_flow_table *next_ft)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+
+	MLX5_SET(modify_flow_table_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_FLOW_TABLE);
+	MLX5_SET(modify_flow_table_in, in, table_type, ft->type);
+	MLX5_SET(modify_flow_table_in, in, table_id, ft->id);
+
+	if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) {
+		MLX5_SET(modify_flow_table_in, in, modify_field_select,
+			 MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID);
+		if (next_ft) {
+			MLX5_SET(modify_flow_table_in, in,
+				 flow_table_context.lag_master_next_table_id, next_ft->id);
+		} else {
+			MLX5_SET(modify_flow_table_in, in,
+				 flow_table_context.lag_master_next_table_id, 0);
+		}
+	} else {
+		MLX5_SET(modify_flow_table_in, in, vport_number, ft->vport);
+		MLX5_SET(modify_flow_table_in, in, other_vport,
+			 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+		MLX5_SET(modify_flow_table_in, in, modify_field_select,
+			 MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
+		if (next_ft) {
+			MLX5_SET(modify_flow_table_in, in,
+				 flow_table_context.table_miss_action,
+				 MLX5_FLOW_TABLE_MISS_ACTION_FWD);
+			MLX5_SET(modify_flow_table_in, in,
+				 flow_table_context.table_miss_id,
+				 next_ft->id);
+		} else {
+			MLX5_SET(modify_flow_table_in, in,
+				 flow_table_context.table_miss_action,
+				 ft->def_miss_action);
+		}
 	}
 
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev, modify_flow_table, in);
 }
 
-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
-			  u32 *in,
-			  u16 vport,
-			  enum fs_ft_type type, unsigned int table_id,
-			  unsigned int *group_id)
+static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns,
+				      struct mlx5_flow_table *ft,
+				      u32 *in,
+				      struct mlx5_flow_group *fg)
 {
-	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0};
+	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
 	int err;
-	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
-	if (!dev)
-		return -EINVAL;
 
 	MLX5_SET(create_flow_group_in, in, opcode,
 		 MLX5_CMD_OP_CREATE_FLOW_GROUP);
-	MLX5_SET(create_flow_group_in, in, table_type, type);
-	MLX5_SET(create_flow_group_in, in, table_id, table_id);
-	if (vport) {
-		MLX5_SET(create_flow_group_in, in, vport_number, vport);
-		MLX5_SET(create_flow_group_in, in, other_vport, 1);
-	}
-
-	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+	MLX5_SET(create_flow_group_in, in, table_type, ft->type);
+	MLX5_SET(create_flow_group_in, in, table_id, ft->id);
+	MLX5_SET(create_flow_group_in, in, vport_number, ft->vport);
+	MLX5_SET(create_flow_group_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+	err = mlx5_cmd_exec_inout(dev, create_flow_group, in, out);
 	if (!err)
-		*group_id = MLX5_GET(create_flow_group_out, out, group_id);
-
+		fg->id = MLX5_GET(create_flow_group_out, out,
+				  group_id);
 	return err;
 }
 
-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_ft_type type, unsigned int table_id,
-			   unsigned int group_id)
+static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
+				       struct mlx5_flow_table *ft,
+				       struct mlx5_flow_group *fg)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0};
-
-	if (!dev)
-		return -EINVAL;
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
 
 	MLX5_SET(destroy_flow_group_in, in, opcode,
 		 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
-	MLX5_SET(destroy_flow_group_in, in, table_type, type);
-	MLX5_SET(destroy_flow_group_in, in, table_id,   table_id);
-	MLX5_SET(destroy_flow_group_in, in, group_id, group_id);
-	if (vport) {
-		MLX5_SET(destroy_flow_group_in, in, vport_number, vport);
-		MLX5_SET(destroy_flow_group_in, in, other_vport, 1);
-	}
-
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	MLX5_SET(destroy_flow_group_in, in, table_type, ft->type);
+	MLX5_SET(destroy_flow_group_in, in, table_id, ft->id);
+	MLX5_SET(destroy_flow_group_in, in, group_id, fg->id);
+	MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport);
+	MLX5_SET(destroy_flow_group_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+	return mlx5_cmd_exec_in(dev, destroy_flow_group, in);
 }
 
-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
-			u16 vport,
-			enum fs_fte_status *fte_status,
-			u32 *match_val,
-			enum fs_ft_type type, unsigned int table_id,
-			unsigned int index, unsigned int group_id,
-			struct mlx5_flow_act *flow_act,
-			u32 sw_action, int dest_size,
-			struct list_head *dests)  /* mlx5_flow_desination */
+static int mlx5_set_extended_dest(struct mlx5_core_dev *dev,
+				  struct fs_fte *fte, bool *extended_dest)
 {
-	u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
-	u32 *in;
-	unsigned int inlen;
+	int fw_log_max_fdb_encap_uplink =
+		MLX5_CAP_ESW(dev, log_max_fdb_encap_uplink);
+	int num_fwd_destinations = 0;
 	struct mlx5_flow_rule *dst;
-	void *in_flow_context;
-	void *in_match_value;
-	void *in_dests;
-	int err;
-	int opmod = 0;
-	int modify_mask = 0;
-	int atomic_mod_cap;
-	u32 prm_action = 0;
-	int count_list = 0;
+	int num_encap = 0;
 
-	if (sw_action != MLX5_FLOW_RULE_FWD_ACTION_DEST)
-		dest_size = 0;
+	*extended_dest = false;
+	if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+		return 0;
 
-	if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_ALLOW)
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+	list_for_each_entry(dst, &fte->node.children, node.list) {
+		if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER ||
+		    dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_NONE)
+			continue;
+		if ((dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+		     dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
+		    dst->dest_attr.vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)
+			num_encap++;
+		num_fwd_destinations++;
+	}
+	if (num_fwd_destinations > 1 && num_encap > 0)
+		*extended_dest = true;
 
-	if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DROP)
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+	if (*extended_dest && !fw_log_max_fdb_encap_uplink) {
+		mlx5_core_warn(dev, "FW does not support extended destination");
+		return -EOPNOTSUPP;
+	}
+	if (num_encap > (1 << fw_log_max_fdb_encap_uplink)) {
+		mlx5_core_warn(dev, "FW does not support more than %d encaps",
+			       1 << fw_log_max_fdb_encap_uplink);
+		return -EOPNOTSUPP;
+	}
 
-	if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DEST)
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	return 0;
+}
 
-	if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
-		count_list = 1;
-	}
+static void
+mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context)
+{
+	void *exe_aso_ctrl;
+	void *execute_aso;
+
+	execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context,
+				   execute_aso[0]);
+	MLX5_SET(execute_aso, execute_aso, valid, 1);
+	MLX5_SET(execute_aso, execute_aso, aso_object_id,
+		 fte->action.exe_aso.object_id);
+
+	exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl);
+	MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id,
+		 fte->action.exe_aso.return_reg_id);
+	MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type,
+		 fte->action.exe_aso.type);
+	MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color,
+		 fte->action.exe_aso.flow_meter.init_color);
+	MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id,
+		 fte->action.exe_aso.flow_meter.meter_idx);
+}
 
-	inlen = MLX5_ST_SZ_BYTES(set_fte_in) +
-		(dest_size + count_list) * MLX5_ST_SZ_BYTES(dest_format_struct);
+static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
+			    int opmod, int modify_mask,
+			    struct mlx5_flow_table *ft,
+			    unsigned group_id,
+			    struct fs_fte *fte)
+{
+	u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
+	bool extended_dest = false;
+	struct mlx5_flow_rule *dst;
+	void *in_flow_context, *vlan;
+	void *in_match_value;
+	unsigned int inlen;
+	int dst_cnt_size;
+	void *in_dests;
+	u32 *in;
+	int err;
 
-	if (!dev)
-		return -EINVAL;
+	if (mlx5_set_extended_dest(dev, fte, &extended_dest))
+		return -EOPNOTSUPP;
 
-	if (*fte_status & FS_FTE_STATUS_EXISTING) {
-		atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
-						    flow_table_properties_nic_receive.
-						    flow_modify_en);
-		if (!atomic_mod_cap)
-			return -ENOTSUPP;
-		opmod = 1;
-		modify_mask = 1 <<
-			MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST;
-	}
+	if (!extended_dest)
+		dst_cnt_size = MLX5_ST_SZ_BYTES(dest_format_struct);
+	else
+		dst_cnt_size = MLX5_ST_SZ_BYTES(extended_dest_format);
 
-	in = mlx5_vzalloc(inlen);
-	if (!in) {
-		mlx5_core_warn(dev, "failed to allocate inbox\n");
+	inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fte->dests_size * dst_cnt_size;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
 		return -ENOMEM;
-	}
 
 	MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
 	MLX5_SET(set_fte_in, in, op_mod, opmod);
 	MLX5_SET(set_fte_in, in, modify_enable_mask, modify_mask);
-	MLX5_SET(set_fte_in, in, table_type, type);
-	MLX5_SET(set_fte_in, in, table_id,   table_id);
-	MLX5_SET(set_fte_in, in, flow_index, index);
-	if (vport) {
-		MLX5_SET(set_fte_in, in, vport_number, vport);
-		MLX5_SET(set_fte_in, in, other_vport, 1);
-	}
+	MLX5_SET(set_fte_in, in, table_type, ft->type);
+	MLX5_SET(set_fte_in, in, table_id,   ft->id);
+	MLX5_SET(set_fte_in, in, flow_index, fte->index);
+	MLX5_SET(set_fte_in, in, ignore_flow_level,
+		 !!(fte->action.flags & FLOW_ACT_IGNORE_FLOW_LEVEL));
+
+	MLX5_SET(set_fte_in, in, vport_number, ft->vport);
+	MLX5_SET(set_fte_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
 
 	in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
 	MLX5_SET(flow_context, in_flow_context, group_id, group_id);
-	if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG)
-		MLX5_SET(flow_context, in_flow_context, flow_tag, flow_act->flow_tag);
-	if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) {
-		MLX5_SET(flow_context, in_flow_context, modify_header_id,
-			 flow_act->modify_hdr->id);
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
-	}
-	if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) {
-		MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
-			 flow_act->pkt_reformat->id);
-		prm_action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+
+	MLX5_SET(flow_context, in_flow_context, flow_tag,
+		 fte->flow_context.flow_tag);
+	MLX5_SET(flow_context, in_flow_context, flow_source,
+		 fte->flow_context.flow_source);
+
+	MLX5_SET(flow_context, in_flow_context, extended_destination,
+		 extended_dest);
+	if (extended_dest) {
+		u32 action;
+
+		action = fte->action.action &
+			~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+		MLX5_SET(flow_context, in_flow_context, action, action);
+	} else {
+		MLX5_SET(flow_context, in_flow_context, action,
+			 fte->action.action);
+		if (fte->action.pkt_reformat)
+			MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
+				 fte->action.pkt_reformat->id);
 	}
-	MLX5_SET(flow_context, in_flow_context, destination_list_size,
-		 dest_size);
+	if (fte->action.modify_hdr)
+		MLX5_SET(flow_context, in_flow_context, modify_header_id,
+			 fte->action.modify_hdr->id);
+
+	MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type,
+		 fte->action.crypto.type);
+	MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id,
+		 fte->action.crypto.obj_id);
+
+	vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan);
+
+	MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype);
+	MLX5_SET(vlan, vlan, vid, fte->action.vlan[0].vid);
+	MLX5_SET(vlan, vlan, prio, fte->action.vlan[0].prio);
+
+	vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan_2);
+
+	MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[1].ethtype);
+	MLX5_SET(vlan, vlan, vid, fte->action.vlan[1].vid);
+	MLX5_SET(vlan, vlan, prio, fte->action.vlan[1].prio);
+
 	in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context,
 				      match_value);
-	memcpy(in_match_value, match_val, MLX5_ST_SZ_BYTES(fte_match_param));
+	memcpy(in_match_value, &fte->val, sizeof(fte->val));
 
 	in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination);
+	if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
+		int list_size = 0;
 
-	if (dest_size) {
-		list_for_each_entry(dst, dests, base.list) {
+		list_for_each_entry(dst, &fte->node.children, node.list) {
+			enum mlx5_flow_destination_type type = dst->dest_attr.type;
+			enum mlx5_ifc_flow_destination_type ifc_type;
 			unsigned int id;
 
-			MLX5_SET(dest_format_struct, in_dests, destination_type,
-				 dst->dest_attr.type);
-			if (dst->dest_attr.type ==
-				MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE)
+			if (type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
+				continue;
+
+			switch (type) {
+			case MLX5_FLOW_DESTINATION_TYPE_NONE:
+				continue;
+			case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM:
+				id = dst->dest_attr.ft_num;
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+				break;
+			case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
 				id = dst->dest_attr.ft->id;
-			else
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+				break;
+			case MLX5_FLOW_DESTINATION_TYPE_UPLINK:
+			case MLX5_FLOW_DESTINATION_TYPE_VPORT:
+				MLX5_SET(dest_format_struct, in_dests,
+					 destination_eswitch_owner_vhca_id_valid,
+					 !!(dst->dest_attr.vport.flags &
+					    MLX5_FLOW_DEST_VPORT_VHCA_ID));
+				MLX5_SET(dest_format_struct, in_dests,
+					 destination_eswitch_owner_vhca_id,
+					 dst->dest_attr.vport.vhca_id);
+				if (type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) {
+					/* destination_id is reserved */
+					id = 0;
+					ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK;
+					break;
+				}
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT;
+				id = dst->dest_attr.vport.num;
+				if (extended_dest &&
+				    dst->dest_attr.vport.pkt_reformat) {
+					MLX5_SET(dest_format_struct, in_dests,
+						 packet_reformat,
+						 !!(dst->dest_attr.vport.flags &
+						    MLX5_FLOW_DEST_VPORT_REFORMAT_ID));
+					MLX5_SET(extended_dest_format, in_dests,
+						 packet_reformat_id,
+						 dst->dest_attr.vport.pkt_reformat->id);
+				}
+				break;
+			case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER:
+				id = dst->dest_attr.sampler_id;
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
+				break;
+			case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE:
+				MLX5_SET(dest_format_struct, in_dests,
+					 destination_table_type, dst->dest_attr.ft->type);
+				id = dst->dest_attr.ft->id;
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+				break;
+			default:
 				id = dst->dest_attr.tir_num;
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR;
+			}
+
+			MLX5_SET(dest_format_struct, in_dests, destination_type,
+				 ifc_type);
 			MLX5_SET(dest_format_struct, in_dests, destination_id, id);
-			in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
+			in_dests += dst_cnt_size;
+			list_size++;
 		}
+
+		MLX5_SET(flow_context, in_flow_context, destination_list_size,
+			 list_size);
 	}
 
-	if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
-		MLX5_SET(dest_format_struct, in_dests, destination_id,
-			 mlx5_fc_id(flow_act->counter));
-		in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
-		MLX5_SET(flow_context, in_flow_context, flow_counter_list_size, 1);
+	if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
+		int max_list_size = BIT(MLX5_CAP_FLOWTABLE_TYPE(dev,
+					log_max_flow_counter,
+					ft->type));
+		int list_size = 0;
+
+		list_for_each_entry(dst, &fte->node.children, node.list) {
+			if (dst->dest_attr.type !=
+			    MLX5_FLOW_DESTINATION_TYPE_COUNTER)
+				continue;
+
+			MLX5_SET(flow_counter_list, in_dests, flow_counter_id,
+				 dst->dest_attr.counter_id);
+			in_dests += dst_cnt_size;
+			list_size++;
+		}
+		if (list_size > max_list_size) {
+			err = -EINVAL;
+			goto err_out;
+		}
+
+		MLX5_SET(flow_context, in_flow_context, flow_counter_list_size,
+			 list_size);
 	}
 
-	MLX5_SET(flow_context, in_flow_context, action, prm_action);
-	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
-	if (!err)
-		*fte_status |= FS_FTE_STATUS_EXISTING;
+	if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) {
+		if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) {
+			mlx5_cmd_set_fte_flow_meter(fte, in_flow_context);
+		} else {
+			err = -EOPNOTSUPP;
+			goto err_out;
+		}
+	}
 
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+err_out:
 	kvfree(in);
+	return err;
+}
+
+static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns,
+			       struct mlx5_flow_table *ft,
+			       struct mlx5_flow_group *group,
+			       struct fs_fte *fte)
+{
+	struct mlx5_core_dev *dev = ns->dev;
+	unsigned int group_id = group->id;
+
+	return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
+}
+
+static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns,
+			       struct mlx5_flow_table *ft,
+			       struct mlx5_flow_group *fg,
+			       int modify_mask,
+			       struct fs_fte *fte)
+{
+	int opmod;
+	struct mlx5_core_dev *dev = ns->dev;
+	int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
+						flow_table_properties_nic_receive.
+						flow_modify_en);
+	if (!atomic_mod_cap)
+		return -EOPNOTSUPP;
+	opmod = 1;
+
+	return	mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte);
+}
+
+static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns,
+			       struct mlx5_flow_table *ft,
+			       struct fs_fte *fte)
+{
+	u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
 
+	MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
+	MLX5_SET(delete_fte_in, in, table_type, ft->type);
+	MLX5_SET(delete_fte_in, in, table_id, ft->id);
+	MLX5_SET(delete_fte_in, in, flow_index, fte->index);
+	MLX5_SET(delete_fte_in, in, vport_number, ft->vport);
+	MLX5_SET(delete_fte_in, in, other_vport,
+		 !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+	return mlx5_cmd_exec_in(dev, delete_fte, in);
+}
+
+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
+			   enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
+			   u32 *id)
+{
+	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
+	int err;
+
+	MLX5_SET(alloc_flow_counter_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
+	MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
+
+	err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
+	if (!err)
+		*id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
 	return err;
 }
 
-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
-			   u16 vport,
-			   enum fs_fte_status *fte_status,
-			   enum fs_ft_type type, unsigned int table_id,
-			   unsigned int index)
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
+{
+	return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
+}
+
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
+
+	MLX5_SET(dealloc_flow_counter_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
+	MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
+	return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
+}
+
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
+		      u64 *packets, u64 *bytes)
+{
+	u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
+		MLX5_ST_SZ_BYTES(traffic_counter)] = {};
+	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
+	void *stats;
+	int err = 0;
+
+	MLX5_SET(query_flow_counter_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
+	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
+	MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
+	*packets = MLX5_GET64(traffic_counter, stats, packets);
+	*bytes = MLX5_GET64(traffic_counter, stats, octets);
+	return 0;
+}
+
+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
 {
-	u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0};
+	return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
+		MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
+}
+
+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
+			   u32 *out)
+{
+	int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
+	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
+
+	MLX5_SET(query_flow_counter_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
+	MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
+	MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
+}
+
+static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
+					  struct mlx5_pkt_reformat_params *params,
+					  enum mlx5_flow_namespace_type namespace,
+					  struct mlx5_pkt_reformat *pkt_reformat)
+{
+	u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+	void *packet_reformat_context_in;
+	int max_encap_size;
+	void *reformat;
+	int inlen;
 	int err;
+	u32 *in;
 
-	if (!(*fte_status & FS_FTE_STATUS_EXISTING))
-		return 0;
+	if (namespace == MLX5_FLOW_NAMESPACE_FDB ||
+	    namespace == MLX5_FLOW_NAMESPACE_FDB_BYPASS)
+		max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
+	else
+		max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);
 
-	if (!dev)
+	if (params->size > max_encap_size) {
+		mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
+			       params->size, max_encap_size);
 		return -EINVAL;
+	}
 
-	MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
-	MLX5_SET(delete_fte_in, in, table_type, type);
-	MLX5_SET(delete_fte_in, in, table_id, table_id);
-	MLX5_SET(delete_fte_in, in, flow_index, index);
-	if (vport) {
-		MLX5_SET(delete_fte_in, in, vport_number,  vport);
-		MLX5_SET(delete_fte_in, in, other_vport, 1);
+	in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
+		     params->size, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
+						  in, packet_reformat_context);
+	reformat = MLX5_ADDR_OF(packet_reformat_context_in,
+				packet_reformat_context_in,
+				reformat_data);
+	inlen = reformat - (void *)in + params->size;
+
+	MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
+	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+		 reformat_data_size, params->size);
+	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+		 reformat_type, params->type);
+	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+		 reformat_param_0, params->param_0);
+	MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+		 reformat_param_1, params->param_1);
+	if (params->data && params->size)
+		memcpy(reformat, params->data, params->size);
+
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+
+	pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
+				    out, packet_reformat_id);
+	kfree(in);
+	return err;
+}
+
+static void mlx5_cmd_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
+					     struct mlx5_pkt_reformat *pkt_reformat)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+
+	MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
+	MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
+		 pkt_reformat->id);
+
+	mlx5_cmd_exec_in(dev, dealloc_packet_reformat_context, in);
+}
+
+static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
+					u8 namespace, u8 num_actions,
+					void *modify_actions,
+					struct mlx5_modify_hdr *modify_hdr)
+{
+	u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
+	int max_actions, actions_size, inlen, err;
+	struct mlx5_core_dev *dev = ns->dev;
+	void *actions_in;
+	u8 table_type;
+	u32 *in;
+
+	switch (namespace) {
+	case MLX5_FLOW_NAMESPACE_FDB:
+	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
+		max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
+		table_type = FS_FT_FDB;
+		break;
+	case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
+	case MLX5_FLOW_NAMESPACE_KERNEL:
+	case MLX5_FLOW_NAMESPACE_BYPASS:
+		max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
+		table_type = FS_FT_NIC_RX;
+		break;
+	case MLX5_FLOW_NAMESPACE_EGRESS:
+	case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+	case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
+		max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions);
+		table_type = FS_FT_NIC_TX;
+		break;
+	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
+		max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
+		table_type = FS_FT_ESW_INGRESS_ACL;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_TX:
+		max_actions = MLX5_CAP_FLOWTABLE_RDMA_TX(dev, max_modify_header_actions);
+		table_type = FS_FT_RDMA_TX;
+		break;
+	default:
+		return -EOPNOTSUPP;
 	}
 
-	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (!err)
-		*fte_status = 0;
+	if (num_actions > max_actions) {
+		mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
+			       num_actions, max_actions);
+		return -EOPNOTSUPP;
+	}
+
+	actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
+	inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
+
+	in = kzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(alloc_modify_header_context_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
+	MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
+	MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
+
+	actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
+	memcpy(actions_in, modify_actions, actions_size);
 
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+
+	modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
+	kfree(in);
 	return err;
 }
 
-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
-				 enum mlx5_flow_namespace_type namespace,
-				 u8 num_actions,
-				 void *modify_actions,
-				 struct mlx5_modify_hdr *modify_hdr)
-{
-        u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
-        int max_actions, actions_size, inlen, err;
-        void *actions_in;
-        u8 table_type;
-        u32 *in;
-
-        switch (namespace) {
-        case MLX5_FLOW_NAMESPACE_FDB:
-                max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
-                table_type = FS_FT_FDB;
-                break;
-        case MLX5_FLOW_NAMESPACE_KERNEL:
-        case MLX5_FLOW_NAMESPACE_BYPASS:
-                max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
-                table_type = FS_FT_NIC_RX;
-                break;
-        case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
-                max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
-                table_type = FS_FT_ESW_INGRESS_ACL;
-                break;
-        default:
-                return -EOPNOTSUPP;
-        }
-
-        if (num_actions > max_actions) {
-                mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
-                               num_actions, max_actions);
-                return -EOPNOTSUPP;
-        }
-
-        actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
-        inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
-
-        in = kzalloc(inlen, GFP_KERNEL);
-        if (!in)
-                return -ENOMEM;
-
-        MLX5_SET(alloc_modify_header_context_in, in, opcode,
-                 MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
-        MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
-        MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
-
-        actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
-        memcpy(actions_in, modify_actions, actions_size);
-
-        err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
-
-        modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
-        kfree(in);
-
-        return err;
-}
-
-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
-				    struct mlx5_modify_hdr *modify_hdr)
-{
-	u32 out[MLX5_ST_SZ_DW(dealloc_modify_header_context_out)] = {};
-        u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
-
-        MLX5_SET(dealloc_modify_header_context_in, in, opcode,
-                 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
-        MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
-                 modify_hdr->id);
-
-        mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-}
-
-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
-				   struct mlx5_pkt_reformat_params *params,
-				   enum mlx5_flow_namespace_type namespace,
-				   struct mlx5_pkt_reformat *pkt_reformat)
-{
-        u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
-        void *packet_reformat_context_in;
-        int max_encap_size;
-        void *reformat;
-        int inlen;
-        int err;
-        u32 *in;
-
-        if (namespace == MLX5_FLOW_NAMESPACE_FDB)
-                max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
-        else
-                max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);
-
-        if (params->size > max_encap_size) {
-                mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
-                               params->size, max_encap_size);
-                return -EINVAL;
-        }
-
-        in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
-                     params->size, GFP_KERNEL);
-        if (!in)
-                return -ENOMEM;
-
-        packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
-                                                  in, packet_reformat_context);
-        reformat = MLX5_ADDR_OF(packet_reformat_context_in,
-                                packet_reformat_context_in,
-                                reformat_data);
-        inlen = reformat - (void *)in + params->size;
-
-        MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
-                 MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
-        MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
-                 reformat_data_size, params->size);
-        MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
-                 reformat_type, params->type);
-        MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
-                 reformat_param_0, params->param_0);
-        MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
-                 reformat_param_1, params->param_1);
-        if (params->data && params->size)
-                memcpy(reformat, params->data, params->size);
-
-        err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
-
-        pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
-                                    out, packet_reformat_id);
-        kfree(in);
+static void mlx5_cmd_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
+					   struct mlx5_modify_hdr *modify_hdr)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
+	struct mlx5_core_dev *dev = ns->dev;
+
+	MLX5_SET(dealloc_modify_header_context_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
+	MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
+		 modify_hdr->id);
 
-        return err;
+	mlx5_cmd_exec_in(dev, dealloc_modify_header_context, in);
 }
 
-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
-				      struct mlx5_pkt_reformat *pkt_reformat)
+static u32 mlx5_cmd_get_capabilities(struct mlx5_flow_root_namespace *ns,
+				     enum fs_flow_table_type ft_type)
 {
-        u32 out[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
+	return 0;
+}
 
-        MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
-                 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
-        MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
-                 pkt_reformat->id);
+static const struct mlx5_flow_cmds mlx5_flow_cmds = {
+	.create_flow_table = mlx5_cmd_create_flow_table,
+	.destroy_flow_table = mlx5_cmd_destroy_flow_table,
+	.modify_flow_table = mlx5_cmd_modify_flow_table,
+	.create_flow_group = mlx5_cmd_create_flow_group,
+	.destroy_flow_group = mlx5_cmd_destroy_flow_group,
+	.create_fte = mlx5_cmd_create_fte,
+	.update_fte = mlx5_cmd_update_fte,
+	.delete_fte = mlx5_cmd_delete_fte,
+	.update_root_ft = mlx5_cmd_update_root_ft,
+	.packet_reformat_alloc = mlx5_cmd_packet_reformat_alloc,
+	.packet_reformat_dealloc = mlx5_cmd_packet_reformat_dealloc,
+	.modify_header_alloc = mlx5_cmd_modify_header_alloc,
+	.modify_header_dealloc = mlx5_cmd_modify_header_dealloc,
+	.set_peer = mlx5_cmd_stub_set_peer,
+	.create_ns = mlx5_cmd_stub_create_ns,
+	.destroy_ns = mlx5_cmd_stub_destroy_ns,
+	.get_capabilities = mlx5_cmd_get_capabilities,
+};
+
+static const struct mlx5_flow_cmds mlx5_flow_cmd_stubs = {
+	.create_flow_table = mlx5_cmd_stub_create_flow_table,
+	.destroy_flow_table = mlx5_cmd_stub_destroy_flow_table,
+	.modify_flow_table = mlx5_cmd_stub_modify_flow_table,
+	.create_flow_group = mlx5_cmd_stub_create_flow_group,
+	.destroy_flow_group = mlx5_cmd_stub_destroy_flow_group,
+	.create_fte = mlx5_cmd_stub_create_fte,
+	.update_fte = mlx5_cmd_stub_update_fte,
+	.delete_fte = mlx5_cmd_stub_delete_fte,
+	.update_root_ft = mlx5_cmd_stub_update_root_ft,
+	.packet_reformat_alloc = mlx5_cmd_stub_packet_reformat_alloc,
+	.packet_reformat_dealloc = mlx5_cmd_stub_packet_reformat_dealloc,
+	.modify_header_alloc = mlx5_cmd_stub_modify_header_alloc,
+	.modify_header_dealloc = mlx5_cmd_stub_modify_header_dealloc,
+	.set_peer = mlx5_cmd_stub_set_peer,
+	.create_ns = mlx5_cmd_stub_create_ns,
+	.destroy_ns = mlx5_cmd_stub_destroy_ns,
+	.get_capabilities = mlx5_cmd_stub_get_capabilities,
+};
+
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void)
+{
+	return &mlx5_flow_cmds;
+}
+
+static const struct mlx5_flow_cmds *mlx5_fs_cmd_get_stub_cmds(void)
+{
+	return &mlx5_flow_cmd_stubs;
+}
 
-        mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type)
+{
+	switch (type) {
+	case FS_FT_NIC_RX:
+	case FS_FT_ESW_EGRESS_ACL:
+	case FS_FT_ESW_INGRESS_ACL:
+	case FS_FT_FDB:
+	case FS_FT_SNIFFER_RX:
+	case FS_FT_SNIFFER_TX:
+	case FS_FT_NIC_TX:
+	case FS_FT_RDMA_RX:
+	case FS_FT_RDMA_TX:
+	case FS_FT_PORT_SEL:
+		return mlx5_fs_cmd_get_fw_cmds();
+	default:
+		return mlx5_fs_cmd_get_stub_cmds();
+	}
 }
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
new file mode 100644
index 000000000000..1a058e0bf711
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
@@ -0,0 +1,3514 @@
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <linux/string.h>
+#include <linux/compiler.h>
+
+#include "eswitch.h"
+#include "fs_ft_pool.h"
+#include "fs_cmd.h"
+
+#define down_write_nested(a, b) down_write(a)
+
+#define INIT_TREE_NODE_ARRAY_SIZE(...)	(sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
+					 sizeof(struct init_tree_node))
+
+#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
+		 ...) {.type = FS_TYPE_PRIO,\
+	.min_ft_level = min_level_val,\
+	.num_levels = num_levels_val,\
+	.num_leaf_prios = num_prios_val,\
+	.caps = caps_val,\
+	.children = (struct init_tree_node[]) {__VA_ARGS__},\
+	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
+}
+
+#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
+	ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
+		 __VA_ARGS__)\
+
+#define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE,	\
+	.def_miss_action = def_miss_act,\
+	.children = (struct init_tree_node[]) {__VA_ARGS__},\
+	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
+}
+
+#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
+				   sizeof(long))
+
+#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
+
+#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
+			       .caps = (long[]) {__VA_ARGS__} }
+
+#define FS_CHAINING_CAPS  FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
+					   FS_CAP(flow_table_properties_nic_receive.modify_root), \
+					   FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
+					   FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
+
+#define FS_CHAINING_CAPS_EGRESS                                                \
+	FS_REQUIRED_CAPS(                                                      \
+		FS_CAP(flow_table_properties_nic_transmit.flow_modify_en),     \
+		FS_CAP(flow_table_properties_nic_transmit.modify_root),        \
+		FS_CAP(flow_table_properties_nic_transmit                      \
+			       .identified_miss_table_mode),                   \
+		FS_CAP(flow_table_properties_nic_transmit.flow_table_modify))
+
+#define FS_CHAINING_CAPS_RDMA_TX                                                \
+	FS_REQUIRED_CAPS(                                                       \
+		FS_CAP(flow_table_properties_nic_transmit_rdma.flow_modify_en), \
+		FS_CAP(flow_table_properties_nic_transmit_rdma.modify_root),    \
+		FS_CAP(flow_table_properties_nic_transmit_rdma                  \
+			       .identified_miss_table_mode),                    \
+		FS_CAP(flow_table_properties_nic_transmit_rdma                  \
+			       .flow_table_modify))
+
+#define LEFTOVERS_NUM_LEVELS 1
+#define LEFTOVERS_NUM_PRIOS 1
+
+#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS 1
+#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS 1
+
+#define BY_PASS_PRIO_NUM_LEVELS 1
+#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
+			   LEFTOVERS_NUM_PRIOS)
+
+#define KERNEL_RX_MACSEC_NUM_PRIOS  1
+#define KERNEL_RX_MACSEC_NUM_LEVELS 2
+#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS)
+
+#define ETHTOOL_PRIO_NUM_LEVELS 1
+#define ETHTOOL_NUM_PRIOS 11
+#define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
+/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy,
+ * IPsec RoCE policy
+ */
+#define KERNEL_NIC_PRIO_NUM_LEVELS 9
+#define KERNEL_NIC_NUM_PRIOS 1
+/* One more level for tc */
+#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
+
+#define KERNEL_NIC_TC_NUM_PRIOS  1
+#define KERNEL_NIC_TC_NUM_LEVELS 13
+
+#define ANCHOR_NUM_LEVELS 1
+#define ANCHOR_NUM_PRIOS 1
+#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
+
+#define OFFLOADS_MAX_FT 2
+#define OFFLOADS_NUM_PRIOS 1
+#define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + OFFLOADS_NUM_PRIOS)
+
+#define LAG_PRIO_NUM_LEVELS 1
+#define LAG_NUM_PRIOS 1
+#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
+
+#define KERNEL_TX_IPSEC_NUM_PRIOS  1
+#define KERNEL_TX_IPSEC_NUM_LEVELS 3
+#define KERNEL_TX_IPSEC_MIN_LEVEL        (KERNEL_TX_IPSEC_NUM_LEVELS)
+
+#define KERNEL_TX_MACSEC_NUM_PRIOS  1
+#define KERNEL_TX_MACSEC_NUM_LEVELS 2
+#define KERNEL_TX_MACSEC_MIN_LEVEL       (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS)
+
+struct node_caps {
+	size_t	arr_sz;
+	long	*caps;
+};
+
+static struct init_tree_node {
+	enum fs_node_type	type;
+	struct init_tree_node *children;
+	int ar_size;
+	struct node_caps caps;
+	int min_ft_level;
+	int num_leaf_prios;
+	int prio;
+	int num_levels;
+	enum mlx5_flow_table_miss_action def_miss_action;
+} root_fs = {
+	.type = FS_TYPE_NAMESPACE,
+	.ar_size = 8,
+	  .children = (struct init_tree_node[]){
+		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
+						    BY_PASS_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS,
+						    KERNEL_RX_MACSEC_NUM_LEVELS))),
+		  ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
+						    LAG_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS,
+						    OFFLOADS_MAX_FT))),
+		  ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
+						    ETHTOOL_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS,
+						    KERNEL_NIC_TC_NUM_LEVELS),
+				  ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
+						    KERNEL_NIC_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS,
+						    LEFTOVERS_NUM_LEVELS))),
+		  ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {},
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS,
+						    ANCHOR_NUM_LEVELS))),
+	}
+};
+
+static struct init_tree_node egress_root_fs = {
+	.type = FS_TYPE_NAMESPACE,
+	.ar_size = 3,
+	.children = (struct init_tree_node[]) {
+		ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0,
+			 FS_CHAINING_CAPS_EGRESS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
+						  BY_PASS_PRIO_NUM_LEVELS))),
+		ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS_EGRESS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS,
+						  KERNEL_TX_IPSEC_NUM_LEVELS))),
+		ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS_EGRESS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS,
+						  KERNEL_TX_MACSEC_NUM_LEVELS))),
+	}
+};
+
+enum {
+	RDMA_RX_IPSEC_PRIO,
+	RDMA_RX_COUNTERS_PRIO,
+	RDMA_RX_BYPASS_PRIO,
+	RDMA_RX_KERNEL_PRIO,
+};
+
+#define RDMA_RX_IPSEC_NUM_PRIOS 1
+#define RDMA_RX_IPSEC_NUM_LEVELS 2
+#define RDMA_RX_IPSEC_MIN_LEVEL  (RDMA_RX_IPSEC_NUM_LEVELS)
+
+#define RDMA_RX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_REGULAR_PRIOS
+#define RDMA_RX_KERNEL_MIN_LEVEL (RDMA_RX_BYPASS_MIN_LEVEL + 1)
+#define RDMA_RX_COUNTERS_MIN_LEVEL (RDMA_RX_KERNEL_MIN_LEVEL + 2)
+
+static struct init_tree_node rdma_rx_root_fs = {
+	.type = FS_TYPE_NAMESPACE,
+	.ar_size = 4,
+	.children = (struct init_tree_node[]) {
+		[RDMA_RX_IPSEC_PRIO] =
+		ADD_PRIO(0, RDMA_RX_IPSEC_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(RDMA_RX_IPSEC_NUM_PRIOS,
+						  RDMA_RX_IPSEC_NUM_LEVELS))),
+		[RDMA_RX_COUNTERS_PRIO] =
+		ADD_PRIO(0, RDMA_RX_COUNTERS_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(MLX5_RDMA_RX_NUM_COUNTERS_PRIOS,
+						  RDMA_RX_COUNTERS_PRIO_NUM_LEVELS))),
+		[RDMA_RX_BYPASS_PRIO] =
+		ADD_PRIO(0, RDMA_RX_BYPASS_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS,
+						  BY_PASS_PRIO_NUM_LEVELS))),
+		[RDMA_RX_KERNEL_PRIO] =
+		ADD_PRIO(0, RDMA_RX_KERNEL_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
+				ADD_MULTIPLE_PRIO(1, 1))),
+	}
+};
+
+enum {
+	RDMA_TX_COUNTERS_PRIO,
+	RDMA_TX_IPSEC_PRIO,
+	RDMA_TX_BYPASS_PRIO,
+};
+
+#define RDMA_TX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_PRIOS
+#define RDMA_TX_COUNTERS_MIN_LEVEL (RDMA_TX_BYPASS_MIN_LEVEL + 1)
+
+#define RDMA_TX_IPSEC_NUM_PRIOS 1
+#define RDMA_TX_IPSEC_PRIO_NUM_LEVELS 1
+#define RDMA_TX_IPSEC_MIN_LEVEL  (RDMA_TX_COUNTERS_MIN_LEVEL + RDMA_TX_IPSEC_NUM_PRIOS)
+
+static struct init_tree_node rdma_tx_root_fs = {
+	.type = FS_TYPE_NAMESPACE,
+	.ar_size = 3,
+	.children = (struct init_tree_node[]) {
+		[RDMA_TX_COUNTERS_PRIO] =
+		ADD_PRIO(0, RDMA_TX_COUNTERS_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(MLX5_RDMA_TX_NUM_COUNTERS_PRIOS,
+						  RDMA_TX_COUNTERS_PRIO_NUM_LEVELS))),
+		[RDMA_TX_IPSEC_PRIO] =
+		ADD_PRIO(0, RDMA_TX_IPSEC_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(RDMA_TX_IPSEC_NUM_PRIOS,
+						  RDMA_TX_IPSEC_PRIO_NUM_LEVELS))),
+
+		[RDMA_TX_BYPASS_PRIO] =
+		ADD_PRIO(0, RDMA_TX_BYPASS_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS_RDMA_TX,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(RDMA_TX_BYPASS_MIN_LEVEL,
+						  BY_PASS_PRIO_NUM_LEVELS))),
+	}
+};
+
+enum fs_i_lock_class {
+	FS_LOCK_GRANDPARENT,
+	FS_LOCK_PARENT,
+	FS_LOCK_CHILD
+};
+
+static void del_hw_flow_table(struct fs_node *node);
+static void del_hw_flow_group(struct fs_node *node);
+static void del_hw_fte(struct fs_node *node);
+static void del_sw_flow_table(struct fs_node *node);
+static void del_sw_flow_group(struct fs_node *node);
+static void del_sw_fte(struct fs_node *node);
+static void del_sw_prio(struct fs_node *node);
+static void del_sw_ns(struct fs_node *node);
+/* Delete rule (destination) is special case that
+ * requires to lock the FTE for all the deletion process.
+ */
+static void del_sw_hw_rule(struct fs_node *node);
+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
+				struct mlx5_flow_destination *d2);
+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns);
+static struct mlx5_flow_rule *
+find_flow_rule(struct fs_fte *fte,
+	       struct mlx5_flow_destination *dest);
+
+static void tree_init_node(struct fs_node *node,
+			   void (*del_hw_func)(struct fs_node *),
+			   void (*del_sw_func)(struct fs_node *))
+{
+	refcount_set(&node->refcount, 1);
+	INIT_LIST_HEAD(&node->list);
+	INIT_LIST_HEAD(&node->children);
+	init_rwsem(&node->lock);
+	node->del_hw_func = del_hw_func;
+	node->del_sw_func = del_sw_func;
+	node->active = false;
+}
+
+static void tree_add_node(struct fs_node *node, struct fs_node *parent)
+{
+	if (parent)
+		refcount_inc(&parent->refcount);
+	node->parent = parent;
+
+	/* Parent is the root */
+	if (!parent)
+		node->root = node;
+	else
+		node->root = parent->root;
+}
+
+static int tree_get_node(struct fs_node *node)
+{
+	return refcount_inc_not_zero(&node->refcount);
+}
+
+static void nested_down_read_ref_node(struct fs_node *node,
+				      enum fs_i_lock_class class)
+{
+	if (node) {
+		down_read_nested(&node->lock, class);
+		refcount_inc(&node->refcount);
+	}
+}
+
+static void nested_down_write_ref_node(struct fs_node *node,
+				       enum fs_i_lock_class class)
+{
+	if (node) {
+		down_write_nested(&node->lock, class);
+		refcount_inc(&node->refcount);
+	}
+}
+
+static void down_write_ref_node(struct fs_node *node, bool locked)
+{
+	if (node) {
+		if (!locked)
+			down_write(&node->lock);
+		refcount_inc(&node->refcount);
+	}
+}
+
+static void up_read_ref_node(struct fs_node *node)
+{
+	refcount_dec(&node->refcount);
+	up_read(&node->lock);
+}
+
+static void up_write_ref_node(struct fs_node *node, bool locked)
+{
+	refcount_dec(&node->refcount);
+	if (!locked)
+		up_write(&node->lock);
+}
+
+static void tree_put_node(struct fs_node *node, bool locked)
+{
+	struct fs_node *parent_node = node->parent;
+
+	if (refcount_dec_and_test(&node->refcount)) {
+		if (node->del_hw_func)
+			node->del_hw_func(node);
+		if (parent_node) {
+			down_write_ref_node(parent_node, locked);
+			list_del_init(&node->list);
+		}
+		node->del_sw_func(node);
+		if (parent_node)
+			up_write_ref_node(parent_node, locked);
+		node = NULL;
+	}
+	if (!node && parent_node)
+		tree_put_node(parent_node, locked);
+}
+
+static int tree_remove_node(struct fs_node *node, bool locked)
+{
+	if (refcount_read(&node->refcount) > 1) {
+		refcount_dec(&node->refcount);
+		return -EEXIST;
+	}
+	tree_put_node(node, locked);
+	return 0;
+}
+
+static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
+				 unsigned int prio)
+{
+	struct fs_prio *iter_prio;
+
+	fs_for_each_prio(iter_prio, ns) {
+		if (iter_prio->prio == prio)
+			return iter_prio;
+	}
+
+	return NULL;
+}
+
+static bool is_fwd_next_action(u32 action)
+{
+	return action & (MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
+			 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
+}
+
+static bool is_fwd_dest_type(enum mlx5_flow_destination_type type)
+{
+	return type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM ||
+		type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE ||
+		type == MLX5_FLOW_DESTINATION_TYPE_UPLINK ||
+		type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+		type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER ||
+		type == MLX5_FLOW_DESTINATION_TYPE_TIR ||
+		type == MLX5_FLOW_DESTINATION_TYPE_RANGE ||
+		type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+}
+
+static bool check_valid_spec(const struct mlx5_flow_spec *spec)
+{
+	int i;
+
+	for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
+		if (spec->match_value[i] & ~spec->match_criteria[i]) {
+			pr_warn("mlx5_core: match_value differs from match_criteria\n");
+			return false;
+		}
+
+	return true;
+}
+
+struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
+{
+	struct fs_node *root;
+	struct mlx5_flow_namespace *ns;
+
+	root = node->root;
+
+	if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) {
+		pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
+		return NULL;
+	}
+
+	ns = container_of(root, struct mlx5_flow_namespace, node);
+	return container_of(ns, struct mlx5_flow_root_namespace, ns);
+}
+
+static inline struct mlx5_flow_steering *get_steering(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root = find_root(node);
+
+	if (root)
+		return root->dev->priv.steering;
+	return NULL;
+}
+
+static inline struct mlx5_core_dev *get_dev(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root = find_root(node);
+
+	if (root)
+		return root->dev;
+	return NULL;
+}
+
+static void del_sw_ns(struct fs_node *node)
+{
+	kfree(node);
+}
+
+static void del_sw_prio(struct fs_node *node)
+{
+	kfree(node);
+}
+
+static void del_hw_flow_table(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_table *ft;
+	struct mlx5_core_dev *dev;
+	int err;
+
+	fs_get_obj(ft, node);
+	dev = get_dev(&ft->node);
+	root = find_root(&ft->node);
+
+	if (node->active) {
+		err = root->cmds->destroy_flow_table(root, ft);
+		if (err)
+			mlx5_core_warn(dev, "flow steering can't destroy ft\n");
+	}
+}
+
+static void del_sw_flow_table(struct fs_node *node)
+{
+	struct mlx5_flow_table *ft;
+	struct fs_prio *prio;
+
+	fs_get_obj(ft, node);
+
+	xa_destroy(&ft->fgs_xa);
+	if (ft->node.parent) {
+		fs_get_obj(prio, ft->node.parent);
+		prio->num_ft--;
+	}
+	kfree(ft);
+}
+
+static void modify_fte(struct fs_fte *fte)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_group *fg;
+	struct mlx5_core_dev *dev;
+	int err;
+
+	fs_get_obj(fg, fte->node.parent);
+	fs_get_obj(ft, fg->node.parent);
+	dev = get_dev(&fte->node);
+
+	root = find_root(&ft->node);
+	err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte);
+	if (err)
+		mlx5_core_warn(dev,
+			       "%s can't del rule fg id=%d fte_index=%d\n",
+			       __func__, fg->id, fte->index);
+	fte->modify_mask = 0;
+}
+
+static void del_sw_hw_rule(struct fs_node *node)
+{
+	struct mlx5_flow_rule *rule;
+	struct fs_fte *fte;
+
+	fs_get_obj(rule, node);
+	fs_get_obj(fte, rule->node.parent);
+	if (is_fwd_next_action(rule->sw_action)) {
+		mutex_lock(&rule->dest_attr.ft->lock);
+		list_del(&rule->next_ft);
+		mutex_unlock(&rule->dest_attr.ft->lock);
+	}
+
+	if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER) {
+		--fte->dests_size;
+		fte->modify_mask |=
+			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) |
+			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
+		fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT;
+		goto out;
+	}
+
+	if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
+		--fte->dests_size;
+		fte->modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
+		fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+		goto out;
+	}
+
+	if (is_fwd_dest_type(rule->dest_attr.type)) {
+		--fte->dests_size;
+		--fte->fwd_dests;
+
+		if (!fte->fwd_dests)
+			fte->action.action &=
+				~MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+		fte->modify_mask |=
+			BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+		goto out;
+	}
+out:
+	kfree(rule);
+}
+
+static void del_hw_fte(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_group *fg;
+	struct mlx5_core_dev *dev;
+	struct fs_fte *fte;
+	int err;
+
+	fs_get_obj(fte, node);
+	fs_get_obj(fg, fte->node.parent);
+	fs_get_obj(ft, fg->node.parent);
+
+	WARN_ON(fte->dests_size);
+	dev = get_dev(&ft->node);
+	root = find_root(&ft->node);
+	if (node->active) {
+		err = root->cmds->delete_fte(root, ft, fte);
+		if (err)
+			mlx5_core_warn(dev,
+				       "flow steering can't delete fte in index %d of flow group id %d\n",
+				       fte->index, fg->id);
+		node->active = false;
+	}
+}
+
+static void del_sw_fte(struct fs_node *node)
+{
+	struct mlx5_flow_steering *steering = get_steering(node);
+	struct mlx5_flow_group *fg;
+	struct fs_fte *deleted_fte;
+	struct fs_fte *fte;
+
+	fs_get_obj(fte, node);
+	fs_get_obj(fg, fte->node.parent);
+
+	deleted_fte = xa_erase(&fg->ftes_xa, fte->index);
+	WARN_ON(deleted_fte != fte);
+	ida_free(&fg->fte_allocator, fte->index - fg->start_index);
+	kmem_cache_free(steering->ftes_cache, fte);
+}
+
+static void del_hw_flow_group(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_group *fg;
+	struct mlx5_flow_table *ft;
+	struct mlx5_core_dev *dev;
+
+	fs_get_obj(fg, node);
+	fs_get_obj(ft, fg->node.parent);
+	dev = get_dev(&ft->node);
+
+	root = find_root(&ft->node);
+	if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg))
+		mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
+			       fg->id, ft->id);
+}
+
+static void del_sw_flow_group(struct fs_node *node)
+{
+	struct mlx5_flow_steering *steering = get_steering(node);
+	struct mlx5_flow_group *deleted_fg;
+	struct mlx5_flow_group *fg;
+	struct mlx5_flow_table *ft;
+
+	fs_get_obj(fg, node);
+	fs_get_obj(ft, fg->node.parent);
+
+	xa_destroy(&fg->ftes_xa);
+	ida_destroy(&fg->fte_allocator);
+	if (ft->autogroup.active &&
+	    fg->max_ftes == ft->autogroup.group_size &&
+	    fg->start_index < ft->autogroup.max_fte)
+		ft->autogroup.num_groups--;
+	deleted_fg = xa_erase(&ft->fgs_xa, fg->start_index);
+	WARN_ON(deleted_fg != fg);
+	kmem_cache_free(steering->fgs_cache, fg);
+}
+
+static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
+{
+	int index;
+	int ret;
+
+	index = ida_alloc_max(&fg->fte_allocator, fg->max_ftes, GFP_KERNEL);
+	if (index < 0)
+		return index;
+
+	fte->index = index + fg->start_index;
+	ret = xa_insert(&fg->ftes_xa, fte->index, fte, GFP_KERNEL);
+	if (ret)
+		goto err_ida_remove;
+
+	tree_add_node(&fte->node, &fg->node);
+	list_add_tail(&fte->node.list, &fg->node.children);
+	return 0;
+
+err_ida_remove:
+	ida_free(&fg->fte_allocator, index);
+	return ret;
+}
+
+static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
+				const struct mlx5_flow_spec *spec,
+				struct mlx5_flow_act *flow_act)
+{
+	struct mlx5_flow_steering *steering = get_steering(&ft->node);
+	struct fs_fte *fte;
+
+	fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);
+	if (!fte)
+		return ERR_PTR(-ENOMEM);
+
+	memcpy(fte->val, &spec->match_value, sizeof(fte->val));
+	fte->node.type =  FS_TYPE_FLOW_ENTRY;
+	fte->action = *flow_act;
+	fte->flow_context = spec->flow_context;
+
+	tree_init_node(&fte->node, del_hw_fte, del_sw_fte);
+
+	return fte;
+}
+
+static void dealloc_flow_group(struct mlx5_flow_steering *steering,
+			       struct mlx5_flow_group *fg)
+{
+	xa_destroy(&fg->ftes_xa);
+	kmem_cache_free(steering->fgs_cache, fg);
+}
+
+static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,
+						u8 match_criteria_enable,
+						const void *match_criteria,
+						int start_index,
+						int end_index)
+{
+	struct mlx5_flow_group *fg;
+
+	fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
+	if (!fg)
+		return ERR_PTR(-ENOMEM);
+
+	xa_init(&fg->ftes_xa);
+
+	ida_init(&fg->fte_allocator);
+	fg->mask.match_criteria_enable = match_criteria_enable;
+	memcpy(&fg->mask.match_criteria, match_criteria,
+	       sizeof(fg->mask.match_criteria));
+	fg->node.type =  FS_TYPE_FLOW_GROUP;
+	fg->start_index = start_index;
+	fg->max_ftes = end_index - start_index + 1;
+
+	return fg;
+}
+
+static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft,
+						       u8 match_criteria_enable,
+						       const void *match_criteria,
+						       int start_index,
+						       int end_index,
+						       struct list_head *prev)
+{
+	struct mlx5_flow_steering *steering = get_steering(&ft->node);
+	struct mlx5_flow_group *fg;
+	int ret;
+
+	fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,
+			      start_index, end_index);
+	if (IS_ERR(fg))
+		return fg;
+
+	/* initialize refcnt, add to parent list */
+	ret = xa_insert(&ft->fgs_xa, fg->start_index, fg, GFP_KERNEL);
+	if (ret) {
+		dealloc_flow_group(steering, fg);
+		return ERR_PTR(ret);
+	}
+
+	tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group);
+	tree_add_node(&fg->node, &ft->node);
+	/* Add node to group list */
+	list_add(&fg->node.list, prev);
+	atomic_inc(&ft->node.version);
+
+	return fg;
+}
+
+static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport,
+						enum fs_flow_table_type table_type,
+						enum fs_flow_table_op_mod op_mod,
+						u32 flags)
+{
+	struct mlx5_flow_table *ft;
+
+	ft  = kzalloc(sizeof(*ft), GFP_KERNEL);
+	if (!ft)
+		return ERR_PTR(-ENOMEM);
+
+	xa_init(&ft->fgs_xa);
+
+	ft->level = level;
+	ft->node.type = FS_TYPE_FLOW_TABLE;
+	ft->op_mod = op_mod;
+	ft->type = table_type;
+	ft->vport = vport;
+	ft->flags = flags;
+	INIT_LIST_HEAD(&ft->fwd_rules);
+	mutex_init(&ft->lock);
+
+	return ft;
+}
+
+/* If reverse is false, then we search for the first flow table in the
+ * root sub-tree from start(closest from right), else we search for the
+ * last flow table in the root sub-tree till start(closest from left).
+ */
+static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node  *root,
+							 struct list_head *start,
+							 bool reverse)
+{
+#define list_advance_entry(pos, reverse)		\
+	((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
+
+#define list_for_each_advance_continue(pos, head, reverse)	\
+	for (pos = list_advance_entry(pos, reverse);		\
+	     &pos->list != (head);				\
+	     pos = list_advance_entry(pos, reverse))
+
+	struct fs_node *iter = list_entry(start, struct fs_node, list);
+	struct mlx5_flow_table *ft = NULL;
+
+	if (!root || root->type == FS_TYPE_PRIO_CHAINS)
+		return NULL;
+
+	list_for_each_advance_continue(iter, &root->children, reverse) {
+		if (iter->type == FS_TYPE_FLOW_TABLE) {
+			fs_get_obj(ft, iter);
+			return ft;
+		}
+		ft = find_closest_ft_recursive(iter, &iter->children, reverse);
+		if (ft)
+			return ft;
+	}
+
+	return ft;
+}
+
+/* If reverse is false then return the first flow table in next priority of
+ * prio in the tree, else return the last flow table in the previous priority
+ * of prio in the tree.
+ */
+static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse)
+{
+	struct mlx5_flow_table *ft = NULL;
+	struct fs_node *curr_node;
+	struct fs_node *parent;
+
+	parent = prio->node.parent;
+	curr_node = &prio->node;
+	while (!ft && parent) {
+		ft = find_closest_ft_recursive(parent, &curr_node->list, reverse);
+		curr_node = parent;
+		parent = curr_node->parent;
+	}
+	return ft;
+}
+
+/* Assuming all the tree is locked by mutex chain lock */
+static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio)
+{
+	return find_closest_ft(prio, false);
+}
+
+/* Assuming all the tree is locked by mutex chain lock */
+static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio)
+{
+	return find_closest_ft(prio, true);
+}
+
+static struct mlx5_flow_table *find_next_fwd_ft(struct mlx5_flow_table *ft,
+						struct mlx5_flow_act *flow_act)
+{
+	struct fs_prio *prio;
+	bool next_ns;
+
+	next_ns = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
+	fs_get_obj(prio, next_ns ? ft->ns->node.parent : ft->node.parent);
+
+	return find_next_chained_ft(prio);
+}
+
+static int connect_fts_in_prio(struct mlx5_core_dev *dev,
+			       struct fs_prio *prio,
+			       struct mlx5_flow_table *ft)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&prio->node);
+	struct mlx5_flow_table *iter;
+	int err;
+
+	fs_for_each_ft(iter, prio) {
+		err = root->cmds->modify_flow_table(root, iter, ft);
+		if (err) {
+			mlx5_core_err(dev,
+				      "Failed to modify flow table id %d, type %d, err %d\n",
+				      iter->id, iter->type, err);
+			/* The driver is out of sync with the FW */
+			return err;
+		}
+	}
+	return 0;
+}
+
+/* Connect flow tables from previous priority of prio to ft */
+static int connect_prev_fts(struct mlx5_core_dev *dev,
+			    struct mlx5_flow_table *ft,
+			    struct fs_prio *prio)
+{
+	struct mlx5_flow_table *prev_ft;
+
+	prev_ft = find_prev_chained_ft(prio);
+	if (prev_ft) {
+		struct fs_prio *prev_prio;
+
+		fs_get_obj(prev_prio, prev_ft->node.parent);
+		return connect_fts_in_prio(dev, prev_prio, ft);
+	}
+	return 0;
+}
+
+static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
+				 *prio)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&prio->node);
+	struct mlx5_ft_underlay_qp *uqp;
+	int min_level = INT_MAX;
+	int err = 0;
+	u32 qpn;
+
+	if (root->root_ft)
+		min_level = root->root_ft->level;
+
+	if (ft->level >= min_level)
+		return 0;
+
+	if (list_empty(&root->underlay_qpns)) {
+		/* Don't set any QPN (zero) in case QPN list is empty */
+		qpn = 0;
+		err = root->cmds->update_root_ft(root, ft, qpn, false);
+	} else {
+		list_for_each_entry(uqp, &root->underlay_qpns, list) {
+			qpn = uqp->qpn;
+			err = root->cmds->update_root_ft(root, ft,
+							 qpn, false);
+			if (err)
+				break;
+		}
+	}
+
+	if (err)
+		mlx5_core_warn(root->dev,
+			       "Update root flow table of id(%u) qpn(%d) failed\n",
+			       ft->id, qpn);
+	else
+		root->root_ft = ft;
+
+	return err;
+}
+
+static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
+					 struct mlx5_flow_destination *dest)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_table *ft;
+	struct mlx5_flow_group *fg;
+	struct fs_fte *fte;
+	int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+	int err = 0;
+
+	fs_get_obj(fte, rule->node.parent);
+	if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+		return -EINVAL;
+	down_write_ref_node(&fte->node, false);
+	fs_get_obj(fg, fte->node.parent);
+	fs_get_obj(ft, fg->node.parent);
+
+	memcpy(&rule->dest_attr, dest, sizeof(*dest));
+	root = find_root(&ft->node);
+	err = root->cmds->update_fte(root, ft, fg,
+				     modify_mask, fte);
+	up_write_ref_node(&fte->node, false);
+
+	return err;
+}
+
+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
+				 struct mlx5_flow_destination *new_dest,
+				 struct mlx5_flow_destination *old_dest)
+{
+	int i;
+
+	if (!old_dest) {
+		if (handle->num_rules != 1)
+			return -EINVAL;
+		return _mlx5_modify_rule_destination(handle->rule[0],
+						     new_dest);
+	}
+
+	for (i = 0; i < handle->num_rules; i++) {
+		if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr))
+			return _mlx5_modify_rule_destination(handle->rule[i],
+							     new_dest);
+	}
+
+	return -EINVAL;
+}
+
+/* Modify/set FWD rules that point on old_next_ft to point on new_next_ft  */
+static int connect_fwd_rules(struct mlx5_core_dev *dev,
+			     struct mlx5_flow_table *new_next_ft,
+			     struct mlx5_flow_table *old_next_ft)
+{
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_rule *iter;
+	int err = 0;
+
+	/* new_next_ft and old_next_ft could be NULL only
+	 * when we create/destroy the anchor flow table.
+	 */
+	if (!new_next_ft || !old_next_ft)
+		return 0;
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft = new_next_ft;
+
+	mutex_lock(&old_next_ft->lock);
+	list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules);
+	mutex_unlock(&old_next_ft->lock);
+	list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) {
+		if ((iter->sw_action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS) &&
+		    iter->ft->ns == new_next_ft->ns)
+			continue;
+
+		err = _mlx5_modify_rule_destination(iter, &dest);
+		if (err)
+			pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
+			       new_next_ft->id);
+	}
+	return 0;
+}
+
+static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
+			      struct fs_prio *prio)
+{
+	struct mlx5_flow_table *next_ft, *first_ft;
+	int err = 0;
+
+	/* Connect_prev_fts and update_root_ft_create are mutually exclusive */
+
+	first_ft = list_first_entry_or_null(&prio->node.children,
+					    struct mlx5_flow_table, node.list);
+	if (!first_ft || first_ft->level > ft->level) {
+		err = connect_prev_fts(dev, ft, prio);
+		if (err)
+			return err;
+
+		next_ft = first_ft ? first_ft : find_next_chained_ft(prio);
+		err = connect_fwd_rules(dev, ft, next_ft);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE(dev,
+			       flow_table_properties_nic_receive.modify_root))
+		err = update_root_ft_create(ft, prio);
+	return err;
+}
+
+static void list_add_flow_table(struct mlx5_flow_table *ft,
+				struct fs_prio *prio)
+{
+	struct list_head *prev = &prio->node.children;
+	struct mlx5_flow_table *iter;
+
+	fs_for_each_ft(iter, prio) {
+		if (iter->level > ft->level)
+			break;
+		prev = &iter->node.list;
+	}
+	list_add(&ft->node.list, prev);
+}
+
+static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+							struct mlx5_flow_table_attr *ft_attr,
+							enum fs_flow_table_op_mod op_mod,
+							u16 vport)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ns->node);
+	bool unmanaged = ft_attr->flags & MLX5_FLOW_TABLE_UNMANAGED;
+	struct mlx5_flow_table *next_ft;
+	struct fs_prio *fs_prio = NULL;
+	struct mlx5_flow_table *ft;
+	int err;
+
+	if (!root) {
+		pr_err("mlx5: flow steering failed to find root of namespace\n");
+		return ERR_PTR(-ENODEV);
+	}
+
+	mutex_lock(&root->chain_lock);
+	fs_prio = find_prio(ns, ft_attr->prio);
+	if (!fs_prio) {
+		err = -EINVAL;
+		goto unlock_root;
+	}
+	if (!unmanaged) {
+		/* The level is related to the
+		 * priority level range.
+		 */
+		if (ft_attr->level >= fs_prio->num_levels) {
+			err = -ENOSPC;
+			goto unlock_root;
+		}
+
+		ft_attr->level += fs_prio->start_level;
+	}
+
+	/* The level is related to the
+	 * priority level range.
+	 */
+	ft = alloc_flow_table(ft_attr->level,
+			      vport,
+			      root->table_type,
+			      op_mod, ft_attr->flags);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto unlock_root;
+	}
+
+	tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table);
+	next_ft = unmanaged ? ft_attr->next_ft :
+			      find_next_chained_ft(fs_prio);
+	ft->def_miss_action = ns->def_miss_action;
+	ft->ns = ns;
+	err = root->cmds->create_flow_table(root, ft, ft_attr, next_ft);
+	if (err)
+		goto free_ft;
+
+	if (!unmanaged) {
+		err = connect_flow_table(root->dev, ft, fs_prio);
+		if (err)
+			goto destroy_ft;
+	}
+
+	ft->node.active = true;
+	down_write_ref_node(&fs_prio->node, false);
+	if (!unmanaged) {
+		tree_add_node(&ft->node, &fs_prio->node);
+		list_add_flow_table(ft, fs_prio);
+	} else {
+		ft->node.root = fs_prio->node.root;
+	}
+	fs_prio->num_ft++;
+	up_write_ref_node(&fs_prio->node, false);
+	mutex_unlock(&root->chain_lock);
+	return ft;
+destroy_ft:
+	root->cmds->destroy_flow_table(root, ft);
+free_ft:
+	xa_destroy(&ft->fgs_xa);
+	kfree(ft);
+unlock_root:
+	mutex_unlock(&root->chain_lock);
+	return ERR_PTR(err);
+}
+
+struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+					       struct mlx5_flow_table_attr *ft_attr)
+{
+	return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0);
+}
+EXPORT_SYMBOL(mlx5_create_flow_table);
+
+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft)
+{
+	return ft->id;
+}
+EXPORT_SYMBOL(mlx5_flow_table_id);
+
+struct mlx5_flow_table *
+mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
+			     struct mlx5_flow_table_attr *ft_attr, u16 vport)
+{
+	return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, vport);
+}
+
+struct mlx5_flow_table*
+mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
+				 int prio, u32 level)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+
+	ft_attr.level = level;
+	ft_attr.prio  = prio;
+	ft_attr.max_fte = 1;
+
+	return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
+}
+EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
+
+#define MAX_FLOW_GROUP_SIZE BIT(24)
+struct mlx5_flow_table*
+mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
+				    struct mlx5_flow_table_attr *ft_attr)
+{
+	int num_reserved_entries = ft_attr->autogroup.num_reserved_entries;
+	int max_num_groups = ft_attr->autogroup.max_num_groups;
+	struct mlx5_flow_table *ft;
+	int autogroups_max_fte;
+
+	ft = mlx5_create_flow_table(ns, ft_attr);
+	if (IS_ERR(ft))
+		return ft;
+
+	autogroups_max_fte = ft->max_fte - num_reserved_entries;
+	if (max_num_groups > autogroups_max_fte)
+		goto err_validate;
+	if (num_reserved_entries > ft->max_fte)
+		goto err_validate;
+
+	/* Align the number of groups according to the largest group size */
+	if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE)
+		max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1;
+
+	ft->autogroup.active = true;
+	ft->autogroup.required_groups = max_num_groups;
+	ft->autogroup.max_fte = autogroups_max_fte;
+	/* We save place for flow groups in addition to max types */
+	ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1);
+
+	return ft;
+
+err_validate:
+	mlx5_destroy_flow_table(ft);
+	return ERR_PTR(-ENOSPC);
+}
+EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
+
+struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
+					       u32 *fg_in)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
+					    fg_in, match_criteria);
+	u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
+					    fg_in,
+					    match_criteria_enable);
+	int start_index = MLX5_GET(create_flow_group_in, fg_in,
+				   start_flow_index);
+	int end_index = MLX5_GET(create_flow_group_in, fg_in,
+				 end_flow_index);
+	struct mlx5_flow_group *fg;
+	int err;
+
+	if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
+		return ERR_PTR(-EPERM);
+
+	down_write_ref_node(&ft->node, false);
+	fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria,
+				     start_index, end_index,
+				     ft->node.children.prev);
+	up_write_ref_node(&ft->node, false);
+	if (IS_ERR(fg))
+		return fg;
+
+	err = root->cmds->create_flow_group(root, ft, fg_in, fg);
+	if (err) {
+		tree_put_node(&fg->node, false);
+		return ERR_PTR(err);
+	}
+	fg->node.active = true;
+
+	return fg;
+}
+EXPORT_SYMBOL(mlx5_create_flow_group);
+
+static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest)
+{
+	struct mlx5_flow_rule *rule;
+
+	rule = kzalloc(sizeof(*rule), GFP_KERNEL);
+	if (!rule)
+		return NULL;
+
+	INIT_LIST_HEAD(&rule->next_ft);
+	rule->node.type = FS_TYPE_FLOW_DEST;
+	if (dest)
+		memcpy(&rule->dest_attr, dest, sizeof(*dest));
+	else
+		rule->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_NONE;
+
+	return rule;
+}
+
+static struct mlx5_flow_handle *alloc_handle(int num_rules)
+{
+	struct mlx5_flow_handle *handle;
+
+	handle = kzalloc(struct_size(handle, rule, num_rules), GFP_KERNEL);
+	if (!handle)
+		return NULL;
+
+	handle->num_rules = num_rules;
+
+	return handle;
+}
+
+static void destroy_flow_handle(struct fs_fte *fte,
+				struct mlx5_flow_handle *handle,
+				struct mlx5_flow_destination *dest,
+				int i)
+{
+	for (; --i >= 0;) {
+		if (refcount_dec_and_test(&handle->rule[i]->node.refcount)) {
+			fte->dests_size--;
+			list_del(&handle->rule[i]->node.list);
+			kfree(handle->rule[i]);
+		}
+	}
+	kfree(handle);
+}
+
+static struct mlx5_flow_handle *
+create_flow_handle(struct fs_fte *fte,
+		   struct mlx5_flow_destination *dest,
+		   int dest_num,
+		   int *modify_mask,
+		   bool *new_rule)
+{
+	struct mlx5_flow_handle *handle;
+	struct mlx5_flow_rule *rule = NULL;
+	static int count = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
+	static int dst = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+	int type;
+	int i = 0;
+
+	handle = alloc_handle((dest_num) ? dest_num : 1);
+	if (!handle)
+		return ERR_PTR(-ENOMEM);
+
+	do {
+		if (dest) {
+			rule = find_flow_rule(fte, dest + i);
+			if (rule) {
+				refcount_inc(&rule->node.refcount);
+				goto rule_found;
+			}
+		}
+
+		*new_rule = true;
+		rule = alloc_rule(dest + i);
+		if (!rule)
+			goto free_rules;
+
+		/* Add dest to dests list- we need flow tables to be in the
+		 * end of the list for forward to next prio rules.
+		 */
+		tree_init_node(&rule->node, NULL, del_sw_hw_rule);
+		if (dest &&
+		    dest[i].type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+			list_add(&rule->node.list, &fte->node.children);
+		else
+			list_add_tail(&rule->node.list, &fte->node.children);
+		if (dest) {
+			fte->dests_size++;
+
+			if (is_fwd_dest_type(dest[i].type))
+				fte->fwd_dests++;
+
+			type = dest[i].type ==
+				MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+			*modify_mask |= type ? count : dst;
+		}
+rule_found:
+		handle->rule[i] = rule;
+	} while (++i < dest_num);
+
+	return handle;
+
+free_rules:
+	destroy_flow_handle(fte, handle, dest, i);
+	return ERR_PTR(-ENOMEM);
+}
+
+/* fte should not be deleted while calling this function */
+static struct mlx5_flow_handle *
+add_rule_fte(struct fs_fte *fte,
+	     struct mlx5_flow_group *fg,
+	     struct mlx5_flow_destination *dest,
+	     int dest_num,
+	     bool update_action)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_handle *handle;
+	struct mlx5_flow_table *ft;
+	int modify_mask = 0;
+	int err;
+	bool new_rule = false;
+
+	handle = create_flow_handle(fte, dest, dest_num, &modify_mask,
+				    &new_rule);
+	if (IS_ERR(handle) || !new_rule)
+		goto out;
+
+	if (update_action)
+		modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
+
+	fs_get_obj(ft, fg->node.parent);
+	root = find_root(&fg->node);
+	if (!(fte->status & FS_FTE_STATUS_EXISTING))
+		err = root->cmds->create_fte(root, ft, fg, fte);
+	else
+		err = root->cmds->update_fte(root, ft, fg, modify_mask, fte);
+	if (err)
+		goto free_handle;
+
+	fte->node.active = true;
+	fte->status |= FS_FTE_STATUS_EXISTING;
+	atomic_inc(&fg->node.version);
+
+out:
+	return handle;
+
+free_handle:
+	destroy_flow_handle(fte, handle, dest, handle->num_rules);
+	return ERR_PTR(err);
+}
+
+static struct mlx5_flow_group *alloc_auto_flow_group(struct mlx5_flow_table  *ft,
+						     const struct mlx5_flow_spec *spec)
+{
+	struct list_head *prev = &ft->node.children;
+	u32 max_fte = ft->autogroup.max_fte;
+	unsigned int candidate_index = 0;
+	unsigned int group_size = 0;
+	struct mlx5_flow_group *fg;
+
+	if (!ft->autogroup.active)
+		return ERR_PTR(-ENOENT);
+
+	if (ft->autogroup.num_groups < ft->autogroup.required_groups)
+		group_size = ft->autogroup.group_size;
+
+	/*  max_fte == ft->autogroup.max_types */
+	if (group_size == 0)
+		group_size = 1;
+
+	/* sorted by start_index */
+	fs_for_each_fg(fg, ft) {
+		if (candidate_index + group_size > fg->start_index)
+			candidate_index = fg->start_index + fg->max_ftes;
+		else
+			break;
+		prev = &fg->node.list;
+	}
+
+	if (candidate_index + group_size > max_fte)
+		return ERR_PTR(-ENOSPC);
+
+	fg = alloc_insert_flow_group(ft,
+				     spec->match_criteria_enable,
+				     spec->match_criteria,
+				     candidate_index,
+				     candidate_index + group_size - 1,
+				     prev);
+	if (IS_ERR(fg))
+		goto out;
+
+	if (group_size == ft->autogroup.group_size)
+		ft->autogroup.num_groups++;
+
+out:
+	return fg;
+}
+
+static int create_auto_flow_group(struct mlx5_flow_table *ft,
+				  struct mlx5_flow_group *fg)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	void *match_criteria_addr;
+	u8 src_esw_owner_mask_on;
+	void *misc;
+	int err;
+	u32 *in;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
+		 fg->mask.match_criteria_enable);
+	MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index);
+	MLX5_SET(create_flow_group_in, in, end_flow_index,   fg->start_index +
+		 fg->max_ftes - 1);
+
+	misc = MLX5_ADDR_OF(fte_match_param, fg->mask.match_criteria,
+			    misc_parameters);
+	src_esw_owner_mask_on = !!MLX5_GET(fte_match_set_misc, misc,
+					 source_eswitch_owner_vhca_id);
+	MLX5_SET(create_flow_group_in, in,
+		 source_eswitch_owner_vhca_id_valid, src_esw_owner_mask_on);
+
+	match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
+					   in, match_criteria);
+	memcpy(match_criteria_addr, fg->mask.match_criteria,
+	       sizeof(fg->mask.match_criteria));
+
+	err = root->cmds->create_flow_group(root, ft, in, fg);
+	if (!err) {
+		fg->node.active = true;
+	}
+
+	kvfree(in);
+	return err;
+}
+
+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
+				struct mlx5_flow_destination *d2)
+{
+	if (d1->type == d2->type) {
+		if (((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+		      d1->type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
+		     d1->vport.num == d2->vport.num &&
+		     d1->vport.flags == d2->vport.flags &&
+		     ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_VHCA_ID) ?
+		      (d1->vport.vhca_id == d2->vport.vhca_id) : true) &&
+		     ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID) ?
+		      (d1->vport.pkt_reformat->id ==
+		       d2->vport.pkt_reformat->id) : true)) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
+		     d1->ft == d2->ft) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
+		     d1->tir_num == d2->tir_num) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM &&
+		     d1->ft_num == d2->ft_num) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER &&
+		     d1->sampler_id == d2->sampler_id) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_RANGE &&
+		     d1->range.field == d2->range.field &&
+		     d1->range.hit_ft == d2->range.hit_ft &&
+		     d1->range.miss_ft == d2->range.miss_ft &&
+		     d1->range.min == d2->range.min &&
+		     d1->range.max == d2->range.max))
+			return true;
+	}
+
+	return false;
+}
+
+static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte,
+					     struct mlx5_flow_destination *dest)
+{
+	struct mlx5_flow_rule *rule;
+
+	list_for_each_entry(rule, &fte->node.children, node.list) {
+		if (mlx5_flow_dests_cmp(&rule->dest_attr, dest))
+			return rule;
+	}
+	return NULL;
+}
+
+static bool check_conflicting_actions_vlan(const struct mlx5_fs_vlan *vlan0,
+					   const struct mlx5_fs_vlan *vlan1)
+{
+	return vlan0->ethtype != vlan1->ethtype ||
+	       vlan0->vid != vlan1->vid ||
+	       vlan0->prio != vlan1->prio;
+}
+
+static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
+				      const struct mlx5_flow_act *act2)
+{
+	u32 action1 = act1->action;
+	u32 action2 = act2->action;
+	u32 xored_actions;
+
+	xored_actions = action1 ^ action2;
+
+	/* if one rule only wants to count, it's ok */
+	if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT ||
+	    action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT)
+		return false;
+
+	if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP  |
+			     MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
+			     MLX5_FLOW_CONTEXT_ACTION_DECAP |
+			     MLX5_FLOW_CONTEXT_ACTION_MOD_HDR  |
+			     MLX5_FLOW_CONTEXT_ACTION_VLAN_POP |
+			     MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
+			     MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 |
+			     MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2))
+		return true;
+
+	if (action1 & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT &&
+	    act1->pkt_reformat != act2->pkt_reformat)
+		return true;
+
+	if (action1 & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
+	    act1->modify_hdr != act2->modify_hdr)
+		return true;
+
+	if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
+	    check_conflicting_actions_vlan(&act1->vlan[0], &act2->vlan[0]))
+		return true;
+
+	if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 &&
+	    check_conflicting_actions_vlan(&act1->vlan[1], &act2->vlan[1]))
+		return true;
+
+	return false;
+}
+
+static int check_conflicting_ftes(struct fs_fte *fte,
+				  const struct mlx5_flow_context *flow_context,
+				  const struct mlx5_flow_act *flow_act)
+{
+	if (check_conflicting_actions(flow_act, &fte->action)) {
+		mlx5_core_warn(get_dev(&fte->node),
+			       "Found two FTEs with conflicting actions\n");
+		return -EEXIST;
+	}
+
+	if ((flow_context->flags & FLOW_CONTEXT_HAS_TAG) &&
+	    fte->flow_context.flow_tag != flow_context->flow_tag) {
+		mlx5_core_warn(get_dev(&fte->node),
+			       "FTE flow tag %u already exists with different flow tag %u\n",
+			       fte->flow_context.flow_tag,
+			       flow_context->flow_tag);
+		return -EEXIST;
+	}
+
+	return 0;
+}
+
+static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg,
+					    const struct mlx5_flow_spec *spec,
+					    struct mlx5_flow_act *flow_act,
+					    struct mlx5_flow_destination *dest,
+					    int dest_num,
+					    struct fs_fte *fte)
+{
+	struct mlx5_flow_handle *handle;
+	int old_action;
+	int i;
+	int ret;
+
+	ret = check_conflicting_ftes(fte, &spec->flow_context, flow_act);
+	if (ret)
+		return ERR_PTR(ret);
+
+	old_action = fte->action.action;
+	fte->action.action |= flow_act->action;
+	handle = add_rule_fte(fte, fg, dest, dest_num,
+			      old_action != flow_act->action);
+	if (IS_ERR(handle)) {
+		fte->action.action = old_action;
+		return handle;
+	}
+
+	for (i = 0; i < handle->num_rules; i++) {
+		if (refcount_read(&handle->rule[i]->node.refcount) == 1) {
+			tree_add_node(&handle->rule[i]->node, &fte->node);
+		}
+	}
+	return handle;
+}
+
+static bool counter_is_valid(u32 action)
+{
+	return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP |
+			  MLX5_FLOW_CONTEXT_ACTION_ALLOW |
+			  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
+}
+
+static bool dest_is_valid(struct mlx5_flow_destination *dest,
+			  struct mlx5_flow_act *flow_act,
+			  struct mlx5_flow_table *ft)
+{
+	bool ignore_level = flow_act->flags & FLOW_ACT_IGNORE_FLOW_LEVEL;
+	u32 action = flow_act->action;
+
+	if (dest && (dest->type == MLX5_FLOW_DESTINATION_TYPE_COUNTER))
+		return counter_is_valid(action);
+
+	if (!(action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+		return true;
+
+	if (ignore_level) {
+		if (ft->type != FS_FT_FDB &&
+		    ft->type != FS_FT_NIC_RX &&
+		    ft->type != FS_FT_NIC_TX)
+			return false;
+
+		if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
+		    ft->type != dest->ft->type)
+			return false;
+	}
+
+	if (!dest || ((dest->type ==
+	    MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) &&
+	    (dest->ft->level <= ft->level && !ignore_level)))
+		return false;
+	return true;
+}
+
+struct match_list {
+	struct list_head	list;
+	struct mlx5_flow_group *g;
+};
+
+static void free_match_list(struct match_list *head, bool ft_locked)
+{
+	struct match_list *iter, *match_tmp;
+
+	list_for_each_entry_safe(iter, match_tmp, &head->list,
+				 list) {
+		tree_put_node(&iter->g->node, ft_locked);
+		list_del(&iter->list);
+		kfree(iter);
+	}
+}
+
+#define xa_for_each_rcu(xa, index, entry) \
+        for ((entry) = NULL, (index) = 0; \
+             ((entry) = xa_next(xa, &index, (entry) != NULL)) != NULL; )
+
+static int build_match_list(struct match_list *match_head,
+			    struct mlx5_flow_table *ft,
+			    const struct mlx5_flow_spec *spec,
+			    struct mlx5_flow_group *fg,
+			    bool ft_locked)
+{
+	struct mlx5_flow_group *g;
+	unsigned long id;
+
+	rcu_read_lock();
+	INIT_LIST_HEAD(&match_head->list);
+	xa_for_each_rcu(&ft->fgs_xa, id, g) {
+		struct match_list *curr_match;
+
+		if (memcmp(&g->mask, spec, sizeof_field(struct mlx5_flow_group,
+							mask)))
+			continue;
+
+		if (fg && fg != g)
+			continue;
+
+		if (unlikely(!tree_get_node(&g->node)))
+			continue;
+
+		curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC);
+		if (!curr_match) {
+			rcu_read_unlock();
+			free_match_list(match_head, ft_locked);
+			return -ENOMEM;
+		}
+		curr_match->g = g;
+		list_add_tail(&curr_match->list, &match_head->list);
+	}
+	rcu_read_unlock();
+	return 0;
+}
+
+static u64 matched_fgs_get_version(struct list_head *match_head)
+{
+	struct match_list *iter;
+	u64 version = 0;
+
+	list_for_each_entry(iter, match_head, list)
+		version += (u64)atomic_read(&iter->g->node.version);
+	return version;
+}
+
+static struct fs_fte *
+lookup_fte_locked(struct mlx5_flow_group *g,
+		  const u32 *match_value,
+		  bool take_write)
+{
+	struct fs_fte *fte_tmp;
+	unsigned long index;
+
+	if (take_write)
+		nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+	else
+		nested_down_read_ref_node(&g->node, FS_LOCK_PARENT);
+	xa_for_each_rcu(&g->ftes_xa, index, fte_tmp) {
+		if (!memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val)))
+			break;
+	}
+	if (!fte_tmp || !tree_get_node(&fte_tmp->node)) {
+		fte_tmp = NULL;
+		goto out;
+	}
+	if (!fte_tmp->node.active) {
+		tree_put_node(&fte_tmp->node, false);
+		fte_tmp = NULL;
+		goto out;
+	}
+
+	nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
+out:
+	if (take_write)
+		up_write_ref_node(&g->node, false);
+	else
+		up_read_ref_node(&g->node);
+	return fte_tmp;
+}
+
+static struct mlx5_flow_handle *
+try_add_to_existing_fg(struct mlx5_flow_table *ft,
+		       struct list_head *match_head,
+		       const struct mlx5_flow_spec *spec,
+		       struct mlx5_flow_act *flow_act,
+		       struct mlx5_flow_destination *dest,
+		       int dest_num,
+		       int ft_version)
+{
+	struct mlx5_flow_steering *steering = get_steering(&ft->node);
+	struct mlx5_flow_group *g;
+	struct mlx5_flow_handle *rule;
+	struct match_list *iter;
+	bool take_write = false;
+	struct fs_fte *fte;
+	u64  version = 0;
+	int err;
+
+	fte = alloc_fte(ft, spec, flow_act);
+	if (IS_ERR(fte))
+		return  ERR_PTR(-ENOMEM);
+
+search_again_locked:
+	if (flow_act->flags & FLOW_ACT_NO_APPEND)
+		goto skip_search;
+	version = matched_fgs_get_version(match_head);
+	/* Try to find an fte with identical match value and attempt update its
+	 * action.
+	 */
+	list_for_each_entry(iter, match_head, list) {
+		struct fs_fte *fte_tmp;
+
+		g = iter->g;
+		fte_tmp = lookup_fte_locked(g, spec->match_value, take_write);
+		if (!fte_tmp)
+			continue;
+		rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte_tmp);
+		/* No error check needed here, because insert_fte() is not called */
+		up_write_ref_node(&fte_tmp->node, false);
+		tree_put_node(&fte_tmp->node, false);
+		kmem_cache_free(steering->ftes_cache, fte);
+		return rule;
+	}
+
+skip_search:
+	/* No group with matching fte found, or we skipped the search.
+	 * Try to add a new fte to any matching fg.
+	 */
+
+	/* Check the ft version, for case that new flow group
+	 * was added while the fgs weren't locked
+	 */
+	if (atomic_read(&ft->node.version) != ft_version) {
+		rule = ERR_PTR(-EAGAIN);
+		goto out;
+	}
+
+	/* Check the fgs version. If version have changed it could be that an
+	 * FTE with the same match value was added while the fgs weren't
+	 * locked.
+	 */
+	if (!(flow_act->flags & FLOW_ACT_NO_APPEND) &&
+	    version != matched_fgs_get_version(match_head)) {
+		take_write = true;
+		goto search_again_locked;
+	}
+
+	list_for_each_entry(iter, match_head, list) {
+		g = iter->g;
+
+		nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+
+		if (!g->node.active) {
+			up_write_ref_node(&g->node, false);
+			continue;
+		}
+
+		err = insert_fte(g, fte);
+		if (err) {
+			up_write_ref_node(&g->node, false);
+			if (err == -ENOSPC)
+				continue;
+			kmem_cache_free(steering->ftes_cache, fte);
+			return ERR_PTR(err);
+		}
+
+		nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
+		up_write_ref_node(&g->node, false);
+		rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
+		up_write_ref_node(&fte->node, false);
+		if (IS_ERR(rule))
+			tree_put_node(&fte->node, false);
+		return rule;
+	}
+	rule = ERR_PTR(-ENOENT);
+out:
+	kmem_cache_free(steering->ftes_cache, fte);
+	return rule;
+}
+
+static struct mlx5_flow_handle *
+_mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+		     const struct mlx5_flow_spec *spec,
+		     struct mlx5_flow_act *flow_act,
+		     struct mlx5_flow_destination *dest,
+		     int dest_num)
+
+{
+	struct mlx5_flow_steering *steering = get_steering(&ft->node);
+	struct mlx5_flow_handle *rule;
+	struct match_list match_head;
+	struct mlx5_flow_group *g;
+	bool take_write = false;
+	struct fs_fte *fte;
+	int version;
+	int err;
+	int i;
+
+	if (!check_valid_spec(spec))
+		return ERR_PTR(-EINVAL);
+
+	if (flow_act->fg && ft->autogroup.active)
+		return ERR_PTR(-EINVAL);
+
+	if (dest && dest_num <= 0)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < dest_num; i++) {
+		if (!dest_is_valid(&dest[i], flow_act, ft))
+			return ERR_PTR(-EINVAL);
+	}
+	nested_down_read_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
+search_again_locked:
+	version = atomic_read(&ft->node.version);
+
+	/* Collect all fgs which has a matching match_criteria */
+	err = build_match_list(&match_head, ft, spec, flow_act->fg, take_write);
+	if (err) {
+		if (take_write)
+			up_write_ref_node(&ft->node, false);
+		else
+			up_read_ref_node(&ft->node);
+		return ERR_PTR(err);
+	}
+
+	if (!take_write)
+		up_read_ref_node(&ft->node);
+
+	rule = try_add_to_existing_fg(ft, &match_head.list, spec, flow_act, dest,
+				      dest_num, version);
+	free_match_list(&match_head, take_write);
+	if (!IS_ERR(rule) ||
+	    (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) {
+		if (take_write)
+			up_write_ref_node(&ft->node, false);
+		return rule;
+	}
+
+	if (!take_write) {
+		nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
+		take_write = true;
+	}
+
+	if (PTR_ERR(rule) == -EAGAIN ||
+	    version != atomic_read(&ft->node.version))
+		goto search_again_locked;
+
+	g = alloc_auto_flow_group(ft, spec);
+	if (IS_ERR(g)) {
+		rule = ERR_CAST(g);
+		up_write_ref_node(&ft->node, false);
+		return rule;
+	}
+
+	fte = alloc_fte(ft, spec, flow_act);
+	if (IS_ERR(fte)) {
+		up_write_ref_node(&ft->node, false);
+		err = PTR_ERR(fte);
+		goto err_alloc_fte;
+	}
+
+	nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+	up_write_ref_node(&ft->node, false);
+
+	err = create_auto_flow_group(ft, g);
+	if (err)
+		goto err_release_fg;
+
+	err = insert_fte(g, fte);
+	if (err)
+		goto err_release_fg;
+
+	nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
+	up_write_ref_node(&g->node, false);
+	rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
+	up_write_ref_node(&fte->node, false);
+	if (IS_ERR(rule))
+		tree_put_node(&fte->node, false);
+	tree_put_node(&g->node, false);
+	return rule;
+
+err_release_fg:
+	up_write_ref_node(&g->node, false);
+	kmem_cache_free(steering->ftes_cache, fte);
+err_alloc_fte:
+	tree_put_node(&g->node, false);
+	return ERR_PTR(err);
+}
+
+static bool fwd_next_prio_supported(struct mlx5_flow_table *ft)
+{
+	return ((ft->type == FS_FT_NIC_RX) &&
+		(MLX5_CAP_FLOWTABLE(get_dev(&ft->node), nic_rx_multi_path_tirs)));
+}
+
+struct mlx5_flow_handle *
+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+		    const struct mlx5_flow_spec *spec,
+		    struct mlx5_flow_act *flow_act,
+		    struct mlx5_flow_destination *dest,
+		    int num_dest)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+	static const struct mlx5_flow_spec zero_spec = {};
+	struct mlx5_flow_destination *gen_dest = NULL;
+	struct mlx5_flow_table *next_ft = NULL;
+	struct mlx5_flow_handle *handle = NULL;
+	u32 sw_action = flow_act->action;
+	int i;
+
+	if (!spec)
+		spec = &zero_spec;
+
+	if (!is_fwd_next_action(sw_action))
+		return _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
+
+	if (!fwd_next_prio_supported(ft))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	mutex_lock(&root->chain_lock);
+	next_ft = find_next_fwd_ft(ft, flow_act);
+	if (!next_ft) {
+		handle = ERR_PTR(-EOPNOTSUPP);
+		goto unlock;
+	}
+
+	gen_dest = kcalloc(num_dest + 1, sizeof(*dest),
+			   GFP_KERNEL);
+	if (!gen_dest) {
+		handle = ERR_PTR(-ENOMEM);
+		goto unlock;
+	}
+	for (i = 0; i < num_dest; i++)
+		gen_dest[i] = dest[i];
+	gen_dest[i].type =
+		MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	gen_dest[i].ft = next_ft;
+	dest = gen_dest;
+	num_dest++;
+	flow_act->action &= ~(MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
+			      MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
+	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	handle = _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
+	if (IS_ERR(handle))
+		goto unlock;
+
+	if (list_empty(&handle->rule[num_dest - 1]->next_ft)) {
+		mutex_lock(&next_ft->lock);
+		list_add(&handle->rule[num_dest - 1]->next_ft,
+			 &next_ft->fwd_rules);
+		mutex_unlock(&next_ft->lock);
+		handle->rule[num_dest - 1]->sw_action = sw_action;
+		handle->rule[num_dest - 1]->ft = ft;
+	}
+unlock:
+	mutex_unlock(&root->chain_lock);
+	kfree(gen_dest);
+	return handle;
+}
+EXPORT_SYMBOL(mlx5_add_flow_rules);
+
+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp)
+{
+	struct mlx5_flow_handle *handle;
+	struct fs_fte *fte;
+	int i;
+
+	handle = *pp;
+	*pp = NULL;
+	if (IS_ERR_OR_NULL(handle))
+		return;
+
+	/* In order to consolidate the HW changes we lock the FTE for other
+	 * changes, and increase its refcount, in order not to perform the
+	 * "del" functions of the FTE. Will handle them here.
+	 * The removal of the rules is done under locked FTE.
+	 * After removing all the handle's rules, if there are remaining
+	 * rules, it means we just need to modify the FTE in FW, and
+	 * unlock/decrease the refcount we increased before.
+	 * Otherwise, it means the FTE should be deleted. First delete the
+	 * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of
+	 * the FTE, which will handle the last decrease of the refcount, as
+	 * well as required handling of its parent.
+	 */
+	fs_get_obj(fte, handle->rule[0]->node.parent);
+	down_write_ref_node(&fte->node, false);
+	for (i = handle->num_rules - 1; i >= 0; i--)
+		tree_remove_node(&handle->rule[i]->node, true);
+	if (list_empty(&fte->node.children)) {
+		fte->node.del_hw_func(&fte->node);
+		/* Avoid double call to del_hw_fte */
+		fte->node.del_hw_func = NULL;
+		up_write_ref_node(&fte->node, false);
+		tree_put_node(&fte->node, false);
+	} else if (fte->dests_size) {
+		if (fte->modify_mask)
+			modify_fte(fte);
+		up_write_ref_node(&fte->node, false);
+	} else {
+		up_write_ref_node(&fte->node, false);
+	}
+	kfree(handle);
+}
+EXPORT_SYMBOL(mlx5_del_flow_rules);
+
+/* Assuming prio->node.children(flow tables) is sorted by level */
+static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft)
+{
+	struct fs_prio *prio;
+
+	fs_get_obj(prio, ft->node.parent);
+
+	if (!list_is_last(&ft->node.list, &prio->node.children))
+		return list_next_entry(ft, node.list);
+	return find_next_chained_ft(prio);
+}
+
+static int update_root_ft_destroy(struct mlx5_flow_table *ft)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+	struct mlx5_ft_underlay_qp *uqp;
+	struct mlx5_flow_table *new_root_ft = NULL;
+	int err = 0;
+	u32 qpn;
+
+	if (root->root_ft != ft)
+		return 0;
+
+	new_root_ft = find_next_ft(ft);
+	if (!new_root_ft) {
+		root->root_ft = NULL;
+		return 0;
+	}
+
+	if (list_empty(&root->underlay_qpns)) {
+		/* Don't set any QPN (zero) in case QPN list is empty */
+		qpn = 0;
+		err = root->cmds->update_root_ft(root, new_root_ft,
+						 qpn, false);
+	} else {
+		list_for_each_entry(uqp, &root->underlay_qpns, list) {
+			qpn = uqp->qpn;
+			err = root->cmds->update_root_ft(root,
+							 new_root_ft, qpn,
+							 false);
+			if (err)
+				break;
+		}
+	}
+
+	if (err)
+		mlx5_core_warn(root->dev,
+			       "Update root flow table of id(%u) qpn(%d) failed\n",
+			       ft->id, qpn);
+	else
+		root->root_ft = new_root_ft;
+
+	return 0;
+}
+
+/* Connect flow table from previous priority to
+ * the next flow table.
+ */
+static int disconnect_flow_table(struct mlx5_flow_table *ft)
+{
+	struct mlx5_core_dev *dev = get_dev(&ft->node);
+	struct mlx5_flow_table *next_ft;
+	struct fs_prio *prio;
+	int err = 0;
+
+	err = update_root_ft_destroy(ft);
+	if (err)
+		return err;
+
+	fs_get_obj(prio, ft->node.parent);
+	if  (!(list_first_entry(&prio->node.children,
+				struct mlx5_flow_table,
+				node.list) == ft))
+		return 0;
+
+	next_ft = find_next_ft(ft);
+	err = connect_fwd_rules(dev, next_ft, ft);
+	if (err)
+		return err;
+
+	err = connect_prev_fts(dev, next_ft, prio);
+	if (err)
+		mlx5_core_warn(dev, "Failed to disconnect flow table %d\n",
+			       ft->id);
+	return err;
+}
+
+int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
+{
+	struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+	int err = 0;
+
+	mutex_lock(&root->chain_lock);
+	if (!(ft->flags & MLX5_FLOW_TABLE_UNMANAGED))
+		err = disconnect_flow_table(ft);
+	if (err) {
+		mutex_unlock(&root->chain_lock);
+		return err;
+	}
+	if (tree_remove_node(&ft->node, false))
+		mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n",
+			       ft->id);
+	mutex_unlock(&root->chain_lock);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_destroy_flow_table);
+
+void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
+{
+	if (tree_remove_node(&fg->node, false))
+		mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
+			       fg->id);
+}
+EXPORT_SYMBOL(mlx5_destroy_flow_group);
+
+struct mlx5_flow_namespace *mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev,
+						int n)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+
+	if (!steering || !steering->fdb_sub_ns)
+		return NULL;
+
+	return steering->fdb_sub_ns[n];
+}
+EXPORT_SYMBOL(mlx5_get_fdb_sub_ns);
+
+static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type)
+{
+	switch (type) {
+	case MLX5_FLOW_NAMESPACE_BYPASS:
+	case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
+	case MLX5_FLOW_NAMESPACE_LAG:
+	case MLX5_FLOW_NAMESPACE_OFFLOADS:
+	case MLX5_FLOW_NAMESPACE_ETHTOOL:
+	case MLX5_FLOW_NAMESPACE_KERNEL:
+	case MLX5_FLOW_NAMESPACE_LEFTOVERS:
+	case MLX5_FLOW_NAMESPACE_ANCHOR:
+		return true;
+	default:
+		return false;
+	}
+}
+
+struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
+						    enum mlx5_flow_namespace_type type)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	struct mlx5_flow_root_namespace *root_ns;
+	int prio = 0;
+	struct fs_prio *fs_prio;
+	struct mlx5_flow_namespace *ns;
+
+	if (!steering)
+		return NULL;
+
+	switch (type) {
+	case MLX5_FLOW_NAMESPACE_FDB:
+		if (steering->fdb_root_ns)
+			return &steering->fdb_root_ns->ns;
+		return NULL;
+	case MLX5_FLOW_NAMESPACE_PORT_SEL:
+		if (steering->port_sel_root_ns)
+			return &steering->port_sel_root_ns->ns;
+		return NULL;
+	case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
+		if (steering->sniffer_rx_root_ns)
+			return &steering->sniffer_rx_root_ns->ns;
+		return NULL;
+	case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
+		if (steering->sniffer_tx_root_ns)
+			return &steering->sniffer_tx_root_ns->ns;
+		return NULL;
+	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
+		root_ns = steering->fdb_root_ns;
+		prio =  FDB_BYPASS_PATH;
+		break;
+	case MLX5_FLOW_NAMESPACE_EGRESS:
+	case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+	case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
+		root_ns = steering->egress_root_ns;
+		prio = type - MLX5_FLOW_NAMESPACE_EGRESS;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_RX:
+		root_ns = steering->rdma_rx_root_ns;
+		prio = RDMA_RX_BYPASS_PRIO;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL:
+		root_ns = steering->rdma_rx_root_ns;
+		prio = RDMA_RX_KERNEL_PRIO;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_TX:
+		root_ns = steering->rdma_tx_root_ns;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS:
+		root_ns = steering->rdma_rx_root_ns;
+		prio = RDMA_RX_COUNTERS_PRIO;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS:
+		root_ns = steering->rdma_tx_root_ns;
+		prio = RDMA_TX_COUNTERS_PRIO;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC:
+		root_ns = steering->rdma_rx_root_ns;
+		prio = RDMA_RX_IPSEC_PRIO;
+		break;
+	case MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC:
+		root_ns = steering->rdma_tx_root_ns;
+		prio = RDMA_TX_IPSEC_PRIO;
+		break;
+	default: /* Must be NIC RX */
+		WARN_ON(!is_nic_rx_ns(type));
+		root_ns = steering->root_ns;
+		prio = type;
+		break;
+	}
+
+	if (!root_ns)
+		return NULL;
+
+	fs_prio = find_prio(&root_ns->ns, prio);
+	if (!fs_prio)
+		return NULL;
+
+	ns = list_first_entry(&fs_prio->node.children,
+			      typeof(*ns),
+			      node.list);
+
+	return ns;
+}
+EXPORT_SYMBOL(mlx5_get_flow_namespace);
+
+struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
+							      enum mlx5_flow_namespace_type type,
+							      int vport)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+
+	if (!steering)
+		return NULL;
+
+	switch (type) {
+	case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
+		if (vport >= steering->esw_egress_acl_vports)
+			return NULL;
+		if (steering->esw_egress_root_ns &&
+		    steering->esw_egress_root_ns[vport])
+			return &steering->esw_egress_root_ns[vport]->ns;
+		else
+			return NULL;
+	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
+		if (vport >= steering->esw_ingress_acl_vports)
+			return NULL;
+		if (steering->esw_ingress_root_ns &&
+		    steering->esw_ingress_root_ns[vport])
+			return &steering->esw_ingress_root_ns[vport]->ns;
+		else
+			return NULL;
+	default:
+		return NULL;
+	}
+}
+
+static struct fs_prio *_fs_create_prio(struct mlx5_flow_namespace *ns,
+				       unsigned int prio,
+				       int num_levels,
+				       enum fs_node_type type)
+{
+	struct fs_prio *fs_prio;
+
+	fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
+	if (!fs_prio)
+		return ERR_PTR(-ENOMEM);
+
+	fs_prio->node.type = type;
+	tree_init_node(&fs_prio->node, NULL, del_sw_prio);
+	tree_add_node(&fs_prio->node, &ns->node);
+	fs_prio->num_levels = num_levels;
+	fs_prio->prio = prio;
+	list_add_tail(&fs_prio->node.list, &ns->node.children);
+
+	return fs_prio;
+}
+
+static struct fs_prio *fs_create_prio_chained(struct mlx5_flow_namespace *ns,
+					      unsigned int prio,
+					      int num_levels)
+{
+	return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO_CHAINS);
+}
+
+static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
+				      unsigned int prio, int num_levels)
+{
+	return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO);
+}
+
+static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
+						     *ns)
+{
+	ns->node.type = FS_TYPE_NAMESPACE;
+
+	return ns;
+}
+
+static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
+						       int def_miss_act)
+{
+	struct mlx5_flow_namespace	*ns;
+
+	ns = kzalloc(sizeof(*ns), GFP_KERNEL);
+	if (!ns)
+		return ERR_PTR(-ENOMEM);
+
+	fs_init_namespace(ns);
+	ns->def_miss_action = def_miss_act;
+	tree_init_node(&ns->node, NULL, del_sw_ns);
+	tree_add_node(&ns->node, &prio->node);
+	list_add_tail(&ns->node.list, &prio->node.children);
+
+	return ns;
+}
+
+static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio,
+			     struct init_tree_node *prio_metadata)
+{
+	struct fs_prio *fs_prio;
+	int i;
+
+	for (i = 0; i < prio_metadata->num_leaf_prios; i++) {
+		fs_prio = fs_create_prio(ns, prio++, prio_metadata->num_levels);
+		if (IS_ERR(fs_prio))
+			return PTR_ERR(fs_prio);
+	}
+	return 0;
+}
+
+#define FLOW_TABLE_BIT_SZ 1
+#define GET_FLOW_TABLE_CAP(dev, offset) \
+	((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) +	\
+			offset / 32)) >>					\
+	  (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
+static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
+{
+	int i;
+
+	for (i = 0; i < caps->arr_sz; i++) {
+		if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
+			return false;
+	}
+	return true;
+}
+
+static int init_root_tree_recursive(struct mlx5_flow_steering *steering,
+				    struct init_tree_node *init_node,
+				    struct fs_node *fs_parent_node,
+				    struct init_tree_node *init_parent_node,
+				    int prio)
+{
+	int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev,
+					      flow_table_properties_nic_receive.
+					      max_ft_level);
+	struct mlx5_flow_namespace *fs_ns;
+	struct fs_prio *fs_prio;
+	struct fs_node *base;
+	int i;
+	int err;
+
+	if (init_node->type == FS_TYPE_PRIO) {
+		if ((init_node->min_ft_level > max_ft_level) ||
+		    !has_required_caps(steering->dev, &init_node->caps))
+			return 0;
+
+		fs_get_obj(fs_ns, fs_parent_node);
+		if (init_node->num_leaf_prios)
+			return create_leaf_prios(fs_ns, prio, init_node);
+		fs_prio = fs_create_prio(fs_ns, prio, init_node->num_levels);
+		if (IS_ERR(fs_prio))
+			return PTR_ERR(fs_prio);
+		base = &fs_prio->node;
+	} else if (init_node->type == FS_TYPE_NAMESPACE) {
+		fs_get_obj(fs_prio, fs_parent_node);
+		fs_ns = fs_create_namespace(fs_prio, init_node->def_miss_action);
+		if (IS_ERR(fs_ns))
+			return PTR_ERR(fs_ns);
+		base = &fs_ns->node;
+	} else {
+		return -EINVAL;
+	}
+	prio = 0;
+	for (i = 0; i < init_node->ar_size; i++) {
+		err = init_root_tree_recursive(steering, &init_node->children[i],
+					       base, init_node, prio);
+		if (err)
+			return err;
+		if (init_node->children[i].type == FS_TYPE_PRIO &&
+		    init_node->children[i].num_leaf_prios) {
+			prio += init_node->children[i].num_leaf_prios;
+		}
+	}
+
+	return 0;
+}
+
+static int init_root_tree(struct mlx5_flow_steering *steering,
+			  struct init_tree_node *init_node,
+			  struct fs_node *fs_parent_node)
+{
+	int err;
+	int i;
+
+	for (i = 0; i < init_node->ar_size; i++) {
+		err = init_root_tree_recursive(steering, &init_node->children[i],
+					       fs_parent_node,
+					       init_node, i);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+static void del_sw_root_ns(struct fs_node *node)
+{
+	struct mlx5_flow_root_namespace *root_ns;
+	struct mlx5_flow_namespace *ns;
+
+	fs_get_obj(ns, node);
+	root_ns = container_of(ns, struct mlx5_flow_root_namespace, ns);
+	mutex_destroy(&root_ns->chain_lock);
+	kfree(node);
+}
+
+static struct mlx5_flow_root_namespace
+*create_root_ns(struct mlx5_flow_steering *steering,
+		enum fs_flow_table_type table_type)
+{
+	const struct mlx5_flow_cmds *cmds = mlx5_fs_cmd_get_default(table_type);
+	struct mlx5_flow_root_namespace *root_ns;
+	struct mlx5_flow_namespace *ns;
+
+	/* Create the root namespace */
+	root_ns = kzalloc(sizeof(*root_ns), GFP_KERNEL);
+	if (!root_ns)
+		return NULL;
+
+	root_ns->dev = steering->dev;
+	root_ns->table_type = table_type;
+	root_ns->cmds = cmds;
+
+	INIT_LIST_HEAD(&root_ns->underlay_qpns);
+
+	ns = &root_ns->ns;
+	fs_init_namespace(ns);
+	mutex_init(&root_ns->chain_lock);
+	tree_init_node(&ns->node, NULL, del_sw_root_ns);
+	tree_add_node(&ns->node, NULL);
+
+	return root_ns;
+}
+
+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level);
+
+static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level)
+{
+	struct fs_prio *prio;
+
+	fs_for_each_prio(prio, ns) {
+		 /* This updates prio start_level and num_levels */
+		set_prio_attrs_in_prio(prio, acc_level);
+		acc_level += prio->num_levels;
+	}
+	return acc_level;
+}
+
+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level)
+{
+	struct mlx5_flow_namespace *ns;
+	int acc_level_ns = acc_level;
+
+	prio->start_level = acc_level;
+	fs_for_each_ns(ns, prio) {
+		/* This updates start_level and num_levels of ns's priority descendants */
+		acc_level_ns = set_prio_attrs_in_ns(ns, acc_level);
+
+		/* If this a prio with chains, and we can jump from one chain
+		 * (namespace) to another, so we accumulate the levels
+		 */
+		if (prio->node.type == FS_TYPE_PRIO_CHAINS)
+			acc_level = acc_level_ns;
+	}
+
+	if (!prio->num_levels)
+		prio->num_levels = acc_level_ns - prio->start_level;
+	WARN_ON(prio->num_levels < acc_level_ns - prio->start_level);
+}
+
+static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns)
+{
+	struct mlx5_flow_namespace *ns = &root_ns->ns;
+	struct fs_prio *prio;
+	int start_level = 0;
+
+	fs_for_each_prio(prio, ns) {
+		set_prio_attrs_in_prio(prio, start_level);
+		start_level += prio->num_levels;
+	}
+}
+
+#define ANCHOR_PRIO 0
+#define ANCHOR_SIZE 1
+#define ANCHOR_LEVEL 0
+static int create_anchor_flow_table(struct mlx5_flow_steering *steering)
+{
+	struct mlx5_flow_namespace *ns = NULL;
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_table *ft;
+
+	ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR);
+	if (WARN_ON(!ns))
+		return -EINVAL;
+
+	ft_attr.max_fte = ANCHOR_SIZE;
+	ft_attr.level   = ANCHOR_LEVEL;
+	ft_attr.prio    = ANCHOR_PRIO;
+
+	ft = mlx5_create_flow_table(ns, &ft_attr);
+	if (IS_ERR(ft)) {
+		mlx5_core_err(steering->dev, "Failed to create last anchor flow table");
+		return PTR_ERR(ft);
+	}
+	return 0;
+}
+
+static int init_root_ns(struct mlx5_flow_steering *steering)
+{
+	int err;
+
+	steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX);
+	if (!steering->root_ns)
+		return -ENOMEM;
+
+	err = init_root_tree(steering, &root_fs, &steering->root_ns->ns.node);
+	if (err)
+		goto out_err;
+
+	set_prio_attrs(steering->root_ns);
+	err = create_anchor_flow_table(steering);
+	if (err)
+		goto out_err;
+
+	return 0;
+
+out_err:
+	cleanup_root_ns(steering->root_ns);
+	steering->root_ns = NULL;
+	return err;
+}
+
+static void clean_tree(struct fs_node *node)
+{
+	if (node) {
+		struct fs_node *iter;
+		struct fs_node *temp;
+
+		tree_get_node(node);
+		list_for_each_entry_safe(iter, temp, &node->children, list)
+			clean_tree(iter);
+		tree_put_node(node, false);
+		tree_remove_node(node, false);
+	}
+}
+
+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
+{
+	if (!root_ns)
+		return;
+
+	clean_tree(&root_ns->ns.node);
+}
+
+static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *prio;
+
+	steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);
+	if (!steering->sniffer_tx_root_ns)
+		return -ENOMEM;
+
+	/* Create single prio */
+	prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);
+	return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *prio;
+
+	steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);
+	if (!steering->sniffer_rx_root_ns)
+		return -ENOMEM;
+
+	/* Create single prio */
+	prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);
+	return PTR_ERR_OR_ZERO(prio);
+}
+
+#define PORT_SEL_NUM_LEVELS 3
+static int init_port_sel_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *prio;
+
+	steering->port_sel_root_ns = create_root_ns(steering, FS_FT_PORT_SEL);
+	if (!steering->port_sel_root_ns)
+		return -ENOMEM;
+
+	/* Create single prio */
+	prio = fs_create_prio(&steering->port_sel_root_ns->ns, 0,
+			      PORT_SEL_NUM_LEVELS);
+	return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering)
+{
+	int err;
+
+	steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX);
+	if (!steering->rdma_rx_root_ns)
+		return -ENOMEM;
+
+	err = init_root_tree(steering, &rdma_rx_root_fs,
+			     &steering->rdma_rx_root_ns->ns.node);
+	if (err)
+		goto out_err;
+
+	set_prio_attrs(steering->rdma_rx_root_ns);
+
+	return 0;
+
+out_err:
+	cleanup_root_ns(steering->rdma_rx_root_ns);
+	steering->rdma_rx_root_ns = NULL;
+	return err;
+}
+
+static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering)
+{
+	int err;
+
+	steering->rdma_tx_root_ns = create_root_ns(steering, FS_FT_RDMA_TX);
+	if (!steering->rdma_tx_root_ns)
+		return -ENOMEM;
+
+	err = init_root_tree(steering, &rdma_tx_root_fs,
+			     &steering->rdma_tx_root_ns->ns.node);
+	if (err)
+		goto out_err;
+
+	set_prio_attrs(steering->rdma_tx_root_ns);
+
+	return 0;
+
+out_err:
+	cleanup_root_ns(steering->rdma_tx_root_ns);
+	steering->rdma_tx_root_ns = NULL;
+	return err;
+}
+
+/* FT and tc chains are stored in the same array so we can re-use the
+ * mlx5_get_fdb_sub_ns() and tc api for FT chains.
+ * When creating a new ns for each chain store it in the first available slot.
+ * Assume tc chains are created and stored first and only then the FT chain.
+ */
+static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
+					struct mlx5_flow_namespace *ns)
+{
+	int chain = 0;
+
+	while (steering->fdb_sub_ns[chain])
+		++chain;
+
+	steering->fdb_sub_ns[chain] = ns;
+}
+
+static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
+					struct fs_prio *maj_prio)
+{
+	struct mlx5_flow_namespace *ns;
+	struct fs_prio *min_prio;
+	int prio;
+
+	ns = fs_create_namespace(maj_prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
+	if (IS_ERR(ns))
+		return PTR_ERR(ns);
+
+	for (prio = 0; prio < FDB_TC_MAX_PRIO; prio++) {
+		min_prio = fs_create_prio(ns, prio, FDB_TC_LEVELS_PER_PRIO);
+		if (IS_ERR(min_prio))
+			return PTR_ERR(min_prio);
+	}
+
+	store_fdb_sub_ns_prio_chain(steering, ns);
+
+	return 0;
+}
+
+static int create_fdb_chains(struct mlx5_flow_steering *steering,
+			     int fs_prio,
+			     int chains)
+{
+	struct fs_prio *maj_prio;
+	int levels;
+	int chain;
+	int err;
+
+	levels = FDB_TC_LEVELS_PER_PRIO * FDB_TC_MAX_PRIO * chains;
+	maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,
+					  fs_prio,
+					  levels);
+	if (IS_ERR(maj_prio))
+		return PTR_ERR(maj_prio);
+
+	for (chain = 0; chain < chains; chain++) {
+		err = create_fdb_sub_ns_prio_chain(steering, maj_prio);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int create_fdb_fast_path(struct mlx5_flow_steering *steering)
+{
+	int err;
+
+	steering->fdb_sub_ns = kcalloc(FDB_NUM_CHAINS,
+				       sizeof(*steering->fdb_sub_ns),
+				       GFP_KERNEL);
+	if (!steering->fdb_sub_ns)
+		return -ENOMEM;
+
+	err = create_fdb_chains(steering, FDB_TC_OFFLOAD, FDB_TC_MAX_CHAIN + 1);
+	if (err)
+		return err;
+
+	err = create_fdb_chains(steering, FDB_FT_OFFLOAD, 1);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int create_fdb_bypass(struct mlx5_flow_steering *steering)
+{
+	struct mlx5_flow_namespace *ns;
+	struct fs_prio *prio;
+	int i;
+
+	prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, 0);
+	if (IS_ERR(prio))
+		return PTR_ERR(prio);
+
+	ns = fs_create_namespace(prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
+	if (IS_ERR(ns))
+		return PTR_ERR(ns);
+
+	for (i = 0; i < MLX5_BY_PASS_NUM_REGULAR_PRIOS; i++) {
+		prio = fs_create_prio(ns, i, 1);
+		if (IS_ERR(prio))
+			return PTR_ERR(prio);
+	}
+	return 0;
+}
+
+static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering)
+{
+	cleanup_root_ns(steering->fdb_root_ns);
+	steering->fdb_root_ns = NULL;
+	kfree(steering->fdb_sub_ns);
+	steering->fdb_sub_ns = NULL;
+}
+
+static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *maj_prio;
+	int err;
+
+	steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB);
+	if (!steering->fdb_root_ns)
+		return -ENOMEM;
+
+	err = create_fdb_bypass(steering);
+	if (err)
+		goto out_err;
+
+	err = create_fdb_fast_path(steering);
+	if (err)
+		goto out_err;
+
+	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1);
+	if (IS_ERR(maj_prio)) {
+		err = PTR_ERR(maj_prio);
+		goto out_err;
+	}
+
+	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4);
+	if (IS_ERR(maj_prio)) {
+		err = PTR_ERR(maj_prio);
+		goto out_err;
+	}
+
+	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);
+	if (IS_ERR(maj_prio)) {
+		err = PTR_ERR(maj_prio);
+		goto out_err;
+	}
+
+	/* We put this priority last, knowing that nothing will get here
+	 * unless explicitly forwarded to. This is possible because the
+	 * slow path tables have catch all rules and nothing gets passed
+	 * those tables.
+	 */
+	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_PER_VPORT, 1);
+	if (IS_ERR(maj_prio)) {
+		err = PTR_ERR(maj_prio);
+		goto out_err;
+	}
+
+	set_prio_attrs(steering->fdb_root_ns);
+	return 0;
+
+out_err:
+	cleanup_fdb_root_ns(steering);
+	return err;
+}
+
+static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
+{
+	struct fs_prio *prio;
+
+	steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
+	if (!steering->esw_egress_root_ns[vport])
+		return -ENOMEM;
+
+	/* create 1 prio*/
+	prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);
+	return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
+{
+	struct fs_prio *prio;
+
+	steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
+	if (!steering->esw_ingress_root_ns[vport])
+		return -ENOMEM;
+
+	/* create 1 prio*/
+	prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);
+	return PTR_ERR_OR_ZERO(prio);
+}
+
+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int err;
+	int i;
+
+	steering->esw_egress_root_ns =
+			kcalloc(total_vports,
+				sizeof(*steering->esw_egress_root_ns),
+				GFP_KERNEL);
+	if (!steering->esw_egress_root_ns)
+		return -ENOMEM;
+
+	for (i = 0; i < total_vports; i++) {
+		err = init_egress_acl_root_ns(steering, i);
+		if (err)
+			goto cleanup_root_ns;
+	}
+	steering->esw_egress_acl_vports = total_vports;
+	return 0;
+
+cleanup_root_ns:
+	for (i--; i >= 0; i--)
+		cleanup_root_ns(steering->esw_egress_root_ns[i]);
+	kfree(steering->esw_egress_root_ns);
+	steering->esw_egress_root_ns = NULL;
+	return err;
+}
+
+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int i;
+
+	if (!steering->esw_egress_root_ns)
+		return;
+
+	for (i = 0; i < steering->esw_egress_acl_vports; i++)
+		cleanup_root_ns(steering->esw_egress_root_ns[i]);
+
+	kfree(steering->esw_egress_root_ns);
+	steering->esw_egress_root_ns = NULL;
+}
+
+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int err;
+	int i;
+
+	steering->esw_ingress_root_ns =
+			kcalloc(total_vports,
+				sizeof(*steering->esw_ingress_root_ns),
+				GFP_KERNEL);
+	if (!steering->esw_ingress_root_ns)
+		return -ENOMEM;
+
+	for (i = 0; i < total_vports; i++) {
+		err = init_ingress_acl_root_ns(steering, i);
+		if (err)
+			goto cleanup_root_ns;
+	}
+	steering->esw_ingress_acl_vports = total_vports;
+	return 0;
+
+cleanup_root_ns:
+	for (i--; i >= 0; i--)
+		cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+	kfree(steering->esw_ingress_root_ns);
+	steering->esw_ingress_root_ns = NULL;
+	return err;
+}
+
+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int i;
+
+	if (!steering->esw_ingress_root_ns)
+		return;
+
+	for (i = 0; i < steering->esw_ingress_acl_vports; i++)
+		cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+
+	kfree(steering->esw_ingress_root_ns);
+	steering->esw_ingress_root_ns = NULL;
+}
+
+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_flow_namespace *ns;
+
+	ns = mlx5_get_flow_namespace(dev, type);
+	if (!ns)
+		return 0;
+
+	root = find_root(&ns->node);
+	if (!root)
+		return 0;
+
+	return root->cmds->get_capabilities(root, root->table_type);
+}
+
+static int init_egress_root_ns(struct mlx5_flow_steering *steering)
+{
+	int err;
+
+	steering->egress_root_ns = create_root_ns(steering,
+						  FS_FT_NIC_TX);
+	if (!steering->egress_root_ns)
+		return -ENOMEM;
+
+	err = init_root_tree(steering, &egress_root_fs,
+			     &steering->egress_root_ns->ns.node);
+	if (err)
+		goto cleanup;
+	set_prio_attrs(steering->egress_root_ns);
+	return 0;
+cleanup:
+	cleanup_root_ns(steering->egress_root_ns);
+	steering->egress_root_ns = NULL;
+	return err;
+}
+
+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+
+	cleanup_root_ns(steering->root_ns);
+	cleanup_fdb_root_ns(steering);
+	cleanup_root_ns(steering->port_sel_root_ns);
+	cleanup_root_ns(steering->sniffer_rx_root_ns);
+	cleanup_root_ns(steering->sniffer_tx_root_ns);
+	cleanup_root_ns(steering->rdma_rx_root_ns);
+	cleanup_root_ns(steering->rdma_tx_root_ns);
+	cleanup_root_ns(steering->egress_root_ns);
+}
+
+int mlx5_fs_core_init(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int err;
+
+	if ((((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
+	      (MLX5_CAP_GEN(dev, nic_flow_table))) ||
+	     ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
+	      MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) &&
+	    MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_support)) {
+		err = init_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_ESWITCH_MANAGER(dev)) {
+		if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ft_support)) {
+			err = init_fdb_root_ns(steering);
+			if (err)
+				goto err;
+		}
+	}
+
+	if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) {
+		err = init_sniffer_rx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) {
+		err = init_sniffer_tx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_PORT_SELECTION(dev, ft_support)) {
+		err = init_port_sel_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) &&
+	    MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain)) {
+		err = init_rdma_rx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_RDMA_TX(dev, ft_support)) {
+		err = init_rdma_tx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_NIC_TX(dev, ft_support)) {
+		err = init_egress_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	mlx5_fs_core_cleanup(dev);
+	return err;
+}
+
+void mlx5_fs_core_free(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+
+	kmem_cache_destroy(steering->ftes_cache);
+	kmem_cache_destroy(steering->fgs_cache);
+	kfree(steering);
+	mlx5_ft_pool_destroy(dev);
+	mlx5_cleanup_fc_stats(dev);
+}
+
+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering;
+	int err = 0;
+
+	err = mlx5_init_fc_stats(dev);
+	if (err)
+		return err;
+
+	err = mlx5_ft_pool_init(dev);
+	if (err)
+		goto err;
+
+	steering = kzalloc(sizeof(*steering), GFP_KERNEL);
+	if (!steering) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	steering->dev = dev;
+	dev->priv.steering = steering;
+
+	steering->mode = MLX5_FLOW_STEERING_MODE_DMFS;
+
+	steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs",
+						sizeof(struct mlx5_flow_group), 0,
+						0, NULL);
+	steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0,
+						 0, NULL);
+	if (!steering->ftes_cache || !steering->fgs_cache) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	return 0;
+
+err:
+	mlx5_fs_core_free(dev);
+	return err;
+}
+
+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
+{
+	struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
+	struct mlx5_ft_underlay_qp *new_uqp;
+	int err = 0;
+
+	new_uqp = kzalloc(sizeof(*new_uqp), GFP_KERNEL);
+	if (!new_uqp)
+		return -ENOMEM;
+
+	mutex_lock(&root->chain_lock);
+
+	if (!root->root_ft) {
+		err = -EINVAL;
+		goto update_ft_fail;
+	}
+
+	err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
+					 false);
+	if (err) {
+		mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n",
+			       underlay_qpn, err);
+		goto update_ft_fail;
+	}
+
+	new_uqp->qpn = underlay_qpn;
+	list_add_tail(&new_uqp->list, &root->underlay_qpns);
+
+	mutex_unlock(&root->chain_lock);
+
+	return 0;
+
+update_ft_fail:
+	mutex_unlock(&root->chain_lock);
+	kfree(new_uqp);
+	return err;
+}
+EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn);
+
+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
+{
+	struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
+	struct mlx5_ft_underlay_qp *uqp;
+	bool found = false;
+	int err = 0;
+
+	mutex_lock(&root->chain_lock);
+	list_for_each_entry(uqp, &root->underlay_qpns, list) {
+		if (uqp->qpn == underlay_qpn) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		mlx5_core_warn(dev, "Failed finding underlay qp (%u) in qpn list\n",
+			       underlay_qpn);
+		err = -EINVAL;
+		goto out;
+	}
+
+	err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
+					 true);
+	if (err)
+		mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n",
+			       underlay_qpn, err);
+
+	list_del(&uqp->list);
+	mutex_unlock(&root->chain_lock);
+	kfree(uqp);
+
+	return 0;
+
+out:
+	mutex_unlock(&root->chain_lock);
+	return err;
+}
+EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn);
+
+static struct mlx5_flow_root_namespace
+*get_root_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type)
+{
+	struct mlx5_flow_namespace *ns;
+
+	if (ns_type == MLX5_FLOW_NAMESPACE_ESW_EGRESS ||
+	    ns_type == MLX5_FLOW_NAMESPACE_ESW_INGRESS)
+		ns = mlx5_get_flow_vport_acl_namespace(dev, ns_type, 0);
+	else
+		ns = mlx5_get_flow_namespace(dev, ns_type);
+	if (!ns)
+		return NULL;
+
+	return find_root(&ns->node);
+}
+
+struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
+						 enum mlx5_flow_namespace_type ns_type,
+						 u8 num_actions,
+						 void *modify_actions)
+{
+	struct mlx5_flow_root_namespace *root;
+	struct mlx5_modify_hdr *modify_hdr;
+	int err;
+
+	root = get_root_namespace(dev, ns_type);
+	if (!root)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
+	if (!modify_hdr)
+		return ERR_PTR(-ENOMEM);
+
+	modify_hdr->ns_type = ns_type;
+	err = root->cmds->modify_header_alloc(root, ns_type, num_actions,
+					      modify_actions, modify_hdr);
+	if (err) {
+		kfree(modify_hdr);
+		return ERR_PTR(err);
+	}
+
+	return modify_hdr;
+}
+EXPORT_SYMBOL(mlx5_modify_header_alloc);
+
+void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
+				struct mlx5_modify_hdr *modify_hdr)
+{
+	struct mlx5_flow_root_namespace *root;
+
+	root = get_root_namespace(dev, modify_hdr->ns_type);
+	if (WARN_ON(!root))
+		return;
+	root->cmds->modify_header_dealloc(root, modify_hdr);
+	kfree(modify_hdr);
+}
+EXPORT_SYMBOL(mlx5_modify_header_dealloc);
+
+struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
+						     struct mlx5_pkt_reformat_params *params,
+						     enum mlx5_flow_namespace_type ns_type)
+{
+	struct mlx5_pkt_reformat *pkt_reformat;
+	struct mlx5_flow_root_namespace *root;
+	int err;
+
+	root = get_root_namespace(dev, ns_type);
+	if (!root)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
+	if (!pkt_reformat)
+		return ERR_PTR(-ENOMEM);
+
+	pkt_reformat->ns_type = ns_type;
+	pkt_reformat->reformat_type = params->type;
+	err = root->cmds->packet_reformat_alloc(root, params, ns_type,
+						pkt_reformat);
+	if (err) {
+		kfree(pkt_reformat);
+		return ERR_PTR(err);
+	}
+
+	return pkt_reformat;
+}
+EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
+
+void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
+				  struct mlx5_pkt_reformat *pkt_reformat)
+{
+	struct mlx5_flow_root_namespace *root;
+
+	root = get_root_namespace(dev, pkt_reformat->ns_type);
+	if (WARN_ON(!root))
+		return;
+	root->cmds->packet_reformat_dealloc(root, pkt_reformat);
+	kfree(pkt_reformat);
+}
+EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
+
+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
+				 struct mlx5_flow_root_namespace *peer_ns)
+{
+	if (peer_ns && ns->mode != peer_ns->mode) {
+		mlx5_core_err(ns->dev,
+			      "Can't peer namespace of different steering mode\n");
+		return -EINVAL;
+	}
+
+	return ns->cmds->set_peer(ns, peer_ns);
+}
+
+/* This function should be called only at init stage of the namespace.
+ * It is not safe to call this function while steering operations
+ * are executed in the namespace.
+ */
+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
+				 enum mlx5_flow_steering_mode mode)
+{
+	struct mlx5_flow_root_namespace *root;
+	const struct mlx5_flow_cmds *cmds;
+	int err;
+
+	root = find_root(&ns->node);
+	if (&root->ns != ns)
+	/* Can't set cmds to non root namespace */
+		return -EINVAL;
+
+	if (root->table_type != FS_FT_FDB)
+		return -EOPNOTSUPP;
+
+	if (root->mode == mode)
+		return 0;
+
+	cmds = mlx5_fs_cmd_get_fw_cmds();
+	if (!cmds)
+		return -EOPNOTSUPP;
+
+	err = cmds->create_ns(root);
+	if (err) {
+		mlx5_core_err(root->dev, "Failed to create flow namespace (%d)\n",
+			      err);
+		return err;
+	}
+
+	root->cmds->destroy_ns(root);
+	root->cmds = cmds;
+	root->mode = mode;
+
+	return 0;
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
index 7214c5256388..f8c7b3adc2c0 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
@@ -31,7 +31,7 @@
 #include <linux/rbtree.h>
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
 #include <dev/mlx5/mlx5_core/fs_core.h>
-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
+#include <dev/mlx5/mlx5_core/fs_cmd.h>
 
 #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
 #define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000)
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
new file mode 100644
index 000000000000..c14590acc772
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#include "fs_ft_pool.h"
+
+/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
+ * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
+ * for each flow table pool. We can allocate up to 16M of each pool,
+ * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz.
+ * Firmware doesn't report any of this for now.
+ * ESW_POOL is expected to be sorted from large to small and match firmware
+ * pools.
+ */
+#define FT_SIZE (16 * 1024 * 1024)
+static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
+					 1 * 1024 * 1024,
+					 64 * 1024,
+					 128,
+					 1 /* size for termination tables */ };
+struct mlx5_ft_pool {
+	int ft_left[ARRAY_SIZE(FT_POOLS)];
+};
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev)
+{
+	struct mlx5_ft_pool *ft_pool;
+	int i;
+
+	ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL);
+	if (!ft_pool)
+		return -ENOMEM;
+
+	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
+		ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i];
+
+	dev->priv.ft_pool = ft_pool;
+	return 0;
+}
+
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev)
+{
+	kfree(dev->priv.ft_pool);
+}
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+			  int desired_size)
+{
+	u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type);
+	int i, found_i = -1;
+
+	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+		if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size &&
+		    FT_POOLS[i] <= max_ft_size) {
+			found_i = i;
+			if (desired_size != POOL_NEXT_SIZE)
+				break;
+		}
+	}
+
+	if (found_i != -1) {
+		--dev->priv.ft_pool->ft_left[found_i];
+		return FT_POOLS[found_i];
+	}
+
+	return 0;
+}
+
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz)
+{
+	int i;
+
+	if (!sz)
+		return;
+
+	for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+		if (sz == FT_POOLS[i]) {
+			++dev->priv.ft_pool->ft_left[i];
+			return;
+		}
+	}
+
+	WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
index d7d63d7932a1..f7ec5af81773 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
@@ -81,12 +81,12 @@ accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
 #endif
 
 void
-mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule)
+mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule)
 {
-	mlx5_del_flow_rule(&rule);
+	mlx5_del_flow_rules(&rule);
 }
 
-struct mlx5_flow_rule *
+struct mlx5_flow_handle *
 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
     struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
     uint16_t vlan_id)
@@ -96,18 +96,17 @@ mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
 #if defined(INET) || defined(INET6)
 	struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
 #endif
-	struct mlx5_flow_rule *flow;
+	struct mlx5_flow_handle *flow;
 	struct mlx5_flow_spec *spec;
-	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = flow_tag,
-	};
+	struct mlx5_flow_act flow_act = {};
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec)
 		return (ERR_PTR(-ENOMEM));
 
 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+	spec->flow_context.flow_tag = flow_tag;
 
 	INP_RLOCK(inp);
 	/* Set VLAN ID to match, if any. */
@@ -160,13 +159,9 @@ mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 	dest.tir_num = tirn;
+	flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
 
-	flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable,
-	    spec->match_criteria,
-	    spec->match_value,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST,
-	    &flow_act,
-	    &dest);
+	flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
 out:
 	kvfree(spec);
 	return (flow);
@@ -175,18 +170,18 @@ out:
 static int
 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
 {
-	static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
-	static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
+	static struct mlx5_flow_spec spec = {};
 	struct mlx5_flow_destination dest = {};
 	struct mlx5e_accel_fs_tcp *fs_tcp;
-	struct mlx5_flow_rule *rule;
+	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
 
 	fs_tcp = &priv->fts.accel_tcp;
 
+	spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+	spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 
 	/*
@@ -197,10 +192,11 @@ accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
 	 * of flow tables.
 	 */
 	dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
-	    priv->fts.vlan.t : fs_tcp->tables[type + 1].t;
+		  ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t)  :
+		  fs_tcp->tables[type + 1].t;
 
-	rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act,
+				   &dest, 1);
 	if (IS_ERR(rule))
 		return (PTR_ERR(rule));
 
@@ -317,11 +313,13 @@ static int
 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
 {
 	struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
+	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
 	ft->num_groups = 0;
-	ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp",
-	    MLX5E_ACCEL_FS_TCP_TABLE_SIZE);
+	ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
+	ft_attr.level = type;
+	ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
@@ -365,7 +363,7 @@ mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
 		return;
 
 	for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
-		mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
+		mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
 		accel_fs_tcp_destroy_table(priv, i);
 	}
 }
@@ -402,7 +400,7 @@ mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
 
 err_destroy_rules:
 	while (i--)
-		mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
+		mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
 	i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;
 
 err_destroy_tables:
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
deleted file mode 100644
index b76ea7b60582..000000000000
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
+++ /dev/null
@@ -1,2874 +0,0 @@
-/*-
- * Copyright (c) 2013-2021, Mellanox Technologies, Ltd.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "opt_rss.h"
-#include "opt_ratelimit.h"
-
-#include <linux/module.h>
-#include <dev/mlx5/driver.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
-#include <dev/mlx5/mlx5_core/fs_core.h>
-#include <linux/string.h>
-#include <linux/compiler.h>
-
-#define INIT_TREE_NODE_ARRAY_SIZE(...)	(sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
-					 sizeof(struct init_tree_node))
-
-#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \
-		 ...) {.type = FS_TYPE_PRIO,\
-	.name = name_val,\
-	.min_ft_level = min_level_val,\
-	.flags = flags_val,\
-	.max_ft = max_ft_val,\
-	.caps = caps_val,\
-	.children = (struct init_tree_node[]) {__VA_ARGS__},\
-	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
-}
-
-#define ADD_FT_PRIO(name_val, flags_val, max_ft_val,  ...)\
-	ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\
-		 __VA_ARGS__)\
-
-#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\
-	.name = name_val,\
-	.children = (struct init_tree_node[]) {__VA_ARGS__},\
-	.ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
-}
-
-#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
-				   sizeof(long))
-
-#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
-
-#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
-			       .caps = (long[]) {__VA_ARGS__}}
-
-/* Flowtable sizes: */
-#define	BYPASS_MAX_FT 5
-#define	BYPASS_PRIO_MAX_FT 1
-#define	OFFLOADS_MAX_FT 2
-#define	KERNEL_MAX_FT 5
-#define	LEFTOVER_MAX_FT 1
-
-/* Flowtable levels: */
-#define	OFFLOADS_MIN_LEVEL 3
-#define	KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
-#define	LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1)
-#define	BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL)
-
-struct node_caps {
-	size_t	arr_sz;
-	long	*caps;
-};
-
-struct init_tree_node {
-	enum fs_type	type;
-	const char	*name;
-	struct init_tree_node *children;
-	int ar_size;
-	struct node_caps caps;
-	u8  flags;
-	int min_ft_level;
-	int prio;
-	int max_ft;
-} root_fs = {
-	.type = FS_TYPE_NAMESPACE,
-	.name = "root",
-	.ar_size = 4,
-	.children = (struct init_tree_node[]) {
-		ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0,
-			 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
-					  FS_CAP(flow_table_properties_nic_receive.modify_root)),
-			 ADD_NS("by_pass_ns",
-				ADD_FT_PRIO("prio0", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio1", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio2", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio3", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio4", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio5", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio6", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio7", 0,
-					    BYPASS_PRIO_MAX_FT),
-				ADD_FT_PRIO("prio-mcast", 0,
-					    BYPASS_PRIO_MAX_FT))),
-		ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {},
-			 ADD_NS("offloads_ns",
-				ADD_FT_PRIO("prio_offloads-0", 0,
-					    OFFLOADS_MAX_FT))),
-		ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {},
-			 ADD_NS("kernel_ns",
-				ADD_FT_PRIO("prio_kernel-0", 0,
-					    KERNEL_MAX_FT))),
-		ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED,
-			 LEFTOVER_MIN_LEVEL, 0,
-			 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
-					  FS_CAP(flow_table_properties_nic_receive.modify_root)),
-			 ADD_NS("leftover_ns",
-				ADD_FT_PRIO("leftovers_prio-0",
-					MLX5_CORE_FS_PRIO_SHARED,
-					LEFTOVER_MAX_FT)))
-	}
-};
-
-/* Tree creation functions */
-
-static struct mlx5_flow_root_namespace *find_root(struct fs_base *node)
-{
-	struct fs_base *parent;
-
-	/* Make sure we only read it once while we go up the tree */
-	while ((parent = node->parent))
-		node = parent;
-
-	if (node->type != FS_TYPE_NAMESPACE) {
-		return NULL;
-	}
-
-	return container_of(container_of(node,
-					 struct mlx5_flow_namespace,
-					 base),
-			    struct mlx5_flow_root_namespace,
-			    ns);
-}
-
-static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node)
-{
-	struct mlx5_flow_root_namespace *root = find_root(node);
-
-	if (root)
-		return root->dev;
-	return NULL;
-}
-
-static void fs_init_node(struct fs_base *node,
-			 unsigned int refcount)
-{
-	kref_init(&node->refcount);
-	atomic_set(&node->users_refcount, refcount);
-	init_completion(&node->complete);
-	INIT_LIST_HEAD(&node->list);
-	mutex_init(&node->lock);
-}
-
-static void _fs_add_node(struct fs_base *node,
-			 const char *name,
-			 struct fs_base *parent)
-{
-	if (parent)
-		atomic_inc(&parent->users_refcount);
-	node->name = kstrdup_const(name, GFP_KERNEL);
-	node->parent = parent;
-}
-
-static void fs_add_node(struct fs_base *node,
-			struct fs_base *parent, const char *name,
-			unsigned int refcount)
-{
-	fs_init_node(node, refcount);
-	_fs_add_node(node, name, parent);
-}
-
-static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
-		    bool parent_locked);
-
-static void fs_del_dst(struct mlx5_flow_rule *dst);
-static void _fs_del_ft(struct mlx5_flow_table *ft);
-static void fs_del_fg(struct mlx5_flow_group *fg);
-static void fs_del_fte(struct fs_fte *fte);
-
-static void cmd_remove_node(struct fs_base *base)
-{
-	switch (base->type) {
-	case FS_TYPE_FLOW_DEST:
-		fs_del_dst(container_of(base, struct mlx5_flow_rule, base));
-		break;
-	case FS_TYPE_FLOW_TABLE:
-		_fs_del_ft(container_of(base, struct mlx5_flow_table, base));
-		break;
-	case FS_TYPE_FLOW_GROUP:
-		fs_del_fg(container_of(base, struct mlx5_flow_group, base));
-		break;
-	case FS_TYPE_FLOW_ENTRY:
-		fs_del_fte(container_of(base, struct fs_fte, base));
-		break;
-	default:
-		break;
-	}
-}
-
-static void __fs_remove_node(struct kref *kref)
-{
-	struct fs_base *node = container_of(kref, struct fs_base, refcount);
-
-	if (node->parent) {
-		if (node->type == FS_TYPE_FLOW_DEST)
-			mutex_lock(&node->parent->parent->lock);
-		mutex_lock(&node->parent->lock);
-	}
-	mutex_lock(&node->lock);
-	cmd_remove_node(node);
-	mutex_unlock(&node->lock);
-	complete(&node->complete);
-	if (node->parent) {
-		mutex_unlock(&node->parent->lock);
-		if (node->type == FS_TYPE_FLOW_DEST)
-			mutex_unlock(&node->parent->parent->lock);
-		_fs_put(node->parent, _fs_remove_node, false);
-	}
-}
-
-void _fs_remove_node(struct kref *kref)
-{
-	struct fs_base *node = container_of(kref, struct fs_base, refcount);
-
-	__fs_remove_node(kref);
-	kfree_const(node->name);
-	kfree(node);
-}
-
-static void fs_get(struct fs_base *node)
-{
-	atomic_inc(&node->users_refcount);
-}
-
-static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
-		    bool parent_locked)
-{
-	struct fs_base *parent_node = node->parent;
-
-	if (parent_node && !parent_locked)
-		mutex_lock(&parent_node->lock);
-	if (atomic_dec_and_test(&node->users_refcount)) {
-		if (parent_node) {
-			/*remove from parent's list*/
-			list_del_init(&node->list);
-			mutex_unlock(&parent_node->lock);
-		}
-		kref_put(&node->refcount, kref_cb);
-		if (parent_node && parent_locked)
-			mutex_lock(&parent_node->lock);
-	} else if (parent_node && !parent_locked) {
-		mutex_unlock(&parent_node->lock);
-	}
-}
-
-static void fs_put(struct fs_base *node)
-{
-	_fs_put(node, __fs_remove_node, false);
-}
-
-static void fs_put_parent_locked(struct fs_base *node)
-{
-	_fs_put(node, __fs_remove_node, true);
-}
-
-static void fs_remove_node(struct fs_base *node)
-{
-	fs_put(node);
-	wait_for_completion(&node->complete);
-	kfree_const(node->name);
-	kfree(node);
-}
-
-static void fs_remove_node_parent_locked(struct fs_base *node)
-{
-	fs_put_parent_locked(node);
-	wait_for_completion(&node->complete);
-	kfree_const(node->name);
-	kfree(node);
-}
-
-static struct fs_fte *fs_alloc_fte(u32 sw_action,
-				   struct mlx5_flow_act *flow_act,
-				   u32 *match_value,
-				   unsigned int index)
-{
-	struct fs_fte *fte;
-
-
-	fte = kzalloc(sizeof(*fte), GFP_KERNEL);
-	if (!fte)
-		return ERR_PTR(-ENOMEM);
-
-	memcpy(fte->val, match_value, sizeof(fte->val));
-	fte->base.type =  FS_TYPE_FLOW_ENTRY;
-	fte->dests_size = 0;
-	fte->index = index;
-	INIT_LIST_HEAD(&fte->dests);
-	fte->flow_act = *flow_act;
-	fte->sw_action = sw_action;
-
-	return fte;
-}
-
-static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft,
-					  struct mlx5_flow_group *fg,
-					  u32 *match_value,
-					  unsigned int index)
-{
-	int err;
-	struct fs_fte *fte;
-	struct mlx5_flow_rule *dst;
-	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
-	};
-
-	if (fg->num_ftes == fg->max_ftes)
-		return ERR_PTR(-ENOSPC);
-
-	fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST,
-			   &flow_act, match_value, index);
-	if (IS_ERR(fte))
-		return fte;
-
-	/*create dst*/
-	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
-	if (!dst) {
-		err = -ENOMEM;
-		goto free_fte;
-	}
-
-	fte->base.parent = &fg->base;
-	fte->dests_size = 1;
-	dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE;
-	dst->base.parent = &fte->base;
-	list_add(&dst->base.list, &fte->dests);
-	/* assumed that the callee creates the star rules sorted by index */
-	list_add_tail(&fte->base.list, &fg->ftes);
-	fg->num_ftes++;
-
-	return fte;
-
-free_fte:
-	kfree(fte);
-	return ERR_PTR(err);
-}
-
-/* assume that fte can't be changed */
-static void free_star_fte_entry(struct fs_fte *fte)
-{
-	struct mlx5_flow_group	*fg;
-	struct mlx5_flow_rule	*dst, *temp;
-
-	fs_get_parent(fg, fte);
-
-	list_for_each_entry_safe(dst, temp, &fte->dests, base.list) {
-		fte->dests_size--;
-		list_del(&dst->base.list);
-		kfree(dst);
-	}
-
-	list_del(&fte->base.list);
-	fg->num_ftes--;
-	kfree(fte);
-}
-
-static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in)
-{
-	struct mlx5_flow_group *fg;
-	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
-					    create_fg_in, match_criteria);
-	u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
-					    create_fg_in,
-					    match_criteria_enable);
-	fg = kzalloc(sizeof(*fg), GFP_KERNEL);
-	if (!fg)
-		return ERR_PTR(-ENOMEM);
-
-	INIT_LIST_HEAD(&fg->ftes);
-	fg->mask.match_criteria_enable = match_criteria_enable;
-	memcpy(&fg->mask.match_criteria, match_criteria,
-	       sizeof(fg->mask.match_criteria));
-	fg->base.type =  FS_TYPE_FLOW_GROUP;
-	fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in,
-				   start_flow_index);
-	fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in,
-				end_flow_index) - fg->start_index + 1;
-	return fg;
-}
-
-static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio);
-static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
-					    struct fs_prio *prio);
-
-/* assumed src_ft and dst_ft can't be freed */
-static int fs_set_star_rule(struct mlx5_core_dev *dev,
-			    struct mlx5_flow_table *src_ft,
-			    struct mlx5_flow_table *dst_ft)
-{
-	struct mlx5_flow_rule *src_dst;
-	struct fs_fte *src_fte;
-	int err = 0;
-	u32 *match_value;
-	int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
-
-	src_dst = list_first_entry(&src_ft->star_rule.fte->dests,
-				   struct mlx5_flow_rule, base.list);
-	match_value = mlx5_vzalloc(match_len);
-	if (!match_value) {
-		mlx5_core_warn(dev, "failed to allocate inbox\n");
-		return -ENOMEM;
-	}
-	/*Create match context*/
-
-	fs_get_parent(src_fte, src_dst);
-
-	src_dst->dest_attr.ft = dst_ft;
-	if (dst_ft) {
-		err = mlx5_cmd_fs_set_fte(dev,
-					  src_ft->vport,
-					  &src_fte->status,
-					  match_value, src_ft->type,
-					  src_ft->id, src_fte->index,
-					  src_ft->star_rule.fg->id,
-					  &src_fte->flow_act,
-					  src_fte->sw_action,
-					  src_fte->dests_size,
-					  &src_fte->dests);
-		if (err)
-			goto free;
-
-		fs_get(&dst_ft->base);
-	} else {
-		mlx5_cmd_fs_delete_fte(dev,
-				       src_ft->vport,
-				       &src_fte->status,
-				       src_ft->type, src_ft->id,
-				       src_fte->index);
-	}
-
-free:
-	kvfree(match_value);
-	return err;
-}
-
-static int connect_prev_fts(struct fs_prio *locked_prio,
-			    struct fs_prio *prev_prio,
-			    struct mlx5_flow_table *next_ft)
-{
-	struct mlx5_flow_table *iter;
-	int err = 0;
-	struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base);
-
-	if (!dev)
-		return -ENODEV;
-
-	mutex_lock(&prev_prio->base.lock);
-	fs_for_each_ft(iter, prev_prio) {
-		struct mlx5_flow_rule *src_dst =
-			list_first_entry(&iter->star_rule.fte->dests,
-					 struct mlx5_flow_rule, base.list);
-		struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft;
-
-		if (prev_ft == next_ft)
-			continue;
-
-		err = fs_set_star_rule(dev, iter, next_ft);
-		if (err) {
-			mlx5_core_warn(dev,
-			    "mlx5: flow steering can't connect prev and next\n");
-			goto unlock;
-		} else {
-			/* Assume ft's prio is locked */
-			if (prev_ft) {
-				struct fs_prio *prio;
-
-				fs_get_parent(prio, prev_ft);
-				if (prio == locked_prio)
-					fs_put_parent_locked(&prev_ft->base);
-				else
-					fs_put(&prev_ft->base);
-			}
-		}
-	}
-
-unlock:
-	mutex_unlock(&prev_prio->base.lock);
-	return 0;
-}
-
-static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
-{
-	struct mlx5_flow_group *fg;
-	int err;
-	u32 *fg_in;
-	u32 *match_value;
-	struct mlx5_flow_table *next_ft;
-	struct mlx5_flow_table *prev_ft;
-	struct mlx5_flow_root_namespace *root = find_root(&prio->base);
-	int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
-	int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
-
-	fg_in = mlx5_vzalloc(fg_inlen);
-	if (!fg_in) {
-		mlx5_core_warn(root->dev, "failed to allocate inbox\n");
-		return -ENOMEM;
-	}
-
-	match_value = mlx5_vzalloc(match_len);
-	if (!match_value) {
-		mlx5_core_warn(root->dev, "failed to allocate inbox\n");
-		kvfree(fg_in);
-		return -ENOMEM;
-	}
-
-	MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte);
-	MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte);
-	fg = fs_alloc_fg(fg_in);
-	if (IS_ERR(fg)) {
-		err = PTR_ERR(fg);
-		goto out;
-	}
-	ft->star_rule.fg = fg;
-	err =  mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base),
-				     fg_in, ft->vport, ft->type,
-				     ft->id,
-				     &fg->id);
-	if (err)
-		goto free_fg;
-
-	ft->star_rule.fte = alloc_star_ft_entry(ft, fg,
-						      match_value,
-						      ft->max_fte);
-	if (IS_ERR(ft->star_rule.fte))
-		goto free_star_rule;
-
-	mutex_lock(&root->fs_chain_lock);
-	next_ft = find_next_ft(prio);
-	err = fs_set_star_rule(root->dev, ft, next_ft);
-	if (err) {
-		mutex_unlock(&root->fs_chain_lock);
-		goto free_star_rule;
-	}
-	if (next_ft) {
-		struct fs_prio *parent;
-
-		fs_get_parent(parent, next_ft);
-		fs_put(&next_ft->base);
-	}
-	prev_ft = find_prev_ft(ft, prio);
-	if (prev_ft) {
-		struct fs_prio *prev_parent;
-
-		fs_get_parent(prev_parent, prev_ft);
-
-		err = connect_prev_fts(NULL, prev_parent, ft);
-		if (err) {
-			mutex_unlock(&root->fs_chain_lock);
-			goto destroy_chained_star_rule;
-		}
-		fs_put(&prev_ft->base);
-	}
-	mutex_unlock(&root->fs_chain_lock);
-	kvfree(fg_in);
-	kvfree(match_value);
-
-	return 0;
-
-destroy_chained_star_rule:
-	fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL);
-	if (next_ft)
-		fs_put(&next_ft->base);
-free_star_rule:
-	free_star_fte_entry(ft->star_rule.fte);
-	mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport,
-			       ft->type, ft->id,
-			       fg->id);
-free_fg:
-	kfree(fg);
-out:
-	kvfree(fg_in);
-	kvfree(match_value);
-	return err;
-}
-
-static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
-{
-	int err;
-	struct mlx5_flow_root_namespace *root;
-	struct mlx5_core_dev *dev = fs_get_dev(&prio->base);
-	struct mlx5_flow_table *prev_ft, *next_ft;
-	struct fs_prio *prev_prio;
-
-	WARN_ON(!dev);
-
-	root = find_root(&prio->base);
-	if (!root)
-		mlx5_core_err(dev,
-		    "flow steering failed to find root of priority %s",
-		    prio->base.name);
-
-	/* In order to ensure atomic deletion, first update
-	 * prev ft to point on the next ft.
-	 */
-	mutex_lock(&root->fs_chain_lock);
-	prev_ft = find_prev_ft(ft, prio);
-	next_ft = find_next_ft(prio);
-	if (prev_ft) {
-		fs_get_parent(prev_prio, prev_ft);
-		/*Prev is connected to ft, only if ft is the first(last) in the prio*/
-		err = connect_prev_fts(prio, prev_prio, next_ft);
-		if (err)
-			mlx5_core_warn(root->dev,
-				       "flow steering can't connect prev and next of flow table\n");
-		fs_put(&prev_ft->base);
-	}
-
-	err = fs_set_star_rule(root->dev, ft, NULL);
-	/*One put is for fs_get in find next ft*/
-	if (next_ft) {
-		fs_put(&next_ft->base);
-		if (!err)
-			fs_put(&next_ft->base);
-	}
-
-	mutex_unlock(&root->fs_chain_lock);
-	err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id,
-				     ft->star_rule.fg->id);
-	if (err)
-		mlx5_core_warn(dev,
-			       "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index,
-			       ft->base.name);
-	free_star_fte_entry(ft->star_rule.fte);
-
-	kfree(ft->star_rule.fg);
-	ft->star_rule.fg = NULL;
-}
-
-static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
-				 unsigned int prio)
-{
-	struct fs_prio *iter_prio;
-
-	fs_for_each_prio(iter_prio, ns) {
-		if (iter_prio->prio == prio)
-			return iter_prio;
-	}
-
-	return NULL;
-}
-
-static unsigned int _alloc_new_level(struct fs_prio *prio,
-				     struct mlx5_flow_namespace *match);
-
-static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns,
-				      struct fs_prio *prio)
-{
-	unsigned int level = 0;
-	struct fs_prio *p;
-
-	if (!ns)
-		return 0;
-
-	mutex_lock(&ns->base.lock);
-	fs_for_each_prio(p, ns) {
-		if (p != prio)
-			level += p->max_ft;
-		else
-			break;
-	}
-	mutex_unlock(&ns->base.lock);
-
-	fs_get_parent(prio, ns);
-	if (prio)
-		WARN_ON(prio->base.type != FS_TYPE_PRIO);
-
-	return level + _alloc_new_level(prio, ns);
-}
-
-/* Called under lock of priority, hence locking all upper objects */
-static unsigned int _alloc_new_level(struct fs_prio *prio,
-				     struct mlx5_flow_namespace *match)
-{
-	struct mlx5_flow_namespace *ns;
-	struct fs_base *it;
-	unsigned int level = 0;
-
-	if (!prio)
-		return 0;
-
-	mutex_lock(&prio->base.lock);
-	fs_for_each_ns_or_ft_reverse(it, prio) {
-		if (it->type == FS_TYPE_NAMESPACE) {
-			struct fs_prio *p;
-
-			fs_get_obj(ns, it);
-
-			if (match != ns) {
-				mutex_lock(&ns->base.lock);
-				fs_for_each_prio(p, ns)
-					level += p->max_ft;
-				mutex_unlock(&ns->base.lock);
-			} else {
-				break;
-			}
-		} else {
-			struct mlx5_flow_table *ft;
-
-			fs_get_obj(ft, it);
-			mutex_unlock(&prio->base.lock);
-			return level + ft->level + 1;
-		}
-	}
-
-	fs_get_parent(ns, prio);
-	mutex_unlock(&prio->base.lock);
-	return __alloc_new_level(ns, prio) + level;
-}
-
-static unsigned int alloc_new_level(struct fs_prio *prio)
-{
-	return _alloc_new_level(prio, NULL);
-}
-
-static int update_root_ft_create(struct mlx5_flow_root_namespace *root,
-				    struct mlx5_flow_table *ft)
-{
-	int err = 0;
-	int min_level = INT_MAX;
-
-	if (root->root_ft)
-		min_level = root->root_ft->level;
-
-	if (ft->level < min_level)
-		err = mlx5_cmd_update_root_ft(root->dev, ft->type,
-					      ft->id);
-	else
-		return err;
-
-	if (err)
-		mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
-			       ft->id);
-	else
-		root->root_ft = ft;
-
-	return err;
-}
-
-static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns,
-						 u16 vport,
-						 struct fs_prio *fs_prio,
-						 int max_fte,
-						 const char *name)
-{
-	struct mlx5_flow_table *ft;
-	int err;
-	int log_table_sz;
-	int ft_size;
-	char gen_name[20];
-	struct mlx5_flow_root_namespace *root = find_root(&ns->base);
-	struct mlx5_core_dev *dev = fs_get_dev(&ns->base);
-
-	if (!root) {
-		mlx5_core_err(dev,
-		    "flow steering failed to find root of namespace %s",
-		    ns->base.name);
-		return ERR_PTR(-ENODEV);
-	}
-
-	if (fs_prio->num_ft == fs_prio->max_ft)
-		return ERR_PTR(-ENOSPC);
-
-	ft  = kzalloc(sizeof(*ft), GFP_KERNEL);
-	if (!ft)
-		return ERR_PTR(-ENOMEM);
-
-	fs_init_node(&ft->base, 1);
-	INIT_LIST_HEAD(&ft->fgs);
-
-	/* Temporarily WA until we expose the level set in the API */
-	if (root->table_type == FS_FT_ESW_EGRESS_ACL ||
-		root->table_type == FS_FT_ESW_INGRESS_ACL)
-		ft->level = 0;
-	else
-		ft->level = alloc_new_level(fs_prio);
-
-	ft->base.type = FS_TYPE_FLOW_TABLE;
-	ft->vport = vport;
-	ft->type = root->table_type;
-	/*Two entries are reserved for star rules*/
-	ft_size = roundup_pow_of_two(max_fte + 2);
-	/*User isn't aware to those rules*/
-	ft->max_fte = ft_size - 2;
-	log_table_sz = ilog2(ft_size);
-
-	if (name == NULL || name[0] == '\0') {
-		snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id);
-		name = gen_name;
-	}
-
-	err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type,
-				    ft->level, log_table_sz, name, &ft->id);
-	if (err)
-		goto free_ft;
-
-	err = create_star_rule(ft, fs_prio);
-	if (err)
-		goto del_ft;
-
-	if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev,
-			       flow_table_properties_nic_receive.modify_root)) {
-		err = update_root_ft_create(root, ft);
-		if (err)
-			goto destroy_star_rule;
-	}
-
-	_fs_add_node(&ft->base, name, &fs_prio->base);
-
-	list_add_tail(&ft->base.list, &fs_prio->objs);
-	fs_prio->num_ft++;
-
-	return ft;
-
-destroy_star_rule:
-	destroy_star_rule(ft, fs_prio);
-del_ft:
-	mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id);
-free_ft:
-	kfree(ft);
-	return ERR_PTR(err);
-}
-
-static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns,
-						u16 vport,
-						unsigned int prio,
-						int max_fte,
-						const char *name)
-{
-	struct fs_prio *fs_prio = NULL;
-	fs_prio = find_prio(ns, prio);
-	if (!fs_prio)
-		return ERR_PTR(-EINVAL);
-
-	return _create_ft_common(ns, vport, fs_prio, max_fte, name);
-}
-
-
-static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
-						   struct list_head *start);
-
-static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
-						     struct list_head *start);
-
-static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio)
-{
-	struct mlx5_flow_table *ft;
-
-	ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs);
-	if (ft) {
-		ft->shared_refcount++;
-		return ft;
-	}
-
-	return NULL;
-}
-
-struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
-							   int prio,
-							   const char *name,
-							   int num_flow_table_entries,
-							   int max_num_groups,
-							   int num_reserved_entries)
-{
-	struct mlx5_flow_table *ft = NULL;
-	struct fs_prio *fs_prio;
-	bool is_shared_prio;
-
-	if (max_num_groups > (num_flow_table_entries - num_reserved_entries))
-		return ERR_PTR(-EINVAL);
-	if (num_reserved_entries > num_flow_table_entries)
-		return ERR_PTR(-EINVAL);
-
-	fs_prio = find_prio(ns, prio);
-	if (!fs_prio)
-		return ERR_PTR(-EINVAL);
-
-	is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED;
-	if (is_shared_prio) {
-		mutex_lock(&fs_prio->shared_lock);
-		ft = mlx5_create_autogrouped_shared_flow_table(fs_prio);
-	}
-
-	if (ft)
-		goto return_ft;
-
-	ft = create_ft_common(ns, 0, prio, num_flow_table_entries,
-			      name);
-	if (IS_ERR(ft))
-		goto return_ft;
-
-	ft->autogroup.active = true;
-	ft->autogroup.max_types = max_num_groups;
-	ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries;
-	/* We save place for flow groups in addition to max types */
-	ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1);
-
-	if (is_shared_prio)
-		ft->shared_refcount = 1;
-
-return_ft:
-	if (is_shared_prio)
-		mutex_unlock(&fs_prio->shared_lock);
-	return ft;
-}
-EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
-
-struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
-						     u16 vport,
-						     int prio,
-						     const char *name,
-						     int num_flow_table_entries)
-{
-	return create_ft_common(ns, vport, prio, num_flow_table_entries, name);
-}
-EXPORT_SYMBOL(mlx5_create_vport_flow_table);
-
-struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
-					       int prio,
-					       const char *name,
-					       int num_flow_table_entries)
-{
-	return create_ft_common(ns, 0, prio, num_flow_table_entries, name);
-}
-EXPORT_SYMBOL(mlx5_create_flow_table);
-
-static void _fs_del_ft(struct mlx5_flow_table *ft)
-{
-	int err;
-	struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
-	struct fs_prio *prio;
-
-	err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id);
-	if (err)
-		mlx5_core_warn(dev, "flow steering can't destroy ft %s\n",
-			       ft->base.name);
-
-	fs_get_parent(prio, ft);
-	prio->num_ft--;
-}
-
-static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root,
-				    struct mlx5_flow_table *ft)
-{
-	int err = 0;
-	struct fs_prio *prio;
-	struct mlx5_flow_table *next_ft = NULL;
-	struct mlx5_flow_table *put_ft = NULL;
-
-	if (root->root_ft != ft)
-		return 0;
-
-	fs_get_parent(prio, ft);
-	/*Assuming objs containis only flow tables and
-	 * flow tables are sorted by level.
-	 */
-	if (!list_is_last(&ft->base.list, &prio->objs)) {
-		next_ft = list_next_entry(ft, base.list);
-	} else {
-		next_ft = find_next_ft(prio);
-		put_ft = next_ft;
-	}
-
-	if (next_ft) {
-		err = mlx5_cmd_update_root_ft(root->dev, next_ft->type,
-					      next_ft->id);
-		if (err)
-			mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
-				       ft->id);
-	}
-	if (!err)
-		root->root_ft = next_ft;
-
-	if (put_ft)
-		fs_put(&put_ft->base);
-
-	return err;
-}
-
-/*Objects in the same prio are destroyed in the reverse order they were createrd*/
-int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
-{
-	int err = 0;
-	struct fs_prio *prio;
-	struct mlx5_flow_root_namespace *root;
-	bool is_shared_prio;
-	struct mlx5_core_dev *dev;
-
-	fs_get_parent(prio, ft);
-	root = find_root(&prio->base);
-	dev = fs_get_dev(&prio->base);
-
-	if (!root) {
-		mlx5_core_err(dev,
-		    "flow steering failed to find root of priority %s",
-		    prio->base.name);
-		return -ENODEV;
-	}
-
-	is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED;
-	if (is_shared_prio) {
-		mutex_lock(&prio->shared_lock);
-		if (ft->shared_refcount > 1) {
-			--ft->shared_refcount;
-			fs_put(&ft->base);
-			mutex_unlock(&prio->shared_lock);
-			return 0;
-		}
-	}
-
-	mutex_lock(&prio->base.lock);
-	mutex_lock(&ft->base.lock);
-
-	err = update_root_ft_destroy(root, ft);
-	if (err)
-		goto unlock_ft;
-
-	/* delete two last entries */
-	destroy_star_rule(ft, prio);
-
-	mutex_unlock(&ft->base.lock);
-	fs_remove_node_parent_locked(&ft->base);
-	mutex_unlock(&prio->base.lock);
-	if (is_shared_prio)
-		mutex_unlock(&prio->shared_lock);
-
-	return err;
-
-unlock_ft:
-	mutex_unlock(&ft->base.lock);
-	mutex_unlock(&prio->base.lock);
-	if (is_shared_prio)
-		mutex_unlock(&prio->shared_lock);
-
-	return err;
-}
-EXPORT_SYMBOL(mlx5_destroy_flow_table);
-
-static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev,
-					    struct mlx5_flow_table *ft,
-					    struct list_head *prev,
-					    u32 *fg_in,
-					    int refcount)
-{
-	struct mlx5_flow_group *fg;
-	unsigned int group_size;
-	int err;
-	char name[20];
-
-	fg = fs_alloc_fg(fg_in);
-	if (IS_ERR(fg))
-		return fg;
-
-	group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) -
-		MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1;
-	err =  mlx5_cmd_fs_create_fg(dev, fg_in,
-				     ft->vport, ft->type, ft->id,
-				     &fg->id);
-	if (err)
-		goto free_fg;
-
-	mutex_lock(&ft->base.lock);
-
-	if (ft->autogroup.active && group_size == ft->autogroup.group_size)
-		ft->autogroup.num_types++;
-
-	snprintf(name, sizeof(name), "group_%u", fg->id);
-	/*Add node to tree*/
-	fs_add_node(&fg->base, &ft->base, name, refcount);
-	/*Add node to group list*/
-	list_add(&fg->base.list, prev);
-	mutex_unlock(&ft->base.lock);
-
-	return fg;
-
-free_fg:
-	kfree(fg);
-	return ERR_PTR(err);
-}
-
-struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
-					       u32 *in)
-{
-	struct mlx5_flow_group *fg;
-	struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
-	unsigned int start_index;
-
-	start_index = MLX5_GET(create_flow_group_in, in, start_flow_index);
-	if (!dev)
-		return ERR_PTR(-ENODEV);
-
-	if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
-		return ERR_PTR(-EPERM);
-
-	fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1);
-
-	return fg;
-}
-EXPORT_SYMBOL(mlx5_create_flow_group);
-
-/*Group is destoyed when all the rules in the group were removed*/
-static void fs_del_fg(struct mlx5_flow_group *fg)
-{
-	struct mlx5_flow_table *parent_ft;
-	struct mlx5_core_dev *dev;
-
-	fs_get_parent(parent_ft, fg);
-	dev = fs_get_dev(&parent_ft->base);
-	WARN_ON(!dev);
-
-	if (parent_ft->autogroup.active &&
-	    fg->max_ftes == parent_ft->autogroup.group_size &&
-	    fg->start_index < parent_ft->autogroup.max_fte)
-		parent_ft->autogroup.num_types--;
-
-	if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport,
-				   parent_ft->type,
-				   parent_ft->id, fg->id))
-		mlx5_core_warn(dev, "flow steering can't destroy fg\n");
-}
-
-void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
-{
-	fs_remove_node(&fg->base);
-}
-EXPORT_SYMBOL(mlx5_destroy_flow_group);
-
-static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size)
-{
-	unsigned int i;
-
-	/* TODO: optimize by comparing 64bits when possible */
-	for (i = 0; i < size; i++, mask++, val1++, val2++)
-		if ((*((u8 *)val1) & (*(u8 *)mask)) !=
-		    ((*(u8 *)val2) & (*(u8 *)mask)))
-			return false;
-
-	return true;
-}
-
-bool fs_match_exact_val(struct mlx5_core_fs_mask *mask,
-			       void *val1, void *val2)
-{
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						val1, outer_headers);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						val2, outer_headers);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					      mask->match_criteria, outer_headers);
-
-		if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
-					 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
-			return false;
-	}
-
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						val1, misc_parameters);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						val2, misc_parameters);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					  mask->match_criteria, misc_parameters);
-
-		if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
-					 MLX5_ST_SZ_BYTES(fte_match_set_misc)))
-			return false;
-	}
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						val1, inner_headers);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						val2, inner_headers);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					  mask->match_criteria, inner_headers);
-
-		if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
-					 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
-			return false;
-	}
-	return true;
-}
-
-bool fs_match_exact_mask(u8 match_criteria_enable1,
-				u8 match_criteria_enable2,
-				void *mask1, void *mask2)
-{
-	return match_criteria_enable1 == match_criteria_enable2 &&
-		!memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param));
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
-							   struct list_head *start);
-
-static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio,
-							      struct list_head *start)
-{
-	struct fs_base *it = container_of(start, struct fs_base, list);
-
-	if (!prio)
-		return NULL;
-
-	fs_for_each_ns_or_ft_continue_reverse(it, prio) {
-		struct mlx5_flow_namespace	*ns;
-		struct mlx5_flow_table		*ft;
-
-		if (it->type == FS_TYPE_FLOW_TABLE) {
-			fs_get_obj(ft, it);
-			fs_get(&ft->base);
-			return ft;
-		}
-
-		fs_get_obj(ns, it);
-		WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
-
-		ft = find_first_ft_in_ns_reverse(ns, &ns->prios);
-		if (ft)
-			return ft;
-	}
-
-	return NULL;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio,
-							     struct list_head *start)
-{
-	struct mlx5_flow_table *ft;
-
-	if (!prio)
-		return NULL;
-
-	mutex_lock(&prio->base.lock);
-	ft = _find_first_ft_in_prio_reverse(prio, start);
-	mutex_unlock(&prio->base.lock);
-
-	return ft;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
-							   struct list_head *start)
-{
-	struct fs_prio *prio;
-
-	if (!ns)
-		return NULL;
-
-	fs_get_obj(prio, container_of(start, struct fs_base, list));
-	mutex_lock(&ns->base.lock);
-	fs_for_each_prio_continue_reverse(prio, ns) {
-		struct mlx5_flow_table *ft;
-
-		ft = find_first_ft_in_prio_reverse(prio, &prio->objs);
-		if (ft) {
-			mutex_unlock(&ns->base.lock);
-			return ft;
-		}
-	}
-	mutex_unlock(&ns->base.lock);
-
-	return NULL;
-}
-
-/* Returned a held ft, assumed curr is protected, assumed curr's parent is
- * locked
- */
-static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
-					    struct fs_prio *prio)
-{
-	struct mlx5_flow_table *ft = NULL;
-	struct fs_base *curr_base;
-
-	if (!curr)
-		return NULL;
-
-	/* prio has either namespace or flow-tables, but not both */
-	if (!list_empty(&prio->objs) &&
-	    list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) !=
-	    curr)
-		return NULL;
-
-	while (!ft && prio) {
-		struct mlx5_flow_namespace *ns;
-
-		fs_get_parent(ns, prio);
-		ft = find_first_ft_in_ns_reverse(ns, &prio->base.list);
-		curr_base = &ns->base;
-		fs_get_parent(prio, ns);
-
-		if (prio && !ft)
-			ft = find_first_ft_in_prio_reverse(prio,
-							   &curr_base->list);
-	}
-	return ft;
-}
-
-static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio,
-						      struct list_head *start)
-{
-	struct fs_base	*it = container_of(start, struct fs_base, list);
-
-	if (!prio)
-		return NULL;
-
-	fs_for_each_ns_or_ft_continue(it, prio) {
-		struct mlx5_flow_namespace	*ns;
-		struct mlx5_flow_table		*ft;
-
-		if (it->type == FS_TYPE_FLOW_TABLE) {
-			fs_get_obj(ft, it);
-			fs_get(&ft->base);
-			return ft;
-		}
-
-		fs_get_obj(ns, it);
-		WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
-
-		ft = find_first_ft_in_ns(ns, &ns->prios);
-		if (ft)
-			return ft;
-	}
-
-	return NULL;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
-						     struct list_head *start)
-{
-	struct mlx5_flow_table *ft;
-
-	if (!prio)
-		return NULL;
-
-	mutex_lock(&prio->base.lock);
-	ft = _find_first_ft_in_prio(prio, start);
-	mutex_unlock(&prio->base.lock);
-
-	return ft;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
-						   struct list_head *start)
-{
-	struct fs_prio *prio;
-
-	if (!ns)
-		return NULL;
-
-	fs_get_obj(prio, container_of(start, struct fs_base, list));
-	mutex_lock(&ns->base.lock);
-	fs_for_each_prio_continue(prio, ns) {
-		struct mlx5_flow_table *ft;
-
-		ft = find_first_ft_in_prio(prio, &prio->objs);
-		if (ft) {
-			mutex_unlock(&ns->base.lock);
-			return ft;
-		}
-	}
-	mutex_unlock(&ns->base.lock);
-
-	return NULL;
-}
-
-/* returned a held ft, assumed curr is protected, assumed curr's parent is
- * locked
- */
-static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio)
-{
-	struct mlx5_flow_table *ft = NULL;
-	struct fs_base *curr_base;
-
-	while (!ft && prio) {
-		struct mlx5_flow_namespace *ns;
-
-		fs_get_parent(ns, prio);
-		ft = find_first_ft_in_ns(ns, &prio->base.list);
-		curr_base = &ns->base;
-		fs_get_parent(prio, ns);
-
-		if (!ft && prio)
-			ft = _find_first_ft_in_prio(prio, &curr_base->list);
-	}
-	return ft;
-}
-
-
-/* called under ft mutex lock */
-static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft,
-						u8 match_criteria_enable,
-						u32 *match_criteria)
-{
-	unsigned int group_size;
-	unsigned int candidate_index = 0;
-	struct mlx5_flow_group *g;
-	struct mlx5_flow_group *ret;
-	struct list_head *prev = &ft->fgs;
-	struct mlx5_core_dev *dev;
-	u32 *in;
-	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
-	void *match_criteria_addr;
-	u32 max_fte = ft->autogroup.max_fte;
-
-	if (!ft->autogroup.active)
-		return ERR_PTR(-ENOENT);
-
-	dev = fs_get_dev(&ft->base);
-	if (!dev)
-		return ERR_PTR(-ENODEV);
-
-	in = mlx5_vzalloc(inlen);
-	if (!in) {
-		mlx5_core_warn(dev, "failed to allocate inbox\n");
-		return ERR_PTR(-ENOMEM);
-	}
-
-
-	if (ft->autogroup.num_types < ft->autogroup.max_types)
-		group_size = ft->autogroup.group_size;
-	else
-		group_size = 1;
-
-	if (group_size == 0) {
-		mlx5_core_warn(dev,
-			       "flow steering can't create group size of 0\n");
-		ret = ERR_PTR(-EINVAL);
-		goto out;
-	}
-
-	/* sorted by start_index */
-	fs_for_each_fg(g, ft) {
-		if (candidate_index + group_size > g->start_index)
-			candidate_index = g->start_index + g->max_ftes;
-		else
-			break;
-		prev = &g->base.list;
-	}
-
-	if (candidate_index + group_size > max_fte) {
-		ret = ERR_PTR(-ENOSPC);
-		goto out;
-	}
-
-	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
-		 match_criteria_enable);
-	MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index);
-	MLX5_SET(create_flow_group_in, in, end_flow_index,   candidate_index +
-		 group_size - 1);
-	match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
-					   in, match_criteria);
-	memcpy(match_criteria_addr, match_criteria,
-	       MLX5_ST_SZ_BYTES(fte_match_param));
-
-	ret = fs_create_fg(dev, ft, prev, in, 0);
-out:
-	kvfree(in);
-	return ret;
-}
-
-static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node)
-{
-	struct mlx5_flow_namespace *ns = NULL;
-
-	while (node  && (node->type != FS_TYPE_NAMESPACE ||
-			      list_empty(&container_of(node, struct
-						       mlx5_flow_namespace,
-						       base)->list_notifiers)))
-		node = node->parent;
-
-	if (node)
-		fs_get_obj(ns, node);
-
-	return ns;
-}
-
-
-/*Assumption- fte is locked*/
-static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst,
-				      struct fs_fte *fte)
-{
-	struct mlx5_flow_namespace *ns;
-	struct mlx5_flow_handler *iter_handler;
-	struct fs_client_priv_data *iter_client;
-	void *data;
-	bool is_new_rule = list_first_entry(&fte->dests,
-					    struct mlx5_flow_rule,
-					    base.list) == dst;
-	int err;
-
-	ns = get_ns_with_notifiers(&fte->base);
-	if (!ns)
-		return;
-
-	down_read(&ns->notifiers_rw_sem);
-	list_for_each_entry(iter_handler, &ns->list_notifiers,
-			    list) {
-		if (iter_handler->add_dst_cb) {
-			data = NULL;
-			mutex_lock(&dst->clients_lock);
-			list_for_each_entry(
-				iter_client, &dst->clients_data, list) {
-				if (iter_client->fs_handler == iter_handler) {
-					data = iter_client->client_dst_data;
-					break;
-				}
-			}
-			mutex_unlock(&dst->clients_lock);
-			err  = iter_handler->add_dst_cb(dst,
-							is_new_rule,
-							data,
-							iter_handler->client_context);
-			if (err)
-				break;
-		}
-	}
-	up_read(&ns->notifiers_rw_sem);
-}
-
-static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst,
-				      struct fs_fte *fte)
-{
-	struct mlx5_flow_namespace *ns;
-	struct mlx5_flow_handler *iter_handler;
-	struct fs_client_priv_data *iter_client;
-	void *data;
-	bool ctx_changed = (fte->dests_size == 0);
-
-	ns = get_ns_with_notifiers(&fte->base);
-	if (!ns)
-		return;
-	down_read(&ns->notifiers_rw_sem);
-	list_for_each_entry(iter_handler, &ns->list_notifiers,
-			    list) {
-		data = NULL;
-		mutex_lock(&dst->clients_lock);
-		list_for_each_entry(iter_client, &dst->clients_data, list) {
-			if (iter_client->fs_handler == iter_handler) {
-				data = iter_client->client_dst_data;
-				break;
-			}
-		}
-		mutex_unlock(&dst->clients_lock);
-		if (iter_handler->del_dst_cb) {
-			iter_handler->del_dst_cb(dst, ctx_changed, data,
-						 iter_handler->client_context);
-		}
-	}
-	up_read(&ns->notifiers_rw_sem);
-}
-
-/* fte should not be deleted while calling this function */
-static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte,
-					      struct mlx5_flow_group *fg,
-					      struct mlx5_flow_destination *dest)
-{
-	struct mlx5_flow_table *ft;
-	struct mlx5_flow_rule *dst;
-	int err;
-
-	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
-	if (!dst)
-		return ERR_PTR(-ENOMEM);
-
-	memcpy(&dst->dest_attr, dest, sizeof(*dest));
-	dst->base.type = FS_TYPE_FLOW_DEST;
-	INIT_LIST_HEAD(&dst->clients_data);
-	mutex_init(&dst->clients_lock);
-	fs_get_parent(ft, fg);
-	/*Add dest to dests list- added as first element after the head*/
-	list_add_tail(&dst->base.list, &fte->dests);
-	fte->dests_size++;
-	err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base),
-				  ft->vport,
-				  &fte->status,
-				  fte->val, ft->type,
-				  ft->id, fte->index, fg->id, &fte->flow_act,
-				  fte->sw_action, fte->dests_size, &fte->dests);
-	if (err)
-		goto free_dst;
-
-	list_del(&dst->base.list);
-
-	return dst;
-
-free_dst:
-	list_del(&dst->base.list);
-	kfree(dst);
-	fte->dests_size--;
-	return ERR_PTR(err);
-}
-
-static char *get_dest_name(struct mlx5_flow_destination *dest)
-{
-	char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL);
-
-	switch (dest->type) {
-	case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE:
-		snprintf(name, 20, "dest_%s_%u", "flow_table",
-			 dest->ft->id);
-		return name;
-	case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT:
-		snprintf(name, 20, "dest_%s_%u", "vport",
-			 dest->vport_num);
-		return name;
-	case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR:
-		snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num);
-		return name;
-	default:
-		kfree(name);
-		return NULL;
-	}
-}
-
-/* assumed fg is locked */
-static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg,
-					 struct list_head **prev)
-{
-	struct fs_fte *fte;
-	unsigned int start = fg->start_index;
-
-	if (prev)
-		*prev = &fg->ftes;
-
-	/* assumed list is sorted by index */
-	fs_for_each_fte(fte, fg) {
-		if (fte->index != start)
-			return start;
-		start++;
-		if (prev)
-			*prev = &fte->base.list;
-	}
-
-	return start;
-}
-
-
-static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg,
-			     u32 *match_value,
-			     u32 sw_action,
-			     struct mlx5_flow_act *flow_act,
-			     struct list_head **prev)
-{
-	struct fs_fte *fte;
-	int index = 0;
-
-	index = fs_get_free_fg_index(fg, prev);
-	fte = fs_alloc_fte(sw_action, flow_act, match_value, index);
-	if (IS_ERR(fte))
-		return fte;
-
-	return fte;
-}
-
-static void add_rule_to_tree(struct mlx5_flow_rule *rule,
-			     struct fs_fte *fte)
-{
-	char *dest_name;
-
-	dest_name = get_dest_name(&rule->dest_attr);
-	fs_add_node(&rule->base, &fte->base, dest_name, 1);
-	/* re-add to list, since fs_add_node reset our list */
-	list_add_tail(&rule->base.list, &fte->dests);
-	kfree(dest_name);
-	call_to_add_rule_notifiers(rule, fte);
-}
-
-static void fs_del_dst(struct mlx5_flow_rule *dst)
-{
-	struct mlx5_flow_table *ft;
-	struct mlx5_flow_group *fg;
-	struct fs_fte *fte;
-	u32	*match_value;
-	struct mlx5_core_dev *dev = fs_get_dev(&dst->base);
-	int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
-	int err;
-
-	WARN_ON(!dev);
-
-	match_value = mlx5_vzalloc(match_len);
-	if (!match_value) {
-		mlx5_core_warn(dev, "failed to allocate inbox\n");
-		return;
-	}
-
-	fs_get_parent(fte, dst);
-	fs_get_parent(fg, fte);
-	sx_assert(&fg->base.lock.sx, SX_XLOCKED);
-	memcpy(match_value, fte->val, sizeof(fte->val));
-	/* ft can't be changed as fg is locked */
-	fs_get_parent(ft, fg);
-	list_del(&dst->base.list);
-	fte->dests_size--;
-	if (fte->dests_size) {
-		err = mlx5_cmd_fs_set_fte(dev, ft->vport,
-					  &fte->status, match_value, ft->type,
-					  ft->id, fte->index, fg->id,
-					  &fte->flow_act, fte->sw_action,
-					  fte->dests_size, &fte->dests);
-		if (err) {
-			mlx5_core_warn(dev, "%s can't delete dst %s\n",
-				       __func__, dst->base.name);
-			goto err;
-		}
-	}
-	call_to_del_rule_notifiers(dst, fte);
-err:
-	kvfree(match_value);
-}
-
-static void fs_del_fte(struct fs_fte *fte)
-{
-	struct mlx5_flow_table *ft;
-	struct mlx5_flow_group *fg;
-	int err;
-	struct mlx5_core_dev *dev;
-
-	fs_get_parent(fg, fte);
-	fs_get_parent(ft, fg);
-
-	dev = fs_get_dev(&ft->base);
-	WARN_ON(!dev);
-
-	err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status,
-				     ft->type, ft->id, fte->index);
-	if (err)
-		mlx5_core_warn(dev, "flow steering can't delete fte %s\n",
-			       fte->base.name);
-
-	fg->num_ftes--;
-}
-
-static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
-				      const struct mlx5_flow_act *act2)
-{
-        u32 action1 = act1->actions;
-        u32 action2 = act2->actions;
-	u32 xored_actions;
-
-	xored_actions = action1 ^ action2;
-
-	if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG))
-		return true;
-
-	if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG &&
-	    act1->flow_tag != act2->flow_tag)
-		return true;
-
-	/* Can even have complex actions in merged rules */
-	if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR)
-		return true;
-
-	if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT)
-		return true;
-
-	if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT)
-		return true;
-
-	return false;
-}
-
-/* assuming parent fg is locked */
-/* Add dst algorithm */
-static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg,
-						   u32 *match_value,
-						   u32 sw_action,
-						   struct mlx5_flow_act *flow_act,
-						   struct mlx5_flow_destination *dest)
-{
-	struct fs_fte *fte;
-	struct mlx5_flow_rule *dst;
-	struct mlx5_flow_table *ft;
-	struct list_head *prev;
-	char fte_name[20];
-
-	mutex_lock(&fg->base.lock);
-	if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND)
-		goto insert_fte;
-
-	fs_for_each_fte(fte, fg) {
-		/* TODO: Check of size against PRM max size */
-		mutex_lock(&fte->base.lock);
-		if (fs_match_exact_val(&fg->mask, match_value, &fte->val) &&
-		    sw_action == fte->sw_action &&
-		    !check_conflicting_actions(flow_act, &fte->flow_act)) {
-			dst = _fs_add_dst_fte(fte, fg, dest);
-			mutex_unlock(&fte->base.lock);
-			if (IS_ERR(dst))
-				goto unlock_fg;
-			goto add_rule;
-		}
-		mutex_unlock(&fte->base.lock);
-	}
-
-insert_fte:
-	fs_get_parent(ft, fg);
-	if (fg->num_ftes == fg->max_ftes) {
-		dst = ERR_PTR(-ENOSPC);
-		goto unlock_fg;
-	}
-
-	fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev);
-	if (IS_ERR(fte)) {
-		dst = (void *)fte;
-		goto unlock_fg;
-	}
-	dst = _fs_add_dst_fte(fte, fg, dest);
-	if (IS_ERR(dst)) {
-		kfree(fte);
-		goto unlock_fg;
-	}
-
-	fg->num_ftes++;
-
-	snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index);
-	/* Add node to tree */
-	fs_add_node(&fte->base, &fg->base, fte_name, 0);
-	list_add(&fte->base.list, prev);
-add_rule:
-	add_rule_to_tree(dst, fte);
-unlock_fg:
-	mutex_unlock(&fg->base.lock);
-	return dst;
-}
-
-static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft,
-					    u8 match_criteria_enable,
-					    u32 *match_criteria,
-					    u32 *match_value,
-					    u32 sw_action,
-					    struct mlx5_flow_act *flow_act,
-					    struct mlx5_flow_destination *dest)
-{
-	/*? where dst_entry is allocated*/
-	struct mlx5_flow_group *g;
-	struct mlx5_flow_rule *dst;
-
-	fs_get(&ft->base);
-	mutex_lock(&ft->base.lock);
-	fs_for_each_fg(g, ft)
-		if (fs_match_exact_mask(g->mask.match_criteria_enable,
-					match_criteria_enable,
-					g->mask.match_criteria,
-					match_criteria)) {
-			mutex_unlock(&ft->base.lock);
-
-			dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest);
-			if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC)
-				goto unlock;
-		}
-	mutex_unlock(&ft->base.lock);
-
-	g = create_autogroup(ft, match_criteria_enable, match_criteria);
-	if (IS_ERR(g)) {
-		dst = (void *)g;
-		goto unlock;
-	}
-
-	dst = fs_add_dst_fg(g, match_value,
-			    sw_action, flow_act, dest);
-	if (IS_ERR(dst)) {
-		/* Remove assumes refcount > 0 and autogroup creates a group
-		 * with a refcount = 0.
-		 */
-		fs_get(&g->base);
-		fs_remove_node(&g->base);
-		goto unlock;
-	}
-
-unlock:
-	fs_put(&ft->base);
-	return dst;
-}
-
-struct mlx5_flow_rule *
-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
-		   u8 match_criteria_enable,
-		   u32 *match_criteria,
-		   u32 *match_value,
-		   u32 sw_action,
-		   struct mlx5_flow_act *flow_act,
-		   struct mlx5_flow_destination *dest)
-{
-	struct mlx5_flow_rule *dst;
-	struct mlx5_flow_namespace *ns;
-
-	ns = get_ns_with_notifiers(&ft->base);
-	if (ns)
-		down_read(&ns->dests_rw_sem);
-	dst =  fs_add_dst_ft(ft, match_criteria_enable, match_criteria,
-			     match_value, sw_action, flow_act, dest);
-	if (ns)
-		up_read(&ns->dests_rw_sem);
-
-	return dst;
-
-
-}
-EXPORT_SYMBOL(mlx5_add_flow_rule);
-
-void mlx5_del_flow_rule(struct mlx5_flow_rule **pp)
-{
-	struct mlx5_flow_namespace *ns;
-	struct mlx5_flow_rule *dst;
-
-	dst = *pp;
-	*pp = NULL;
-
-	if (IS_ERR_OR_NULL(dst))
-		return;
-	ns = get_ns_with_notifiers(&dst->base);
-	if (ns)
-		down_read(&ns->dests_rw_sem);
-	fs_remove_node(&dst->base);
-	if (ns)
-		up_read(&ns->dests_rw_sem);
-}
-EXPORT_SYMBOL(mlx5_del_flow_rule);
-
-#define MLX5_CORE_FS_ROOT_NS_NAME "root"
-#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root"
-#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root"
-#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root"
-#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root"
-#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root"
-#define MLX5_CORE_FS_PRIO_MAX_FT 4
-#define MLX5_CORE_FS_PRIO_MAX_NS 1
-
-static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
-				      unsigned prio, int max_ft,
-				      const char *name, u8 flags)
-{
-	struct fs_prio *fs_prio;
-
-	fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
-	if (!fs_prio)
-		return ERR_PTR(-ENOMEM);
-
-	fs_prio->base.type = FS_TYPE_PRIO;
-	fs_add_node(&fs_prio->base, &ns->base, name, 1);
-	fs_prio->max_ft = max_ft;
-	fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS;
-	fs_prio->prio = prio;
-	fs_prio->flags = flags;
-	list_add_tail(&fs_prio->base.list, &ns->prios);
-	INIT_LIST_HEAD(&fs_prio->objs);
-	mutex_init(&fs_prio->shared_lock);
-
-	return fs_prio;
-}
-
-static void cleanup_root_ns(struct mlx5_core_dev *dev)
-{
-	struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
-	struct fs_prio *iter_prio;
-
-	if (!root_ns)
-		return;
-
-	/* stage 1 */
-	fs_for_each_prio(iter_prio, &root_ns->ns) {
-		struct mlx5_flow_namespace *iter_ns;
-
-		fs_for_each_ns(iter_ns, iter_prio) {
-			while (!list_empty(&iter_ns->prios)) {
-				struct fs_base *iter_prio2 =
-					list_first_entry(&iter_ns->prios,
-							 struct fs_base,
-							 list);
-
-				fs_remove_node(iter_prio2);
-			}
-		}
-	}
-
-	/* stage 2 */
-	fs_for_each_prio(iter_prio, &root_ns->ns) {
-		while (!list_empty(&iter_prio->objs)) {
-			struct fs_base *iter_ns =
-				list_first_entry(&iter_prio->objs,
-						 struct fs_base,
-						 list);
-
-				fs_remove_node(iter_ns);
-		}
-	}
-	/* stage 3 */
-	while (!list_empty(&root_ns->ns.prios)) {
-		struct fs_base *iter_prio =
-			list_first_entry(&root_ns->ns.prios,
-					 struct fs_base,
-					 list);
-
-		fs_remove_node(iter_prio);
-	}
-
-	fs_remove_node(&root_ns->ns.base);
-	dev->root_ns = NULL;
-}
-
-static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev,
-					struct mlx5_flow_root_namespace *root_ns)
-{
-	struct fs_base *prio;
-
-	if (!root_ns)
-		return;
-
-	if (!list_empty(&root_ns->ns.prios)) {
-		prio = list_first_entry(&root_ns->ns.prios,
-					struct fs_base,
-				 list);
-		fs_remove_node(prio);
-	}
-	fs_remove_node(&root_ns->ns.base);
-	root_ns = NULL;
-}
-
-void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
-{
-	mlx5_cleanup_fc_stats(dev);
-	cleanup_root_ns(dev);
-	cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns);
-	cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns);
-	cleanup_single_prio_root_ns(dev, dev->fdb_root_ns);
-	cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns);
-	cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns);
-}
-
-static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
-						 *ns)
-{
-	ns->base.type = FS_TYPE_NAMESPACE;
-	init_rwsem(&ns->dests_rw_sem);
-	init_rwsem(&ns->notifiers_rw_sem);
-	INIT_LIST_HEAD(&ns->prios);
-	INIT_LIST_HEAD(&ns->list_notifiers);
-
-	return ns;
-}
-
-static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev,
-							  enum fs_ft_type
-							  table_type,
-							  char *name)
-{
-	struct mlx5_flow_root_namespace *root_ns;
-	struct mlx5_flow_namespace *ns;
-
-	/* create the root namespace */
-	root_ns = mlx5_vzalloc(sizeof(*root_ns));
-	if (!root_ns)
-		goto err;
-
-	root_ns->dev = dev;
-	root_ns->table_type = table_type;
-	mutex_init(&root_ns->fs_chain_lock);
-
-	ns = &root_ns->ns;
-	fs_init_namespace(ns);
-	fs_add_node(&ns->base, NULL, name, 1);
-
-	return root_ns;
-err:
-	return NULL;
-}
-
-static int init_fdb_root_ns(struct mlx5_core_dev *dev)
-{
-	struct fs_prio *prio;
-
-	dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB,
-					  MLX5_CORE_FS_FDB_ROOT_NS_NAME);
-	if (!dev->fdb_root_ns)
-		return -ENOMEM;
-
-	/* create 1 prio*/
-	prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0);
-	if (IS_ERR(prio))
-		return PTR_ERR(prio);
-	else
-		return 0;
-}
-
-#define MAX_VPORTS 128
-
-static int init_egress_acl_root_ns(struct mlx5_core_dev *dev)
-{
-	struct fs_prio *prio;
-
-	dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL,
-						 MLX5_CORE_FS_ESW_EGRESS_ACL);
-	if (!dev->esw_egress_root_ns)
-		return -ENOMEM;
-
-	/* create 1 prio*/
-	prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS,
-			      "esw_egress_prio", 0);
-	if (IS_ERR(prio))
-		return PTR_ERR(prio);
-	else
-		return 0;
-}
-
-static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev)
-{
-	struct fs_prio *prio;
-
-	dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL,
-						  MLX5_CORE_FS_ESW_INGRESS_ACL);
-	if (!dev->esw_ingress_root_ns)
-		return -ENOMEM;
-
-	/* create 1 prio*/
-	prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS,
-			      "esw_ingress_prio", 0);
-	if (IS_ERR(prio))
-		return PTR_ERR(prio);
-	else
-		return 0;
-}
-
-static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev)
-{
-	struct fs_prio *prio;
-
-	dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX,
-				     MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME);
-	if (!dev->sniffer_rx_root_ns)
-		return  -ENOMEM;
-
-	/* create 1 prio*/
-	prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1,
-			      "sniffer_prio", 0);
-	if (IS_ERR(prio))
-		return PTR_ERR(prio);
-	else
-		return 0;
-}
-
-
-static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev)
-{
-	struct fs_prio *prio;
-
-	dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX,
-						 MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME);
-	if (!dev->sniffer_tx_root_ns)
-		return  -ENOMEM;
-
-	/* create 1 prio*/
-	prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1,
-			      "sniffer_prio", 0);
-	if (IS_ERR(prio))
-		return PTR_ERR(prio);
-	else
-		return 0;
-}
-
-static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
-						       const char *name)
-{
-	struct mlx5_flow_namespace	*ns;
-
-	ns = kzalloc(sizeof(*ns), GFP_KERNEL);
-	if (!ns)
-		return ERR_PTR(-ENOMEM);
-
-	fs_init_namespace(ns);
-	fs_add_node(&ns->base, &prio->base, name, 1);
-	list_add_tail(&ns->base.list, &prio->objs);
-
-	return ns;
-}
-
-#define FLOW_TABLE_BIT_SZ 1
-#define GET_FLOW_TABLE_CAP(dev, offset) \
-	((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) +	\
-			offset / 32)) >>					\
-	  (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
-
-static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
-{
-	int i;
-
-	for (i = 0; i < caps->arr_sz; i++) {
-		if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
-			return false;
-	}
-	return true;
-}
-
-static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
-		    struct init_tree_node *node, struct fs_base *base_parent,
-		    struct init_tree_node *tree_parent)
-{
-	struct mlx5_flow_namespace *fs_ns;
-	struct fs_prio *fs_prio;
-	int priority;
-	struct fs_base *base;
-	int i;
-	int err = 0;
-
-	if (node->type == FS_TYPE_PRIO) {
-		if ((node->min_ft_level > max_ft_level) ||
-		    !has_required_caps(dev, &node->caps))
-			goto out;
-
-		fs_get_obj(fs_ns, base_parent);
-		priority = node - tree_parent->children;
-		fs_prio = fs_create_prio(fs_ns, priority,
-					 node->max_ft,
-					 node->name, node->flags);
-		if (IS_ERR(fs_prio)) {
-			err = PTR_ERR(fs_prio);
-			goto out;
-		}
-		base = &fs_prio->base;
-	} else if (node->type == FS_TYPE_NAMESPACE) {
-		fs_get_obj(fs_prio, base_parent);
-		fs_ns = fs_create_namespace(fs_prio, node->name);
-		if (IS_ERR(fs_ns)) {
-			err = PTR_ERR(fs_ns);
-			goto out;
-		}
-		base = &fs_ns->base;
-	} else {
-		return -EINVAL;
-	}
-	for (i = 0; i < node->ar_size; i++) {
-		err = _init_root_tree(dev, max_ft_level, &node->children[i], base,
-				      node);
-		if (err)
-			break;
-	}
-out:
-	return err;
-}
-
-static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
-		   struct init_tree_node *node, struct fs_base *parent)
-{
-	int i;
-	struct mlx5_flow_namespace *fs_ns;
-	int err = 0;
-
-	fs_get_obj(fs_ns, parent);
-	for (i = 0; i < node->ar_size; i++) {
-		err = _init_root_tree(dev, max_ft_level,
-				      &node->children[i], &fs_ns->base, node);
-		if (err)
-			break;
-	}
-	return err;
-}
-
-static int sum_max_ft_in_prio(struct fs_prio *prio);
-static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns)
-{
-	struct fs_prio *prio;
-	int sum = 0;
-
-	fs_for_each_prio(prio, ns) {
-		sum += sum_max_ft_in_prio(prio);
-	}
-	return  sum;
-}
-
-static int sum_max_ft_in_prio(struct fs_prio *prio)
-{
-	int sum = 0;
-	struct fs_base *it;
-	struct mlx5_flow_namespace	*ns;
-
-	if (prio->max_ft)
-		return prio->max_ft;
-
-	fs_for_each_ns_or_ft(it, prio) {
-		if (it->type == FS_TYPE_FLOW_TABLE)
-			continue;
-
-		fs_get_obj(ns, it);
-		sum += sum_max_ft_in_ns(ns);
-	}
-	prio->max_ft = sum;
-	return  sum;
-}
-
-static void set_max_ft(struct mlx5_flow_namespace *ns)
-{
-	struct fs_prio *prio;
-
-	if (!ns)
-		return;
-
-	fs_for_each_prio(prio, ns)
-		sum_max_ft_in_prio(prio);
-}
-
-static int init_root_ns(struct mlx5_core_dev *dev)
-{
-	int max_ft_level = MLX5_CAP_FLOWTABLE(dev,
-					      flow_table_properties_nic_receive.
-					      max_ft_level);
-
-	dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX,
-				      MLX5_CORE_FS_ROOT_NS_NAME);
-	if (IS_ERR_OR_NULL(dev->root_ns))
-		goto err;
-
-
-	if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base))
-		goto err;
-
-	set_max_ft(&dev->root_ns->ns);
-
-	return 0;
-err:
-	return -ENOMEM;
-}
-
-u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule)
-{
-	struct fs_base *pbase;
-	struct mlx5_flow_group *fg;
-
-	pbase = rule->base.parent;
-	WARN_ON(!pbase);
-	pbase = pbase->parent;
-	WARN_ON(!pbase);
-
-	fs_get_obj(fg, pbase);
-	return fg->mask.match_criteria_enable;
-}
-
-void mlx5_get_match_value(u32 *match_value,
-			  struct mlx5_flow_rule *rule)
-{
-	struct fs_base *pbase;
-	struct fs_fte *fte;
-
-	pbase = rule->base.parent;
-	WARN_ON(!pbase);
-	fs_get_obj(fte, pbase);
-
-	memcpy(match_value, fte->val, sizeof(fte->val));
-}
-
-void mlx5_get_match_criteria(u32 *match_criteria,
-			     struct mlx5_flow_rule *rule)
-{
-	struct fs_base *pbase;
-	struct mlx5_flow_group *fg;
-
-	pbase = rule->base.parent;
-	WARN_ON(!pbase);
-	pbase = pbase->parent;
-	WARN_ON(!pbase);
-
-	fs_get_obj(fg, pbase);
-	memcpy(match_criteria, &fg->mask.match_criteria,
-	       sizeof(fg->mask.match_criteria));
-}
-
-int mlx5_init_fs(struct mlx5_core_dev *dev)
-{
-	int err;
-
-	if (MLX5_CAP_GEN(dev, nic_flow_table)) {
-		err = init_root_ns(dev);
-		if (err)
-			goto err;
-	}
-
-	err = init_fdb_root_ns(dev);
-	if (err)
-		goto err;
-
-	err = init_egress_acl_root_ns(dev);
-	if (err)
-		goto err;
-
-	err = init_ingress_acl_root_ns(dev);
-	if (err)
-		goto err;
-
-	err = init_sniffer_tx_root_ns(dev);
-	if (err)
-		goto err;
-
-	err = init_sniffer_rx_root_ns(dev);
-	if (err)
-		goto err;
-
-	err = mlx5_init_fc_stats(dev);
-	if (err)
-		goto err;
-
-	return 0;
-err:
-	mlx5_cleanup_fs(dev);
-	return err;
-}
-
-struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
-						  enum mlx5_flow_namespace_type type)
-{
-	struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
-	int prio;
-	static struct fs_prio *fs_prio;
-	struct mlx5_flow_namespace *ns;
-
-	switch (type) {
-	case MLX5_FLOW_NAMESPACE_BYPASS:
-		prio = 0;
-		break;
-	case MLX5_FLOW_NAMESPACE_OFFLOADS:
-		prio = 1;
-		break;
-	case MLX5_FLOW_NAMESPACE_KERNEL:
-		prio = 2;
-		break;
-	case MLX5_FLOW_NAMESPACE_LEFTOVERS:
-		prio = 3;
-		break;
-	case MLX5_FLOW_NAMESPACE_FDB:
-		if (dev->fdb_root_ns)
-			return &dev->fdb_root_ns->ns;
-		else
-			return NULL;
-	case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
-		if (dev->esw_egress_root_ns)
-			return &dev->esw_egress_root_ns->ns;
-		else
-			return NULL;
-	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
-		if (dev->esw_ingress_root_ns)
-			return &dev->esw_ingress_root_ns->ns;
-		else
-			return NULL;
-	case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
-		if (dev->sniffer_rx_root_ns)
-			return &dev->sniffer_rx_root_ns->ns;
-		else
-			return NULL;
-	case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
-		if (dev->sniffer_tx_root_ns)
-			return &dev->sniffer_tx_root_ns->ns;
-		else
-			return NULL;
-	default:
-		return NULL;
-	}
-
-	if (!root_ns)
-		return NULL;
-
-	fs_prio = find_prio(&root_ns->ns, prio);
-	if (!fs_prio)
-		return NULL;
-
-	ns = list_first_entry(&fs_prio->objs,
-			      typeof(*ns),
-			      base.list);
-
-	return ns;
-}
-EXPORT_SYMBOL(mlx5_get_flow_namespace);
-
-
-int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule,
-				  struct mlx5_flow_handler *fs_handler,
-				  void  *client_data)
-{
-	struct fs_client_priv_data *priv_data;
-
-	mutex_lock(&rule->clients_lock);
-	/*Check that hanlder isn't exists in the list already*/
-	list_for_each_entry(priv_data, &rule->clients_data, list) {
-		if (priv_data->fs_handler == fs_handler) {
-			priv_data->client_dst_data = client_data;
-			goto unlock;
-		}
-	}
-	priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL);
-	if (!priv_data) {
-		mutex_unlock(&rule->clients_lock);
-		return -ENOMEM;
-	}
-
-	priv_data->client_dst_data = client_data;
-	priv_data->fs_handler = fs_handler;
-	list_add(&priv_data->list, &rule->clients_data);
-
-unlock:
-	mutex_unlock(&rule->clients_lock);
-
-	return 0;
-}
-
-static int remove_from_clients(struct mlx5_flow_rule *rule,
-			bool ctx_changed,
-			void *client_data,
-			void *context)
-{
-	struct fs_client_priv_data *iter_client;
-	struct fs_client_priv_data *temp_client;
-	struct mlx5_flow_handler *handler = (struct
-						mlx5_flow_handler*)context;
-
-	mutex_lock(&rule->clients_lock);
-	list_for_each_entry_safe(iter_client, temp_client,
-				 &rule->clients_data, list) {
-		if (iter_client->fs_handler == handler) {
-			list_del(&iter_client->list);
-			kfree(iter_client);
-			break;
-		}
-	}
-	mutex_unlock(&rule->clients_lock);
-
-	return 0;
-}
-
-struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev,
-								enum mlx5_flow_namespace_type ns_type,
-								rule_event_fn add_cb,
-								rule_event_fn del_cb,
-								void *context)
-{
-	struct mlx5_flow_namespace *ns;
-	struct mlx5_flow_handler *handler;
-
-	ns = mlx5_get_flow_namespace(dev, ns_type);
-	if (!ns)
-		return ERR_PTR(-EINVAL);
-
-	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
-	if (!handler)
-		return ERR_PTR(-ENOMEM);
-
-	handler->add_dst_cb = add_cb;
-	handler->del_dst_cb = del_cb;
-	handler->client_context = context;
-	handler->ns = ns;
-	down_write(&ns->notifiers_rw_sem);
-	list_add_tail(&handler->list, &ns->list_notifiers);
-	up_write(&ns->notifiers_rw_sem);
-
-	return handler;
-}
-
-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
-				rule_event_fn add_rule_cb,
-				void *context);
-
-void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler)
-{
-	struct mlx5_flow_namespace *ns = handler->ns;
-
-	/*Remove from dst's clients*/
-	down_write(&ns->dests_rw_sem);
-	down_write(&ns->notifiers_rw_sem);
-	iterate_rules_in_ns(ns, remove_from_clients, handler);
-	list_del(&handler->list);
-	up_write(&ns->notifiers_rw_sem);
-	up_write(&ns->dests_rw_sem);
-	kfree(handler);
-}
-
-static void iterate_rules_in_ft(struct mlx5_flow_table *ft,
-				rule_event_fn add_rule_cb,
-				void *context)
-{
-	struct mlx5_flow_group *iter_fg;
-	struct fs_fte *iter_fte;
-	struct mlx5_flow_rule *iter_rule;
-	int err = 0;
-	bool is_new_rule;
-
-	mutex_lock(&ft->base.lock);
-	fs_for_each_fg(iter_fg, ft) {
-		mutex_lock(&iter_fg->base.lock);
-		fs_for_each_fte(iter_fte, iter_fg) {
-			mutex_lock(&iter_fte->base.lock);
-			is_new_rule = true;
-			fs_for_each_dst(iter_rule, iter_fte) {
-				fs_get(&iter_rule->base);
-				err = add_rule_cb(iter_rule,
-						 is_new_rule,
-						 NULL,
-						 context);
-				fs_put_parent_locked(&iter_rule->base);
-				if (err)
-					break;
-				is_new_rule = false;
-			}
-			mutex_unlock(&iter_fte->base.lock);
-			if (err)
-				break;
-		}
-		mutex_unlock(&iter_fg->base.lock);
-		if (err)
-			break;
-	}
-	mutex_unlock(&ft->base.lock);
-}
-
-static void iterate_rules_in_prio(struct fs_prio *prio,
-				  rule_event_fn add_rule_cb,
-				  void *context)
-{
-	struct fs_base *it;
-
-	mutex_lock(&prio->base.lock);
-	fs_for_each_ns_or_ft(it, prio) {
-		if (it->type == FS_TYPE_FLOW_TABLE) {
-			struct mlx5_flow_table	      *ft;
-
-			fs_get_obj(ft, it);
-			iterate_rules_in_ft(ft, add_rule_cb, context);
-		} else {
-			struct mlx5_flow_namespace *ns;
-
-			fs_get_obj(ns, it);
-			iterate_rules_in_ns(ns, add_rule_cb, context);
-		}
-	}
-	mutex_unlock(&prio->base.lock);
-}
-
-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
-				rule_event_fn add_rule_cb,
-				void *context)
-{
-	struct fs_prio *iter_prio;
-
-	mutex_lock(&ns->base.lock);
-	fs_for_each_prio(iter_prio, ns) {
-		iterate_rules_in_prio(iter_prio, add_rule_cb, context);
-	}
-	mutex_unlock(&ns->base.lock);
-}
-
-void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns,
-					 rule_event_fn add_rule_cb,
-					 void *context)
-{
-	down_write(&ns->dests_rw_sem);
-	down_read(&ns->notifiers_rw_sem);
-	iterate_rules_in_ns(ns, add_rule_cb, context);
-	up_read(&ns->notifiers_rw_sem);
-	up_write(&ns->dests_rw_sem);
-}
-
-
-void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list)
-{
-	struct mlx5_flow_rule_node *iter_node;
-	struct mlx5_flow_rule_node *temp_node;
-
-	list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) {
-		list_del(&iter_node->list);
-		kfree(iter_node);
-	}
-
-	kfree(rules_list);
-}
-
-#define ROCEV1_ETHERTYPE 0x8915
-static int set_rocev1_rules(struct list_head *rules_list)
-{
-	struct mlx5_flow_rule_node *rocev1_rule;
-
-	rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL);
-	if (!rocev1_rule)
-		return -ENOMEM;
-
-	rocev1_rule->match_criteria_enable =
-		1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
-	MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype,
-		 0xffff);
-	MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype,
-		 ROCEV1_ETHERTYPE);
-
-	list_add_tail(&rocev1_rule->list, rules_list);
-
-	return 0;
-}
-
-#define ROCEV2_UDP_PORT 4791
-static int set_rocev2_rules(struct list_head *rules_list)
-{
-	struct mlx5_flow_rule_node *ipv4_rule;
-	struct mlx5_flow_rule_node *ipv6_rule;
-
-	ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL);
-	if (!ipv4_rule)
-		return -ENOMEM;
-
-	ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL);
-	if (!ipv6_rule) {
-		kfree(ipv4_rule);
-		return -ENOMEM;
-	}
-
-	ipv4_rule->match_criteria_enable =
-		1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype,
-		 0xffff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype,
-		 0x0800);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol,
-		 0xff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol,
-		 IPPROTO_UDP);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport,
-		 0xffff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport,
-		 ROCEV2_UDP_PORT);
-
-	ipv6_rule->match_criteria_enable =
-		1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype,
-		 0xffff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype,
-		 0x86dd);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol,
-		 0xff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol,
-		 IPPROTO_UDP);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport,
-		 0xffff);
-	MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport,
-		 ROCEV2_UDP_PORT);
-
-	list_add_tail(&ipv4_rule->list, rules_list);
-	list_add_tail(&ipv6_rule->list, rules_list);
-
-	return 0;
-}
-
-
-struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode)
-{
-	int err = 0;
-	struct mlx5_flow_rules_list *rules_list =
-		kzalloc(sizeof(*rules_list), GFP_KERNEL);
-
-	if (!rules_list)
-		return NULL;
-
-	INIT_LIST_HEAD(&rules_list->head);
-
-	if (roce_mode & MLX5_ROCE_VERSION_1_CAP) {
-		err = set_rocev1_rules(&rules_list->head);
-		if (err)
-			goto free_list;
-	}
-	if (roce_mode & MLX5_ROCE_VERSION_2_CAP)
-		err = set_rocev2_rules(&rules_list->head);
-	if (err)
-		goto free_list;
-
-	return rules_list;
-
-free_list:
-	mlx5_del_flow_rules_list(rules_list);
-	return NULL;
-}
-
-struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
-						 enum mlx5_flow_namespace_type ns_type,
-						 u8 num_actions,
-						 void *modify_actions)
-{
-	struct mlx5_modify_hdr *modify_hdr;
-	int err;
-
-	modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
-	if (!modify_hdr)
-		return ERR_PTR(-ENOMEM);
-
-	modify_hdr->ns_type = ns_type;
-	err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions,
-					   modify_actions, modify_hdr);
-	if (err) {
-		kfree(modify_hdr);
-		return ERR_PTR(err);
-	}
-
-	return modify_hdr;
-}
-EXPORT_SYMBOL(mlx5_modify_header_alloc);
-
-void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
-                                struct mlx5_modify_hdr *modify_hdr)
-{
-        mlx5_cmd_modify_header_dealloc(dev, modify_hdr);
-        kfree(modify_hdr);
-}
-EXPORT_SYMBOL(mlx5_modify_header_dealloc);
-
-struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
-                                                     struct mlx5_pkt_reformat_params *params,
-                                                     enum mlx5_flow_namespace_type ns_type)
-{
-        struct mlx5_pkt_reformat *pkt_reformat;
-        int err;
-
-        pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
-        if (!pkt_reformat)
-                return ERR_PTR(-ENOMEM);
-
-        pkt_reformat->ns_type = ns_type;
-        pkt_reformat->reformat_type = params->type;
-	err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type,
-					     pkt_reformat);
-        if (err) {
-                kfree(pkt_reformat);
-                return ERR_PTR(err);
-        }
-
-        return pkt_reformat;
-}
-EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
-
-void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
-                                  struct mlx5_pkt_reformat *pkt_reformat)
-{
-        mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat);
-        kfree(pkt_reformat);
-}
-EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
-
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fw.c b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
index 1a4956b09d32..b3a7ffb732f9 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fw.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
@@ -240,6 +240,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 			return err;
 	}
 
+	if (MLX5_CAP_GEN(dev, ipsec_offload)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_IPSEC);
+		if (err)
+			return err;
+	}
+
 	err = mlx5_core_query_special_contexts(dev);
 	if (err)
 		return err;
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
index 95d88c728306..7400a8eafaa4 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -26,6 +26,7 @@
 
 #include "opt_rss.h"
 #include "opt_ratelimit.h"
+#include "opt_ipsec.h"
 
 #include <linux/kmod.h>
 #include <linux/module.h>
@@ -67,6 +68,9 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1);
 MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1);
 MODULE_DEPEND(mlx5, firmware, 1, 1, 1);
+#ifdef IPSEC_OFFLOAD
+MODULE_DEPEND(mlx5, ipsec, 1, 1, 1);
+#endif
 MODULE_VERSION(mlx5, 1);
 
 SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
@@ -1209,7 +1213,7 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto err_stop_eqs;
 	}
 
-	err = mlx5_init_fs(dev);
+	err = mlx5_fs_core_init(dev);
 	if (err) {
 		mlx5_core_err(dev, "flow steering init %d\n", err);
 		goto err_free_comp_eqs;
@@ -1327,7 +1331,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	mlx5_diag_cnt_cleanup(dev);
 	mlx5_fpga_device_stop(dev);
 	mlx5_mpfs_destroy(dev);
-	mlx5_cleanup_fs(dev);
+	mlx5_fs_core_cleanup(dev);
 	mlx5_wait_for_reclaim_vfs_pages(dev);
 	free_comp_eqs(dev);
 	mlx5_stop_eqs(dev);
@@ -1694,10 +1698,17 @@ static int init_one(struct pci_dev *pdev,
 
 	mlx5_pagealloc_init(dev);
 
+	pr_info("%s - MARK BLOCH WAS HERE\n", __func__);
+	err = mlx5_fs_core_alloc(dev);
+	if (err) {
+		mlx5_core_err(dev, "Failed to alloc flow steering\n");
+		goto clean_health;
+	}
+
 	err = mlx5_load_one(dev, priv, true);
 	if (err) {
 		mlx5_core_err(dev, "mlx5_load_one failed %d\n", err);
-		goto clean_health;
+		goto clean_fs;
 	}
 
 	mlx5_fwdump_prep(dev);
@@ -1743,6 +1754,8 @@ static int init_one(struct pci_dev *pdev,
 	pci_save_state(pdev);
 	return 0;
 
+clean_fs:
+	mlx5_fs_core_free(dev);
 clean_health:
 	mlx5_pagealloc_cleanup(dev);
 	mlx5_health_cleanup(dev);
@@ -1774,6 +1787,7 @@ static void remove_one(struct pci_dev *pdev)
 		    (long long)(dev->priv.fw_pages * MLX5_ADAPTER_PAGE_SIZE));
 	}
 
+	mlx5_fs_core_free(dev);
 	mlx5_pagealloc_cleanup(dev);
 	mlx5_health_cleanup(dev);
 	mlx5_fwdump_clean(dev);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_tls.c b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
index b3a49c603fed..3ed209e2028d 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_tls.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
@@ -33,66 +33,6 @@
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
 #include <dev/mlx5/mlx5_core/transobj.h>
 
-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
-    const void *p_key, u32 key_len, u32 *p_obj_id)
-{
-	u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
-	u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
-	u64 general_obj_types;
-	int err;
-
-	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
-	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
-		return -EINVAL;
-
-	switch (key_len) {
-	case 128 / 8:
-		memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
-		    encryption_key_object.key[4]), p_key, 128 / 8);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
-			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
-			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
-		break;
-	case 256 / 8:
-		memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
-		    encryption_key_object.key[0]), p_key, 256 / 8);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
-			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
-		MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
-			 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
-	MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
-
-	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-	if (err == 0)
-		*p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
-
-	/* avoid leaking key on the stack */
-	memset(in, 0, sizeof(in));
-
-	return err;
-}
-
-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
-{
-	u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
-	u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
-
-	MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
-	MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
-	MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
-
-	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-}
-
 int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32 *p_tisn)
 {
 	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h
index c996eca41114..2e1d6a6fcff0 100644
--- a/sys/dev/mlx5/mlx5_core/wq.h
+++ b/sys/dev/mlx5/mlx5_core/wq.h
@@ -27,6 +27,7 @@
 #define __MLX5_WQ_H__
 
 #include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/cq.h>
 
 struct mlx5_wq_param {
 	int		linear;
@@ -136,6 +137,22 @@ static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq)
 	*wq->db = cpu_to_be32(wq->cc & 0xffffff);
 }
 
+static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq)
+{
+        u32 ci = mlx5_cqwq_get_ci(wq);
+        struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
+        u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK;
+        u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1;
+
+        if (cqe_ownership_bit != sw_ownership_val)
+                return NULL;
+
+        /* ensure cqe content is read after cqe ownership bit */
+	atomic_thread_fence_acq();
+
+        return cqe;
+}
+
 static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
 {
 	return wq->cur_sz == wq->sz_m1;
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 502c1c19af6f..cdc8caa838d6 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -70,6 +70,7 @@
 #include <dev/mlx5/mlx5_core/wq.h>
 #include <dev/mlx5/mlx5_core/transobj.h>
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
 
 #define	MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
 
@@ -956,7 +957,7 @@ struct mlx5_flow_rule;
 struct mlx5e_eth_addr_info {
 	u8	addr [ETH_ALEN + 2];
 	/* flow table rule per traffic type */
-	struct mlx5_flow_rule	*ft_rule[MLX5E_NUM_TT];
+	struct mlx5_flow_handle	*ft_rule[MLX5E_NUM_TT];
 };
 
 #define	MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)
@@ -992,10 +993,10 @@ enum {
 
 struct mlx5e_vlan_db {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
-	struct mlx5_flow_rule	*active_vlans_ft_rule[VLAN_N_VID];
-	struct mlx5_flow_rule	*untagged_ft_rule;
-	struct mlx5_flow_rule	*any_cvlan_ft_rule;
-	struct mlx5_flow_rule	*any_svlan_ft_rule;
+	struct mlx5_flow_handle	*active_vlans_ft_rule[VLAN_N_VID];
+	struct mlx5_flow_handle	*untagged_ft_rule;
+	struct mlx5_flow_handle	*any_cvlan_ft_rule;
+	struct mlx5_flow_handle	*any_svlan_ft_rule;
 	bool	filter_disabled;
 };
 
@@ -1004,7 +1005,7 @@ struct mlx5e_vxlan_db_el {
 	u_int proto;
 	u_int port;
 	bool installed;
-	struct mlx5_flow_rule *vxlan_ft_rule;
+	struct mlx5_flow_handle *vxlan_ft_rule;
 	TAILQ_ENTRY(mlx5e_vxlan_db_el) link;
 };
 
@@ -1027,19 +1028,20 @@ enum accel_fs_tcp_type {
 struct mlx5e_accel_fs_tcp {
 	struct mlx5_flow_namespace *ns;
 	struct mlx5e_flow_table tables[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
-	struct mlx5_flow_rule *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
+	struct mlx5_flow_handle *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
 };
 
 struct mlx5e_flow_tables {
 	struct mlx5_flow_namespace *ns;
 	struct mlx5e_flow_table vlan;
 	struct mlx5e_flow_table vxlan;
-	struct mlx5_flow_rule *vxlan_catchall_ft_rule;
+	struct mlx5_flow_handle *vxlan_catchall_ft_rule;
 	struct mlx5e_flow_table main;
 	struct mlx5e_flow_table main_vxlan;
-	struct mlx5_flow_rule *main_vxlan_rule[MLX5E_NUM_TT];
+	struct mlx5_flow_handle *main_vxlan_rule[MLX5E_NUM_TT];
 	struct mlx5e_flow_table inner_rss;
 	struct mlx5e_accel_fs_tcp accel_tcp;
+	struct mlx5_flow_table *ipsec_ft;
 };
 
 struct mlx5e_xmit_args {
@@ -1067,6 +1069,7 @@ struct mlx5e_dcbx {
 	u32	xoff;
 };
 
+struct mlx5e_ipsec;
 struct mlx5e_priv {
 	struct mlx5_core_dev *mdev;     /* must be first */
 
@@ -1145,6 +1148,7 @@ struct mlx5e_priv {
 	bool	sw_is_port_buf_owner;
 
 	struct pfil_head *pfil;
+	struct mlx5e_ipsec *ipsec;
 	struct mlx5e_channel channel[];
 };
 
diff --git a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
index b824ca686e2c..e185aceb4b26 100644
--- a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
+++ b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
@@ -61,7 +61,7 @@ struct mlx5e_tls_rx_tag {
 	uint32_t tirn;		/* HW TIR context number */
 	uint32_t dek_index;	/* HW TLS context number */
 	struct mlx5e_tls_rx *tls_rx; /* parent pointer */
-	struct mlx5_flow_rule *flow_rule;
+	struct mlx5_flow_handle *flow_rule;
 	struct mtx mtx;
 	struct completion progress_complete;
 	uint32_t state;	/* see MLX5E_TLS_RX_ST_XXX */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
index 4939deb4c122..1601557e52cc 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
@@ -143,17 +143,17 @@ static void
 mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
     struct mlx5e_eth_addr_info *ai)
 {
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]);
-	mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4]);
+	mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_ANY]);
 }
 
 static int
@@ -248,24 +248,30 @@ mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type)
 static int
 mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
     struct mlx5e_eth_addr_info *ai, int type,
-    u32 *mc, u32 *mv)
+    struct mlx5_flow_spec *spec)
 {
 	struct mlx5_flow_destination dest = {};
 	u8 mc_enable = 0;
-	struct mlx5_flow_rule **rule_p;
+	struct mlx5_flow_handle **rule_p;
 	struct mlx5_flow_table *ft = priv->fts.main.t;
-	u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc,
+	u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 				   outer_headers.dmac_47_16);
-	u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv,
+	u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 				   outer_headers.dmac_47_16);
 	u32 *tirn = priv->tirn;
 	u32 tt_vec;
 	int err = 0;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_ETH_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
+	u8 *mc;
+	u8 *mv;
 
+	mv = (u8 *)spec->match_value;
+	mc = (u8 *)spec->match_criteria;
+
+	spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 
 	switch (type) {
@@ -289,12 +295,11 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 
 	tt_vec = mlx5e_get_tt_vec(ai, type);
 
+	spec->match_criteria_enable = mc_enable;
 	if (tt_vec & BIT(MLX5E_TT_ANY)) {
 		rule_p = &ai->ft_rule[MLX5E_TT_ANY];
 		dest.tir_num = tirn[MLX5E_TT_ANY];
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -302,14 +307,13 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 	mc_enable = MLX5_MATCH_OUTER_HEADERS;
 	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 
+	spec->match_criteria_enable = mc_enable;
 	if (tt_vec & BIT(MLX5E_TT_IPV4)) {
 		rule_p = &ai->ft_rule[MLX5E_TT_IPV4];
 		dest.tir_num = tirn[MLX5E_TT_IPV4];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IP);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -319,9 +323,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV6];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IPV6);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -334,9 +336,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IP);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -346,9 +346,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IPV6);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -360,9 +358,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IP);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -372,9 +368,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IPV6);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -386,9 +380,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IP);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -398,9 +390,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IPV6);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -412,9 +402,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IP);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -424,9 +412,7 @@ mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 		dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 		MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 			 ETHERTYPE_IPV6);
-		*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-					     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-					     &flow_act, &dest);
+		*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 		if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 	}
@@ -445,23 +431,19 @@ static int
 mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
     struct mlx5e_eth_addr_info *ai, int type)
 {
-	u32 *match_criteria;
-	u32 *match_value;
+	struct mlx5_flow_spec *spec;
 	int err = 0;
 
-	match_value	= mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	match_criteria	= mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	if (!match_value || !match_criteria) {
+	spec = mlx5_vzalloc(sizeof(*spec));
+	if (!spec) {
 		mlx5_en_err(priv->ifp, "alloc failed\n");
 		err = -ENOMEM;
 		goto add_eth_addr_rule_out;
 	}
-	err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria,
-	    match_value);
+	err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, spec);
 
 add_eth_addr_rule_out:
-	kvfree(match_criteria);
-	kvfree(match_value);
+	kvfree(spec);
 
 	return (err);
 }
@@ -469,51 +451,56 @@ add_eth_addr_rule_out:
 static void
 mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv)
 {
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
-	mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
+	mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
 }
 
 static int
-mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
+mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv,
+			       struct mlx5_flow_spec *spec)
 {
 	struct mlx5_flow_destination dest = {};
-	u8 mc_enable = 0;
-	struct mlx5_flow_rule **rule_p;
+	struct mlx5_flow_handle **rule_p;
 	struct mlx5_flow_table *ft = priv->fts.main_vxlan.t;
 	u32 *tirn = priv->tirn_inner_vxlan;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_ETH_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
 	int err = 0;
+	u8 *mc;
+	u8 *mv;
+
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+	spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+
+	mc = (u8 *)spec->match_criteria;
+	mv = (u8 *)spec->match_value;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 
-	mc_enable = MLX5_MATCH_INNER_HEADERS;
+	spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
 	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
 
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4];
 	dest.tir_num = tirn[MLX5E_TT_IPV4];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6];
 	dest.tir_num = tirn[MLX5E_TT_IPV6];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	     MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
@@ -523,16 +510,14 @@ mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP];
 	dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP];
 	dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
@@ -541,16 +526,14 @@ mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP];
 	dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP];
 	dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
@@ -559,16 +542,14 @@ mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH];
 	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH];
 	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
@@ -577,8 +558,7 @@ mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP];
 	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 			goto err_del_ai;
 
@@ -586,18 +566,16 @@ mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 	MLX5_SET(fte_match_param, mv, inner_headers.ethertype,
 		 ETHERTYPE_IPV6);
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
-	mc_enable = 0;
+	spec->match_criteria_enable = 0;
 	memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 	memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY];
 	dest.tir_num = tirn[MLX5E_TT_ANY];
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
 	if (IS_ERR_OR_NULL(*rule_p))
 		goto err_del_ai;
 
@@ -614,22 +592,19 @@ err_del_ai:
 static int
 mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv)
 {
-	u32 *match_criteria;
-	u32 *match_value;
+	struct mlx5_flow_spec *spec;
 	int err = 0;
 
-	match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	if (match_value == NULL || match_criteria == NULL) {
+	spec = mlx5_vzalloc(sizeof(*spec));
+	if (!spec) {
 		mlx5_en_err(priv->ifp, "alloc failed\n");
 		err = -ENOMEM;
 		goto add_main_vxlan_rules_out;
 	}
-	err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value);
+	err = mlx5e_add_main_vxlan_rules_sub(priv, spec);
 
 add_main_vxlan_rules_out:
-	kvfree(match_criteria);
-	kvfree(match_value);
+	kvfree(spec);
 
 	return (err);
 }
@@ -687,22 +662,27 @@ enum mlx5e_vlan_rule_type {
 static int
 mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
     enum mlx5e_vlan_rule_type rule_type, u16 vid,
-    u32 *mc, u32 *mv)
+    struct mlx5_flow_spec *spec)
 {
 	struct mlx5_flow_table *ft = priv->fts.vlan.t;
 	struct mlx5_flow_destination dest = {};
-	u8 mc_enable = 0;
-	struct mlx5_flow_rule **rule_p;
+	struct mlx5_flow_handle **rule_p;
 	int err = 0;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_ETH_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
+	u8 *mv;
+	u8 *mc;
 
+	mv = (u8 *)spec->match_value;
+	mc = (u8 *)spec->match_criteria;
+
+	spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = priv->fts.vxlan.t;
 
-	mc_enable = MLX5_MATCH_OUTER_HEADERS;
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 
 	switch (rule_type) {
 	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
@@ -729,11 +709,7 @@ mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
 		break;
 	}
 
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-				     MLX5_FLOW_RULE_FWD_ACTION_DEST,
-				     &flow_act,
-				     &dest);
-
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR(*rule_p)) {
 		err = PTR_ERR(*rule_p);
 		*rule_p = NULL;
@@ -747,24 +723,20 @@ static int
 mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
     enum mlx5e_vlan_rule_type rule_type, u16 vid)
 {
-	u32 *match_criteria;
-	u32 *match_value;
+	struct mlx5_flow_spec *spec;
 	int err = 0;
 
-	match_value	= mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	match_criteria	= mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	if (!match_value || !match_criteria) {
+	spec = mlx5_vzalloc(sizeof(*spec));
+	if (!spec) {
 		mlx5_en_err(priv->ifp, "alloc failed\n");
 		err = -ENOMEM;
 		goto add_vlan_rule_out;
 	}
 
-	err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria,
-				    match_value);
+	err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, spec);
 
 add_vlan_rule_out:
-	kvfree(match_criteria);
-	kvfree(match_value);
+	kvfree(spec);
 
 	return (err);
 }
@@ -775,16 +747,16 @@ mlx5e_del_vlan_rule(struct mlx5e_priv *priv,
 {
 	switch (rule_type) {
 	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
-		mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule);
+		mlx5_del_flow_rules(&priv->vlan.untagged_ft_rule);
 		break;
 	case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
-		mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule);
+		mlx5_del_flow_rules(&priv->vlan.any_cvlan_ft_rule);
 		break;
 	case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
-		mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule);
+		mlx5_del_flow_rules(&priv->vlan.any_svlan_ft_rule);
 		break;
 	case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
-		mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]);
+		mlx5_del_flow_rules(&priv->vlan.active_vlans_ft_rule[vid]);
 		mlx5e_vport_context_update_vlans(priv);
 		break;
 	default:
@@ -1518,11 +1490,16 @@ mlx5e_create_main_flow_table(struct mlx5e_priv *priv, bool inner_vxlan)
 {
 	struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan :
 	    &priv->fts.main;
+	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
 	ft->num_groups = 0;
-	ft->t = mlx5_create_flow_table(priv->fts.ns, 0,
-	    inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE);
+	ft_attr.max_fte = MLX5E_MAIN_TABLE_SIZE;
+	if (priv->ipsec)
+		ft_attr.level = inner_vxlan ? 10 : 12;
+	else
+		ft_attr.level = inner_vxlan ? 2 : 4;
+	ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
 
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
@@ -1643,11 +1620,13 @@ static int
 mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
 {
 	struct mlx5e_flow_table *ft = &priv->fts.vlan;
+	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
 	ft->num_groups = 0;
-	ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan",
-				       MLX5E_VLAN_TABLE_SIZE);
+	ft_attr.max_fte = MLX5E_VLAN_TABLE_SIZE;
+	ft_attr.level = (priv->ipsec) ? 8 : 0;
+	ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
 
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
@@ -1683,23 +1662,29 @@ mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv)
 }
 
 static int
-mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv,
+mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec,
     struct mlx5e_vxlan_db_el *el)
 {
 	struct mlx5_flow_table *ft = priv->fts.vxlan.t;
 	struct mlx5_flow_destination dest = {};
-	u8 mc_enable;
-	struct mlx5_flow_rule **rule_p;
+	struct mlx5_flow_handle **rule_p;
 	int err = 0;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_ETH_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
+	u8 *mc;
+	u8 *mv;
+
+	mv = (u8 *)spec->match_value;
+	mc = (u8 *)spec->match_criteria;
+
+	spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = priv->fts.main_vxlan.t;
 
-	mc_enable = MLX5_MATCH_OUTER_HEADERS;
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 	rule_p = &el->vxlan_ft_rule;
 	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 	MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto);
@@ -1708,8 +1693,7 @@ mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv,
 	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
 	MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port);
 
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 
 	if (IS_ERR(*rule_p)) {
 		err = PTR_ERR(*rule_p);
@@ -1764,23 +1748,20 @@ static int
 mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv,
     struct mlx5e_vxlan_db_el *el)
 {
-	u32 *match_criteria;
-	u32 *match_value;
+	struct mlx5_flow_spec *spec;
 	int err;
 
-	match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	if (match_value == NULL || match_criteria == NULL) {
+	spec = mlx5_vzalloc(sizeof(*spec));
+	if (!spec) {
 		mlx5_en_err(priv->ifp, "alloc failed\n");
 		err = -ENOMEM;
 		goto add_vxlan_rule_out;
 	}
 
-	err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el);
+	err = mlx5e_add_vxlan_rule_sub(priv, spec, el);
 
 add_vxlan_rule_out:
-	kvfree(match_criteria);
-	kvfree(match_value);
+	kvfree(spec);
 
 	return (err);
 }
@@ -1818,24 +1799,25 @@ mlx5e_add_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
 }
 
 static int
-mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
+mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv,
+				  struct mlx5_flow_spec *spec)
 {
 	struct mlx5_flow_table *ft = priv->fts.vxlan.t;
 	struct mlx5_flow_destination dest = {};
-	u8 mc_enable = 0;
-	struct mlx5_flow_rule **rule_p;
+	struct mlx5_flow_handle **rule_p;
 	int err = 0;
 	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_ETH_FLOW_TAG,
+		.action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
 	};
 
+	spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = priv->fts.main.t;
 
 	rule_p = &priv->fts.vxlan_catchall_ft_rule;
-	*rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
-	    MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+	*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 
 	if (IS_ERR(*rule_p)) {
 		err = PTR_ERR(*rule_p);
@@ -1850,24 +1832,20 @@ mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 static int
 mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv)
 {
-	u32 *match_criteria;
-	u32 *match_value;
+	struct mlx5_flow_spec *spec;
 	int err;
 
-	match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
-	if (match_value == NULL || match_criteria == NULL) {
+	spec = mlx5_vzalloc(sizeof(*spec));
+	if (!spec) {
 		mlx5_en_err(priv->ifp, "alloc failed\n");
 		err = -ENOMEM;
 		goto add_vxlan_rule_out;
 	}
 
-	err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria,
-	    match_value);
+	err = mlx5e_add_vxlan_catchall_rule_sub(priv, spec);
 
 add_vxlan_rule_out:
-	kvfree(match_criteria);
-	kvfree(match_value);
+	kvfree(spec);
 
 	return (err);
 }
@@ -1911,7 +1889,7 @@ mlx5e_del_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
 	}
 
 	if (el->installed)
-		mlx5_del_flow_rule(&el->vxlan_ft_rule);
+		mlx5_del_flow_rules(&el->vxlan_ft_rule);
 	TAILQ_REMOVE(&priv->vxlan.head, el, link);
 	kvfree(el);
 	return (0);
@@ -1925,7 +1903,7 @@ mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv)
 	TAILQ_FOREACH(el, &priv->vxlan.head, link) {
 		if (!el->installed)
 			continue;
-		mlx5_del_flow_rule(&el->vxlan_ft_rule);
+		mlx5_del_flow_rules(&el->vxlan_ft_rule);
 		el->installed = false;
 	}
 }
@@ -1933,7 +1911,7 @@ mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv)
 static void
 mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv)
 {
-	mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule);
+	mlx5_del_flow_rules(&priv->fts.vxlan_catchall_ft_rule);
 }
 
 void
@@ -2030,11 +2008,13 @@ static int
 mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv)
 {
 	struct mlx5e_flow_table *ft = &priv->fts.vxlan;
+	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
 	ft->num_groups = 0;
-	ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan",
-				       MLX5E_VXLAN_TABLE_SIZE);
+	ft_attr.max_fte = MLX5E_VXLAN_TABLE_SIZE;
+	ft_attr.level = (priv->ipsec) ? 9 : 1;
+	ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
 
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
@@ -2144,11 +2124,13 @@ static int
 mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv)
 {
 	struct mlx5e_flow_table *ft = &priv->fts.inner_rss;
+	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
 	ft->num_groups = 0;
-	ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss",
-				       MLX5E_INNER_RSS_TABLE_SIZE);
+	ft_attr.max_fte = MLX5E_INNER_RSS_TABLE_SIZE;
+	ft_attr.level = (priv->ipsec) ? 11 : 3;
+	ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
 
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
@@ -2198,9 +2180,13 @@ mlx5e_open_flow_tables(struct mlx5e_priv *priv)
 	priv->fts.ns = mlx5_get_flow_namespace(
 	    priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
 
+	err = mlx5e_accel_ipsec_fs_rx_tables_create(priv);
+	if (err)
+		return err;
+
 	err = mlx5e_create_vlan_flow_table(priv);
 	if (err)
-		return (err);
+		goto err_destroy_ipsec_flow_table;
 
 	err = mlx5e_create_vxlan_flow_table(priv);
 	if (err)
@@ -2222,13 +2208,19 @@ mlx5e_open_flow_tables(struct mlx5e_priv *priv)
 	if (err)
 		goto err_destroy_main_flow_table_false;
 
+	err = mlx5e_accel_ipsec_fs_rx_catchall_rules(priv);
+	if (err)
+		goto err_destroy_vxlan_catchall_rule;
+
 	err = mlx5e_accel_fs_tcp_create(priv);
 	if (err)
-		goto err_del_vxlan_catchall_rule;
+		goto err_destroy_ipsec_catchall_rules;
 
 	return (0);
 
-err_del_vxlan_catchall_rule:
+err_destroy_ipsec_catchall_rules:
+	mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
+err_destroy_vxlan_catchall_rule:
 	mlx5e_del_vxlan_catchall_rule(priv);
 err_destroy_main_flow_table_false:
 	mlx5e_destroy_main_flow_table(priv);
@@ -2240,6 +2232,8 @@ err_destroy_vxlan_flow_table:
 	mlx5e_destroy_vxlan_flow_table(priv);
 err_destroy_vlan_flow_table:
 	mlx5e_destroy_vlan_flow_table(priv);
+err_destroy_ipsec_flow_table:
+	mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);
 
 	return (err);
 }
@@ -2248,12 +2242,14 @@ void
 mlx5e_close_flow_tables(struct mlx5e_priv *priv)
 {
 	mlx5e_accel_fs_tcp_destroy(priv);
+	mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
 	mlx5e_del_vxlan_catchall_rule(priv);
 	mlx5e_destroy_main_flow_table(priv);
 	mlx5e_destroy_inner_rss_flow_table(priv);
 	mlx5e_destroy_main_vxlan_flow_table(priv);
 	mlx5e_destroy_vxlan_flow_table(priv);
 	mlx5e_destroy_vlan_flow_table(priv);
+	mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);
 }
 
 int
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
index f2ed45826843..a8522d68d5aa 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
@@ -31,6 +31,7 @@
 #include <dev/mlx5/mlx5_en/en.h>
 
 #include <dev/mlx5/tls.h>
+#include <dev/mlx5/crypto.h>
 
 #include <linux/delay.h>
 #include <sys/ktls.h>
@@ -237,6 +238,7 @@ mlx5e_tls_work(struct work_struct *work)
 
 		/* try to allocate a DEK context ID */
 		err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
+		    MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
 		    MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
 		    MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
 		    &ptag->dek_index);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
index 19c6adf18809..55768038c20f 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
@@ -29,6 +29,7 @@
 
 #include <dev/mlx5/mlx5_en/en.h>
 
+#include <dev/mlx5/crypto.h>
 #include <dev/mlx5/tls.h>
 
 #include <dev/mlx5/fs.h>
@@ -551,6 +552,7 @@ mlx5e_tls_rx_work(struct work_struct *work)
 
 		/* try to allocate a DEK context ID */
 		err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
+		    MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
 		    MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
 		    MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
 		    &ptag->dek_index);
@@ -659,7 +661,7 @@ mlx5e_tls_rx_snd_tag_alloc(if_t ifp,
 	struct mlx5e_iq *iq;
 	struct mlx5e_priv *priv;
 	struct mlx5e_tls_rx_tag *ptag;
-	struct mlx5_flow_rule *flow_rule;
+	struct mlx5_flow_handle *flow_rule;
 	const struct tls_session_params *en;
 	uint32_t value;
 	int error;
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 36a2a37a6e8b..791e278daaf8 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -24,6 +24,7 @@
  * SUCH DAMAGE.
  */
 
+#include "opt_ipsec.h"
 #include "opt_kern_tls.h"
 #include "opt_rss.h"
 #include "opt_ratelimit.h"
@@ -35,6 +36,8 @@
 #include <machine/atomic.h>
 
 #include <net/debugnet.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec_offload.h>
 
 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
 static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
@@ -3640,6 +3643,18 @@ siocsifcap_driver:
 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS4));
 		if ((mask & IFCAP2_BIT(IFCAP2_RXTLS6)) != 0)
 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS6));
+#ifdef IPSEC_OFFLOAD
+		if ((mask & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
+			bool was_enabled = (if_getcapenable2(ifp) &
+			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0;
+			mlx5e_close_locked(ifp);
+			if (was_enabled)
+				ipsec_accel_on_ifdown(priv->ifp);
+			if_togglecapenable2(ifp,
+			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD));
+			mlx5e_open_locked(ifp);
+		}
+#endif
 out:
 		PRIV_UNLOCK(priv);
 		break;
@@ -4521,6 +4536,11 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
 	if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
 	if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) |
 	    IFCAP2_BIT(IFCAP2_RXTLS6), 0);
+
+	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
+		if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD),
+		    0);
+
 	if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
 #ifdef RATELIMIT
 	if_setratelimitqueryfn(ifp, mlx5e_ratelimit_query);
@@ -4620,10 +4640,18 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
 		goto err_rl_init;
 	}
 
+	if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
+		err = mlx5e_ipsec_init(priv);
+		if (err) {
+			if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
+			goto err_tls_init;
+		}
+	}
+
 	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
 	if (err) {
 		if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
-		goto err_tls_init;
+		goto err_ipsec_init;
 	}
 
 	err = mlx5e_open_rqts(priv);
@@ -4799,6 +4827,9 @@ err_open_rqts:
 err_open_drop_rq:
 	mlx5e_close_drop_rq(&priv->drop_rq);
 
+err_ipsec_init:
+	mlx5e_ipsec_cleanup(priv);
+
 err_tls_init:
 	mlx5e_tls_cleanup(priv);
 
@@ -4905,10 +4936,14 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
 	ether_ifdetach(ifp);
 
 	mlx5e_tls_rx_cleanup(priv);
+#ifdef IPSEC_OFFLOAD
+	ipsec_accel_on_ifdown(priv->ifp);
+#endif
 	mlx5e_close_flow_tables(priv);
 	mlx5e_close_tirs(priv);
 	mlx5e_close_rqts(priv);
 	mlx5e_close_drop_rq(&priv->drop_rq);
+	mlx5e_ipsec_cleanup(priv);
 	mlx5e_tls_cleanup(priv);
 	mlx5e_rl_cleanup(priv);
 
@@ -5023,6 +5058,7 @@ mlx5e_cleanup(void)
 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
 
+MODULE_DEPEND(mlx5en, ipsec, 1, 1, 1);
 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
 MODULE_VERSION(mlx5en, 1);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index b52dee102a3d..3d4b75884354 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -28,6 +28,7 @@
 
 #include <dev/mlx5/mlx5_en/en.h>
 #include <machine/in_cksum.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
 
 static inline int
 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
@@ -69,6 +70,9 @@ mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
 	/* get IP header aligned */
 	m_adj(mb, MLX5E_NET_IP_ALIGN);
 
+	err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, mb);
+	if (err)
+		goto err_free_mbuf;
 	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
 	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
 	if (err != 0)
@@ -418,6 +422,8 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
 	default:
 		break;
 	}
+
+	mlx5e_accel_ipsec_handle_rx(mb, cqe);
 }
 
 static inline void
@@ -563,7 +569,9 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
 					("Filter returned %d!\n", rv));
 			}
 		}
-		if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
+		if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
+						    to rq->mbuf */ &&
+		    MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
 		    (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
 			/* set maximum mbuf length */
 			mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index 3f70e8a818ea..14f797dc1b1f 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -30,6 +30,7 @@
 
 #include <dev/mlx5/mlx5_en/en.h>
 #include <machine/atomic.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
 
 static inline bool
 mlx5e_do_send_cqe_inline(struct mlx5e_sq *sq)
@@ -744,6 +745,8 @@ top:
 	/* get pointer to mbuf */
 	mb = *mbp;
 
+	mlx5e_accel_ipsec_handle_tx(mb, wqe);
+
 	/* Send a copy of the frame to the BPF listener, if any */
 	if (ifp != NULL)
 		ETHER_BPF_MTAP(ifp, mb);
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
index 5dbde72d0c5b..c2c4bc5d2791 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
@@ -170,7 +170,7 @@ struct mlx5_ib_flow_handler {
 	struct list_head		list;
 	struct ib_flow			ibflow;
 	struct mlx5_ib_flow_prio	*prio;
-	struct mlx5_flow_rule	*rule;
+	struct mlx5_flow_handle	*rule;
 };
 
 struct mlx5_ib_flow_db {
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
index 13908f75669a..81beadd263f7 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
@@ -201,7 +201,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 	case MLX5_CQE_RESP_WR_IMM:
 		wc->opcode	= IB_WC_RECV_RDMA_WITH_IMM;
 		wc->wc_flags	= IB_WC_WITH_IMM;
-		wc->ex.imm_data = cqe->imm_inval_pkey;
+		wc->ex.imm_data = cqe->immediate;
 		break;
 	case MLX5_CQE_RESP_SEND:
 		wc->opcode   = IB_WC_RECV;
@@ -213,12 +213,12 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 	case MLX5_CQE_RESP_SEND_IMM:
 		wc->opcode	= IB_WC_RECV;
 		wc->wc_flags	= IB_WC_WITH_IMM;
-		wc->ex.imm_data = cqe->imm_inval_pkey;
+		wc->ex.imm_data = cqe->immediate;
 		break;
 	case MLX5_CQE_RESP_SEND_INV:
 		wc->opcode	= IB_WC_RECV;
 		wc->wc_flags	= IB_WC_WITH_INVALIDATE;
-		wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
+		wc->ex.invalidate_rkey = be32_to_cpu(cqe->inval_rkey);
 		break;
 	}
 	wc->src_qp	   = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
@@ -226,7 +226,7 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 	g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
 	wc->wc_flags |= g ? IB_WC_GRH : 0;
 	if (unlikely(is_qp1(qp->ibqp.qp_type))) {
-		u16 pkey = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
+		u16 pkey = be32_to_cpu(cqe->pkey) & 0xffff;
 
 		ib_find_cached_pkey(&dev->ib_dev, qp->port, pkey,
 				    &wc->pkey_index);
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
index 027f8ded3faf..1b688b70f3dd 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
@@ -2072,13 +2072,13 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
 	mutex_lock(&dev->flow_db.lock);
 
 	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
-		mlx5_del_flow_rule(&iter->rule);
+		mlx5_del_flow_rules(&iter->rule);
 		put_flow_table(dev, iter->prio, true);
 		list_del(&iter->list);
 		kfree(iter);
 	}
 
-	mlx5_del_flow_rule(&handler->rule);
+	mlx5_del_flow_rules(&handler->rule);
 	put_flow_table(dev, handler->prio, true);
 	mutex_unlock(&dev->flow_db.lock);
 
@@ -2107,6 +2107,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 						enum flow_table_type ft_type)
 {
 	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_flow_namespace *ns = NULL;
 	struct mlx5_ib_flow_prio *prio;
 	struct mlx5_flow_table *ft;
@@ -2155,10 +2156,11 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 
 	ft = prio->flow_table;
 	if (!ft) {
-		ft = mlx5_create_auto_grouped_flow_table(ns, priority, "bypass",
-							 num_entries,
-							 num_groups,
-							 0);
+		ft_attr.prio = priority;
+		ft_attr.max_fte = num_entries;
+		ft_attr.autogroup.max_num_groups = num_groups;
+
+		ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
 
 		if (!IS_ERR(ft)) {
 			prio->refcount = 0;
@@ -2181,10 +2183,8 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 	struct mlx5_flow_spec *spec;
 	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
 	unsigned int spec_index;
-	struct mlx5_flow_act flow_act = {
-		.actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
-		.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
-	};
+	struct mlx5_flow_act flow_act = {};
+
 	u32 action;
 	int err = 0;
 
@@ -2198,6 +2198,9 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 		goto free;
 	}
 
+	spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+	spec->flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
+
 	INIT_LIST_HEAD(&handler->list);
 
 	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
@@ -2211,12 +2214,8 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 
 	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
 	action = dst ? MLX5_FLOW_RULE_FWD_ACTION_DEST : 0;
-	handler->rule = mlx5_add_flow_rule(ft, spec->match_criteria_enable,
-					   spec->match_criteria,
-					   spec->match_value,
-					   action,
-					   &flow_act,
-					   dst);
+	flow_act.action = action;
+	handler->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dst, 1);
 
 	if (IS_ERR(handler->rule)) {
 		err = PTR_ERR(handler->rule);
@@ -2247,7 +2246,7 @@ static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *de
 		handler_dst = create_flow_rule(dev, ft_prio,
 					       flow_attr, dst);
 		if (IS_ERR(handler_dst)) {
-			mlx5_del_flow_rule(&handler->rule);
+			mlx5_del_flow_rules(&handler->rule);
 			ft_prio->refcount--;
 			kfree(handler);
 			handler = handler_dst;
@@ -2310,7 +2309,7 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
 						 &leftovers_specs[LEFTOVERS_UC].flow_attr,
 						 dst);
 		if (IS_ERR(handler_ucast)) {
-			mlx5_del_flow_rule(&handler->rule);
+			mlx5_del_flow_rules(&handler->rule);
 			ft_prio->refcount--;
 			kfree(handler);
 			handler = handler_ucast;
@@ -2353,7 +2352,7 @@ static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
 	return handler_rx;
 
 err_tx:
-	mlx5_del_flow_rule(&handler_rx->rule);
+	mlx5_del_flow_rules(&handler_rx->rule);
 	ft_rx->refcount--;
 	kfree(handler_rx);
 err:
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
index a10cb60dbfdd..20deb783f174 100644
--- a/sys/dev/mlx5/mlx5_ifc.h
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -63,7 +63,7 @@ enum {
 	MLX5_EVENT_TYPE_NIC_VPORT_CHANGE                           = 0xd,
 	MLX5_EVENT_TYPE_FPGA_ERROR                                 = 0x20,
 	MLX5_EVENT_TYPE_FPGA_QP_ERROR                              = 0x21,
-	MLX5_EVENT_TYPE_CODING_GENERAL_OBJ_EVENT                   = 0x27,
+	MLX5_EVENT_TYPE_OBJECT_CHANGE                              = 0x27,
 };
 
 enum {
@@ -323,7 +323,12 @@ enum {
 };
 
 enum {
+	MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC = 1ULL << 0x13,
+};
+
+enum {
 	MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = 0xc,
+	MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13,
 };
 
 enum {
@@ -336,7 +341,8 @@ enum {
 };
 
 enum {
-	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK = 0x1,
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1,
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2,
 };
 
 struct mlx5_ifc_flow_table_fields_supported_bits {
@@ -463,39 +469,70 @@ struct mlx5_ifc_eth_discard_cntrs_grp_bits {
 
 	u8         reserved_at_340[0x440];
 };
+
 struct mlx5_ifc_flow_table_prop_layout_bits {
 	u8         ft_support[0x1];
-	u8         flow_tag[0x1];
+	u8         reserved_at_1[0x1];
 	u8         flow_counter[0x1];
 	u8         flow_modify_en[0x1];
 	u8         modify_root[0x1];
-	u8         identified_miss_table[0x1];
+	u8         identified_miss_table_mode[0x1];
 	u8         flow_table_modify[0x1];
-	u8         encap[0x1];
+	u8         reformat[0x1];
 	u8         decap[0x1];
-	u8         reset_root_to_default[0x1];
-	u8         reserved_at_a[0x16];
-
-	u8         reserved_at_20[0x2];
+	u8         reserved_at_9[0x1];
+	u8         pop_vlan[0x1];
+	u8         push_vlan[0x1];
+	u8         reserved_at_c[0x1];
+	u8         pop_vlan_2[0x1];
+	u8         push_vlan_2[0x1];
+	u8         reformat_and_vlan_action[0x1];
+	u8         reserved_at_10[0x1];
+	u8         sw_owner[0x1];
+	u8         reformat_l3_tunnel_to_l2[0x1];
+	u8         reformat_l2_to_l3_tunnel[0x1];
+	u8         reformat_and_modify_action[0x1];
+	u8         ignore_flow_level[0x1];
+	u8         reserved_at_16[0x1];
+	u8         table_miss_action_domain[0x1];
+	u8         termination_table[0x1];
+	u8         reformat_and_fwd_to_table[0x1];
+	u8         reserved_at_1a[0x2];
+	u8         ipsec_encrypt[0x1];
+	u8         ipsec_decrypt[0x1];
+	u8         sw_owner_v2[0x1];
+	u8         reserved_at_1f[0x1];
+	u8         termination_table_raw_traffic[0x1];
+	u8         reserved_at_21[0x1];
 	u8         log_max_ft_size[0x6];
 	u8         log_max_modify_header_context[0x8];
         u8         max_modify_header_actions[0x8];
 	u8         max_ft_level[0x8];
 
-	u8         reserved_at_40[0x20];
-
-	u8         reserved_at_60[0x18];
+	u8         reformat_add_esp_trasport[0x1];
+	u8         reformat_l2_to_l3_esp_tunnel[0x1];
+	u8         reformat_add_esp_transport_over_udp[0x1];
+	u8         reformat_del_esp_trasport[0x1];
+	u8         reformat_l3_esp_tunnel_to_l2[0x1];
+	u8         reformat_del_esp_transport_over_udp[0x1];
+	u8         execute_aso[0x1];
+	u8         reserved_at_47[0x19];
+	u8         reserved_at_60[0x2];
+	u8         reformat_insert[0x1];
+	u8         reformat_remove[0x1];
+	u8         macsec_encrypt[0x1];
+	u8         macsec_decrypt[0x1];
+	u8         reserved_at_66[0x2];
+	u8         reformat_add_macsec[0x1];
+	u8         reformat_remove_macsec[0x1];
+	u8         reserved_at_6a[0xe];
 	u8         log_max_ft_num[0x8];
-
 	u8         reserved_at_80[0x10];
 	u8         log_max_flow_counter[0x8];
 	u8         log_max_destination[0x8];
-
 	u8         reserved_at_a0[0x18];
 	u8         log_max_flow[0x8];
-
 	u8         reserved_at_c0[0x40];
-
 	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_support;
 
 	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_bitmask_support;
@@ -523,15 +560,18 @@ enum {
 	MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE               = 0x1,
 	MLX5_FLOW_CONTEXT_DEST_TYPE_TIR                      = 0x2,
 	MLX5_FLOW_CONTEXT_DEST_TYPE_QP                       = 0x3,
+	MLX5_FLOW_CONTEXT_DEST_TYPE_TABLE_TYPE               = 0xA,
 };
 
 struct mlx5_ifc_dest_format_struct_bits {
-	u8         destination_type[0x8];
-	u8         destination_id[0x18];
+        u8         destination_type[0x8];
+        u8         destination_id[0x18];
 
-	u8         reserved_0[0x8];
-	u8         destination_table_type[0x8];
-	u8         reserved_at_1[0x10];
+        u8         destination_eswitch_owner_vhca_id_valid[0x1];
+        u8         packet_reformat[0x1];
+        u8         reserved_at_22[0x6];
+        u8         destination_table_type[0x8];
+        u8         destination_eswitch_owner_vhca_id[0x10];
 };
 
 struct mlx5_ifc_ipv4_layout_bits {
@@ -585,11 +625,25 @@ struct mlx5_ifc_fte_match_set_lyr_2_4_bits {
 	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits dst_ipv4_dst_ipv6;
 };
 
+struct mlx5_ifc_nvgre_key_bits {
+	u8 hi[0x18];
+	u8 lo[0x8];
+};
+
+union mlx5_ifc_gre_key_bits {
+	struct mlx5_ifc_nvgre_key_bits nvgre;
+	u8 key[0x20];
+};
+
 struct mlx5_ifc_fte_match_set_misc_bits {
-	u8         reserved_0[0x8];
+	u8         gre_c_present[0x1];
+	u8         reserved_at_1[0x1];
+	u8         gre_k_present[0x1];
+	u8         gre_s_present[0x1];
+	u8         source_vhca_port[0x4];
 	u8         source_sqn[0x18];
 
-	u8         reserved_1[0x10];
+	u8         source_eswitch_owner_vhca_id[0x10];
 	u8         source_port[0x10];
 
 	u8         outer_second_prio[0x3];
@@ -599,35 +653,163 @@ struct mlx5_ifc_fte_match_set_misc_bits {
 	u8         inner_second_cfi[0x1];
 	u8         inner_second_vid[0xc];
 
-	u8         outer_second_vlan_tag[0x1];
-	u8         inner_second_vlan_tag[0x1];
-	u8         reserved_2[0xe];
+	u8         outer_second_cvlan_tag[0x1];
+	u8         inner_second_cvlan_tag[0x1];
+	u8         outer_second_svlan_tag[0x1];
+	u8         inner_second_svlan_tag[0x1];
+	u8         reserved_at_64[0xc];
 	u8         gre_protocol[0x10];
 
-	u8         gre_key_h[0x18];
-	u8         gre_key_l[0x8];
+	union mlx5_ifc_gre_key_bits gre_key;
 
 	u8         vxlan_vni[0x18];
-	u8         reserved_3[0x8];
+	u8         bth_opcode[0x8];
 
 	u8         geneve_vni[0x18];
-	u8         reserved4[0x7];
+	u8         reserved_at_d8[0x6];
+	u8         geneve_tlv_option_0_exist[0x1];
 	u8         geneve_oam[0x1];
 
-	u8         reserved_5[0xc];
+	u8         reserved_at_e0[0xc];
 	u8         outer_ipv6_flow_label[0x14];
 
-	u8         reserved_6[0xc];
+	u8         reserved_at_100[0xc];
 	u8         inner_ipv6_flow_label[0x14];
 
-	u8         reserved_7[0xa];
+	u8         reserved_at_120[0xa];
 	u8         geneve_opt_len[0x6];
 	u8         geneve_protocol_type[0x10];
 
-	u8         reserved_8[0x8];
+	u8         reserved_at_140[0x8];
 	u8         bth_dst_qp[0x18];
+	u8         inner_esp_spi[0x20];
+	u8         outer_esp_spi[0x20];
+	u8         reserved_at_1a0[0x60];
+};
 
-	u8         reserved_9[0xa0];
+struct mlx5_ifc_fte_match_mpls_bits {
+	u8         mpls_label[0x14];
+	u8         mpls_exp[0x3];
+	u8         mpls_s_bos[0x1];
+	u8         mpls_ttl[0x8];
+};
+
+struct mlx5_ifc_fte_match_set_misc2_bits {
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls;
+
+	struct mlx5_ifc_fte_match_mpls_bits inner_first_mpls;
+
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_gre;
+
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_udp;
+
+	u8         metadata_reg_c_7[0x20];
+
+	u8         metadata_reg_c_6[0x20];
+
+	u8         metadata_reg_c_5[0x20];
+
+	u8         metadata_reg_c_4[0x20];
+
+	u8         metadata_reg_c_3[0x20];
+
+	u8         metadata_reg_c_2[0x20];
+
+	u8         metadata_reg_c_1[0x20];
+
+	u8         metadata_reg_c_0[0x20];
+
+	u8         metadata_reg_a[0x20];
+
+	u8         reserved_at_1a0[0x8];
+
+	u8         macsec_syndrome[0x8];
+	u8         ipsec_syndrome[0x8];
+	u8         reserved_at_1b8[0x8];
+
+	u8         reserved_at_1c0[0x40];
+};
+
+struct mlx5_ifc_fte_match_set_misc3_bits {
+	u8         inner_tcp_seq_num[0x20];
+
+	u8         outer_tcp_seq_num[0x20];
+
+	u8         inner_tcp_ack_num[0x20];
+
+	u8         outer_tcp_ack_num[0x20];
+
+	u8         reserved_at_80[0x8];
+	u8         outer_vxlan_gpe_vni[0x18];
+
+	u8         outer_vxlan_gpe_next_protocol[0x8];
+	u8         outer_vxlan_gpe_flags[0x8];
+	u8         reserved_at_b0[0x10];
+
+	u8         icmp_header_data[0x20];
+
+	u8         icmpv6_header_data[0x20];
+
+	u8         icmp_type[0x8];
+	u8         icmp_code[0x8];
+	u8         icmpv6_type[0x8];
+	u8         icmpv6_code[0x8];
+
+	u8         geneve_tlv_option_0_data[0x20];
+
+	u8         gtpu_teid[0x20];
+
+	u8         gtpu_msg_type[0x8];
+	u8         gtpu_msg_flags[0x8];
+	u8         reserved_at_170[0x10];
+
+	u8         gtpu_dw_2[0x20];
+
+	u8         gtpu_first_ext_dw_0[0x20];
+
+	u8         gtpu_dw_0[0x20];
+
+	u8         reserved_at_1e0[0x20];
+};
+
+struct mlx5_ifc_fte_match_set_misc4_bits {
+        u8         prog_sample_field_value_0[0x20];
+
+        u8         prog_sample_field_id_0[0x20];
+
+        u8         prog_sample_field_value_1[0x20];
+
+        u8         prog_sample_field_id_1[0x20];
+
+        u8         prog_sample_field_value_2[0x20];
+
+        u8         prog_sample_field_id_2[0x20];
+
+        u8         prog_sample_field_value_3[0x20];
+
+        u8         prog_sample_field_id_3[0x20];
+
+        u8         reserved_at_100[0x100];
+};
+
+struct mlx5_ifc_fte_match_set_misc5_bits {
+        u8         macsec_tag_0[0x20];
+
+        u8         macsec_tag_1[0x20];
+
+        u8         macsec_tag_2[0x20];
+
+        u8         macsec_tag_3[0x20];
+
+        u8         tunnel_header_0[0x20];
+
+        u8         tunnel_header_1[0x20];
+
+        u8         tunnel_header_2[0x20];
+
+        u8         tunnel_header_3[0x20];
+
+        u8         reserved_at_100[0x100];
 };
 
 struct mlx5_ifc_cmd_pas_bits {
@@ -863,6 +1045,20 @@ struct mlx5_ifc_flow_table_nic_cap_bits {
 	u8         reserved_1[0x7200];
 };
 
+struct mlx5_ifc_port_selection_cap_bits {
+        u8         reserved_at_0[0x10];
+        u8         port_select_flow_table[0x1];
+        u8         reserved_at_11[0x1];
+        u8         port_select_flow_table_bypass[0x1];
+        u8         reserved_at_13[0xd];
+
+        u8         reserved_at_20[0x1e0];
+
+        struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_port_selection;
+
+        u8         reserved_at_400[0x7c00];
+};
+
 struct mlx5_ifc_pddr_module_info_bits {
 	u8         cable_technology[0x8];
 	u8         cable_breakout[0x8];
@@ -1154,7 +1350,15 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         reserved_0[0x20];
 
 	u8         hca_cap_2[0x1];
-	u8         reserved_at_21[0x1f];
+	u8         create_lag_when_not_master_up[0x1];
+        u8         dtor[0x1];
+        u8         event_on_vhca_state_teardown_request[0x1];
+        u8         event_on_vhca_state_in_use[0x1];
+        u8         event_on_vhca_state_active[0x1];
+        u8         event_on_vhca_state_allocated[0x1];
+        u8         event_on_vhca_state_invalid[0x1];
+        u8         reserved_at_28[0x8];
+        u8         vhca_id[0x10];
 
 	u8         reserved_at_40[0x40];
 
@@ -1404,7 +1608,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 
 	u8         reserved_at_460[0x3];
 	u8         log_max_uctx[0x5];
-	u8         reserved_at_468[0x3];
+	u8         reserved_at_468[0x2];
+	u8         ipsec_offload[0x1];
 	u8         log_max_umem[0x5];
 	u8         max_num_eqs[0x10];
 
@@ -1488,11 +1693,27 @@ struct mlx5_ifc_cmd_hca_cap_2_bits {
 	u8	   reserved_at_260[0x5a0];
 };
 
-enum mlx5_flow_destination_type {
-	MLX5_FLOW_DESTINATION_TYPE_VPORT	= 0x0,
-	MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE	= 0x1,
-	MLX5_FLOW_DESTINATION_TYPE_TIR		= 0x2,
-	MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE   = 0xA,
+enum mlx5_ifc_flow_destination_type {
+        MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT        = 0x0,
+        MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE   = 0x1,
+        MLX5_IFC_FLOW_DESTINATION_TYPE_TIR          = 0x2,
+        MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6,
+        MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK       = 0x8,
+        MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE   = 0xA,
+};
+
+enum mlx5_flow_table_miss_action {
+        MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+        MLX5_FLOW_TABLE_MISS_ACTION_FWD,
+        MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
+};
+
+struct mlx5_ifc_extended_dest_format_bits {
+        struct mlx5_ifc_dest_format_struct_bits destination_entry;
+
+        u8         packet_reformat_id[0x20];
+
+        u8         reserved_at_60[0x20];
 };
 
 union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits {
@@ -1502,13 +1723,21 @@ union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits {
 };
 
 struct mlx5_ifc_fte_match_param_bits {
-	struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+        struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+
+        struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+
+        struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
 
-	struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+        struct mlx5_ifc_fte_match_set_misc2_bits misc_parameters_2;
 
-	struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
+        struct mlx5_ifc_fte_match_set_misc3_bits misc_parameters_3;
 
-	u8         reserved_0[0xa00];
+        struct mlx5_ifc_fte_match_set_misc4_bits misc_parameters_4;
+
+        struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5;
+
+        u8         reserved_at_e00[0x200];
 };
 
 enum {
@@ -2310,43 +2539,85 @@ struct mlx5_ifc_rdbc_bits {
 	u8         atomic_resp[32][0x8];
 };
 
+struct mlx5_ifc_vlan_bits {
+	u8         ethtype[0x10];
+	u8         prio[0x3];
+	u8         cfi[0x1];
+	u8         vid[0xc];
+};
+
 enum {
-	MLX5_FLOW_CONTEXT_ACTION_ALLOW     = 0x1,
-	MLX5_FLOW_CONTEXT_ACTION_DROP      = 0x2,
-	MLX5_FLOW_CONTEXT_ACTION_FWD_DEST  = 0x4,
-	MLX5_FLOW_CONTEXT_ACTION_COUNT     = 0x8,
-	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
-	MLX5_FLOW_CONTEXT_ACTION_MOD_HDR   = 0x40,
+	MLX5_FLOW_METER_COLOR_RED       = 0x0,
+	MLX5_FLOW_METER_COLOR_YELLOW    = 0x1,
+	MLX5_FLOW_METER_COLOR_GREEN     = 0x2,
+	MLX5_FLOW_METER_COLOR_UNDEFINED = 0x3,
+};
+
+enum {
+	MLX5_EXE_ASO_FLOW_METER         = 0x2,
+};
+
+struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits {
+	u8        return_reg_id[0x4];
+	u8        aso_type[0x4];
+	u8        reserved_at_8[0x14];
+	u8        action[0x1];
+	u8        init_color[0x2];
+	u8        meter_id[0x1];
+};
+
+union mlx5_ifc_exe_aso_ctrl {
+	struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits exe_aso_ctrl_flow_meter;
+};
+
+struct mlx5_ifc_execute_aso_bits {
+	u8        valid[0x1];
+	u8        reserved_at_1[0x7];
+	u8        aso_object_id[0x18];
+
+	union mlx5_ifc_exe_aso_ctrl exe_aso_ctrl;
+};
+
+enum {
+	MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC   = 0x0,
 };
 
 struct mlx5_ifc_flow_context_bits {
-	u8         reserved_0[0x20];
+	struct mlx5_ifc_vlan_bits push_vlan;
 
 	u8         group_id[0x20];
 
-	u8         reserved_1[0x8];
+	u8         reserved_at_40[0x8];
 	u8         flow_tag[0x18];
 
-	u8         reserved_2[0x10];
+	u8         reserved_at_60[0x10];
 	u8         action[0x10];
 
-	u8         reserved_3[0x8];
+	u8         extended_destination[0x1];
+	u8         reserved_at_81[0x1];
+	u8         flow_source[0x2];
+	u8         encrypt_decrypt_type[0x4];
 	u8         destination_list_size[0x18];
 
-	u8         reserved_4[0x8];
+	u8         reserved_at_a0[0x8];
 	u8         flow_counter_list_size[0x18];
 
 	u8         packet_reformat_id[0x20];
 
-	u8	   modify_header_id[0x20];
+	u8         modify_header_id[0x20];
+
+	struct mlx5_ifc_vlan_bits push_vlan_2;
 
-	u8	   reserved_6[0x100];
+	u8         encrypt_decrypt_obj_id[0x20];
+	u8         reserved_at_140[0xc0];
 
 	struct mlx5_ifc_fte_match_param_bits match_value;
 
-	u8         reserved_7[0x600];
+	struct mlx5_ifc_execute_aso_bits execute_aso[4];
 
-	union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[0];
+	u8         reserved_at_1300[0x500];
+
+	union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[];
 };
 
 enum {
@@ -3008,21 +3279,27 @@ enum {
 };
 
 struct mlx5_ifc_flow_table_context_bits {
-	u8         reformat_en[0x1];
-	u8         decap_en[0x1];
-	u8         reserved_at_2[0x2];
-	u8         table_miss_action[0x4];
-	u8         level[0x8];
-	u8         reserved_at_10[0x8];
-	u8         log_size[0x8];
+        u8         reformat_en[0x1];
+        u8         decap_en[0x1];
+        u8         sw_owner[0x1];
+        u8         termination_table[0x1];
+        u8         table_miss_action[0x4];
+        u8         level[0x8];
+        u8         reserved_at_10[0x8];
+        u8         log_size[0x8];
 
-	u8         reserved_at_20[0x8];
-	u8         table_miss_id[0x18];
+        u8         reserved_at_20[0x8];
+        u8         table_miss_id[0x18];
 
-	u8         reserved_at_40[0x8];
-	u8         lag_master_next_table_id[0x18];
+        u8         reserved_at_40[0x8];
+        u8         lag_master_next_table_id[0x18];
+
+        u8         reserved_at_60[0x60];
+
+        u8         sw_owner_icm_root_1[0x40];
+
+        u8         sw_owner_icm_root_0[0x40];
 
-	u8         reserved_at_60[0xe0];
 };
 
 struct mlx5_ifc_esw_vport_context_bits {
@@ -3980,28 +4257,32 @@ struct mlx5_ifc_set_flow_table_root_out_bits {
 };
 
 struct mlx5_ifc_set_flow_table_root_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_0[0x10];
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
 
-	u8         reserved_1[0x10];
-	u8         op_mod[0x10];
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
 
-	u8         other_vport[0x1];
-	u8         reserved_2[0xf];
-	u8         vport_number[0x10];
+        u8         other_vport[0x1];
+        u8         reserved_at_41[0xf];
+        u8         vport_number[0x10];
 
-	u8         reserved_3[0x20];
+        u8         reserved_at_60[0x20];
 
-	u8         table_type[0x8];
-	u8         reserved_4[0x18];
+        u8         table_type[0x8];
+        u8         reserved_at_88[0x7];
+        u8         table_of_other_vport[0x1];
+        u8         table_vport_number[0x10];
 
-	u8         reserved_5[0x8];
-	u8         table_id[0x18];
+        u8         reserved_at_a0[0x8];
+        u8         table_id[0x18];
 
-	u8         reserved_6[0x8];
-	u8         underlay_qpn[0x18];
-
-	u8         reserved_7[0x120];
+        u8         reserved_at_c0[0x8];
+        u8         underlay_qpn[0x18];
+        u8         table_eswitch_owner_vhca_id_valid[0x1];
+        u8         reserved_at_e1[0xf];
+        u8         table_eswitch_owner_vhca_id[0x10];
+        u8         reserved_at_100[0x100];
 };
 
 struct mlx5_ifc_set_fte_out_bits {
@@ -4014,34 +4295,35 @@ struct mlx5_ifc_set_fte_out_bits {
 };
 
 struct mlx5_ifc_set_fte_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_0[0x10];
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
 
-	u8         reserved_1[0x10];
-	u8         op_mod[0x10];
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
 
-	u8         other_vport[0x1];
-	u8         reserved_2[0xf];
-	u8         vport_number[0x10];
+        u8         other_vport[0x1];
+        u8         reserved_at_41[0xf];
+        u8         vport_number[0x10];
 
-	u8         reserved_3[0x20];
+        u8         reserved_at_60[0x20];
 
-	u8         table_type[0x8];
-	u8         reserved_4[0x18];
+        u8         table_type[0x8];
+        u8         reserved_at_88[0x18];
 
-	u8         reserved_5[0x8];
-	u8         table_id[0x18];
+        u8         reserved_at_a0[0x8];
+        u8         table_id[0x18];
 
-	u8         reserved_6[0x18];
-	u8         modify_enable_mask[0x8];
+        u8         ignore_flow_level[0x1];
+        u8         reserved_at_c1[0x17];
+        u8         modify_enable_mask[0x8];
 
-	u8         reserved_7[0x20];
+        u8         reserved_at_e0[0x20];
 
-	u8         flow_index[0x20];
+        u8         flow_index[0x20];
 
-	u8         reserved_8[0xe0];
+        u8         reserved_at_120[0xe0];
 
-	struct mlx5_ifc_flow_context_bits flow_context;
+        struct mlx5_ifc_flow_context_bits flow_context;
 };
 
 struct mlx5_ifc_set_driver_version_out_bits {
@@ -4322,6 +4604,23 @@ enum {
 	MLX5_QUERY_VPORT_STATE_IN_OP_MOD_UPLINK      = 0x2,
 };
 
+enum {
+        MLX5_FLOW_CONTEXT_ACTION_ALLOW     = 0x1,
+        MLX5_FLOW_CONTEXT_ACTION_DROP      = 0x2,
+        MLX5_FLOW_CONTEXT_ACTION_FWD_DEST  = 0x4,
+        MLX5_FLOW_CONTEXT_ACTION_COUNT     = 0x8,
+        MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
+        MLX5_FLOW_CONTEXT_ACTION_DECAP     = 0x20,
+        MLX5_FLOW_CONTEXT_ACTION_MOD_HDR   = 0x40,
+        MLX5_FLOW_CONTEXT_ACTION_VLAN_POP  = 0x80,
+        MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH = 0x100,
+        MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2  = 0x400,
+        MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800,
+        MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT = 0x1000,
+        MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT = 0x2000,
+        MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000,
+};
+
 struct mlx5_ifc_query_vport_state_in_bits {
 	u8         opcode[0x10];
 	u8         reserved_0[0x10];
@@ -5542,6 +5841,12 @@ enum mlx5_reformat_ctx_type {
 	MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2,
 	MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3,
 	MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4,
+	MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5,
+	MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4 = 0x7,
+	MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8,
+	MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP = 0xa,
+	MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb,
+	MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6 = 0xc,
 };
 
 struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
@@ -6252,6 +6557,11 @@ struct mlx5_ifc_modify_hca_vport_context_in_bits {
 	struct mlx5_ifc_hca_vport_context_bits hca_vport_context;
 };
 
+enum {
+        MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID     = (1UL << 0),
+        MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID = (1UL << 15),
+};
+
 struct mlx5_ifc_modify_flow_table_out_bits {
 	u8         status[0x8];
 	u8         reserved_at_8[0x18];
@@ -7895,24 +8205,24 @@ struct mlx5_ifc_create_flow_table_out_bits {
 };
 
 struct mlx5_ifc_create_flow_table_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
+        u8         opcode[0x10];
+        u8         uid[0x10];
 
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
 
-	u8         other_vport[0x1];
-	u8         reserved_at_41[0xf];
-	u8         vport_number[0x10];
+        u8         other_vport[0x1];
+        u8         reserved_at_41[0xf];
+        u8         vport_number[0x10];
 
-	u8         reserved_at_60[0x20];
+        u8         reserved_at_60[0x20];
 
-	u8         table_type[0x8];
-	u8         reserved_at_88[0x18];
+        u8         table_type[0x8];
+        u8         reserved_at_88[0x18];
 
-	u8         reserved_at_a0[0x20];
+        u8         reserved_at_a0[0x20];
 
-	struct mlx5_ifc_flow_table_context_bits flow_table_context;
+        struct mlx5_ifc_flow_table_context_bits flow_table_context;
 };
 
 struct mlx5_ifc_create_flow_group_out_bits {
@@ -7928,46 +8238,54 @@ struct mlx5_ifc_create_flow_group_out_bits {
 };
 
 enum {
-	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS    = 0x0,
-	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS  = 0x1,
-	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS    = 0x2,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS     = 0x0,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS   = 0x1,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS     = 0x2,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2 = 0x3,
 };
 
 struct mlx5_ifc_create_flow_group_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_0[0x10];
+        u8         opcode[0x10];
+        u8         reserved_at_10[0x10];
 
-	u8         reserved_1[0x10];
-	u8         op_mod[0x10];
+        u8         reserved_at_20[0x10];
+        u8         op_mod[0x10];
 
-	u8         other_vport[0x1];
-	u8         reserved_2[0xf];
-	u8         vport_number[0x10];
+        u8         other_vport[0x1];
+        u8         reserved_at_41[0xf];
+        u8         vport_number[0x10];
 
-	u8         reserved_3[0x20];
+        u8         reserved_at_60[0x20];
 
-	u8         table_type[0x8];
-	u8         reserved_4[0x18];
+        u8         table_type[0x8];
+        u8         reserved_at_88[0x4];
+        u8         group_type[0x4];
+        u8         reserved_at_90[0x10];
 
-	u8         reserved_5[0x8];
-	u8         table_id[0x18];
+        u8         reserved_at_a0[0x8];
+        u8         table_id[0x18];
 
-	u8         reserved_6[0x20];
+        u8         source_eswitch_owner_vhca_id_valid[0x1];
 
-	u8         start_flow_index[0x20];
+        u8         reserved_at_c1[0x1f];
 
-	u8         reserved_7[0x20];
+        u8         start_flow_index[0x20];
 
-	u8         end_flow_index[0x20];
+        u8         reserved_at_100[0x20];
 
-	u8         reserved_8[0xa0];
+        u8         end_flow_index[0x20];
 
-	u8         reserved_9[0x18];
-	u8         match_criteria_enable[0x8];
+        u8         reserved_at_140[0x10];
+        u8         match_definer_id[0x10];
 
-	struct mlx5_ifc_fte_match_param_bits match_criteria;
+        u8         reserved_at_160[0x80];
+
+        u8         reserved_at_1e0[0x18];
+        u8         match_criteria_enable[0x8];
+
+        struct mlx5_ifc_fte_match_param_bits match_criteria;
 
-	u8         reserved_10[0xe00];
+        u8         reserved_at_1200[0xe00];
 };
 
 struct mlx5_ifc_create_encryption_key_out_bits {
@@ -11618,5 +11936,120 @@ enum mlx5_fc_bulk_alloc_bitmask {
 
 #define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum))
 
+struct mlx5_ifc_ipsec_cap_bits {
+	u8         ipsec_full_offload[0x1];
+	u8         ipsec_crypto_offload[0x1];
+	u8         ipsec_esn[0x1];
+	u8         ipsec_crypto_esp_aes_gcm_256_encrypt[0x1];
+	u8         ipsec_crypto_esp_aes_gcm_128_encrypt[0x1];
+	u8         ipsec_crypto_esp_aes_gcm_256_decrypt[0x1];
+	u8         ipsec_crypto_esp_aes_gcm_128_decrypt[0x1];
+	u8         reserved_at_7[0x4];
+	u8         log_max_ipsec_offload[0x5];
+	u8         reserved_at_10[0x10];
+
+	u8         min_log_ipsec_full_replay_window[0x8];
+	u8         max_log_ipsec_full_replay_window[0x8];
+	u8         reserved_at_30[0x7d0];
+};
+
+enum {
+	MLX5_IPSEC_OBJECT_ICV_LEN_16B,
+};
+
+enum {
+	MLX5_IPSEC_ASO_REG_C_0_1 = 0x0,
+	MLX5_IPSEC_ASO_REG_C_2_3 = 0x1,
+	MLX5_IPSEC_ASO_REG_C_4_5 = 0x2,
+	MLX5_IPSEC_ASO_REG_C_6_7 = 0x3,
+};
+
+enum {
+	MLX5_IPSEC_ASO_MODE              = 0x0,
+	MLX5_IPSEC_ASO_REPLAY_PROTECTION = 0x1,
+	MLX5_IPSEC_ASO_INC_SN            = 0x2,
+};
+
+enum {
+	MLX5_IPSEC_ASO_REPLAY_WIN_32BIT  = 0x0,
+	MLX5_IPSEC_ASO_REPLAY_WIN_64BIT  = 0x1,
+	MLX5_IPSEC_ASO_REPLAY_WIN_128BIT = 0x2,
+	MLX5_IPSEC_ASO_REPLAY_WIN_256BIT = 0x3,
+};
+
+struct mlx5_ifc_ipsec_aso_bits {
+	u8         valid[0x1];
+	u8         reserved_at_201[0x1];
+	u8         mode[0x2];
+	u8         window_sz[0x2];
+	u8         soft_lft_arm[0x1];
+	u8         hard_lft_arm[0x1];
+	u8         remove_flow_enable[0x1];
+	u8         esn_event_arm[0x1];
+	u8         reserved_at_20a[0x16];
+
+	u8         remove_flow_pkt_cnt[0x20];
+
+	u8         remove_flow_soft_lft[0x20];
+
+	u8         reserved_at_260[0x80];
+
+	u8         mode_parameter[0x20];
+
+	u8         replay_protection_window[0x100];
+};
+
+struct mlx5_ifc_ipsec_obj_bits {
+	u8         modify_field_select[0x40];
+	u8         full_offload[0x1];
+	u8         reserved_at_41[0x1];
+	u8         esn_en[0x1];
+	u8         esn_overlap[0x1];
+	u8         reserved_at_44[0x2];
+	u8         icv_length[0x2];
+	u8         reserved_at_48[0x4];
+	u8         aso_return_reg[0x4];
+	u8         reserved_at_50[0x10];
+
+	u8         esn_msb[0x20];
+
+	u8         reserved_at_80[0x8];
+	u8         dekn[0x18];
+
+	u8         salt[0x20];
+
+	u8         implicit_iv[0x40];
 
+	u8         reserved_at_100[0x8];
+	u8         ipsec_aso_access_pd[0x18];
+	u8         reserved_at_120[0xe0];
+
+	struct mlx5_ifc_ipsec_aso_bits ipsec_aso;
+};
+
+struct mlx5_ifc_create_ipsec_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+enum {
+	MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP = 1 << 0,
+	MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB = 1 << 1,
+};
+
+struct mlx5_ifc_query_ipsec_obj_out_bits {
+	struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr;
+	struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+struct mlx5_ifc_modify_ipsec_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+enum {
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS = 0x1,
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC = 0x2,
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC = 0x4,
+};
 #endif /* MLX5_IFC_H */
diff --git a/sys/dev/mlx5/mlx5_lib/aso.h b/sys/dev/mlx5/mlx5_lib/aso.h
new file mode 100644
index 000000000000..e880a5e51db1
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_lib/aso.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_LIB_ASO_H__
+#define __MLX5_LIB_ASO_H__
+
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+
+#define MLX5_ASO_WQEBBS \
+	(DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB))
+#define MLX5_ASO_WQEBBS_DATA \
+	(DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB))
+#define ASO_CTRL_READ_EN BIT(0)
+#define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24
+#define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS))
+
+#define ASO_CTRL_READ_EN BIT(0)
+struct mlx5_wqe_aso_ctrl_seg {
+	__be32  va_h;
+	__be32  va_l; /* include read_enable */
+	__be32  l_key;
+	u8      data_mask_mode;
+	u8      condition_1_0_operand;
+	u8      condition_1_0_offset;
+	u8      data_offset_condition_operand;
+	__be32  condition_0_data;
+	__be32  condition_0_mask;
+	__be32  condition_1_data;
+	__be32  condition_1_mask;
+	__be64  bitwise_data;
+	__be64  data_mask;
+};
+
+struct mlx5_wqe_aso_data_seg {
+	__be32  bytewise_data[16];
+};
+
+struct mlx5_aso_wqe {
+	struct mlx5_wqe_ctrl_seg      ctrl;
+	struct mlx5_wqe_aso_ctrl_seg  aso_ctrl;
+};
+
+struct mlx5_aso_wqe_data {
+	struct mlx5_wqe_ctrl_seg      ctrl;
+	struct mlx5_wqe_aso_ctrl_seg  aso_ctrl;
+	struct mlx5_wqe_aso_data_seg  aso_data;
+};
+
+enum {
+	MLX5_ASO_LOGICAL_AND,
+	MLX5_ASO_LOGICAL_OR,
+};
+
+enum {
+	MLX5_ASO_ALWAYS_FALSE,
+	MLX5_ASO_ALWAYS_TRUE,
+	MLX5_ASO_EQUAL,
+	MLX5_ASO_NOT_EQUAL,
+	MLX5_ASO_GREATER_OR_EQUAL,
+	MLX5_ASO_LESSER_OR_EQUAL,
+	MLX5_ASO_LESSER,
+	MLX5_ASO_GREATER,
+	MLX5_ASO_CYCLIC_GREATER,
+	MLX5_ASO_CYCLIC_LESSER,
+};
+
+enum {
+	MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT,
+	MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE,
+	MLX5_ASO_DATA_MASK_MODE_CALCULATED_64BYTE,
+};
+
+enum {
+	MLX5_ACCESS_ASO_OPC_MOD_IPSEC = 0x0,
+	MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2,
+	MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5,
+};
+
+struct mlx5_aso;
+
+struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso);
+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
+			struct mlx5_aso_wqe *aso_wqe,
+			u32 obj_id, u32 opc_mode);
+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
+		       struct mlx5_wqe_ctrl_seg *doorbell_cseg);
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data);
+
+struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn);
+void mlx5_aso_destroy(struct mlx5_aso *aso);
+#endif /* __MLX5_LIB_ASO_H__ */
diff --git a/sys/dev/mlx5/mlx5_lib/mlx5_aso.c b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
new file mode 100644
index 000000000000..7040c9e148c8
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
@@ -0,0 +1,428 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+#include <linux/printk.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/transobj.h>
+#include "aso.h"
+#include <dev/mlx5/mlx5_core/wq.h>
+#include <dev/mlx5/cq.h>
+
+struct mlx5_aso_cq {
+	/* data path - accessed per cqe */
+	struct mlx5_cqwq           wq;
+
+	/* data path - accessed per napi poll */
+	struct mlx5_core_cq        mcq;
+
+	/* control */
+	struct mlx5_core_dev      *mdev;
+	struct mlx5_wq_ctrl        wq_ctrl;
+} ____cacheline_aligned_in_smp;
+
+struct mlx5_aso {
+	/* data path */
+	u16                        cc;
+	u16                        pc;
+
+	struct mlx5_wqe_ctrl_seg  *doorbell_cseg;
+	struct mlx5_aso_cq         cq;
+
+	/* read only */
+	struct mlx5_wq_cyc         wq;
+	void __iomem               *uar_map;
+	u32                        sqn;
+
+	/* control path */
+	struct mlx5_wq_ctrl        wq_ctrl;
+
+} ____cacheline_aligned_in_smp;
+
+static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
+{
+	mlx5_wq_destroy(&cq->wq_ctrl);
+}
+
+static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
+			     void *cqc_data, struct mlx5_aso_cq *cq)
+{
+	struct mlx5_core_cq *mcq = &cq->mcq;
+	struct mlx5_wq_param param;
+	int err;
+	u32 i;
+
+	param.linear = 1;
+	err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
+	if (err)
+		return err;
+
+	mcq->cqe_sz     = 64;
+	mcq->set_ci_db  = cq->wq_ctrl.db.db;
+	mcq->arm_db     = cq->wq_ctrl.db.db + 1;
+
+	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
+		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
+
+		cqe->op_own = 0xf1;
+	}
+
+	cq->mdev = mdev;
+
+	return 0;
+}
+
+static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
+{
+	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+	struct mlx5_core_dev *mdev = cq->mdev;
+	struct mlx5_core_cq *mcq = &cq->mcq;
+	int inlen, eqn, irqn_not_used;
+	void *in, *cqc;
+	int err;
+
+	err = mlx5_vector2eqn(mdev, 0, &eqn, &irqn_not_used);
+	if (err)
+		return err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+		sizeof(u64) * cq->wq_ctrl.buf.npages;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
+
+	memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));
+
+	mlx5_fill_page_array(&cq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
+
+	MLX5_SET(cqc,   cqc, cq_period_mode, 0);
+	MLX5_SET(cqc,   cqc, c_eqn, eqn);
+	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
+	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
+					    MLX5_ADAPTER_PAGE_SHIFT);
+	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
+
+	err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
+
+	kvfree(in);
+
+	return err;
+}
+
+static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
+{
+	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
+	mlx5_wq_destroy(&cq->wq_ctrl);
+}
+
+static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
+			      struct mlx5_aso_cq *cq)
+{
+	void *cqc_data;
+	int err;
+
+	cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
+	if (!cqc_data)
+		return -ENOMEM;
+
+	MLX5_SET(cqc, cqc_data, log_cq_size, 1);
+	MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
+	if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
+		MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
+
+	err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
+	if (err) {
+		mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
+		goto err_out;
+	}
+
+	err = create_aso_cq(cq, cqc_data);
+	if (err) {
+		mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
+		goto err_free_cq;
+	}
+
+	kvfree(cqc_data);
+	return 0;
+
+err_free_cq:
+	mlx5_aso_free_cq(cq);
+err_out:
+	kvfree(cqc_data);
+	return err;
+}
+
+static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
+			     void *sqc_data, struct mlx5_aso *sq)
+{
+	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
+	struct mlx5_wq_cyc *wq = &sq->wq;
+	struct mlx5_wq_param param;
+	int err;
+
+	sq->uar_map = mdev->priv.uar->map;
+
+	param.linear = 1;
+	err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
+	if (err)
+		return err;
+
+	wq->db = &wq->db[MLX5_SND_DBR];
+
+	return 0;
+}
+
+static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
+			 void *sqc_data, struct mlx5_aso *sq)
+{
+	void *in, *sqc, *wq;
+	int inlen, err;
+	u8 ts_format;
+
+	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
+		sizeof(u64) * sq->wq_ctrl.buf.npages;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
+	wq = MLX5_ADDR_OF(sqc, sqc, wq);
+
+	memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
+	MLX5_SET(sqc,  sqc, cqn, sq->cq.mcq.cqn);
+
+	MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
+	MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
+
+	ts_format = mlx5_get_sq_default_ts(mdev);
+	MLX5_SET(sqc, sqc, ts_format, ts_format);
+
+	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
+	MLX5_SET(wq,   wq, uar_page,      mdev->priv.uar->index);
+	MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
+					  MLX5_ADAPTER_PAGE_SHIFT);
+	MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
+
+	mlx5_fill_page_array(&sq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
+
+	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
+
+	kvfree(in);
+
+	return err;
+}
+
+static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
+{
+	void *in, *sqc;
+	int inlen, err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
+	MLX5_SET(modify_sq_in, in, sqn, sqn);
+	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
+	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
+
+	err = mlx5_core_modify_sq(mdev, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+
+static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
+				  void *sqc_data, struct mlx5_aso *sq)
+{
+	int err;
+
+	err = create_aso_sq(mdev, pdn, sqc_data, sq);
+	if (err)
+		return err;
+
+	err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
+	if (err)
+		mlx5_core_destroy_sq(mdev, sq->sqn);
+
+	return err;
+}
+
+static void mlx5_aso_free_sq(struct mlx5_aso *sq)
+{
+	mlx5_wq_destroy(&sq->wq_ctrl);
+}
+
+static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
+{
+	mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
+	mlx5_aso_free_sq(sq);
+}
+
+static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
+			      u32 pdn, struct mlx5_aso *sq)
+{
+	void *sqc_data, *wq;
+	int err;
+
+	sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
+	if (!sqc_data)
+		return -ENOMEM;
+
+	wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
+	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
+	MLX5_SET(wq, wq, pd, pdn);
+	MLX5_SET(wq, wq, log_wq_sz, 1);
+
+	err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
+	if (err) {
+		mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
+		goto err_out;
+	}
+
+	err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
+	if (err) {
+		mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
+		goto err_free_asosq;
+	}
+
+	mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);
+
+	kvfree(sqc_data);
+	return 0;
+
+err_free_asosq:
+	mlx5_aso_free_sq(sq);
+err_out:
+	kvfree(sqc_data);
+	return err;
+}
+
+struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn)
+{
+	int numa_node = dev_to_node(&mdev->pdev->dev);
+	struct mlx5_aso *aso;
+	int err;
+
+	aso = kzalloc(sizeof(*aso), GFP_KERNEL);
+	if (!aso)
+		return ERR_PTR(-ENOMEM);
+
+	err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
+	if (err)
+		goto err_cq;
+
+	err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
+	if (err)
+		goto err_sq;
+
+	return aso;
+
+err_sq:
+	mlx5_aso_destroy_cq(&aso->cq);
+err_cq:
+	kfree(aso);
+	return ERR_PTR(err);
+}
+
+void mlx5_aso_destroy(struct mlx5_aso *aso)
+{
+	mlx5_aso_destroy_sq(aso);
+	mlx5_aso_destroy_cq(&aso->cq);
+	kfree(aso);
+}
+
+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
+			struct mlx5_aso_wqe *aso_wqe,
+			u32 obj_id, u32 opc_mode)
+{
+	struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;
+
+	cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) |
+					     (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
+					     MLX5_OPCODE_ACCESS_ASO);
+	cseg->qpn_ds     = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
+	cseg->fm_ce_se   = MLX5_WQE_CTRL_CQ_UPDATE;
+	cseg->general_id = cpu_to_be32(obj_id);
+}
+
+struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso)
+{
+	struct mlx5_aso_wqe *wqe;
+	u16 pi;
+
+	pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
+	wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
+	memset(wqe, 0, sizeof(*wqe));
+	return wqe;
+}
+
+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
+		       struct mlx5_wqe_ctrl_seg *doorbell_cseg)
+{
+	doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
+	/* ensure wqe is visible to device before updating doorbell record */
+	wmb();
+
+	if (with_data)
+		aso->pc += MLX5_ASO_WQEBBS_DATA;
+	else
+		aso->pc += MLX5_ASO_WQEBBS;
+	*aso->wq.db = cpu_to_be32(aso->pc);
+
+	/* ensure doorbell record is visible to device before ringing the
+	 * doorbell
+	 */
+	wmb();
+
+	mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map + MLX5_BF_OFFSET, NULL);
+
+	/* Ensure doorbell is written on uar_page before poll_cq */
+	WRITE_ONCE(doorbell_cseg, NULL);
+}
+
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
+{
+	struct mlx5_aso_cq *cq = &aso->cq;
+	struct mlx5_cqe64 *cqe;
+
+	cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	if (!cqe)
+		return -ETIMEDOUT;
+
+	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
+	 * otherwise a cq overrun may occur
+	 */
+	mlx5_cqwq_pop(&cq->wq);
+
+	if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
+		struct mlx5_err_cqe *err_cqe;
+
+		mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
+			      get_cqe_opcode(cqe));
+
+		err_cqe = (struct mlx5_err_cqe *)cqe;
+		mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
+			      err_cqe->vendor_err_synd);
+		mlx5_core_err(cq->mdev, "syndrome=%x\n",
+			      err_cqe->syndrome);
+		print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
+			       16, 1, err_cqe,
+			       sizeof(*err_cqe), false);
+	}
+
+	mlx5_cqwq_update_db_record(&cq->wq);
+
+	/* ensure cq space is freed before enabling more cqes */
+	wmb();
+
+	if (with_data)
+		aso->cc += MLX5_ASO_WQEBBS_DATA;
+	else
+		aso->cc += MLX5_ASO_WQEBBS;
+
+	return 0;
+}
diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h
index 50e964c8f912..11acb94e7dd6 100644
--- a/sys/dev/mlx5/qp.h
+++ b/sys/dev/mlx5/qp.h
@@ -144,6 +144,8 @@ enum {
 #define	MLX5_SEND_WQE_DS	16
 #define	MLX5_SEND_WQE_BB	64
 #define MLX5_SEND_WQEBB_NUM_DS	(MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
+#define MLX5_WQE_CTRL_QPN_SHIFT 8
+#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
 
 enum {
 	MLX5_SEND_WQE_MAX_WQEBBS	= 16,
@@ -192,7 +194,10 @@ struct mlx5_wqe_ctrl_seg {
 	u8			signature;
 	u8			rsvd[2];
 	u8			fm_ce_se;
-	__be32			imm;
+	union {
+		__be32		imm;
+		__be32          general_id;
+	};
 };
 
 #define MLX5_WQE_CTRL_DS_MASK 0x3f
@@ -226,6 +231,10 @@ enum {
 	MLX5_ETH_WQE_SWP_OUTER_L4_TYPE = 1 << 5,
 };
 
+enum {
+	MLX5_ETH_WQE_FT_META_IPSEC = BIT(0),
+};
+
 struct mlx5_wqe_eth_seg {
 	u8              swp_outer_l4_offset;
 	u8		swp_outer_l3_offset;
@@ -234,7 +243,7 @@ struct mlx5_wqe_eth_seg {
 	u8		cs_flags;
 	u8		swp_flags;
 	__be16		mss;
-	__be32		rsvd2;
+	__be32		flow_table_metadata;
 	union {
 		struct {
 			__be16		inline_hdr_sz;
diff --git a/sys/dev/mlx5/tls.h b/sys/dev/mlx5/tls.h
index 67bd7ef686c5..93682a90861c 100644
--- a/sys/dev/mlx5/tls.h
+++ b/sys/dev/mlx5/tls.h
@@ -28,9 +28,6 @@
 
 struct mlx5_core_dev;
 
-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
-    const void *p_key, u32 key_len, u32 * p_obj_id);
-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
 int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32 *p_tisn);
 void mlx5_tls_close_tis(struct mlx5_core_dev *mdev, u32 tisn);
 int mlx5_tls_open_tir(struct mlx5_core_dev *mdev, int tdn, int rqtn, u32 *p_tirn);
diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile
index 339b0bef9382..506c045ab0ce 100644
--- a/sys/modules/mlx5/Makefile
+++ b/sys/modules/mlx5/Makefile
@@ -1,21 +1,25 @@
 .PATH:	${SRCTOP}/sys/dev/mlx5/mlx5_core \
 	${SRCTOP}/sys/dev/mlx5/mlx5_lib \
-	${SRCTOP}/sys/dev/mlx5/mlx5_fpga
+	${SRCTOP}/sys/dev/mlx5/mlx5_fpga \
+	${SRCTOP}/sys/dev/mlx5/mlx5_accel
 
 KMOD=mlx5
 SRCS= \
 mlx5_alloc.c \
+mlx5_aso.c \
 mlx5_cmd.c \
+mlx5_crypto.c \
 mlx5_cq.c \
 mlx5_diag_cnt.c \
 mlx5_diagnostics.c \
 mlx5_eq.c \
 mlx5_eswitch.c \
-mlx5_fc_cmd.c \
+mlx5_fs_chains.c \
 mlx5_fs_cmd.c \
-mlx5_fs_tcp.c \
-mlx5_fs_tree.c \
+mlx5_fs_core.c \
 mlx5_fs_counters.c \
+mlx5_fs_ft_pool.c \
+mlx5_fs_tcp.c \
 mlx5_fw.c \
 mlx5_fwdump.c \
 mlx5_health.c \
@@ -36,7 +40,11 @@ mlx5_uar.c \
 mlx5_vport.c \
 mlx5_vsc.c \
 mlx5_wq.c \
-mlx5_gid.c
+mlx5_gid.c \
+mlx5_ipsec_fs.c \
+mlx5_ipsec_offload.c \
+mlx5_ipsec.c \
+mlx5_ipsec_rxtx.c
 SRCS+=	${LINUXKPI_GENSRCS}
 SRCS+=	opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h