mlx5ib: allocate IB queue counters as a shared resource

A QP owned by a DEVX uid references the port's queue counter.  The
counter was allocated with uid 0, so RST2INIT_QP on a uid-owned QP
failed with "bad resource state".

Allocate and free the IB queue counters directly and, on devices that
support user contexts, stamp them with MLX5_SHARED_RESOURCE_UID so
uid-owned QPs can use them.

The code follows the Linux commit d2c8a1554c10d5e0443b1f97f480d7dacd55cf55
("IB/mlx5: Enable UAR to have DevX UID").

Reviewed by:	kib
Tested by:	Wafa Hamzah <wafah@nvidia.com>
Sponsored by:	Nvidia networking
MFC after:	1 month

1 files changed, 26 insertions, 6 deletions

diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
index fc9af189f3fa..daf5cb217001 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
@@ -3223,9 +3223,17 @@ static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
 
 static void mlx5_ib_dealloc_q_port_counter(struct mlx5_ib_dev *dev, u8 port_num)
 {
-	mlx5_vport_dealloc_q_counter(dev->mdev,
-				     MLX5_INTERFACE_PROTOCOL_IB,
-				     dev->port[port_num].q_cnt_id);
+	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(dealloc_q_counter_out)] = {};
+
+	if (!dev->port[port_num].q_cnt_id)
+		return;
+
+	MLX5_SET(dealloc_q_counter_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
+	MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
+		 dev->port[port_num].q_cnt_id);
+	mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 	dev->port[port_num].q_cnt_id = 0;
 }
 
@@ -3239,19 +3247,31 @@ static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
 
 static int mlx5_ib_alloc_q_counters(struct mlx5_ib_dev *dev)
 {
+	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
 	int i;
 	int ret;
 
+	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
+	/*
+	 * On devices that support user contexts, allocate the queue counters
+	 * as a shared resource so QPs owned by a user context (DEVX uid) can
+	 * reference them; otherwise RST2INIT_QP fails with a firmware "bad
+	 * resource state" error.
+	 */
+	if (MLX5_CAP_GEN(dev->mdev, log_max_uctx))
+		MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
+
 	for (i = 0; i < dev->num_ports; i++) {
-		ret = mlx5_vport_alloc_q_counter(dev->mdev,
-						 MLX5_INTERFACE_PROTOCOL_IB,
-						 &dev->port[i].q_cnt_id);
+		ret = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 		if (ret) {
 			mlx5_ib_warn(dev,
 				     "couldn't allocate queue counter for port %d, err %d\n",
 				     i + 1, ret);
 			goto dealloc_counters;
 		}
+		dev->port[i].q_cnt_id = MLX5_GET(alloc_q_counter_out, out,
+						 counter_set_id);
 	}
 
 	return 0;