mlx5en: Add more error checks in the transmit path.

- Upon error more completion events than requested may be generated,
  particularly when using the completion event factor feature.
- Count number of event errors in the transmit path.

MFC after:	1 week
Reviewed by:	kib
Sponsored by:	Mellanox Technologies // NVIDIA Networking

3 files changed, 28 insertions, 4 deletions

diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index 64d4ed87d58f..e59fb6771d83 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -692,6 +692,11 @@ struct mlx5_cqe64 {
 
 #define	MLX5_CQE_TSTMP_PTP	(1ULL << 63)
 
+static inline u8 get_cqe_opcode(struct mlx5_cqe64 *cqe)
+{
+	return (cqe->op_own >> 4);
+}
+
 static inline bool get_cqe_lro_timestamp_valid(struct mlx5_cqe64 *cqe)
 {
 	return (cqe->lro_tcppsh_abort_dupack >> 7) & 1;
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index e4b66bea8f60..b249a82d30ef 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -627,6 +627,7 @@ struct mlx5e_rq_stats {
   m(+1, u64, defragged, "defragged", "Transmitted packets")		\
   m(+1, u64, dropped, "dropped", "Transmitted packets")			\
   m(+1, u64, enobuf, "enobuf", "Transmitted packets")			\
+  m(+1, u64, cqe_err, "cqe_err", "Transmit CQE errors")			\
   m(+1, u64, nop, "nop", "Transmitted packets")
 
 #define	MLX5E_SQ_STATS_NUM (0 MLX5E_SQ_STATS(MLX5E_STATS_COUNT))
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index 437910ee7964..753b7ea20e5b 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -1038,6 +1038,9 @@ mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
 	while (budget > 0) {
 		struct mlx5_cqe64 *cqe;
 		struct mbuf *mb;
+		bool match;
+		u16 sqcc_this;
+		u16 delta;
 		u16 x;
 		u16 ci;
 
@@ -1047,11 +1050,28 @@ mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
 
 		mlx5_cqwq_pop(&sq->cq.wq);
 
+		/* check if the completion event indicates an error */
+		if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ))
+			sq->stats.cqe_err++;
+
+		/* setup local variables */
+		sqcc_this = be16toh(cqe->wqe_counter);
+		match = false;
+
 		/* update budget according to the event factor */
 		budget -= sq->cev_factor;
 
-		for (x = 0; x != sq->cev_factor; x++) {
+		for (x = 0;; x++) {
+			if (unlikely(match != false)) {
+				break;
+			} else if (unlikely(x == sq->cev_factor)) {
+				/* WQE counter match not found */
+				sq->stats.cqe_err++;
+				break;
+			}
 			ci = sqcc & sq->wq.sz_m1;
+			delta = sqcc_this - sqcc;
+			match = (delta < sq->mbuf[ci].num_wqebbs);
 			mb = sq->mbuf[ci].mbuf;
 			sq->mbuf[ci].mbuf = NULL;
 
@@ -1061,10 +1081,8 @@ mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
 			}
 
 			if (mb == NULL) {
-				if (sq->mbuf[ci].num_bytes == 0) {
-					/* NOP */
+				if (unlikely(sq->mbuf[ci].num_bytes == 0))
 					sq->stats.nop++;
-				}
 			} else {
 				bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
 				    BUS_DMASYNC_POSTWRITE);