ena: add support for the large LLQ headers in ENA

Default LLQ (Low-latency queue) maximum header size is 96 bytes and can
be too small for some types of packets - like IPv6 packets with multiple
extension. This can be fixed, by using large LLQ headers.

If the device supports larger LLQ headers, the user can activate this
feature by setting sysctl tunable 'hw.ena.force_large_llq_header' to '1'
in the /boot/loader.conf file.

In case the device isn't supporting this feature, the default value (96B)
will be used.

Submitted by: Michal Krawczyk <mk@semihalf.com>
Obtained from: Semihalf
MFC after: 2 weeks
Sponsored by: Amazon, Inc.

3 files changed, 54 insertions, 7 deletions

diff --git a/sys/dev/ena/ena.c b/sys/dev/ena/ena.c
index 680eb0e9d049..20da750bcac0 100644
--- a/sys/dev/ena/ena.c
+++ b/sys/dev/ena/ena.c
@@ -1,7 +1,7 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
- * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -2606,14 +2606,25 @@ ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev,
 }
 
 static inline
-void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
+void set_default_llq_configurations(struct ena_llq_configurations *llq_config,
+	struct ena_admin_feature_llq_desc *llq)
 {
+
 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
-	llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
 	llq_config->llq_num_decs_before_header =
 	    ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
-	llq_config->llq_ring_entry_size_value = 128;
+	if ((llq->entry_size_ctrl_supported &
+	     ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0 &&
+	    ena_force_large_llq_header) {
+		llq_config->llq_ring_entry_size =
+		    ENA_ADMIN_LIST_ENTRY_SIZE_256B;
+		llq_config->llq_ring_entry_size_value = 256;
+	} else {
+		llq_config->llq_ring_entry_size =
+		    ENA_ADMIN_LIST_ENTRY_SIZE_128B;
+		llq_config->llq_ring_entry_size_value = 128;
+	}
 }
 
 static int
@@ -2672,6 +2683,26 @@ ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx)
 	max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1);
 	max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1);
 
+	/*
+	 * When forcing large headers, we multiply the entry size by 2,
+	 * and therefore divide the queue size by 2, leaving the amount
+	 * of memory used by the queues unchanged.
+	 */
+	if (ena_force_large_llq_header) {
+		if ((llq->entry_size_ctrl_supported &
+		     ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0 &&
+		    ena_dev->tx_mem_queue_type ==
+		     ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+			max_tx_queue_size /= 2;
+			device_printf(ctx->pdev,
+			    "Forcing large headers and decreasing maximum Tx queue size to %d\n",
+			    max_tx_queue_size);
+		} else {
+			device_printf(ctx->pdev,
+			    "Forcing large headers failed: LLQ is disabled or device does not support large headers\n");
+		}
+	}
+
 	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
 	    max_tx_queue_size);
 	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
@@ -3618,7 +3649,7 @@ ena_attach(device_t pdev)
 		goto err_bus_free;
 	}
 
-	set_default_llq_configurations(&llq_config);
+	set_default_llq_configurations(&llq_config, &get_feat_ctx.llq);
 
 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq,
 	     &llq_config);
diff --git a/sys/dev/ena/ena_sysctl.c b/sys/dev/ena/ena_sysctl.c
index 3bc49790b450..26fc79379e33 100644
--- a/sys/dev/ena/ena_sysctl.c
+++ b/sys/dev/ena/ena_sysctl.c
@@ -1,7 +1,7 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
- * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,6 +69,19 @@ int ena_enable_9k_mbufs = 0;
 SYSCTL_INT(_hw_ena, OID_AUTO, enable_9k_mbufs, CTLFLAG_RDTUN,
     &ena_enable_9k_mbufs, 0, "Use 9 kB mbufs for Rx descriptors");
 
+/*
+ * Force the driver to use large LLQ (Low Latency Queue) header. Defaults to
+ * false. This option may be important for platforms, which often handle packet
+ * headers on Tx with total header size greater than 96B, as it may
+ * reduce the latency.
+ * It also reduces the maximum Tx queue size by half, so it may cause more Tx
+ * packet drops.
+ */
+bool ena_force_large_llq_header = false;
+SYSCTL_BOOL(_hw_ena, OID_AUTO, force_large_llq_header, CTLFLAG_RDTUN,
+    &ena_force_large_llq_header, 0,
+    "Increases maximum supported header size in LLQ mode to 224 bytes, while reducing the maximum Tx queue size by half.\n");
+
 void
 ena_sysctl_add_nodes(struct ena_adapter *adapter)
 {
diff --git a/sys/dev/ena/ena_sysctl.h b/sys/dev/ena/ena_sysctl.h
index 5aa0b795fbb5..360e32c3ad18 100644
--- a/sys/dev/ena/ena_sysctl.h
+++ b/sys/dev/ena/ena_sysctl.h
@@ -1,7 +1,7 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
- * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
+ * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,4 +44,7 @@ void	ena_sysctl_add_nodes(struct ena_adapter *adapter);
 extern int ena_enable_9k_mbufs;
 #define ena_mbuf_sz (ena_enable_9k_mbufs ? MJUM9BYTES : MJUMPAGESIZE)
 
+/* Force the driver to use large LLQ (Low Latency Queue) headers. */
+extern bool ena_force_large_llq_header;
+
 #endif /* !(ENA_SYSCTL_H) */