Make igb and ixgbe check tunables at probe time.

This allows one to make a kernel module to tune the
number of queues before the driver loads.

This is needed so that a module at SI_SUB_CPU can set
tunables for these drivers to take.  Otherwise getenv
is called too early by the TUNABLE macros.

Reviewed by: smh
Phabric: https://reviews.freebsd.org/D1149

1 files changed, 46 insertions, 11 deletions

diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c
index 4796a915f2e0..335aa01d95c9 100644
--- a/sys/dev/e1000/if_igb.c
+++ b/sys/dev/e1000/if_igb.c
@@ -188,6 +188,7 @@ static char *igb_strings[] = {
 /*********************************************************************
  *  Function prototypes
  *********************************************************************/
+static int	igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS);
 static int	igb_probe(device_t);
 static int	igb_attach(device_t);
 static int	igb_detach(device_t);
@@ -493,6 +494,11 @@ igb_attach(device_t dev)
 	    OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 	    igb_sysctl_nvm_info, "I", "NVM Information");
 
+        SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
+                        SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+			OID_AUTO, "num_queues", CTLTYPE_INT | CTLFLAG_RD,
+			adapter, 0, igb_per_unit_num_queues, "I", "Number of Queues");
+
 	igb_set_sysctl_value(adapter, "enable_aim",
 	    "Interrupt Moderation", &adapter->enable_aim,
 	    igb_enable_aim);
@@ -2831,6 +2837,7 @@ igb_setup_msix(struct adapter *adapter)
 {
 	device_t	dev = adapter->dev;
 	int		bar, want, queues, msgs, maxqueues;
+	int		n_queues;
 
 	/* tuneable override */
 	if (igb_enable_msix == 0)
@@ -2858,8 +2865,18 @@ igb_setup_msix(struct adapter *adapter)
 		goto msi;
 	}
 
-	/* Figure out a reasonable auto config value */
-	queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
+	n_queues = 0;
+	/* try more specific tunable, then global, then finally default to boot time tunable if set. */
+	if (device_getenv_int(dev, "num_queues", &n_queues) != 0) {
+		device_printf(dev, "using specific tunable num_queues=%d", n_queues);
+	} else if (TUNABLE_INT_FETCH("hw.igb.num_queues", &n_queues) != 0) {
+		if (igb_num_queues != n_queues) {
+			device_printf(dev, "using global tunable hw.igb.num_queues=%d", n_queues);
+			igb_num_queues = n_queues;
+		}
+	} else {
+		n_queues = igb_num_queues;
+	}
 
 #ifdef	RSS
 	/* If we're doing RSS, clamp at the number of RSS buckets */
@@ -2867,10 +2884,12 @@ igb_setup_msix(struct adapter *adapter)
 		queues = rss_getnumbuckets();
 #endif
 
-
-	/* Manual override */
-	if (igb_num_queues != 0)
-		queues = igb_num_queues;
+	if (n_queues != 0) {
+		queues = n_queues;
+	} else {
+		/* Figure out a reasonable auto config value */
+		queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
+	}
 
 	/* Sanity check based on HW */
 	switch (adapter->hw.mac.type) {
@@ -2893,12 +2912,17 @@ igb_setup_msix(struct adapter *adapter)
 			maxqueues = 1;
 			break;
 	}
-	if (queues > maxqueues)
+	if (queues > maxqueues) {
+		device_printf(adapter->dev, "requested %d queues, but max for this adapter is %d\n",
+		    queues, maxqueues);
 		queues = maxqueues;
-
-	/* Manual override */
-	if (igb_num_queues != 0)
-		queues = igb_num_queues;
+	} else if (queues == 0) {
+		queues = 1;
+	} else if (queues < 0) {
+		device_printf(adapter->dev, "requested %d queues, but min for this adapter is %d\n",
+		    queues, 1);
+		queues = 1;
+	}
 
 	/*
 	** One vector (RX/TX pair) per queue
@@ -6384,3 +6408,14 @@ igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
 	IGB_CORE_UNLOCK(adapter);
 	return (0);
 }
+
+static int
+igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS)
+{
+	struct adapter          *adapter;
+
+	adapter = (struct adapter *) arg1;
+
+	return sysctl_handle_int(oidp, &adapter->num_queues, 0, req);
+}
+