Few more minor if_vmx tweaks

 - Allow the Rx/Tx queue sizes to be configured by tunables
 - Bail out earlier if the Tx queue unlikely has enough free
   descriptors to hold the frame
 - Cleanup some of the offloading capabilities handling

2 files changed, 72 insertions, 24 deletions

diff --git a/sys/dev/vmware/vmxnet3/if_vmx.c b/sys/dev/vmware/vmxnet3/if_vmx.c
index 80707ffe6ccd..1cab2d557d8e 100644
--- a/sys/dev/vmware/vmxnet3/if_vmx.c
+++ b/sys/dev/vmware/vmxnet3/if_vmx.c
@@ -199,6 +199,8 @@ static int	vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t,
 		    bus_size_t, struct vmxnet3_dma_alloc *);
 static void	vmxnet3_dma_free(struct vmxnet3_softc *,
 		    struct vmxnet3_dma_alloc *);
+static int	vmxnet3_tunable_int(struct vmxnet3_softc *,
+		    const char *, int);
 
 typedef enum {
 	VMXNET3_BARRIER_RD,
@@ -208,6 +210,12 @@ typedef enum {
 
 static void	vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
 
+/* Tunables. */
+static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC;
+TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc);
+static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC;
+TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc);
+
 static device_method_t vmxnet3_methods[] = {
 	/* Device interface. */
 	DEVMETHOD(device_probe,		vmxnet3_probe),
@@ -453,11 +461,28 @@ vmxnet3_check_version(struct vmxnet3_softc *sc)
 static void
 vmxnet3_initial_config(struct vmxnet3_softc *sc)
 {
+	int ndesc;
 
-	sc->vmx_ntxqueues = 1;
-	sc->vmx_nrxqueues = 1;
-	sc->vmx_ntxdescs = VMXNET3_MAX_TX_NDESC;
-	sc->vmx_nrxdescs = VMXNET3_MAX_RX_NDESC;
+	/*
+	 * BMV Much of the work is already done, but this driver does
+	 * not support multiqueue yet.
+	 */
+	sc->vmx_ntxqueues = VMXNET3_TX_QUEUES;
+	sc->vmx_nrxqueues = VMXNET3_RX_QUEUES;
+
+	ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc);
+	if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC)
+		ndesc = VMXNET3_DEF_TX_NDESC;
+	if (ndesc & VMXNET3_MASK_TX_NDESC)
+		ndesc &= ~VMXNET3_MASK_TX_NDESC;
+	sc->vmx_ntxdescs = ndesc;
+
+	ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc);
+	if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC)
+		ndesc = VMXNET3_DEF_RX_NDESC;
+	if (ndesc & VMXNET3_MASK_RX_NDESC)
+		ndesc &= ~VMXNET3_MASK_RX_NDESC;
+	sc->vmx_nrxdescs = ndesc;
 	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
 }
 
@@ -1382,10 +1407,10 @@ vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
 	ds = sc->vmx_ds;
 
 	ds->upt_features = 0;
-	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
-		ds->upt_features |= UPT1_F_VLAN;
 	if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
 		ds->upt_features |= UPT1_F_CSUM;
+	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
+		ds->upt_features |= UPT1_F_VLAN;
 	if (ifp->if_capenable & IFCAP_LRO)
 		ds->upt_features |= UPT1_F_LRO;
 
@@ -1464,18 +1489,13 @@ vmxnet3_setup_interface(struct vmxnet3_softc *sc)
 	ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM;
 	ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6;
 	ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
-	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
-	ifp->if_hwassist |= VMXNET3_CSUM_ALL_OFFLOAD;
-
+	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
+	    IFCAP_VLAN_HWCSUM;
 	ifp->if_capenable = ifp->if_capabilities;
 
-	/*
-	 * Capabilities after here are not enabled by default.
-	 */
-
-	ifp->if_capabilities |= IFCAP_LRO;
+	/* These capabilities are not enabled by default. */
+	ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
 
-	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 	sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 	    vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST);
 	sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config,
@@ -2517,7 +2537,7 @@ vmxnet3_start_locked(struct ifnet *ifp)
 	struct vmxnet3_txqueue *txq;
 	struct vmxnet3_txring *txr;
 	struct mbuf *m_head;
-	int tx;
+	int tx, avail;
 
 	sc = ifp->if_softc;
 	txq = &sc->vmx_txq[0];
@@ -2530,11 +2550,20 @@ vmxnet3_start_locked(struct ifnet *ifp)
 	    sc->vmx_link_active == 0)
 		return;
 
-	while (VMXNET3_TXRING_AVAIL(txr) > 0) {
+	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+		if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2)
+			break;
+
 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 		if (m_head == NULL)
 			break;
 
+		/* Assume worse case if this mbuf is the head of a chain. */
+		if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) {
+			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
+			break;
+		}
+
 		if (vmxnet3_txq_encap(txq, &m_head) != 0) {
 			if (m_head != NULL)
 				IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
@@ -2752,8 +2781,8 @@ vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			ifp->if_capenable ^= IFCAP_TSO6;
 
 		if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
-		    IFCAP_VLAN_HWFILTER)) {
-			/* These Rx features require us to renegotiate. */
+		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) {
+			/* Changing these features requires us to reinit. */
 			reinit = 1;
 
 			if (mask & IFCAP_RXCSUM)
@@ -2762,6 +2791,8 @@ vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 				ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 			if (mask & IFCAP_LRO)
 				ifp->if_capenable ^= IFCAP_LRO;
+			if (mask & IFCAP_VLAN_HWTAGGING)
+				ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			if (mask & IFCAP_VLAN_HWFILTER)
 				ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 		} else
@@ -2769,8 +2800,6 @@ vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 		if (mask & IFCAP_VLAN_HWTSO)
 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
-		if (mask & IFCAP_VLAN_HWTAGGING)
-			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 
 		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
@@ -3282,6 +3311,18 @@ vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
 	bzero(dma, sizeof(struct vmxnet3_dma_alloc));
 }
 
+static int
+vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def)
+{
+	char path[64];
+
+	snprintf(path, sizeof(path),
+	    "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob);
+	TUNABLE_INT_FETCH(path, &def);
+
+	return (def);
+}
+
 /*
  * Since this is a purely paravirtualized device, we do not have
  * to worry about DMA coherency. But at times, we must make sure
diff --git a/sys/dev/vmware/vmxnet3/if_vmxvar.h b/sys/dev/vmware/vmxnet3/if_vmxvar.h
index f2dd52b0b737..bbe5a4a547a5 100644
--- a/sys/dev/vmware/vmxnet3/if_vmxvar.h
+++ b/sys/dev/vmware/vmxnet3/if_vmxvar.h
@@ -42,10 +42,17 @@ struct vmxnet3_dma_alloc {
 #define VMXNET3_RXRINGS_PERQ	2
 
 /*
- * The maximum number of descriptors in each Rx/Tx ring.
+ * The number of descriptors in each Rx/Tx ring.
  */
-#define VMXNET3_MAX_TX_NDESC		512
-#define VMXNET3_MAX_RX_NDESC		256
+#define VMXNET3_DEF_TX_NDESC		512
+#define VMXNET3_MAX_TX_NDESC		4096
+#define VMXNET3_MIN_TX_NDESC		32
+#define VMXNET3_MASK_TX_NDESC		0x1F
+#define VMXNET3_DEF_RX_NDESC		256
+#define VMXNET3_MAX_RX_NDESC		2048
+#define VMXNET3_MIN_RX_NDESC		32
+#define VMXNET3_MASK_RX_NDESC		0x1F
+
 #define VMXNET3_MAX_TX_NCOMPDESC	VMXNET3_MAX_TX_NDESC
 #define VMXNET3_MAX_RX_NCOMPDESC \
     (VMXNET3_MAX_RX_NDESC * VMXNET3_RXRINGS_PERQ)