rge: add disable_aspm tunable for PCIe power management

Add a per-interface loader tunable dev.rge.%d.disable_aspm to
disable PCIe ASPM (L0s/L1) and ECPM on the RTL8125/8126/8127.

Disabling ASPM reduces latency at the cost of higher power
consumption. Default is off (ASPM left as configured by BIOS).

Signed-off-by: Christos Longros <chris.longros@gmail.com>

Reviewed by:	adrian
Differential Revision:	https://reviews.freebsd.org/D56103

4 files changed, 27 insertions, 16 deletions

diff --git a/share/man/man4/rge.4 b/share/man/man4/rge.4
index f9077a56f28b..24c42f9106dc 100644
--- a/share/man/man4/rge.4
+++ b/share/man/man4/rge.4
@@ -152,6 +152,15 @@ Maximum number of RX packets to process per interrupt.
 The default value is 16.
 Increasing this value may improve throughput on high-speed links at the
 cost of increased interrupt latency.
+.It Va dev.rge.%d.disable_aspm
+Disable PCIe Active State Power Management (ASPM) and Extended
+Configuration Power Management (ECPM).
+The default value is 0 (leave ASPM enabled).
+Setting this to 1 reduces latency at the cost of increased power
+consumption.
+This tunable can only be set in
+.Xr loader.conf 5
+and requires a reboot to take effect.
 .El
 .Sh DIAGNOSTICS
 .Bl -diag
diff --git a/sys/dev/rge/if_rge.c b/sys/dev/rge/if_rge.c
index 17225e065482..8887e8d39ae4 100644
--- a/sys/dev/rge/if_rge.c
+++ b/sys/dev/rge/if_rge.c
@@ -445,23 +445,19 @@ rge_attach(device_t dev)
 
 	rge_config_imtype(sc, RGE_IMTYPE_SIM);
 
-	/* TODO: disable ASPM/ECPM? */
-
-#if 0
-	/*
-	 * PCI Express check.
-	 */
-	if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_PCIEXPRESS,
-	    &offset, NULL)) {
-		/* Disable PCIe ASPM and ECPM. */
-		reg = pci_conf_read(pa->pa_pc, pa->pa_tag,
-		    offset + PCI_PCIE_LCSR);
-		reg &= ~(PCI_PCIE_LCSR_ASPM_L0S | PCI_PCIE_LCSR_ASPM_L1 |
-		    PCI_PCIE_LCSR_ECPM);
-		pci_conf_write(pa->pa_pc, pa->pa_tag, offset + PCI_PCIE_LCSR,
-		    reg);
+	/* Disable PCIe ASPM and ECPM if requested. */
+	if (sc->sc_disable_aspm) {
+		int ecap;
+		if (pci_find_cap(dev, PCIY_EXPRESS, &ecap) == 0) {
+			uint16_t lctl;
+			lctl = pci_read_config(dev,
+			    ecap + PCIER_LINK_CTL, 2);
+			lctl &= ~(PCIEM_LINK_CTL_ASPMC |
+			    PCIEM_LINK_CTL_ECPM);
+			pci_write_config(dev,
+			    ecap + PCIER_LINK_CTL, lctl, 2);
+		}
 	}
-#endif
 
 	RGE_LOCK(sc);
 	if (rge_chipinit(sc)) {
diff --git a/sys/dev/rge/if_rge_sysctl.c b/sys/dev/rge/if_rge_sysctl.c
index 16001b4c1d94..75e2316042ea 100644
--- a/sys/dev/rge/if_rge_sysctl.c
+++ b/sys/dev/rge/if_rge_sysctl.c
@@ -237,6 +237,11 @@ rge_sysctl_attach(struct rge_softc *sc)
 	    "rx_process_limit", CTLFLAG_RW, &sc->sc_rx_process_limit, 0,
 	    "max number of RX packets to process per interrupt");
 
+	sc->sc_disable_aspm = 0;
+	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+	    "disable_aspm", CTLFLAG_RDTUN, &sc->sc_disable_aspm, 0,
+	    "disable PCIe ASPM and ECPM (requires reboot)");
+
 	/* Stats */
 	rge_sysctl_drv_stats_attach(sc);
 	rge_sysctl_mac_stats_attach(sc);
diff --git a/sys/dev/rge/if_rgevar.h b/sys/dev/rge/if_rgevar.h
index 89d02e8acb72..2e80dcf42187 100644
--- a/sys/dev/rge/if_rgevar.h
+++ b/sys/dev/rge/if_rgevar.h
@@ -205,6 +205,7 @@ struct rge_softc {
 	uint32_t		sc_debug;
 
 	int			sc_rx_process_limit;
+	int			sc_disable_aspm;
 
 	struct rge_drv_stats	sc_drv_stats;