cxgbe(4): internal knob for flexible control over FEC selection.

Recent firmwares have support for autonomous FEC selection and a "force"
knob to let the driver control this behavior (or not) in a fine grained
manner. This change adds a driver knob so that all the different ways of
configuring the link FEC can be exercised. Note that this controls the
internal driver/firmware interaction for link configuration and is not
meant for general use.

Sponsored by:	Chelsio Communications

(cherry picked from commit 448bcd01dc5edcbbf23975588f131e13cd09778f)

3 files changed, 85 insertions, 7 deletions

diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h
index 50859e868b9d..bee4f58f693c 100644
--- a/sys/dev/cxgbe/common/common.h
+++ b/sys/dev/cxgbe/common/common.h
@@ -443,9 +443,11 @@ struct link_config {
 	int8_t requested_aneg;	/* link autonegotiation */
 	int8_t requested_fc;	/* flow control */
 	int8_t requested_fec;	/* FEC */
+	int8_t force_fec;	/* FORCE_FEC in L1_CFG32 command. */
 	u_int requested_speed;	/* speed (Mbps) */
 	uint32_t requested_caps;/* rcap in last l1cfg issued by the driver. */
 
+	/* These are populated with information from the firmware. */
 	uint32_t pcaps;		/* link capabilities */
 	uint32_t acaps;		/* advertised capabilities */
 	uint32_t lpacaps;	/* peer advertised capabilities */
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index b9bf5df5ccc6..d6f85a1fcd34 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -3917,9 +3917,19 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
 		speed = fwcap_top_speed(lc->pcaps);
 
 	fec = 0;
+#ifdef INVARIANTS
+	if (lc->force_fec != 0)
+		MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_FEC);
+#endif
 	if (fec_supported(speed)) {
 		if (lc->requested_fec == FEC_AUTO) {
-			if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC) {
+			if (lc->force_fec > 0) {
+				/*
+				 * Must use FORCE_FEC even though requested FEC
+				 * is AUTO. Set all the FEC bits valid for the
+				 * speed and let the firmware pick one.
+				 */
+				fec |= FW_PORT_CAP32_FORCE_FEC;
 				if (speed & FW_PORT_CAP32_SPEED_100G) {
 					fec |= FW_PORT_CAP32_FEC_RS;
 					fec |= FW_PORT_CAP32_FEC_NO_FEC;
@@ -3929,20 +3939,26 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
 					fec |= FW_PORT_CAP32_FEC_NO_FEC;
 				}
 			} else {
-				/* Set only 1b with old firmwares. */
+				/*
+				 * Set only 1b. Old firmwares can't deal with
+				 * multiple bits and new firmwares are free to
+				 * ignore this and try whatever FECs they want
+				 * because we aren't setting FORCE_FEC here.
+				 */
 				fec |= fec_to_fwcap(lc->fec_hint);
 			}
 		} else {
+			/*
+			 * User has explicitly requested some FEC(s). Set
+			 * FORCE_FEC unless prohibited from using it.
+			 */
+			if (lc->force_fec != 0)
+				fec |= FW_PORT_CAP32_FORCE_FEC;
 			fec |= fec_to_fwcap(lc->requested_fec &
 			    M_FW_PORT_CAP32_FEC);
 			if (lc->requested_fec & FEC_MODULE)
 				fec |= fec_to_fwcap(lc->fec_hint);
 		}
-
-		if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC)
-			fec |= FW_PORT_CAP32_FORCE_FEC;
-		else if (fec == FW_PORT_CAP32_FEC_NO_FEC)
-			fec = 0;
 	}
 
 	/* Force AN on for BT cards. */
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 71be2c1db71d..0ccca0469a8a 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -523,6 +523,21 @@ SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0,
     "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");
 
 /*
+ * Controls when the driver sets the FORCE_FEC bit in the L1_CFG32 that it
+ * issues to the firmware.  If the firmware doesn't support FORCE_FEC then the
+ * driver runs as if this is set to 0.
+ * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay.
+ *  0 to never set FORCE_FEC. requested_fec = AUTO means use the hint from the
+ *    transceiver. Multiple FEC bits may not be okay but will be passed on to
+ *    the firmware anyway (may result in l1cfg errors with old firmwares).
+ *  1 to always set FORCE_FEC. Multiple FEC bits are okay. requested_fec = AUTO
+ *    means set all FEC bits that are valid for the speed.
+ */
+static int t4_force_fec = -1;
+SYSCTL_INT(_hw_cxgbe, OID_AUTO, force_fec, CTLFLAG_RDTUN, &t4_force_fec, 0,
+    "Controls the use of FORCE_FEC bit in L1 configuration.");
+
+/*
  * Link autonegotiation.
  * -1 to run with the firmware default.
  *  0 to disable.
@@ -780,6 +795,7 @@ static int sysctl_link_fec(SYSCTL_HANDLER_ARGS);
 static int sysctl_requested_fec(SYSCTL_HANDLER_ARGS);
 static int sysctl_module_fec(SYSCTL_HANDLER_ARGS);
 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
+static int sysctl_force_fec(SYSCTL_HANDLER_ARGS);
 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 static int sysctl_vdd(SYSCTL_HANDLER_ARGS);
@@ -5762,6 +5778,7 @@ init_link_config(struct port_info *pi)
 	struct link_config *lc = &pi->link_cfg;
 
 	PORT_LOCK_ASSERT_OWNED(pi);
+	MPASS(lc->pcaps != 0);
 
 	lc->requested_caps = 0;
 	lc->requested_speed = 0;
@@ -5787,6 +5804,13 @@ init_link_config(struct port_info *pi)
 		if (lc->requested_fec == 0)
 			lc->requested_fec = FEC_AUTO;
 	}
+	lc->force_fec = 0;
+	if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC) {
+		if (t4_force_fec < 0)
+			lc->force_fec = -1;
+		else if (t4_force_fec > 0)
+			lc->force_fec = 1;
+	}
 }
 
 /*
@@ -7827,6 +7851,9 @@ cxgbe_sysctls(struct port_info *pi)
 	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
 	    sysctl_autoneg, "I",
 	    "autonegotiation (-1 = not supported)");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "force_fec",
+	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
+	    sysctl_force_fec, "I", "when to use FORCE_FEC bit for link config");
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rcaps", CTLFLAG_RD,
 	    &pi->link_cfg.requested_caps, 0, "L1 config requested by driver");
@@ -8546,6 +8573,39 @@ done:
 }
 
 static int
+sysctl_force_fec(SYSCTL_HANDLER_ARGS)
+{
+	struct port_info *pi = arg1;
+	struct adapter *sc = pi->adapter;
+	struct link_config *lc = &pi->link_cfg;
+	int rc, val;
+
+	val = lc->force_fec;
+	MPASS(val >= -1 && val <= 1);
+	rc = sysctl_handle_int(oidp, &val, 0, req);
+	if (rc != 0 || req->newptr == NULL)
+		return (rc);
+	if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC))
+		return (ENOTSUP);
+	if (val < -1 || val > 1)
+		return (EINVAL);
+
+	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff");
+	if (rc)
+		return (rc);
+	PORT_LOCK(pi);
+	lc->force_fec = val;
+	if (!hw_off_limits(sc)) {
+		fixup_link_config(pi);
+		if (pi->up_vis > 0)
+			rc = apply_link_config(pi);
+	}
+	PORT_UNLOCK(pi);
+	end_synchronized_op(sc, 0);
+	return (rc);
+}
+
+static int
 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *sc = arg1;