aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/cxgbe
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/cxgbe')
-rw-r--r--sys/dev/cxgbe/adapter.h26
-rw-r--r--sys/dev/cxgbe/common/common.h3
-rw-r--r--sys/dev/cxgbe/common/t4_hw.c1592
-rw-r--r--sys/dev/cxgbe/common/t4_msg.h566
-rw-r--r--sys/dev/cxgbe/common/t4_regs.h100
-rw-r--r--sys/dev/cxgbe/crypto/t7_kern_tls.c18
-rw-r--r--sys/dev/cxgbe/firmware/t4fw_interface.h7
-rw-r--r--sys/dev/cxgbe/nvmf/nvmf_che.c3331
-rw-r--r--sys/dev/cxgbe/offload.h3
-rw-r--r--sys/dev/cxgbe/t4_main.c162
-rw-r--r--sys/dev/cxgbe/t4_sge.c86
-rw-r--r--sys/dev/cxgbe/tom/t4_cpl_io.c347
-rw-r--r--sys/dev/cxgbe/tom/t4_tom.c6
13 files changed, 5754 insertions, 493 deletions
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 55f09fefb7e3..38875b535067 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -184,7 +184,16 @@ enum {
DF_LOAD_FW_ANYTIME = (1 << 1), /* Allow LOAD_FW after init */
DF_DISABLE_TCB_CACHE = (1 << 2), /* Disable TCB cache (T6+) */
DF_DISABLE_CFG_RETRY = (1 << 3), /* Disable fallback config */
- DF_VERBOSE_SLOWINTR = (1 << 4), /* Chatty slow intr handler */
+
+ /* adapter intr handler flags */
+ IHF_INTR_CLEAR_ON_INIT = (1 << 0), /* Driver calls t4_intr_clear */
+ IHF_NO_SHOW = (1 << 1), /* Do not display intr info */
+ IHF_VERBOSE = (1 << 2), /* Display extra intr info */
+ IHF_FATAL_IFF_ENABLED = (1 << 3), /* Fatal only if enabled */
+ IHF_IGNORE_IF_DISABLED = (1 << 4), /* Ignore if disabled */
+ IHF_CLR_ALL_SET = (1 << 5), /* Clear all set bits */
+ IHF_CLR_ALL_UNIGNORED = (1 << 6), /* Clear all unignored bits */
+ IHF_RUN_ALL_ACTIONS = (1 << 7), /* As if all cause are set */
};
#define IS_DETACHING(vi) ((vi)->flags & VI_DETACHING)
@@ -723,6 +732,16 @@ struct sge_ofld_rxq {
uint64_t rx_iscsi_padding_errors;
uint64_t rx_iscsi_header_digest_errors;
uint64_t rx_iscsi_data_digest_errors;
+ counter_u64_t rx_nvme_ddp_setup_ok;
+ counter_u64_t rx_nvme_ddp_setup_no_stag;
+ counter_u64_t rx_nvme_ddp_setup_error;
+ counter_u64_t rx_nvme_ddp_pdus;
+ counter_u64_t rx_nvme_ddp_octets;
+ counter_u64_t rx_nvme_fl_pdus;
+ counter_u64_t rx_nvme_fl_octets;
+ counter_u64_t rx_nvme_invalid_headers;
+ counter_u64_t rx_nvme_header_digest_errors;
+ counter_u64_t rx_nvme_data_digest_errors;
uint64_t rx_aio_ddp_jobs;
uint64_t rx_aio_ddp_octets;
u_long rx_toe_tls_records;
@@ -795,6 +814,9 @@ struct sge_ofld_txq {
counter_u64_t tx_iscsi_pdus;
counter_u64_t tx_iscsi_octets;
counter_u64_t tx_iscsi_iso_wrs;
+ counter_u64_t tx_nvme_pdus;
+ counter_u64_t tx_nvme_octets;
+ counter_u64_t tx_nvme_iso_wrs;
counter_u64_t tx_aio_jobs;
counter_u64_t tx_aio_octets;
counter_u64_t tx_toe_tls_records;
@@ -997,6 +1019,7 @@ struct adapter {
void *iwarp_softc; /* (struct c4iw_dev *) */
struct iw_tunables iwt;
void *iscsi_ulp_softc; /* (struct cxgbei_data *) */
+ void *nvme_ulp_softc; /* (struct nvmf_che_adapter *) */
struct l2t_data *l2t; /* L2 table */
struct smt_data *smt; /* Source MAC Table */
struct tid_info tids;
@@ -1013,6 +1036,7 @@ struct adapter {
int flags;
int debug_flags;
int error_flags; /* Used by error handler and live reset. */
+ int intr_flags; /* Used by interrupt setup/handlers. */
char ifp_lockname[16];
struct mtx ifp_lock;
diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h
index 6b36832a7464..2033967ffb94 100644
--- a/sys/dev/cxgbe/common/common.h
+++ b/sys/dev/cxgbe/common/common.h
@@ -684,9 +684,10 @@ u32 t4_hw_pci_read_cfg4(adapter_t *adapter, int reg);
struct fw_filter_wr;
+void t4_intr_clear(struct adapter *adapter);
void t4_intr_enable(struct adapter *adapter);
void t4_intr_disable(struct adapter *adapter);
-bool t4_slow_intr_handler(struct adapter *adapter, bool verbose);
+bool t4_slow_intr_handler(struct adapter *adapter, int flags);
int t4_hash_mac_addr(const u8 *addr);
int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index eb7ea9acc108..65292486cbc8 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -84,6 +84,41 @@ static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
delay, NULL);
}
+ /**
+ * t7_wait_sram_done - wait until an operation is completed
+ * @adapter: the adapter performing the operation
+ * @reg: the register to check for completion
+ * @result_reg: register that holds the result value
+ * @attempts: number of check iterations
+ * @delay: delay in usecs between iterations
+ * @valp: where to store the value of the result register at completion time
+ *
+ * Waits until a specific bit in @reg is cleared, checking up to
+ * @attempts times.Once the bit is cleared, reads from @result_reg
+ * and stores the value in @valp if it is not NULL. Returns 0 if the
+ * operation completes successfully and -EAGAIN if it times out.
+ */
+static int t7_wait_sram_done(struct adapter *adap, int reg, int result_reg,
+ int attempts, int delay, u32 *valp)
+{
+ while (1) {
+ u32 val = t4_read_reg(adap, reg);
+
+ /* Check if SramStart (bit 19) is cleared */
+ if (!(val & (1 << 19))) {
+ if (valp)
+ *valp = t4_read_reg(adap, result_reg);
+ return 0;
+ }
+
+ if (--attempts == 0)
+ return -EAGAIN;
+
+ if (delay)
+ udelay(delay);
+ }
+}
+
/**
* t4_set_reg_field - set a register field to a value
* @adapter: the adapter to program
@@ -535,11 +570,11 @@ static int t4_edc_err_read(struct adapter *adap, int idx)
edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx);
CH_WARN(adap,
- "edc%d err addr 0x%x: 0x%x.\n",
+ " edc%d err addr 0x%x: 0x%x.\n",
idx, edc_ecc_err_addr_reg,
t4_read_reg(adap, edc_ecc_err_addr_reg));
CH_WARN(adap,
- "bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
+ " bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
edc_bist_status_rdata_reg,
(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg),
(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8),
@@ -578,14 +613,15 @@ int t4_mc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
mc_bist_cmd_len_reg = A_MC_BIST_CMD_LEN;
mc_bist_status_rdata_reg = A_MC_BIST_STATUS_RDATA;
mc_bist_data_pattern_reg = A_MC_BIST_DATA_PATTERN;
- } else {
+ } else if (chip_id(adap) < CHELSIO_T7) {
mc_bist_cmd_reg = MC_REG(A_MC_P_BIST_CMD, idx);
mc_bist_cmd_addr_reg = MC_REG(A_MC_P_BIST_CMD_ADDR, idx);
mc_bist_cmd_len_reg = MC_REG(A_MC_P_BIST_CMD_LEN, idx);
- mc_bist_status_rdata_reg = MC_REG(A_MC_P_BIST_STATUS_RDATA,
- idx);
- mc_bist_data_pattern_reg = MC_REG(A_MC_P_BIST_DATA_PATTERN,
- idx);
+ mc_bist_status_rdata_reg = MC_REG(A_MC_P_BIST_STATUS_RDATA, idx);
+ mc_bist_data_pattern_reg = MC_REG(A_MC_P_BIST_DATA_PATTERN, idx);
+ } else {
+ /* Need to figure out split mode and the rest. */
+ return (-ENOTSUP);
}
if (t4_read_reg(adap, mc_bist_cmd_reg) & F_START_BIST)
@@ -636,21 +672,13 @@ int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
edc_bist_status_rdata_reg = EDC_REG(A_EDC_BIST_STATUS_RDATA,
idx);
} else {
-/*
- * These macro are missing in t4_regs.h file.
- * Added temporarily for testing.
- */
-#define EDC_STRIDE_T5 (EDC_T51_BASE_ADDR - EDC_T50_BASE_ADDR)
-#define EDC_REG_T5(reg, idx) (reg + EDC_STRIDE_T5 * idx)
- edc_bist_cmd_reg = EDC_REG_T5(A_EDC_H_BIST_CMD, idx);
- edc_bist_cmd_addr_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_ADDR, idx);
- edc_bist_cmd_len_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_LEN, idx);
- edc_bist_cmd_data_pattern = EDC_REG_T5(A_EDC_H_BIST_DATA_PATTERN,
+ edc_bist_cmd_reg = EDC_T5_REG(A_EDC_H_BIST_CMD, idx);
+ edc_bist_cmd_addr_reg = EDC_T5_REG(A_EDC_H_BIST_CMD_ADDR, idx);
+ edc_bist_cmd_len_reg = EDC_T5_REG(A_EDC_H_BIST_CMD_LEN, idx);
+ edc_bist_cmd_data_pattern = EDC_T5_REG(A_EDC_H_BIST_DATA_PATTERN,
idx);
- edc_bist_status_rdata_reg = EDC_REG_T5(A_EDC_H_BIST_STATUS_RDATA,
+ edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA,
idx);
-#undef EDC_REG_T5
-#undef EDC_STRIDE_T5
}
if (t4_read_reg(adap, edc_bist_cmd_reg) & F_START_BIST)
@@ -2662,10 +2690,9 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x173c, 0x1760,
0x1800, 0x18fc,
0x3000, 0x3044,
- 0x3060, 0x3064,
0x30a4, 0x30b0,
0x30b8, 0x30d8,
- 0x30e0, 0x30fc,
+ 0x30e0, 0x30e8,
0x3140, 0x357c,
0x35a8, 0x35cc,
0x35e0, 0x35ec,
@@ -2680,7 +2707,7 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x480c, 0x4814,
0x4890, 0x489c,
0x48a4, 0x48ac,
- 0x48b8, 0x48c4,
+ 0x48b8, 0x48bc,
0x4900, 0x4924,
0x4ffc, 0x4ffc,
0x5500, 0x5624,
@@ -2698,8 +2725,10 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x5a60, 0x5a6c,
0x5a80, 0x5a8c,
0x5a94, 0x5a9c,
- 0x5b94, 0x5bfc,
- 0x5c10, 0x5e48,
+ 0x5b94, 0x5bec,
+ 0x5bf8, 0x5bfc,
+ 0x5c10, 0x5c40,
+ 0x5c4c, 0x5e48,
0x5e50, 0x5e94,
0x5ea0, 0x5eb0,
0x5ec0, 0x5ec0,
@@ -2708,7 +2737,8 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x5ef0, 0x5ef0,
0x5f00, 0x5f04,
0x5f0c, 0x5f10,
- 0x5f20, 0x5f88,
+ 0x5f20, 0x5f78,
+ 0x5f84, 0x5f88,
0x5f90, 0x5fd8,
0x6000, 0x6020,
0x6028, 0x6030,
@@ -3084,7 +3114,7 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x38140, 0x38140,
0x38150, 0x38154,
0x38160, 0x381c4,
- 0x381f0, 0x38204,
+ 0x381d0, 0x38204,
0x3820c, 0x38214,
0x3821c, 0x3822c,
0x38244, 0x38244,
@@ -3156,6 +3186,10 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
0x3a000, 0x3a004,
0x3a050, 0x3a084,
0x3a090, 0x3a09c,
+ 0x3a93c, 0x3a93c,
+ 0x3b93c, 0x3b93c,
+ 0x3c93c, 0x3c93c,
+ 0x3d93c, 0x3d93c,
0x3e000, 0x3e020,
0x3e03c, 0x3e05c,
0x3e100, 0x3e120,
@@ -4743,10 +4777,9 @@ struct intr_details {
struct intr_action {
u32 mask;
int arg;
- bool (*action)(struct adapter *, int, bool);
+ bool (*action)(struct adapter *, int, int);
};
-#define NONFATAL_IF_DISABLED 1
struct intr_info {
const char *name; /* name of the INT_CAUSE register */
int cause_reg; /* INT_CAUSE register */
@@ -4769,73 +4802,78 @@ intr_alert_char(u32 cause, u32 enable, u32 fatal)
}
static void
-t4_show_intr_info(struct adapter *adap, const struct intr_info *ii, u32 cause)
+show_intr_info(struct adapter *sc, const struct intr_info *ii, uint32_t cause,
+ uint32_t ucause, uint32_t enabled, uint32_t fatal, int flags)
{
- u32 enable, fatal, leftover;
+ uint32_t leftover, msgbits;
const struct intr_details *details;
char alert;
+ const bool verbose = flags & IHF_VERBOSE;
- enable = t4_read_reg(adap, ii->enable_reg);
- if (ii->flags & NONFATAL_IF_DISABLED)
- fatal = ii->fatal & t4_read_reg(adap, ii->enable_reg);
- else
- fatal = ii->fatal;
- alert = intr_alert_char(cause, enable, fatal);
- CH_ALERT(adap, "%c %s 0x%x = 0x%08x, E 0x%08x, F 0x%08x\n",
- alert, ii->name, ii->cause_reg, cause, enable, fatal);
+ if (verbose || ucause != 0 || flags & IHF_RUN_ALL_ACTIONS) {
+ alert = intr_alert_char(cause, enabled, fatal);
+ CH_ALERT(sc, "%c %s 0x%x = 0x%08x, E 0x%08x, F 0x%08x\n", alert,
+ ii->name, ii->cause_reg, cause, enabled, fatal);
+ }
- leftover = cause;
+ leftover = verbose ? cause : ucause;
for (details = ii->details; details && details->mask != 0; details++) {
- u32 msgbits = details->mask & cause;
+ msgbits = details->mask & leftover;
if (msgbits == 0)
continue;
- alert = intr_alert_char(msgbits, enable, ii->fatal);
- CH_ALERT(adap, " %c [0x%08x] %s\n", alert, msgbits,
- details->msg);
+ alert = intr_alert_char(msgbits, enabled, fatal);
+ CH_ALERT(sc, " %c [0x%08x] %s\n", alert, msgbits, details->msg);
leftover &= ~msgbits;
}
- if (leftover != 0 && leftover != cause)
- CH_ALERT(adap, " ? [0x%08x]\n", leftover);
+ if (leftover != 0 && leftover != (verbose ? cause : ucause))
+ CH_ALERT(sc, " ? [0x%08x]\n", leftover);
}
/*
* Returns true for fatal error.
*/
static bool
-t4_handle_intr(struct adapter *adap, const struct intr_info *ii,
- u32 additional_cause, bool verbose)
+t4_handle_intr(struct adapter *sc, const struct intr_info *ii, uint32_t acause,
+ int flags)
{
- u32 cause, fatal;
+ uint32_t cause, ucause, enabled, fatal;
bool rc;
const struct intr_action *action;
- /*
- * Read and display cause. Note that the top level PL_INT_CAUSE is a
- * bit special and we need to completely ignore the bits that are not in
- * PL_INT_ENABLE.
- */
- cause = t4_read_reg(adap, ii->cause_reg);
- if (ii->cause_reg == A_PL_INT_CAUSE)
- cause &= t4_read_reg(adap, ii->enable_reg);
- if (verbose || cause != 0)
- t4_show_intr_info(adap, ii, cause);
- fatal = cause & ii->fatal;
- if (fatal != 0 && ii->flags & NONFATAL_IF_DISABLED)
- fatal &= t4_read_reg(adap, ii->enable_reg);
- cause |= additional_cause;
- if (cause == 0)
- return (false);
+ cause = t4_read_reg(sc, ii->cause_reg);
+ enabled = t4_read_reg(sc, ii->enable_reg);
+ flags |= ii->flags;
+ fatal = ii->fatal & cause;
+ if (flags & IHF_FATAL_IFF_ENABLED)
+ fatal &= enabled;
+ ucause = cause;
+ if (flags & IHF_IGNORE_IF_DISABLED)
+ ucause &= enabled;
+ if (!(flags & IHF_NO_SHOW))
+ show_intr_info(sc, ii, cause, ucause, enabled, fatal, flags);
rc = fatal != 0;
for (action = ii->actions; action && action->mask != 0; action++) {
- if (!(action->mask & cause))
+ if (action->action == NULL)
continue;
- rc |= (action->action)(adap, action->arg, verbose);
+ if (action->mask & (ucause | acause) ||
+ flags & IHF_RUN_ALL_ACTIONS) {
+ bool rc1 = (action->action)(sc, action->arg, flags);
+ if (action->mask & ucause)
+ rc |= rc1;
+ }
}
/* clear */
- t4_write_reg(adap, ii->cause_reg, cause);
- (void)t4_read_reg(adap, ii->cause_reg);
+ if (cause != 0) {
+ if (flags & IHF_CLR_ALL_SET) {
+ t4_write_reg(sc, ii->cause_reg, cause);
+ (void)t4_read_reg(sc, ii->cause_reg);
+ } else if (ucause != 0 && flags & IHF_CLR_ALL_UNIGNORED) {
+ t4_write_reg(sc, ii->cause_reg, ucause);
+ (void)t4_read_reg(sc, ii->cause_reg);
+ }
+ }
return (rc);
}
@@ -4843,7 +4881,7 @@ t4_handle_intr(struct adapter *adap, const struct intr_info *ii,
/*
* Interrupt handler for the PCIE module.
*/
-static bool pcie_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool pcie_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details sysbus_intr_details[] = {
{ F_RNPP, "RXNP array parity error" },
@@ -4956,21 +4994,43 @@ static bool pcie_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_PCIE_INT_CAUSE,
.enable_reg = A_PCIE_INT_ENABLE,
.fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ struct intr_info pcie_int_cause_ext = {
+ .name = "PCIE_INT_CAUSE_EXT",
+ .cause_reg = A_PCIE_INT_CAUSE_EXT,
+ .enable_reg = A_PCIE_INT_ENABLE_EXT,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ struct intr_info pcie_int_cause_x8 = {
+ .name = "PCIE_INT_CAUSE_X8",
+ .cause_reg = A_PCIE_INT_CAUSE_X8,
+ .enable_reg = A_PCIE_INT_ENABLE_X8,
+ .fatal = 0,
+ .flags = 0,
.details = NULL,
.actions = NULL,
};
bool fatal = false;
if (is_t4(adap)) {
- fatal |= t4_handle_intr(adap, &sysbus_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &pcie_port_intr_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &sysbus_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &pcie_port_intr_info, 0, flags);
pcie_intr_info.details = pcie_intr_details;
} else {
pcie_intr_info.details = t5_pcie_intr_details;
}
- fatal |= t4_handle_intr(adap, &pcie_intr_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &pcie_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ fatal |= t4_handle_intr(adap, &pcie_int_cause_ext, 0, flags);
+ fatal |= t4_handle_intr(adap, &pcie_int_cause_x8, 0, flags);
+ }
return (fatal);
}
@@ -4978,7 +5038,7 @@ static bool pcie_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* TP interrupt handler.
*/
-static bool tp_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool tp_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details tp_intr_details[] = {
{ 0x3fffffff, "TP parity error" },
@@ -4990,25 +5050,90 @@ static bool tp_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_TP_INT_CAUSE,
.enable_reg = A_TP_INT_ENABLE,
.fatal = 0x7fffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = tp_intr_details,
.actions = NULL,
};
+ static const struct intr_info tp_inic_perr_cause = {
+ .name = "TP_INIC_PERR_CAUSE",
+ .cause_reg = A_TP_INIC_PERR_CAUSE,
+ .enable_reg = A_TP_INIC_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info tp_c_perr_cause = {
+ .name = "TP_C_PERR_CAUSE",
+ .cause_reg = A_TP_C_PERR_CAUSE,
+ .enable_reg = A_TP_C_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info tp_e_eg_perr_cause = {
+ .name = "TP_E_EG_PERR_CAUSE",
+ .cause_reg = A_TP_E_EG_PERR_CAUSE,
+ .enable_reg = A_TP_E_EG_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info tp_e_in0_perr_cause = {
+ .name = "TP_E_IN0_PERR_CAUSE",
+ .cause_reg = A_TP_E_IN0_PERR_CAUSE,
+ .enable_reg = A_TP_E_IN0_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info tp_e_in1_perr_cause = {
+ .name = "TP_E_IN1_PERR_CAUSE",
+ .cause_reg = A_TP_E_IN1_PERR_CAUSE,
+ .enable_reg = A_TP_E_IN1_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info tp_o_perr_cause = {
+ .name = "TP_O_PERR_CAUSE",
+ .cause_reg = A_TP_O_PERR_CAUSE,
+ .enable_reg = A_TP_O_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ bool fatal;
+
+ fatal = t4_handle_intr(adap, &tp_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ fatal |= t4_handle_intr(adap, &tp_inic_perr_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &tp_c_perr_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &tp_e_eg_perr_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &tp_e_in0_perr_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &tp_e_in1_perr_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &tp_o_perr_cause, 0, flags);
+ }
- return (t4_handle_intr(adap, &tp_intr_info, 0, verbose));
+ return (fatal);
}
/*
* SGE interrupt handler.
*/
-static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool sge_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_info sge_int1_info = {
.name = "SGE_INT_CAUSE1",
.cause_reg = A_SGE_INT_CAUSE1,
.enable_reg = A_SGE_INT_ENABLE1,
.fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = NULL,
.actions = NULL,
};
@@ -5017,7 +5142,7 @@ static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_SGE_INT_CAUSE2,
.enable_reg = A_SGE_INT_ENABLE2,
.fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = NULL,
.actions = NULL,
};
@@ -5115,7 +5240,7 @@ static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_SGE_INT_CAUSE5,
.enable_reg = A_SGE_INT_ENABLE5,
.fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = NULL,
.actions = NULL,
};
@@ -5128,7 +5253,24 @@ static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
.details = NULL,
.actions = NULL,
};
-
+ static const struct intr_info sge_int7_info = {
+ .name = "SGE_INT_CAUSE7",
+ .cause_reg = A_SGE_INT_CAUSE7,
+ .enable_reg = A_SGE_INT_ENABLE7,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info sge_int8_info = {
+ .name = "SGE_INT_CAUSE8",
+ .cause_reg = A_SGE_INT_CAUSE8,
+ .enable_reg = A_SGE_INT_ENABLE8,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
bool fatal;
u32 v;
@@ -5139,14 +5281,18 @@ static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
}
fatal = false;
- fatal |= t4_handle_intr(adap, &sge_int1_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &sge_int2_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &sge_int3_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &sge_int4_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &sge_int1_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &sge_int2_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &sge_int3_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &sge_int4_info, 0, flags);
if (chip_id(adap) >= CHELSIO_T5)
- fatal |= t4_handle_intr(adap, &sge_int5_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &sge_int5_info, 0, flags);
if (chip_id(adap) >= CHELSIO_T6)
- fatal |= t4_handle_intr(adap, &sge_int6_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &sge_int6_info, 0, flags);
+ if (chip_id(adap) >= CHELSIO_T7) {
+ fatal |= t4_handle_intr(adap, &sge_int7_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &sge_int8_info, 0, flags);
+ }
v = t4_read_reg(adap, A_SGE_ERROR_STATS);
if (v & F_ERROR_QID_VALID) {
@@ -5163,7 +5309,7 @@ static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* CIM interrupt handler.
*/
-static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool cim_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details cim_host_intr_details[] = {
/* T6+ */
@@ -5208,7 +5354,7 @@ static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_CIM_HOST_INT_CAUSE,
.enable_reg = A_CIM_HOST_INT_ENABLE,
.fatal = 0x007fffe6,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = cim_host_intr_details,
.actions = NULL,
};
@@ -5259,7 +5405,7 @@ static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_CIM_HOST_UPACC_INT_CAUSE,
.enable_reg = A_CIM_HOST_UPACC_INT_ENABLE,
.fatal = 0x3fffeeff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = cim_host_upacc_intr_details,
.actions = NULL,
};
@@ -5272,6 +5418,15 @@ static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
.details = NULL,
.actions = NULL,
};
+ static const struct intr_info cim_perr_cause = {
+ .name = "CIM_PERR_CAUSE",
+ .cause_reg = A_CIM_PERR_CAUSE,
+ .enable_reg = A_CIM_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
u32 val, fw_err;
bool fatal;
@@ -5290,9 +5445,11 @@ static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
}
fatal = (fw_err & F_PCIE_FW_ERR) != 0;
- fatal |= t4_handle_intr(adap, &cim_host_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &cim_host_upacc_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &cim_pf_host_intr_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &cim_host_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &cim_host_upacc_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &cim_pf_host_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6)
+ fatal |= t4_handle_intr(adap, &cim_perr_cause, 0, flags);
if (fatal)
t4_os_cim_err(adap);
@@ -5302,7 +5459,7 @@ static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* ULP RX interrupt handler.
*/
-static bool ulprx_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool ulprx_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details ulprx_intr_details[] = {
/* T5+ */
@@ -5320,7 +5477,7 @@ static bool ulprx_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_ULP_RX_INT_CAUSE,
.enable_reg = A_ULP_RX_INT_ENABLE,
.fatal = 0x07ffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = ulprx_intr_details,
.actions = NULL,
};
@@ -5333,10 +5490,53 @@ static bool ulprx_intr_handler(struct adapter *adap, int arg, bool verbose)
.details = NULL,
.actions = NULL,
};
+ static const struct intr_info ulprx_int_cause_pcmd = {
+ .name = "ULP_RX_INT_CAUSE_PCMD",
+ .cause_reg = A_ULP_RX_INT_CAUSE_PCMD,
+ .enable_reg = A_ULP_RX_INT_ENABLE_PCMD,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulprx_int_cause_data = {
+ .name = "ULP_RX_INT_CAUSE_DATA",
+ .cause_reg = A_ULP_RX_INT_CAUSE_DATA,
+ .enable_reg = A_ULP_RX_INT_ENABLE_DATA,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulprx_int_cause_arb = {
+ .name = "ULP_RX_INT_CAUSE_ARB",
+ .cause_reg = A_ULP_RX_INT_CAUSE_ARB,
+ .enable_reg = A_ULP_RX_INT_ENABLE_ARB,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulprx_int_cause_intf = {
+ .name = "ULP_RX_INT_CAUSE_INTERFACE",
+ .cause_reg = A_ULP_RX_INT_CAUSE_INTERFACE,
+ .enable_reg = A_ULP_RX_INT_ENABLE_INTERFACE,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
bool fatal = false;
- fatal |= t4_handle_intr(adap, &ulprx_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &ulprx_intr2_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &ulprx_intr_info, 0, flags);
+ if (chip_id(adap) < CHELSIO_T7)
+ fatal |= t4_handle_intr(adap, &ulprx_intr2_info, 0, flags);
+ else {
+ fatal |= t4_handle_intr(adap, &ulprx_int_cause_pcmd, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulprx_int_cause_data, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulprx_int_cause_arb, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulprx_int_cause_intf, 0, flags);
+ }
return (fatal);
}
@@ -5344,7 +5544,7 @@ static bool ulprx_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* ULP TX interrupt handler.
*/
-static bool ulptx_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool ulptx_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details ulptx_intr_details[] = {
{ F_PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds" },
@@ -5359,32 +5559,98 @@ static bool ulptx_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_ULP_TX_INT_CAUSE,
.enable_reg = A_ULP_TX_INT_ENABLE,
.fatal = 0x0fffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = ulptx_intr_details,
.actions = NULL,
};
- static const struct intr_info ulptx_intr2_info = {
+ static const struct intr_info ulptx_intr_info2 = {
.name = "ULP_TX_INT_CAUSE_2",
.cause_reg = A_ULP_TX_INT_CAUSE_2,
.enable_reg = A_ULP_TX_INT_ENABLE_2,
- .fatal = 0xf0,
- .flags = NONFATAL_IF_DISABLED,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info3 = {
+ .name = "ULP_TX_INT_CAUSE_3",
+ .cause_reg = A_ULP_TX_INT_CAUSE_3,
+ .enable_reg = A_ULP_TX_INT_ENABLE_3,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info4 = {
+ .name = "ULP_TX_INT_CAUSE_4",
+ .cause_reg = A_ULP_TX_INT_CAUSE_4,
+ .enable_reg = A_ULP_TX_INT_ENABLE_4,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info5 = {
+ .name = "ULP_TX_INT_CAUSE_5",
+ .cause_reg = A_ULP_TX_INT_CAUSE_5,
+ .enable_reg = A_ULP_TX_INT_ENABLE_5,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info6 = {
+ .name = "ULP_TX_INT_CAUSE_6",
+ .cause_reg = A_ULP_TX_INT_CAUSE_6,
+ .enable_reg = A_ULP_TX_INT_ENABLE_6,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info7 = {
+ .name = "ULP_TX_INT_CAUSE_7",
+ .cause_reg = A_ULP_TX_INT_CAUSE_7,
+ .enable_reg = A_ULP_TX_INT_ENABLE_7,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info ulptx_intr_info8 = {
+ .name = "ULP_TX_INT_CAUSE_8",
+ .cause_reg = A_ULP_TX_INT_CAUSE_8,
+ .enable_reg = A_ULP_TX_INT_ENABLE_8,
+ .fatal = 0,
+ .flags = 0,
.details = NULL,
.actions = NULL,
};
bool fatal = false;
- fatal |= t4_handle_intr(adap, &ulptx_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &ulptx_intr2_info, 0, verbose);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T4)
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info2, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info3, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info4, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info5, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info6, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info7, 0, flags);
+ fatal |= t4_handle_intr(adap, &ulptx_intr_info8, 0, flags);
+ }
return (fatal);
}
-static bool pmtx_dump_dbg_stats(struct adapter *adap, int arg, bool verbose)
+static bool pmtx_dump_dbg_stats(struct adapter *adap, int arg, int flags)
{
int i;
u32 data[17];
+ if (flags & IHF_NO_SHOW)
+ return (false);
+
t4_read_indirect(adap, A_PM_TX_DBG_CTRL, A_PM_TX_DBG_DATA, &data[0],
ARRAY_SIZE(data), A_PM_TX_DBG_STAT0);
for (i = 0; i < ARRAY_SIZE(data); i++) {
@@ -5398,13 +5664,9 @@ static bool pmtx_dump_dbg_stats(struct adapter *adap, int arg, bool verbose)
/*
* PM TX interrupt handler.
*/
-static bool pmtx_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool pmtx_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_action pmtx_intr_actions[] = {
- { 0xffffffff, 0, pmtx_dump_dbg_stats },
- { 0 },
- };
- static const struct intr_details pmtx_intr_details[] = {
+ static const struct intr_details pmtx_int_cause_fields[] = {
{ F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large" },
{ F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large" },
{ F_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large" },
@@ -5421,25 +5683,29 @@ static bool pmtx_intr_handler(struct adapter *adap, int arg, bool verbose)
{ F_C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error" },
{ 0 }
};
- static const struct intr_info pmtx_intr_info = {
+ static const struct intr_action pmtx_int_cause_actions[] = {
+ { 0xffffffff, -1, pmtx_dump_dbg_stats },
+ { 0 },
+ };
+ static const struct intr_info pmtx_int_cause = {
.name = "PM_TX_INT_CAUSE",
.cause_reg = A_PM_TX_INT_CAUSE,
.enable_reg = A_PM_TX_INT_ENABLE,
.fatal = 0xffffffff,
.flags = 0,
- .details = pmtx_intr_details,
- .actions = pmtx_intr_actions,
+ .details = pmtx_int_cause_fields,
+ .actions = pmtx_int_cause_actions,
};
- return (t4_handle_intr(adap, &pmtx_intr_info, 0, verbose));
+ return (t4_handle_intr(adap, &pmtx_int_cause, 0, flags));
}
/*
* PM RX interrupt handler.
*/
-static bool pmrx_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool pmrx_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_details pmrx_intr_details[] = {
+ static const struct intr_details pmrx_int_cause_fields[] = {
/* T6+ */
{ 0x18000000, "PMRX ospi overflow" },
{ F_MA_INTF_SDC_ERR, "PMRX MA interface SDC parity error" },
@@ -5461,25 +5727,25 @@ static bool pmrx_intr_handler(struct adapter *adap, int arg, bool verbose)
{ F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error"},
{ 0 }
};
- static const struct intr_info pmrx_intr_info = {
+ static const struct intr_info pmrx_int_cause = {
.name = "PM_RX_INT_CAUSE",
.cause_reg = A_PM_RX_INT_CAUSE,
.enable_reg = A_PM_RX_INT_ENABLE,
.fatal = 0x1fffffff,
- .flags = NONFATAL_IF_DISABLED,
- .details = pmrx_intr_details,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = pmrx_int_cause_fields,
.actions = NULL,
};
- return (t4_handle_intr(adap, &pmrx_intr_info, 0, verbose));
+ return (t4_handle_intr(adap, &pmrx_int_cause, 0, flags));
}
/*
* CPL switch interrupt handler.
*/
-static bool cplsw_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool cplsw_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_details cplsw_intr_details[] = {
+ static const struct intr_details cplsw_int_cause_fields[] = {
/* T5+ */
{ F_PERR_CPL_128TO128_1, "CPLSW 128TO128 FIFO1 parity error" },
{ F_PERR_CPL_128TO128_0, "CPLSW 128TO128 FIFO0 parity error" },
@@ -5493,17 +5759,17 @@ static bool cplsw_intr_handler(struct adapter *adap, int arg, bool verbose)
{ F_ZERO_SWITCH_ERROR, "CPLSW no-switch error" },
{ 0 }
};
- static const struct intr_info cplsw_intr_info = {
+ static const struct intr_info cplsw_int_cause = {
.name = "CPL_INTR_CAUSE",
.cause_reg = A_CPL_INTR_CAUSE,
.enable_reg = A_CPL_INTR_ENABLE,
- .fatal = 0xff,
- .flags = NONFATAL_IF_DISABLED,
- .details = cplsw_intr_details,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = cplsw_int_cause_fields,
.actions = NULL,
};
- return (t4_handle_intr(adap, &cplsw_intr_info, 0, verbose));
+ return (t4_handle_intr(adap, &cplsw_int_cause, 0, flags));
}
#define T4_LE_FATAL_MASK (F_PARITYERR | F_UNKNOWNCMD | F_REQQPARERR)
@@ -5515,11 +5781,12 @@ static bool cplsw_intr_handler(struct adapter *adap, int arg, bool verbose)
#define T6_LE_FATAL_MASK (T6_LE_PERRCRC_MASK | F_T6_UNKNOWNCMD | \
F_TCAMACCFAIL | F_HASHTBLACCFAIL | F_CMDTIDERR | F_CMDPRSRINTERR | \
F_TOTCNTERR | F_CLCAMFIFOERR | F_CLIPSUBERR)
+#define T7_LE_FATAL_MASK (T6_LE_FATAL_MASK | F_CACHESRAMPERR | F_CACHEINTPERR)
/*
* LE interrupt handler.
*/
-static bool le_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool le_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details le_intr_details[] = {
{ F_REQQPARERR, "LE request queue parity error" },
@@ -5556,7 +5823,7 @@ static bool le_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_LE_DB_INT_CAUSE,
.enable_reg = A_LE_DB_INT_ENABLE,
.fatal = 0,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = NULL,
.actions = NULL,
};
@@ -5566,16 +5833,19 @@ static bool le_intr_handler(struct adapter *adap, int arg, bool verbose)
le_intr_info.fatal = T5_LE_FATAL_MASK;
} else {
le_intr_info.details = t6_le_intr_details;
- le_intr_info.fatal = T6_LE_FATAL_MASK;
+ if (chip_id(adap) < CHELSIO_T7)
+ le_intr_info.fatal = T6_LE_FATAL_MASK;
+ else
+ le_intr_info.fatal = T7_LE_FATAL_MASK;
}
- return (t4_handle_intr(adap, &le_intr_info, 0, verbose));
+ return (t4_handle_intr(adap, &le_intr_info, 0, flags));
}
/*
* MPS interrupt handler.
*/
-static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool mps_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_details mps_rx_perr_intr_details[] = {
{ 0xffffffff, "MPS Rx parity error" },
@@ -5586,10 +5856,55 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_MPS_RX_PERR_INT_CAUSE,
.enable_reg = A_MPS_RX_PERR_INT_ENABLE,
.fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = mps_rx_perr_intr_details,
.actions = NULL,
};
+ static const struct intr_info mps_rx_perr_intr_info2 = {
+ .name = "MPS_RX_PERR_INT_CAUSE2",
+ .cause_reg = A_MPS_RX_PERR_INT_CAUSE2,
+ .enable_reg = A_MPS_RX_PERR_INT_ENABLE2,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_rx_perr_intr_info3 = {
+ .name = "MPS_RX_PERR_INT_CAUSE3",
+ .cause_reg = A_MPS_RX_PERR_INT_CAUSE3,
+ .enable_reg = A_MPS_RX_PERR_INT_ENABLE3,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_rx_perr_intr_info4 = {
+ .name = "MPS_RX_PERR_INT_CAUSE4",
+ .cause_reg = A_MPS_RX_PERR_INT_CAUSE4,
+ .enable_reg = A_MPS_RX_PERR_INT_ENABLE4,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_rx_perr_intr_info5 = {
+ .name = "MPS_RX_PERR_INT_CAUSE5",
+ .cause_reg = A_MPS_RX_PERR_INT_CAUSE5,
+ .enable_reg = A_MPS_RX_PERR_INT_ENABLE5,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_rx_perr_intr_info6 = {
+ .name = "MPS_RX_PERR_INT_CAUSE6",
+ .cause_reg = A_MPS_RX_PERR_INT_CAUSE6,
+ .enable_reg = A_MPS_RX_PERR_INT_ENABLE6,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
static const struct intr_details mps_tx_intr_details[] = {
{ F_PORTERR, "MPS Tx destination port is disabled" },
{ F_FRMERR, "MPS Tx framing error" },
@@ -5606,10 +5921,37 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_MPS_TX_INT_CAUSE,
.enable_reg = A_MPS_TX_INT_ENABLE,
.fatal = 0x1ffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = mps_tx_intr_details,
.actions = NULL,
};
+ static const struct intr_info mps_tx_intr_info2 = {
+ .name = "MPS_TX_INT2_CAUSE",
+ .cause_reg = A_MPS_TX_INT2_CAUSE,
+ .enable_reg = A_MPS_TX_INT2_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_tx_intr_info3 = {
+ .name = "MPS_TX_INT3_CAUSE",
+ .cause_reg = A_MPS_TX_INT3_CAUSE,
+ .enable_reg = A_MPS_TX_INT3_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mps_tx_intr_info4 = {
+ .name = "MPS_TX_INT4_CAUSE",
+ .cause_reg = A_MPS_TX_INT4_CAUSE,
+ .enable_reg = A_MPS_TX_INT4_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
static const struct intr_details mps_trc_intr_details[] = {
{ F_MISCPERR, "MPS TRC misc parity error" },
{ V_PKTFIFO(M_PKTFIFO), "MPS TRC packet FIFO parity error" },
@@ -5626,14 +5968,23 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.actions = NULL,
};
static const struct intr_info t7_mps_trc_intr_info = {
- .name = "T7_MPS_TRC_INT_CAUSE",
+ .name = "MPS_TRC_INT_CAUSE",
.cause_reg = A_T7_MPS_TRC_INT_CAUSE,
.enable_reg = A_T7_MPS_TRC_INT_ENABLE,
- .fatal = F_MISCPERR | V_PKTFIFO(M_PKTFIFO) | V_FILTMEM(M_FILTMEM),
- .flags = 0,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = mps_trc_intr_details,
.actions = NULL,
};
+ static const struct intr_info t7_mps_trc_intr_info2 = {
+ .name = "MPS_TRC_INT_CAUSE2",
+ .cause_reg = A_MPS_TRC_INT_CAUSE2,
+ .enable_reg = A_MPS_TRC_INT_ENABLE2,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
static const struct intr_details mps_stat_sram_intr_details[] = {
{ 0xffffffff, "MPS statistics SRAM parity error" },
{ 0 }
@@ -5643,7 +5994,7 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_SRAM,
.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_SRAM,
.fatal = 0x1fffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = mps_stat_sram_intr_details,
.actions = NULL,
};
@@ -5656,7 +6007,7 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_TX_FIFO,
.fatal = 0xffffff,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = mps_stat_tx_intr_details,
.actions = NULL,
};
@@ -5701,24 +6052,31 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
.details = mps_stat_sram1_intr_details,
.actions = NULL,
};
+ bool fatal = false;
- bool fatal;
-
- fatal = false;
- fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &mps_tx_intr_info, 0, verbose);
- if (chip_id(adap) > CHELSIO_T6)
- fatal |= t4_handle_intr(adap, &t7_mps_trc_intr_info, 0, verbose);
- else
- fatal |= t4_handle_intr(adap, &mps_trc_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &mps_stat_sram_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &mps_stat_tx_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &mps_stat_rx_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &mps_cls_intr_info, 0, verbose);
- if (chip_id(adap) > CHELSIO_T4) {
- fatal |= t4_handle_intr(adap, &mps_stat_sram1_intr_info, 0,
- verbose);
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info2, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info3, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info4, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info5, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info6, 0, flags);
}
+ fatal |= t4_handle_intr(adap, &mps_tx_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ fatal |= t4_handle_intr(adap, &mps_tx_intr_info2, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_tx_intr_info3, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_tx_intr_info4, 0, flags);
+ fatal |= t4_handle_intr(adap, &t7_mps_trc_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &t7_mps_trc_intr_info2, 0, flags);
+ } else
+ fatal |= t4_handle_intr(adap, &mps_trc_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_stat_sram_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_stat_tx_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_stat_rx_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &mps_cls_intr_info, 0, flags);
+ if (chip_id(adap) > CHELSIO_T4)
+ fatal |= t4_handle_intr(adap, &mps_stat_sram1_intr_info, 0, flags);
t4_write_reg(adap, A_MPS_INT_CAUSE, is_t4(adap) ? 0 : 0xffffffff);
t4_read_reg(adap, A_MPS_INT_CAUSE); /* flush */
@@ -5730,7 +6088,7 @@ static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* EDC/MC interrupt handler.
*/
-static bool mem_intr_handler(struct adapter *adap, int idx, bool verbose)
+static bool mem_intr_handler(struct adapter *adap, int idx, int flags)
{
static const char name[4][5] = { "EDC0", "EDC1", "MC0", "MC1" };
unsigned int count_reg, v;
@@ -5740,61 +6098,106 @@ static bool mem_intr_handler(struct adapter *adap, int idx, bool verbose)
{ F_PERR_INT_CAUSE, "FIFO parity error" },
{ 0 }
};
+ char rname[32];
struct intr_info ii = {
+ .name = &rname[0],
.fatal = F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE,
.details = mem_intr_details,
.flags = 0,
.actions = NULL,
};
- bool fatal;
+ bool fatal = false;
+ int i = 0;
switch (idx) {
+ case MEM_EDC1: i = 1;
+ /* fall through */
case MEM_EDC0:
- ii.name = "EDC0_INT_CAUSE";
- ii.cause_reg = EDC_REG(A_EDC_INT_CAUSE, 0);
- ii.enable_reg = EDC_REG(A_EDC_INT_ENABLE, 0);
- count_reg = EDC_REG(A_EDC_ECC_STATUS, 0);
- break;
- case MEM_EDC1:
- ii.name = "EDC1_INT_CAUSE";
- ii.cause_reg = EDC_REG(A_EDC_INT_CAUSE, 1);
- ii.enable_reg = EDC_REG(A_EDC_INT_ENABLE, 1);
- count_reg = EDC_REG(A_EDC_ECC_STATUS, 1);
+ snprintf(rname, sizeof(rname), "EDC%u_INT_CAUSE", i);
+ if (is_t4(adap)) {
+ ii.cause_reg = EDC_REG(A_EDC_INT_CAUSE, i);
+ ii.enable_reg = EDC_REG(A_EDC_INT_ENABLE, i);
+ count_reg = EDC_REG(A_EDC_ECC_STATUS, i);
+ } else {
+ ii.cause_reg = EDC_T5_REG(A_EDC_H_INT_CAUSE, i);
+ ii.enable_reg = EDC_T5_REG(A_EDC_H_INT_ENABLE, i);
+ count_reg = EDC_T5_REG(A_EDC_H_ECC_STATUS, i);
+ }
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6) {
+ snprintf(rname, sizeof(rname), "EDC%u_PAR_CAUSE", i);
+ ii.cause_reg = EDC_T5_REG(A_EDC_H_PAR_CAUSE, i);
+ ii.enable_reg = EDC_T5_REG(A_EDC_H_PAR_ENABLE, i);
+ ii.fatal = 0xffffffff;
+ ii.details = NULL;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ }
break;
+ case MEM_MC1:
+ if (is_t4(adap) || is_t6(adap))
+ return (false);
+ i = 1;
+ /* fall through */
case MEM_MC0:
- ii.name = "MC0_INT_CAUSE";
+ snprintf(rname, sizeof(rname), "MC%u_INT_CAUSE", i);
if (is_t4(adap)) {
ii.cause_reg = A_MC_INT_CAUSE;
ii.enable_reg = A_MC_INT_ENABLE;
count_reg = A_MC_ECC_STATUS;
+ } else if (chip_id(adap) < CHELSIO_T7) {
+ ii.cause_reg = MC_REG(A_MC_P_INT_CAUSE, i);
+ ii.enable_reg = MC_REG(A_MC_P_INT_ENABLE, i);
+ count_reg = MC_REG(A_MC_P_ECC_STATUS, i);
} else {
- ii.cause_reg = A_MC_P_INT_CAUSE;
- ii.enable_reg = A_MC_P_INT_ENABLE;
- count_reg = A_MC_P_ECC_STATUS;
+ ii.cause_reg = MC_T7_REG(A_T7_MC_P_INT_CAUSE, i);
+ ii.enable_reg = MC_T7_REG(A_T7_MC_P_INT_ENABLE, i);
+ count_reg = MC_T7_REG(A_T7_MC_P_ECC_STATUS, i);
+ }
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+
+ snprintf(rname, sizeof(rname), "MC%u_PAR_CAUSE", i);
+ if (is_t4(adap)) {
+ ii.cause_reg = A_MC_PAR_CAUSE;
+ ii.enable_reg = A_MC_PAR_ENABLE;
+ } else if (chip_id(adap) < CHELSIO_T7) {
+ ii.cause_reg = MC_REG(A_MC_P_PAR_CAUSE, i);
+ ii.enable_reg = MC_REG(A_MC_P_PAR_ENABLE, i);
+ } else {
+ ii.cause_reg = MC_T7_REG(A_T7_MC_P_PAR_CAUSE, i);
+ ii.enable_reg = MC_T7_REG(A_T7_MC_P_PAR_ENABLE, i);
+ }
+ ii.fatal = 0xffffffff;
+ ii.details = NULL;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+
+ if (chip_id(adap) > CHELSIO_T6) {
+ snprintf(rname, sizeof(rname), "MC%u_DDRCTL_INT_CAUSE", i);
+ ii.cause_reg = MC_T7_REG(A_MC_P_DDRCTL_INT_CAUSE, i);
+ ii.enable_reg = MC_T7_REG(A_MC_P_DDRCTL_INT_ENABLE, i);
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+
+ snprintf(rname, sizeof(rname), "MC%u_ECC_UE_INT_CAUSE", i);
+ ii.cause_reg = MC_T7_REG(A_MC_P_ECC_UE_INT_CAUSE, i);
+ ii.enable_reg = MC_T7_REG(A_MC_P_ECC_UE_INT_ENABLE, i);
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
}
- break;
- case MEM_MC1:
- ii.name = "MC1_INT_CAUSE";
- ii.cause_reg = MC_REG(A_MC_P_INT_CAUSE, 1);
- ii.enable_reg = MC_REG(A_MC_P_INT_ENABLE, 1);
- count_reg = MC_REG(A_MC_P_ECC_STATUS, 1);
break;
}
- fatal = t4_handle_intr(adap, &ii, 0, verbose);
-
v = t4_read_reg(adap, count_reg);
if (v != 0) {
- if (G_ECC_UECNT(v) != 0) {
+ if (G_ECC_UECNT(v) != 0 && !(flags & IHF_NO_SHOW)) {
CH_ALERT(adap,
- "%s: %u uncorrectable ECC data error(s)\n",
+ " %s: %u uncorrectable ECC data error(s)\n",
name[idx], G_ECC_UECNT(v));
}
- if (G_ECC_CECNT(v) != 0) {
+ if (G_ECC_CECNT(v) != 0 && !(flags & IHF_NO_SHOW)) {
if (idx <= MEM_EDC1)
t4_edc_err_read(adap, idx);
CH_WARN_RATELIMIT(adap,
- "%s: %u correctable ECC data error(s)\n",
+ " %s: %u correctable ECC data error(s)\n",
name[idx], G_ECC_CECNT(v));
}
t4_write_reg(adap, count_reg, 0xffffffff);
@@ -5803,14 +6206,16 @@ static bool mem_intr_handler(struct adapter *adap, int idx, bool verbose)
return (fatal);
}
-static bool ma_wrap_status(struct adapter *adap, int arg, bool verbose)
+static bool ma_wrap_status(struct adapter *adap, int arg, int flags)
{
u32 v;
v = t4_read_reg(adap, A_MA_INT_WRAP_STATUS);
- CH_ALERT(adap,
- "MA address wrap-around error by client %u to address %#x\n",
- G_MEM_WRAP_CLIENT_NUM(v), G_MEM_WRAP_ADDRESS(v) << 4);
+ if (!(flags & IHF_NO_SHOW)) {
+ CH_ALERT(adap,
+ " MA address wrap-around by client %u to address %#x\n",
+ G_MEM_WRAP_CLIENT_NUM(v), G_MEM_WRAP_ADDRESS(v) << 4);
+ }
t4_write_reg(adap, A_MA_INT_WRAP_STATUS, v);
return (false);
@@ -5820,7 +6225,7 @@ static bool ma_wrap_status(struct adapter *adap, int arg, bool verbose)
/*
* MA interrupt handler.
*/
-static bool ma_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool ma_intr_handler(struct adapter *adap, int arg, int flags)
{
static const struct intr_action ma_intr_actions[] = {
{ F_MEM_WRAP_INT_CAUSE, 0, ma_wrap_status },
@@ -5831,7 +6236,7 @@ static bool ma_intr_handler(struct adapter *adap, int arg, bool verbose)
.cause_reg = A_MA_INT_CAUSE,
.enable_reg = A_MA_INT_ENABLE,
.fatal = F_MEM_PERR_INT_CAUSE | F_MEM_TO_INT_CAUSE,
- .flags = NONFATAL_IF_DISABLED,
+ .flags = IHF_FATAL_IFF_ENABLED,
.details = NULL,
.actions = ma_intr_actions,
};
@@ -5856,10 +6261,10 @@ static bool ma_intr_handler(struct adapter *adap, int arg, bool verbose)
bool fatal;
fatal = false;
- fatal |= t4_handle_intr(adap, &ma_intr_info, 0, verbose);
- fatal |= t4_handle_intr(adap, &ma_perr_status1, 0, verbose);
+ fatal |= t4_handle_intr(adap, &ma_intr_info, 0, flags);
+ fatal |= t4_handle_intr(adap, &ma_perr_status1, 0, flags);
if (chip_id(adap) > CHELSIO_T4)
- fatal |= t4_handle_intr(adap, &ma_perr_status2, 0, verbose);
+ fatal |= t4_handle_intr(adap, &ma_perr_status2, 0, flags);
return (fatal);
}
@@ -5867,58 +6272,115 @@ static bool ma_intr_handler(struct adapter *adap, int arg, bool verbose)
/*
* SMB interrupt handler.
*/
-static bool smb_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool smb_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_details smb_intr_details[] = {
+ static const struct intr_details smb_int_cause_fields[] = {
{ F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error" },
{ F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error" },
{ F_SLVFIFOPARINT, "SMB slave FIFO parity error" },
{ 0 }
};
- static const struct intr_info smb_intr_info = {
+ static const struct intr_info smb_int_cause = {
.name = "SMB_INT_CAUSE",
.cause_reg = A_SMB_INT_CAUSE,
.enable_reg = A_SMB_INT_ENABLE,
.fatal = F_SLVFIFOPARINT | F_MSTRXFIFOPARINT | F_MSTTXFIFOPARINT,
.flags = 0,
- .details = smb_intr_details,
+ .details = smb_int_cause_fields,
.actions = NULL,
};
-
- return (t4_handle_intr(adap, &smb_intr_info, 0, verbose));
+ return (t4_handle_intr(adap, &smb_int_cause, 0, flags));
}
/*
* NC-SI interrupt handler.
*/
-static bool ncsi_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool ncsi_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_details ncsi_intr_details[] = {
+ static const struct intr_details ncsi_int_cause_fields[] = {
{ F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error" },
{ F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error" },
{ F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error" },
{ F_RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error" },
{ 0 }
};
- static const struct intr_info ncsi_intr_info = {
+ static const struct intr_info ncsi_int_cause = {
.name = "NCSI_INT_CAUSE",
.cause_reg = A_NCSI_INT_CAUSE,
.enable_reg = A_NCSI_INT_ENABLE,
.fatal = F_RXFIFO_PRTY_ERR | F_TXFIFO_PRTY_ERR |
F_MPS_DM_PRTY_ERR | F_CIM_DM_PRTY_ERR,
.flags = 0,
- .details = ncsi_intr_details,
+ .details = ncsi_int_cause_fields,
+ .actions = NULL,
+ };
+ static const struct intr_info ncsi_xgmac0_int_cause = {
+ .name = "NCSI_XGMAC0_INT_CAUSE",
+ .cause_reg = A_NCSI_XGMAC0_INT_CAUSE,
+ .enable_reg = A_NCSI_XGMAC0_INT_ENABLE,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
.actions = NULL,
};
+ bool fatal = false;
- return (t4_handle_intr(adap, &ncsi_intr_info, 0, verbose));
+ fatal |= t4_handle_intr(adap, &ncsi_int_cause, 0, flags);
+ if (chip_id(adap) > CHELSIO_T6)
+ fatal |= t4_handle_intr(adap, &ncsi_xgmac0_int_cause, 0, flags);
+ return (fatal);
}
/*
* MAC interrupt handler.
*/
-static bool mac_intr_handler(struct adapter *adap, int port, bool verbose)
+static bool mac_intr_handler(struct adapter *adap, int port, int flags)
{
+ static const struct intr_info mac_int_cause_cmn = {
+ .name = "MAC_INT_CAUSE_CMN",
+ .cause_reg = A_MAC_INT_CAUSE_CMN,
+ .enable_reg = A_MAC_INT_EN_CMN,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mac_perr_cause_mtip = {
+ .name = "MAC_PERR_INT_CAUSE_MTIP",
+ .cause_reg = A_MAC_PERR_INT_CAUSE_MTIP,
+ .enable_reg = A_MAC_PERR_INT_EN_MTIP,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED | IHF_IGNORE_IF_DISABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mac_cerr_cause_mtip = {
+ .name = "MAC_CERR_INT_CAUSE_MTIP",
+ .cause_reg = A_MAC_CERR_INT_CAUSE_MTIP,
+ .enable_reg = A_MAC_CERR_INT_EN_MTIP,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mac_ios_int_cause_quad0 = {
+ .name = "MAC_IOS_INTR_CAUSE_QUAD0",
+ .cause_reg = A_MAC_IOS_INTR_CAUSE_QUAD0,
+ .enable_reg = A_MAC_IOS_INTR_EN_QUAD0,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info mac_ios_int_cause_quad1 = {
+ .name = "MAC_IOS_INTR_CAUSE_QUAD1",
+ .cause_reg = A_MAC_IOS_INTR_CAUSE_QUAD1,
+ .enable_reg = A_MAC_IOS_INTR_EN_QUAD1,
+ .fatal = 0,
+ .flags = 0,
+ .details = NULL,
+ .actions = NULL,
+ };
static const struct intr_details mac_intr_details[] = {
{ F_TXFIFO_PRTY_ERR, "MAC Tx FIFO parity error" },
{ F_RXFIFO_PRTY_ERR, "MAC Rx FIFO parity error" },
@@ -5928,6 +6390,9 @@ static bool mac_intr_handler(struct adapter *adap, int port, bool verbose)
struct intr_info ii;
bool fatal = false;
+ if (port > 1 && is_t6(adap))
+ return (false);
+
if (is_t4(adap)) {
snprintf(name, sizeof(name), "XGMAC_PORT%u_INT_CAUSE", port);
ii.name = &name[0];
@@ -5947,66 +6412,79 @@ static bool mac_intr_handler(struct adapter *adap, int port, bool verbose)
ii.details = mac_intr_details;
ii.actions = NULL;
} else {
- snprintf(name, sizeof(name), "T7_MAC_PORT%u_INT_CAUSE", port);
+ snprintf(name, sizeof(name), "MAC_PORT%u_INT_CAUSE", port);
ii.name = &name[0];
ii.cause_reg = T7_PORT_REG(port, A_T7_MAC_PORT_INT_CAUSE);
ii.enable_reg = T7_PORT_REG(port, A_T7_MAC_PORT_INT_EN);
- ii.fatal = F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR;
- ii.flags = 0;
- ii.details = mac_intr_details;
+ ii.fatal = 0xffffffff;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
+ ii.details = NULL;
ii.actions = NULL;
}
- fatal |= t4_handle_intr(adap, &ii, 0, verbose);
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ if (is_t4(adap))
+ return (fatal);
+ MPASS(chip_id(adap) >= CHELSIO_T5);
+ snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE", port);
if (chip_id(adap) > CHELSIO_T6) {
- snprintf(name, sizeof(name), "T7_MAC_PORT%u_PERR_INT_CAUSE", port);
ii.name = &name[0];
ii.cause_reg = T7_PORT_REG(port, A_T7_MAC_PORT_PERR_INT_CAUSE);
ii.enable_reg = T7_PORT_REG(port, A_T7_MAC_PORT_PERR_INT_EN);
- ii.fatal = 0;
- ii.flags = 0;
+ ii.fatal = 0xffffffff;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
ii.details = NULL;
ii.actions = NULL;
- fatal |= t4_handle_intr(adap, &ii, 0, verbose);
- } else if (chip_id(adap) >= CHELSIO_T5) {
- snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE", port);
+ } else {
ii.name = &name[0];
ii.cause_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_CAUSE);
ii.enable_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_EN);
- ii.fatal = 0;
- ii.flags = 0;
+ ii.fatal = 0xffffffff;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
ii.details = NULL;
ii.actions = NULL;
- fatal |= t4_handle_intr(adap, &ii, 0, verbose);
}
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ if (is_t5(adap))
+ return (fatal);
+ MPASS(chip_id(adap) >= CHELSIO_T6);
+ snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE_100G", port);
if (chip_id(adap) > CHELSIO_T6) {
- snprintf(name, sizeof(name), "T7_MAC_PORT%u_PERR_INT_CAUSE_100G", port);
ii.name = &name[0];
ii.cause_reg = T7_PORT_REG(port, A_T7_MAC_PORT_PERR_INT_CAUSE_100G);
ii.enable_reg = T7_PORT_REG(port, A_T7_MAC_PORT_PERR_INT_EN_100G);
- ii.fatal = 0;
- ii.flags = 0;
+ ii.fatal = 0xffffffff;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
ii.details = NULL;
ii.actions = NULL;
- fatal |= t4_handle_intr(adap, &ii, 0, verbose);
- } else if (is_t6(adap)) {
- snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE_100G", port);
+ } else {
ii.name = &name[0];
ii.cause_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_CAUSE_100G);
ii.enable_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_EN_100G);
- ii.fatal = 0;
- ii.flags = 0;
+ ii.fatal = 0xffffffff;
+ ii.flags = IHF_FATAL_IFF_ENABLED;
ii.details = NULL;
ii.actions = NULL;
- fatal |= t4_handle_intr(adap, &ii, 0, verbose);
}
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ if (is_t6(adap))
+ return (fatal);
+
+ MPASS(chip_id(adap) >= CHELSIO_T7);
+ fatal |= t4_handle_intr(adap, &mac_int_cause_cmn, 0, flags);
+ fatal |= t4_handle_intr(adap, &mac_perr_cause_mtip, 0, flags);
+ fatal |= t4_handle_intr(adap, &mac_cerr_cause_mtip, 0, flags);
+ fatal |= t4_handle_intr(adap, &mac_ios_int_cause_quad0, 0, flags);
+ fatal |= t4_handle_intr(adap, &mac_ios_int_cause_quad1, 0, flags);
return (fatal);
}
-static bool pl_timeout_status(struct adapter *adap, int arg, bool verbose)
+static bool pl_timeout_status(struct adapter *adap, int arg, int flags)
{
+ if (flags & IHF_NO_SHOW)
+ return (false);
CH_ALERT(adap, " PL_TIMEOUT_STATUS 0x%08x 0x%08x\n",
t4_read_reg(adap, A_PL_TIMEOUT_STATUS0),
@@ -6015,13 +6493,9 @@ static bool pl_timeout_status(struct adapter *adap, int arg, bool verbose)
return (false);
}
-static bool plpl_intr_handler(struct adapter *adap, int arg, bool verbose)
+static bool plpl_intr_handler(struct adapter *adap, int arg, int flags)
{
- static const struct intr_action plpl_intr_actions[] = {
- { F_TIMEOUT, 0, pl_timeout_status },
- { 0 },
- };
- static const struct intr_details plpl_intr_details[] = {
+ static const struct intr_details plpl_int_cause_fields[] = {
{ F_PL_BUSPERR, "Bus parity error" },
{ F_FATALPERR, "Fatal parity error" },
{ F_INVALIDACCESS, "Global reserved memory access" },
@@ -6030,31 +6504,397 @@ static bool plpl_intr_handler(struct adapter *adap, int arg, bool verbose)
{ F_PERRVFID, "VFID_MAP parity error" },
{ 0 }
};
- static const struct intr_info plpl_intr_info = {
+ static const struct intr_action plpl_int_cause_actions[] = {
+ { F_TIMEOUT, -1, pl_timeout_status },
+ { 0 },
+ };
+ static const struct intr_info plpl_int_cause = {
.name = "PL_PL_INT_CAUSE",
.cause_reg = A_PL_PL_INT_CAUSE,
.enable_reg = A_PL_PL_INT_ENABLE,
.fatal = F_FATALPERR | F_PERRVFID,
- .flags = NONFATAL_IF_DISABLED,
- .details = plpl_intr_details,
- .actions = plpl_intr_actions,
+ .flags = IHF_FATAL_IFF_ENABLED | IHF_IGNORE_IF_DISABLED,
+ .details = plpl_int_cause_fields,
+ .actions = plpl_int_cause_actions,
+ };
+
+ return (t4_handle_intr(adap, &plpl_int_cause, 0, flags));
+}
+
+/* similar to t4_port_reg */
+static inline u32
+t7_tlstx_reg(u8 instance, u8 channel, u32 reg)
+{
+ MPASS(instance <= 1);
+ MPASS(channel < NUM_TLS_TX_CH_INSTANCES);
+ return (instance * (CRYPTO_1_BASE_ADDR - CRYPTO_0_BASE_ADDR) +
+ TLS_TX_CH_REG(reg, channel));
+}
+
+/*
+ * CRYPTO (aka TLS_TX) interrupt handler.
+ */
+static bool tlstx_intr_handler(struct adapter *adap, int idx, int flags)
+{
+ static const struct intr_details tlstx_int_cause_fields[] = {
+ { F_KEX_CERR, "KEX SRAM Correctable error" },
+ { F_KEYLENERR, "IPsec Key length error" },
+ { F_INTF1_PERR, "Input Interface1 parity error" },
+ { F_INTF0_PERR, "Input Interface0 parity error" },
+ { F_KEX_PERR, "KEX SRAM Parity error" },
+ { 0 }
+ };
+ struct intr_info ii = {
+ .fatal = F_KEX_PERR | F_INTF0_PERR | F_INTF1_PERR,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = tlstx_int_cause_fields,
+ .actions = NULL,
+ };
+ char name[32];
+ int ch;
+ bool fatal = false;
+
+ for (ch = 0; ch < NUM_TLS_TX_CH_INSTANCES; ch++) {
+ snprintf(name, sizeof(name), "TLSTX%u_CH%u_INT_CAUSE", idx, ch);
+ ii.name = &name[0];
+ ii.cause_reg = t7_tlstx_reg(idx, ch, A_TLS_TX_CH_INT_CAUSE);
+ ii.enable_reg = t7_tlstx_reg(idx, ch, A_TLS_TX_CH_INT_ENABLE);
+ fatal |= t4_handle_intr(adap, &ii, 0, flags);
+ }
+
+ return (fatal);
+}
+
+/*
+ * HMA interrupt handler.
+ */
+static bool hma_intr_handler(struct adapter *adap, int idx, int flags)
+{
+ static const struct intr_details hma_int_cause_fields[] = {
+ { F_GK_UF_INT_CAUSE, "Gatekeeper underflow" },
+ { F_IDTF_INT_CAUSE, "Invalid descriptor fault" },
+ { F_OTF_INT_CAUSE, "Offset translation fault" },
+ { F_RTF_INT_CAUSE, "Region translation fault" },
+ { F_PCIEMST_INT_CAUSE, "PCIe master access error" },
+ { F_MAMST_INT_CAUSE, "MA master access error" },
+ { 1, "FIFO parity error" },
+ { 0 }
+ };
+ static const struct intr_info hma_int_cause = {
+ .name = "HMA_INT_CAUSE",
+ .cause_reg = A_HMA_INT_CAUSE,
+ .enable_reg = A_HMA_INT_ENABLE,
+ .fatal = 7,
+ .flags = 0,
+ .details = hma_int_cause_fields,
+ .actions = NULL,
+ };
+
+ return (t4_handle_intr(adap, &hma_int_cause, 0, flags));
+}
+
+/*
+ * CRYPTO_KEY interrupt handler.
+ */
+static bool cryptokey_intr_handler(struct adapter *adap, int idx, int flags)
+{
+ static const struct intr_details cryptokey_int_cause_fields[] = {
+ { F_MA_FIFO_PERR, "MA arbiter FIFO parity error" },
+ { F_MA_RSP_PERR, "MA response IF parity error" },
+ { F_ING_CACHE_DATA_PERR, "Ingress key cache data parity error" },
+ { F_ING_CACHE_TAG_PERR, "Ingress key cache tag parity error" },
+ { F_LKP_KEY_REQ_PERR, "Ingress key req parity error" },
+ { F_LKP_CLIP_TCAM_PERR, "Ingress LKP CLIP TCAM parity error" },
+ { F_LKP_MAIN_TCAM_PERR, "Ingress LKP main TCAM parity error" },
+ { F_EGR_KEY_REQ_PERR, "Egress key req or FIFO3 parity error" },
+ { F_EGR_CACHE_DATA_PERR, "Egress key cache data parity error" },
+ { F_EGR_CACHE_TAG_PERR, "Egress key cache tag parity error" },
+ { F_CIM_PERR, "CIM interface parity error" },
+ { F_MA_INV_RSP_TAG, "MA invalid response tag" },
+ { F_ING_KEY_RANGE_ERR, "Ingress key range error" },
+ { F_ING_MFIFO_OVFL, "Ingress MFIFO overflow" },
+ { F_LKP_REQ_OVFL, "Ingress lookup FIFO overflow" },
+ { F_EOK_WAIT_ERR, "EOK wait error" },
+ { F_EGR_KEY_RANGE_ERR, "Egress key range error" },
+ { F_EGR_MFIFO_OVFL, "Egress MFIFO overflow" },
+ { F_SEQ_WRAP_HP_OVFL, "Sequence wrap (hi-pri)" },
+ { F_SEQ_WRAP_LP_OVFL, "Sequence wrap (lo-pri)" },
+ { F_EGR_SEQ_WRAP_HP, "Egress sequence wrap (hi-pri)" },
+ { F_EGR_SEQ_WRAP_LP, "Egress sequence wrap (lo-pri)" },
+ { 0 }
+ };
+ static const struct intr_info cryptokey_int_cause = {
+ .name = "CRYPTO_KEY_INT_CAUSE",
+ .cause_reg = A_CRYPTO_KEY_INT_CAUSE,
+ .enable_reg = A_CRYPTO_KEY_INT_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = cryptokey_int_cause_fields,
+ .actions = NULL,
+ };
+
+ return (t4_handle_intr(adap, &cryptokey_int_cause, 0, flags));
+}
+
+/*
+ * GCACHE interrupt handler.
+ */
+static bool gcache_intr_handler(struct adapter *adap, int idx, int flags)
+{
+ static const struct intr_details gcache_int_cause_fields[] = {
+ { F_GC1_SRAM_RSP_DATAQ_PERR_INT_CAUSE, "GC1 SRAM rsp dataq perr" },
+ { F_GC0_SRAM_RSP_DATAQ_PERR_INT_CAUSE, "GC0 SRAM rsp dataq perr" },
+ { F_GC1_WQDATA_FIFO_PERR_INT_CAUSE, "GC1 wqdata FIFO perr" },
+ { F_GC0_WQDATA_FIFO_PERR_INT_CAUSE, "GC0 wqdata FIFO perr" },
+ { F_GC1_RDTAG_QUEUE_PERR_INT_CAUSE, "GC1 rdtag queue perr" },
+ { F_GC0_RDTAG_QUEUE_PERR_INT_CAUSE, "GC0 rdtag queue perr" },
+ { F_GC1_SRAM_RDTAG_QUEUE_PERR_INT_CAUSE, "GC1 SRAM rdtag queue perr" },
+ { F_GC0_SRAM_RDTAG_QUEUE_PERR_INT_CAUSE, "GC0 SRAM rdtag queue perr" },
+ { F_GC1_RSP_PERR_INT_CAUSE, "GC1 rsp perr" },
+ { F_GC0_RSP_PERR_INT_CAUSE, "GC0 rsp perr" },
+ { F_GC1_LRU_UERR_INT_CAUSE, "GC1 lru uerr" },
+ { F_GC0_LRU_UERR_INT_CAUSE, "GC0 lru uerr" },
+ { F_GC1_TAG_UERR_INT_CAUSE, "GC1 tag uerr" },
+ { F_GC0_TAG_UERR_INT_CAUSE, "GC0 tag uerr" },
+ { F_GC1_LRU_CERR_INT_CAUSE, "GC1 lru cerr" },
+ { F_GC0_LRU_CERR_INT_CAUSE, "GC0 lru cerr" },
+ { F_GC1_TAG_CERR_INT_CAUSE, "GC1 tag cerr" },
+ { F_GC0_TAG_CERR_INT_CAUSE, "GC0 tag cerr" },
+ { F_GC1_CE_INT_CAUSE, "GC1 correctable error" },
+ { F_GC0_CE_INT_CAUSE, "GC0 correctable error" },
+ { F_GC1_UE_INT_CAUSE, "GC1 uncorrectable error" },
+ { F_GC0_UE_INT_CAUSE, "GC0 uncorrectable error" },
+ { F_GC1_CMD_PAR_INT_CAUSE, "GC1 cmd perr" },
+ { F_GC1_DATA_PAR_INT_CAUSE, "GC1 data perr" },
+ { F_GC0_CMD_PAR_INT_CAUSE, "GC0 cmd perr" },
+ { F_GC0_DATA_PAR_INT_CAUSE, "GC0 data perr" },
+ { F_ILLADDRACCESS1_INT_CAUSE, "GC1 illegal address access" },
+ { F_ILLADDRACCESS0_INT_CAUSE, "GC0 illegal address access" },
+ { 0 }
+ };
+ static const struct intr_info gcache_perr_cause = {
+ .name = "GCACHE_PAR_CAUSE",
+ .cause_reg = A_GCACHE_PAR_CAUSE,
+ .enable_reg = A_GCACHE_PAR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info gcache_int_cause = {
+ .name = "GCACHE_INT_CAUSE",
+ .cause_reg = A_GCACHE_INT_CAUSE,
+ .enable_reg = A_GCACHE_INT_ENABLE,
+ .fatal = 0,
+ .flags = 0,
+ .details = gcache_int_cause_fields,
+ .actions = NULL,
+ };
+ bool fatal = false;
+
+ fatal |= t4_handle_intr(adap, &gcache_int_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &gcache_perr_cause, 0, flags);
+
+ return (fatal);
+}
+
+/*
+ * ARM interrupt handler.
+ */
+static bool arm_intr_handler(struct adapter *adap, int idx, int flags)
+{
+ static const struct intr_info arm_perr_cause0 = {
+ .name = "ARM_PERR_INT_CAUSE0",
+ .cause_reg = A_ARM_PERR_INT_CAUSE0,
+ .enable_reg = A_ARM_PERR_INT_ENB0,
+ .fatal = 0xffffffff,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info arm_perr_cause1 = {
+ .name = "ARM_PERR_INT_CAUSE1",
+ .cause_reg = A_ARM_PERR_INT_CAUSE1,
+ .enable_reg = A_ARM_PERR_INT_ENB1,
+ .fatal = 0xffffffff,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info arm_perr_cause2 = {
+ .name = "ARM_PERR_INT_CAUSE2",
+ .cause_reg = A_ARM_PERR_INT_CAUSE2,
+ .enable_reg = A_ARM_PERR_INT_ENB2,
+ .fatal = 0xffffffff,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info arm_cerr_cause0 = {
+ .name = "ARM_CERR_INT_CAUSE",
+ .cause_reg = A_ARM_CERR_INT_CAUSE0,
+ .enable_reg = A_ARM_CERR_INT_ENB0,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
};
+ static const struct intr_info arm_err_cause0 = {
+ .name = "ARM_ERR_INT_CAUSE",
+ .cause_reg = A_ARM_ERR_INT_CAUSE0,
+ .enable_reg = A_ARM_ERR_INT_ENB0,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info arm_periph_cause = {
+ .name = "ARM_PERIPHERAL_INT_CAUSE",
+ .cause_reg = A_ARM_PERIPHERAL_INT_CAUSE,
+ .enable_reg = A_ARM_PERIPHERAL_INT_ENB,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ static const struct intr_info arm_nvme_db_emu_cause = {
+ .name = "ARM_NVME_DB_EMU_INT_CAUSE",
+ .cause_reg = A_ARM_NVME_DB_EMU_INT_CAUSE,
+ .enable_reg = A_ARM_NVME_DB_EMU_INT_ENABLE,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = NULL,
+ .actions = NULL,
+ };
+ bool fatal = false;
+
+ fatal |= t4_handle_intr(adap, &arm_perr_cause0, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_perr_cause1, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_perr_cause2, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_cerr_cause0, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_err_cause0, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_periph_cause, 0, flags);
+ fatal |= t4_handle_intr(adap, &arm_nvme_db_emu_cause, 0, flags);
+
+ return (fatal);
+}
+
+static inline uint32_t
+get_perr_ucause(struct adapter *sc, const struct intr_info *ii)
+{
+ uint32_t cause;
- return (t4_handle_intr(adap, &plpl_intr_info, 0, verbose));
+ cause = t4_read_reg(sc, ii->cause_reg);
+ if (ii->flags & IHF_IGNORE_IF_DISABLED)
+ cause &= t4_read_reg(sc, ii->enable_reg);
+ return (cause);
+}
+
+static uint32_t
+t4_perr_to_ic(struct adapter *adap, uint32_t perr)
+{
+ uint32_t mask;
+
+ if (adap->chip_params->nchan > 2)
+ mask = F_MAC0 | F_MAC1 | F_MAC2 | F_MAC3;
+ else
+ mask = F_MAC0 | F_MAC1;
+ return (perr & mask ? perr | mask : perr);
+}
+
+static uint32_t
+t7_perr_to_ic1(uint32_t perr)
+{
+ uint32_t cause = 0;
+
+ if (perr & F_T7_PL_PERR_ULP_TX)
+ cause |= F_T7_ULP_TX;
+ if (perr & F_T7_PL_PERR_SGE)
+ cause |= F_T7_SGE;
+ if (perr & F_T7_PL_PERR_HMA)
+ cause |= F_T7_HMA;
+ if (perr & F_T7_PL_PERR_CPL_SWITCH)
+ cause |= F_T7_CPL_SWITCH;
+ if (perr & F_T7_PL_PERR_ULP_RX)
+ cause |= F_T7_ULP_RX;
+ if (perr & F_T7_PL_PERR_PM_RX)
+ cause |= F_T7_PM_RX;
+ if (perr & F_T7_PL_PERR_PM_TX)
+ cause |= F_T7_PM_TX;
+ if (perr & F_T7_PL_PERR_MA)
+ cause |= F_T7_MA;
+ if (perr & F_T7_PL_PERR_TP)
+ cause |= F_T7_TP;
+ if (perr & F_T7_PL_PERR_LE)
+ cause |= F_T7_LE;
+ if (perr & F_T7_PL_PERR_EDC1)
+ cause |= F_T7_EDC1;
+ if (perr & F_T7_PL_PERR_EDC0)
+ cause |= F_T7_EDC0;
+ if (perr & F_T7_PL_PERR_MC1)
+ cause |= F_T7_MC1;
+ if (perr & F_T7_PL_PERR_MC0)
+ cause |= F_T7_MC0;
+ if (perr & F_T7_PL_PERR_PCIE)
+ cause |= F_T7_PCIE;
+ if (perr & F_T7_PL_PERR_UART)
+ cause |= F_T7_UART;
+ if (perr & F_T7_PL_PERR_PMU)
+ cause |= F_PMU;
+ if (perr & F_T7_PL_PERR_MAC)
+ cause |= F_MAC0 | F_MAC1 | F_MAC2 | F_MAC3;
+ if (perr & F_T7_PL_PERR_SMB)
+ cause |= F_SMB;
+ if (perr & F_T7_PL_PERR_SF)
+ cause |= F_SF;
+ if (perr & F_T7_PL_PERR_PL)
+ cause |= F_PL;
+ if (perr & F_T7_PL_PERR_NCSI)
+ cause |= F_NCSI;
+ if (perr & F_T7_PL_PERR_MPS)
+ cause |= F_MPS;
+ if (perr & F_T7_PL_PERR_MI)
+ cause |= F_MI;
+ if (perr & F_T7_PL_PERR_DBG)
+ cause |= F_DBG;
+ if (perr & F_T7_PL_PERR_I2CM)
+ cause |= F_I2CM;
+ if (perr & F_T7_PL_PERR_CIM)
+ cause |= F_CIM;
+
+ return (cause);
+}
+
+static uint32_t
+t7_perr_to_ic2(uint32_t perr)
+{
+ uint32_t cause = 0;
+
+ if (perr & F_T7_PL_PERR_CRYPTO_KEY)
+ cause |= F_CRYPTO_KEY;
+ if (perr & F_T7_PL_PERR_CRYPTO1)
+ cause |= F_CRYPTO1;
+ if (perr & F_T7_PL_PERR_CRYPTO0)
+ cause |= F_CRYPTO0;
+ if (perr & F_T7_PL_PERR_GCACHE)
+ cause |= F_GCACHE;
+ if (perr & F_T7_PL_PERR_ARM)
+ cause |= F_ARM;
+
+ return (cause);
}
/**
* t4_slow_intr_handler - control path interrupt handler
* @adap: the adapter
- * @verbose: increased verbosity, for debug
*
* T4 interrupt handler for non-data global interrupt events, e.g., errors.
* The designation 'slow' is because it involves register reads, while
* data interrupts typically don't involve any MMIOs.
*/
-bool t4_slow_intr_handler(struct adapter *adap, bool verbose)
+bool t4_slow_intr_handler(struct adapter *adap, int flags)
{
- static const struct intr_details pl_intr_details[] = {
+ static const struct intr_details pl_int_cause_fields[] = {
{ F_MC1, "MC1" },
{ F_UART, "UART" },
{ F_ULP_TX, "ULP TX" },
@@ -6087,10 +6927,56 @@ bool t4_slow_intr_handler(struct adapter *adap, bool verbose)
{ F_CIM, "CIM" },
{ 0 }
};
- static const struct intr_details t7_pl_intr_details[] = {
- { F_T7_MC1, "MC1" },
+ static const struct intr_action pl_int_cause_actions[] = {
+ { F_ULP_TX, -1, ulptx_intr_handler },
+ { F_SGE, -1, sge_intr_handler },
+ { F_CPL_SWITCH, -1, cplsw_intr_handler },
+ { F_ULP_RX, -1, ulprx_intr_handler },
+ { F_PM_RX, -1, pmtx_intr_handler },
+ { F_PM_TX, -1, pmtx_intr_handler },
+ { F_MA, -1, ma_intr_handler },
+ { F_TP, -1, tp_intr_handler },
+ { F_LE, -1, le_intr_handler },
+ { F_EDC0, MEM_EDC0, mem_intr_handler },
+ { F_EDC1, MEM_EDC1, mem_intr_handler },
+ { F_MC0, MEM_MC0, mem_intr_handler },
+ { F_MC1, MEM_MC1, mem_intr_handler },
+ { F_PCIE, -1, pcie_intr_handler },
+ { F_MAC0, 0, mac_intr_handler },
+ { F_MAC1, 1, mac_intr_handler },
+ { F_MAC2, 2, mac_intr_handler },
+ { F_MAC3, 3, mac_intr_handler },
+ { F_SMB, -1, smb_intr_handler },
+ { F_PL, -1, plpl_intr_handler },
+ { F_NCSI, -1, ncsi_intr_handler },
+ { F_MPS, -1, mps_intr_handler },
+ { F_CIM, -1, cim_intr_handler },
+ { 0 }
+ };
+ static const struct intr_info pl_int_cause = {
+ .name = "PL_INT_CAUSE",
+ .cause_reg = A_PL_INT_CAUSE,
+ .enable_reg = A_PL_INT_ENABLE,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED,
+ .details = pl_int_cause_fields,
+ .actions = pl_int_cause_actions,
+ };
+ static const struct intr_info pl_perr_cause = {
+ .name = "PL_PERR_CAUSE",
+ .cause_reg = A_PL_PERR_CAUSE,
+ .enable_reg = A_PL_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = pl_int_cause_fields,
+ .actions = NULL,
+ };
+ static const struct intr_details t7_pl_int_cause_fields[] = {
+ { F_T7_FLR, "FLR" },
+ { F_T7_SW_CIM, "SW CIM" },
{ F_T7_ULP_TX, "ULP TX" },
{ F_T7_SGE, "SGE" },
+ { F_T7_HMA, "HMA" },
{ F_T7_CPL_SWITCH, "CPL Switch" },
{ F_T7_ULP_RX, "ULP RX" },
{ F_T7_PM_RX, "PM RX" },
@@ -6100,117 +6986,165 @@ bool t4_slow_intr_handler(struct adapter *adap, bool verbose)
{ F_T7_LE, "LE" },
{ F_T7_EDC1, "EDC1" },
{ F_T7_EDC0, "EDC0" },
+ { F_T7_MC1, "MC1" },
{ F_T7_MC0, "MC0" },
{ F_T7_PCIE, "PCIE" },
+ { F_T7_UART, "UART" },
+ { F_PMU, "PMU" },
{ F_MAC3, "MAC3" },
{ F_MAC2, "MAC2" },
{ F_MAC1, "MAC1" },
{ F_MAC0, "MAC0" },
{ F_SMB, "SMB" },
+ { F_SF, "SF" },
{ F_PL, "PL" },
{ F_NCSI, "NC-SI" },
{ F_MPS, "MPS" },
+ { F_MI, "MI" },
{ F_DBG, "DBG" },
{ F_I2CM, "I2CM" },
- { F_MI, "MI" },
{ F_CIM, "CIM" },
{ 0 }
};
- struct intr_info pl_perr_cause = {
- .name = "PL_PERR_CAUSE",
- .cause_reg = A_PL_PERR_CAUSE,
- .enable_reg = A_PL_PERR_ENABLE,
- .fatal = 0xffffffff,
- .flags = NONFATAL_IF_DISABLED,
- .details = NULL,
- .actions = NULL,
- };
- static const struct intr_action pl_intr_action[] = {
- { F_MC1, MEM_MC1, mem_intr_handler },
- { F_ULP_TX, -1, ulptx_intr_handler },
- { F_SGE, -1, sge_intr_handler },
- { F_CPL_SWITCH, -1, cplsw_intr_handler },
- { F_ULP_RX, -1, ulprx_intr_handler },
- { F_PM_RX, -1, pmrx_intr_handler},
- { F_PM_TX, -1, pmtx_intr_handler},
- { F_MA, -1, ma_intr_handler },
- { F_TP, -1, tp_intr_handler },
- { F_LE, -1, le_intr_handler },
- { F_EDC1, MEM_EDC1, mem_intr_handler },
- { F_EDC0, MEM_EDC0, mem_intr_handler },
- { F_MC0, MEM_MC0, mem_intr_handler },
- { F_PCIE, -1, pcie_intr_handler },
- { F_MAC3, 3, mac_intr_handler},
- { F_MAC2, 2, mac_intr_handler},
- { F_MAC1, 1, mac_intr_handler},
- { F_MAC0, 0, mac_intr_handler},
- { F_SMB, -1, smb_intr_handler},
- { F_PL, -1, plpl_intr_handler },
- { F_NCSI, -1, ncsi_intr_handler},
- { F_MPS, -1, mps_intr_handler },
- { F_CIM, -1, cim_intr_handler },
- { 0 }
- };
- static const struct intr_action t7_pl_intr_action[] = {
+ static const struct intr_action t7_pl_int_cause_actions[] = {
{ F_T7_ULP_TX, -1, ulptx_intr_handler },
{ F_T7_SGE, -1, sge_intr_handler },
+ { F_T7_HMA, -1, hma_intr_handler },
{ F_T7_CPL_SWITCH, -1, cplsw_intr_handler },
{ F_T7_ULP_RX, -1, ulprx_intr_handler },
- { F_T7_PM_RX, -1, pmrx_intr_handler},
- { F_T7_PM_TX, -1, pmtx_intr_handler},
+ { F_T7_PM_RX, -1, pmrx_intr_handler },
+ { F_T7_PM_TX, -1, pmtx_intr_handler },
{ F_T7_MA, -1, ma_intr_handler },
{ F_T7_TP, -1, tp_intr_handler },
{ F_T7_LE, -1, le_intr_handler },
- { F_T7_EDC1, MEM_EDC1, mem_intr_handler },
{ F_T7_EDC0, MEM_EDC0, mem_intr_handler },
- { F_T7_MC1, MEM_MC1, mem_intr_handler },
+ { F_T7_EDC1, MEM_EDC1, mem_intr_handler },
{ F_T7_MC0, MEM_MC0, mem_intr_handler },
+ { F_T7_MC1, MEM_MC1, mem_intr_handler },
{ F_T7_PCIE, -1, pcie_intr_handler },
- { F_MAC3, 3, mac_intr_handler},
- { F_MAC2, 2, mac_intr_handler},
- { F_MAC1, 1, mac_intr_handler},
- { F_MAC0, 0, mac_intr_handler},
- { F_SMB, -1, smb_intr_handler},
+ { F_MAC0, 0, mac_intr_handler },
+ { F_MAC1, 1, mac_intr_handler },
+ { F_MAC2, 2, mac_intr_handler },
+ { F_MAC3, 3, mac_intr_handler },
+ { F_SMB, -1, smb_intr_handler },
{ F_PL, -1, plpl_intr_handler },
- { F_NCSI, -1, ncsi_intr_handler},
+ { F_NCSI, -1, ncsi_intr_handler },
{ F_MPS, -1, mps_intr_handler },
{ F_CIM, -1, cim_intr_handler },
{ 0 }
};
- struct intr_info pl_intr_info = {
+ static const struct intr_info t7_pl_int_cause = {
.name = "PL_INT_CAUSE",
.cause_reg = A_PL_INT_CAUSE,
.enable_reg = A_PL_INT_ENABLE,
.fatal = 0,
- .flags = 0,
- .details = NULL,
+ .flags = IHF_IGNORE_IF_DISABLED,
+ .details = t7_pl_int_cause_fields,
+ .actions = t7_pl_int_cause_actions,
+ };
+ static const struct intr_details t7_pl_int_cause2_fields[] = {
+ { F_CRYPTO_KEY, "CRYPTO KEY" },
+ { F_CRYPTO1, "CRYPTO1" },
+ { F_CRYPTO0, "CRYPTO0" },
+ { F_GCACHE, "GCACHE" },
+ { F_ARM, "ARM" },
+ { 0 }
+ };
+ static const struct intr_action t7_pl_int_cause2_actions[] = {
+ { F_CRYPTO_KEY, -1, cryptokey_intr_handler },
+ { F_CRYPTO1, 1, tlstx_intr_handler },
+ { F_CRYPTO0, 0, tlstx_intr_handler },
+ { F_GCACHE, -1, gcache_intr_handler },
+ { F_ARM, -1, arm_intr_handler },
+ { 0 }
+ };
+ static const struct intr_info t7_pl_int_cause2 = {
+ .name = "PL_INT_CAUSE2",
+ .cause_reg = A_PL_INT_CAUSE2,
+ .enable_reg = A_PL_INT_ENABLE2,
+ .fatal = 0,
+ .flags = IHF_IGNORE_IF_DISABLED,
+ .details = t7_pl_int_cause2_fields,
+ .actions = t7_pl_int_cause2_actions,
+ };
+ static const struct intr_details t7_pl_perr_cause_fields[] = {
+ { F_T7_PL_PERR_CRYPTO_KEY, "CRYPTO KEY" },
+ { F_T7_PL_PERR_CRYPTO1, "CRYPTO1" },
+ { F_T7_PL_PERR_CRYPTO0, "CRYPTO0" },
+ { F_T7_PL_PERR_GCACHE, "GCACHE" },
+ { F_T7_PL_PERR_ARM, "ARM" },
+ { F_T7_PL_PERR_ULP_TX, "ULP TX" },
+ { F_T7_PL_PERR_SGE, "SGE" },
+ { F_T7_PL_PERR_HMA, "HMA" },
+ { F_T7_PL_PERR_CPL_SWITCH, "CPL Switch" },
+ { F_T7_PL_PERR_ULP_RX, "ULP RX" },
+ { F_T7_PL_PERR_PM_RX, "PM RX" },
+ { F_T7_PL_PERR_PM_TX, "PM TX" },
+ { F_T7_PL_PERR_MA, "MA" },
+ { F_T7_PL_PERR_TP, "TP" },
+ { F_T7_PL_PERR_LE, "LE" },
+ { F_T7_PL_PERR_EDC1, "EDC1" },
+ { F_T7_PL_PERR_EDC0, "EDC0" },
+ { F_T7_PL_PERR_MC1, "MC1" },
+ { F_T7_PL_PERR_MC0, "MC0" },
+ { F_T7_PL_PERR_PCIE, "PCIE" },
+ { F_T7_PL_PERR_UART, "UART" },
+ { F_T7_PL_PERR_PMU, "PMU" },
+ { F_T7_PL_PERR_MAC, "MAC" },
+ { F_T7_PL_PERR_SMB, "SMB" },
+ { F_T7_PL_PERR_SF, "SF" },
+ { F_T7_PL_PERR_PL, "PL" },
+ { F_T7_PL_PERR_NCSI, "NC-SI" },
+ { F_T7_PL_PERR_MPS, "MPS" },
+ { F_T7_PL_PERR_MI, "MI" },
+ { F_T7_PL_PERR_DBG, "DBG" },
+ { F_T7_PL_PERR_I2CM, "I2CM" },
+ { F_T7_PL_PERR_CIM, "CIM" },
+ { 0 }
+ };
+ static const struct intr_info t7_pl_perr_cause = {
+ .name = "PL_PERR_CAUSE",
+ .cause_reg = A_PL_PERR_CAUSE,
+ .enable_reg = A_PL_PERR_ENABLE,
+ .fatal = 0xffffffff,
+ .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED,
+ .details = t7_pl_perr_cause_fields,
.actions = NULL,
};
- u32 perr;
-
- if (chip_id(adap) >= CHELSIO_T7) {
- pl_perr_cause.details = t7_pl_intr_details;
- pl_intr_info.details = t7_pl_intr_details;
- pl_intr_info.actions = t7_pl_intr_action;
+ bool fatal = false;
+ uint32_t perr;
+
+ if (chip_id(adap) < CHELSIO_T7) {
+ perr = get_perr_ucause(adap, &pl_perr_cause);
+ fatal |= t4_handle_intr(adap, &pl_perr_cause, 0,
+ flags & ~(IHF_CLR_ALL_SET | IHF_CLR_ALL_UNIGNORED));
+ fatal |= t4_handle_intr(adap, &pl_int_cause,
+ t4_perr_to_ic(adap, perr), flags);
+ t4_write_reg(adap, pl_perr_cause.cause_reg, perr);
+ (void)t4_read_reg(adap, pl_perr_cause.cause_reg);
} else {
- pl_perr_cause.details = pl_intr_details;
- pl_intr_info.details = pl_intr_details;
- pl_intr_info.actions = pl_intr_action;
+ perr = get_perr_ucause(adap, &t7_pl_perr_cause);
+ fatal |= t4_handle_intr(adap, &t7_pl_perr_cause, 0,
+ flags & ~(IHF_CLR_ALL_SET | IHF_CLR_ALL_UNIGNORED));
+ fatal |= t4_handle_intr(adap, &t7_pl_int_cause,
+ t7_perr_to_ic1(perr), flags);
+ fatal |= t4_handle_intr(adap, &t7_pl_int_cause2,
+ t7_perr_to_ic2(perr), flags);
+ t4_write_reg(adap, t7_pl_perr_cause.cause_reg, perr);
+ (void)t4_read_reg(adap, t7_pl_perr_cause.cause_reg);
}
-
- perr = t4_read_reg(adap, pl_perr_cause.cause_reg);
- if (verbose || perr != 0) {
- t4_show_intr_info(adap, &pl_perr_cause, perr);
- if (perr != 0)
- t4_write_reg(adap, pl_perr_cause.cause_reg, perr);
- if (verbose)
- perr |= t4_read_reg(adap, pl_intr_info.enable_reg);
- }
-
- return (t4_handle_intr(adap, &pl_intr_info, perr, verbose));
+ return (fatal);
}
-#define PF_INTR_MASK (F_PFSW | F_PFCIM)
+void t4_intr_clear(struct adapter *adap)
+{
+#if 1
+ if (chip_id(adap) >= CHELSIO_T7)
+ t4_write_reg(adap, A_SGE_INT_CAUSE8, 0xffffffff);
+#endif
+ (void)t4_slow_intr_handler(adap,
+ IHF_NO_SHOW | IHF_RUN_ALL_ACTIONS | IHF_CLR_ALL_SET);
+}
/**
* t4_intr_enable - enable interrupts
@@ -6229,6 +7163,8 @@ void t4_intr_enable(struct adapter *adap)
{
u32 mask, val;
+ if (adap->intr_flags & IHF_INTR_CLEAR_ON_INIT)
+ t4_intr_clear(adap);
if (chip_id(adap) <= CHELSIO_T5)
val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT |
F_DBFIFO_LP_INT;
@@ -6241,8 +7177,14 @@ void t4_intr_enable(struct adapter *adap)
F_ERR_BAD_DB_PIDX0 | F_ERR_ING_CTXT_PRIO | F_EGRESS_SIZE_ERR;
mask = val;
t4_set_reg_field(adap, A_SGE_INT_ENABLE3, mask, val);
- t4_write_reg(adap, MYPF_REG(A_PL_PF_INT_ENABLE), PF_INTR_MASK);
+ if (chip_id(adap) >= CHELSIO_T7)
+ t4_write_reg(adap, A_SGE_INT_ENABLE4, 0xffffffff);
+ t4_write_reg(adap, MYPF_REG(A_PL_PF_INT_ENABLE), F_PFSW | F_PFCIM);
t4_set_reg_field(adap, A_PL_INT_ENABLE, F_SF | F_I2CM, 0);
+#if 1
+ if (chip_id(adap) >= CHELSIO_T7)
+ t4_set_reg_field(adap, A_PL_INT_ENABLE, F_MAC0 | F_MAC1 | F_MAC2 | F_MAC3, 0);
+#endif
t4_set_reg_field(adap, A_PL_INT_MAP0, 0, 1 << adap->pf);
}
@@ -6439,9 +7381,15 @@ int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
/* Read an RSS table row */
static int rd_rss_row(struct adapter *adap, int row, u32 *val)
{
- t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row);
- return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE, F_LKPTBLROWVLD, 1,
- 5, 0, val);
+ if (chip_id(adap) < CHELSIO_T7) {
+ t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row);
+ return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE,
+ F_LKPTBLROWVLD, 1, 5, 0, val);
+ } else {
+ t4_write_reg(adap, A_TP_RSS_CONFIG_SRAM, 0xB0000 | row);
+ return t7_wait_sram_done(adap, A_TP_RSS_CONFIG_SRAM,
+ A_TP_RSS_LKP_TABLE, 5, 0, val);
+ }
}
/**
@@ -10178,7 +11126,7 @@ const struct chip_params *t4_get_chip_params(int chipid)
.vfcount = 256,
.sge_fl_db = 0,
.sge_ctxt_size = SGE_CTXT_SIZE_T7,
- .mps_tcam_size = NUM_MPS_T5_CLS_SRAM_L_INSTANCES,
+ .mps_tcam_size = NUM_MPS_T5_CLS_SRAM_L_INSTANCES * 3,
.rss_nentries = T7_RSS_NENTRIES,
.cim_la_size = CIMLA_SIZE_T6,
},
diff --git a/sys/dev/cxgbe/common/t4_msg.h b/sys/dev/cxgbe/common/t4_msg.h
index 0d12ccf2e910..214080964fbb 100644
--- a/sys/dev/cxgbe/common/t4_msg.h
+++ b/sys/dev/cxgbe/common/t4_msg.h
@@ -30,6 +30,7 @@
#define T4_MSG_H
enum cpl_opcodes {
+ CPL_TLS_TX_SCMD_FMT = 0x0,
CPL_PASS_OPEN_REQ = 0x1,
CPL_PASS_ACCEPT_RPL = 0x2,
CPL_ACT_OPEN_REQ = 0x3,
@@ -48,6 +49,8 @@ enum cpl_opcodes {
CPL_RTE_READ_REQ = 0x11,
CPL_L2T_WRITE_REQ = 0x12,
CPL_L2T_READ_REQ = 0x13,
+ CPL_GRE_TABLE_REQ = 0x1b,
+ CPL_GRE_TABLE_RPL = 0xbb,
CPL_SMT_WRITE_REQ = 0x14,
CPL_SMT_READ_REQ = 0x15,
CPL_TAG_WRITE_REQ = 0x16,
@@ -130,6 +133,7 @@ enum cpl_opcodes {
CPL_TX_TLS_SFO = 0x89,
CPL_TX_SEC_PDU = 0x8A,
CPL_TX_TLS_ACK = 0x8B,
+ CPL_TX_QUIC_ENC = 0x8d,
CPL_RCB_UPD = 0x8C,
CPL_SGE_FLR_FLUSH = 0xA0,
@@ -258,6 +262,7 @@ enum {
ULP_MODE_TCPDDP = 5,
ULP_MODE_FCOE = 6,
ULP_MODE_TLS = 8,
+ ULP_MODE_DTLS = 9,
ULP_MODE_RDMA_V2 = 10,
ULP_MODE_NVMET = 11,
};
@@ -1149,23 +1154,36 @@ struct cpl_get_tcb {
#define V_QUEUENO(x) ((x) << S_QUEUENO)
#define G_QUEUENO(x) (((x) >> S_QUEUENO) & M_QUEUENO)
-#define S_T7_QUEUENO 0
-#define M_T7_QUEUENO 0xFFF
-#define V_T7_QUEUENO(x) ((x) << S_T7_QUEUENO)
-#define G_T7_QUEUENO(x) (((x) >> S_T7_QUEUENO) & M_T7_QUEUENO)
-
#define S_REPLY_CHAN 14
#define V_REPLY_CHAN(x) ((x) << S_REPLY_CHAN)
#define F_REPLY_CHAN V_REPLY_CHAN(1U)
+#define S_NO_REPLY 15
+#define V_NO_REPLY(x) ((x) << S_NO_REPLY)
+#define F_NO_REPLY V_NO_REPLY(1U)
+
+struct cpl_t7_get_tcb {
+ WR_HDR;
+ union opcode_tid ot;
+ __be16 rxchan_queue;
+ __be16 cookie_pkd;
+};
+
#define S_T7_REPLY_CHAN 12
#define M_T7_REPLY_CHAN 0x7
#define V_T7_REPLY_CHAN(x) ((x) << S_T7_REPLY_CHAN)
#define G_T7_REPLY_CHAN(x) (((x) >> S_T7_REPLY_CHAN) & M_T7_REPLY_CHAN)
-#define S_NO_REPLY 15
-#define V_NO_REPLY(x) ((x) << S_NO_REPLY)
-#define F_NO_REPLY V_NO_REPLY(1U)
+#define S_T7_QUEUENO 0
+#define M_T7_QUEUENO 0xFFF
+#define V_T7_QUEUENO(x) ((x) << S_T7_QUEUENO)
+#define G_T7_QUEUENO(x) (((x) >> S_T7_QUEUENO) & M_T7_QUEUENO)
+
+#define S_CPL_GET_TCB_COOKIE 0
+#define M_CPL_GET_TCB_COOKIE 0xff
+#define V_CPL_GET_TCB_COOKIE(x) ((x) << S_CPL_GET_TCB_COOKIE)
+#define G_CPL_GET_TCB_COOKIE(x) \
+ (((x) >> S_CPL_GET_TCB_COOKIE) & M_CPL_GET_TCB_COOKIE)
struct cpl_get_tcb_rpl {
RSS_HDR
@@ -1234,6 +1252,16 @@ struct cpl_close_con_rpl {
__be32 rcv_nxt;
};
+struct cpl_t7_close_con_rpl {
+ RSS_HDR
+ union opcode_tid ot;
+ __be16 rto;
+ __u8 rsvd;
+ __u8 status;
+ __be32 snd_nxt;
+ __be32 rcv_nxt;
+};
+
struct cpl_close_listsvr_req {
WR_HDR;
union opcode_tid ot;
@@ -1340,6 +1368,24 @@ struct cpl_abort_rpl_rss {
__u8 status;
};
+struct cpl_t7_abort_rpl_rss {
+ RSS_HDR
+ union opcode_tid ot;
+ __be32 idx_status;
+};
+
+#define S_CPL_ABORT_RPL_RSS_IDX 8
+#define M_CPL_ABORT_RPL_RSS_IDX 0xffffff
+#define V_CPL_ABORT_RPL_RSS_IDX(x) ((x) << S_CPL_ABORT_RPL_RSS_IDX)
+#define G_CPL_ABORT_RPL_RSS_IDX(x) \
+ (((x) >> S_CPL_ABORT_RPL_RSS_IDX) & M_CPL_ABORT_RPL_RSS_IDX)
+
+#define S_CPL_ABORT_RPL_RSS_STATUS 0
+#define M_CPL_ABORT_RPL_RSS_STATUS 0xff
+#define V_CPL_ABORT_RPL_RSS_STATUS(x) ((x) << S_CPL_ABORT_RPL_RSS_STATUS)
+#define G_CPL_ABORT_RPL_RSS_STATUS(x) \
+ (((x) >> S_CPL_ABORT_RPL_RSS_STATUS) & M_CPL_ABORT_RPL_RSS_STATUS)
+
struct cpl_abort_rpl_rss6 {
RSS_HDR
union opcode_tid ot;
@@ -1444,6 +1490,11 @@ struct cpl_tx_data {
#define V_TX_ULP_MODE(x) ((x) << S_TX_ULP_MODE)
#define G_TX_ULP_MODE(x) (((x) >> S_TX_ULP_MODE) & M_TX_ULP_MODE)
+#define S_T7_TX_ULP_MODE 10
+#define M_T7_TX_ULP_MODE 0xf
+#define V_T7_TX_ULP_MODE(x) ((x) << S_T7_TX_ULP_MODE)
+#define G_T7_TX_ULP_MODE(x) (((x) >> S_T7_TX_ULP_MODE) & M_T7_TX_ULP_MODE)
+
#define S_TX_FORCE 13
#define V_TX_FORCE(x) ((x) << S_TX_FORCE)
#define F_TX_FORCE V_TX_FORCE(1U)
@@ -1881,14 +1932,6 @@ struct cpl_tx_pkt_xt {
(((x) >> S_CPL_TX_PKT_XT_ROCEIPHDRLEN_HI) & \
M_CPL_TX_PKT_XT_ROCEIPHDRLEN_HI)
-#define S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO 30
-#define M_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO 0x3
-#define V_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO(x) \
- ((x) << S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO)
-#define G_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO(x) \
- (((x) >> S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO) & \
- M_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO)
-
/* cpl_tx_pkt_xt.core.ctrl2 fields */
#define S_CPL_TX_PKT_XT_CHKINSRTOFFSET_LO 30
#define M_CPL_TX_PKT_XT_CHKINSRTOFFSET_LO 0x3
@@ -1898,6 +1941,14 @@ struct cpl_tx_pkt_xt {
(((x) >> S_CPL_TX_PKT_XT_CHKINSRTOFFSET_LO) & \
M_CPL_TX_PKT_XT_CHKINSRTOFFSET_LO)
+#define S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO 30
+#define M_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO 0x3
+#define V_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO(x) \
+ ((x) << S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO)
+#define G_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO(x) \
+ (((x) >> S_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO) & \
+ M_CPL_TX_PKT_XT_ROCEIPHDRLEN_LO)
+
#define S_CPL_TX_PKT_XT_CHKSTARTOFFSET 20
#define M_CPL_TX_PKT_XT_CHKSTARTOFFSET 0x3ff
#define V_CPL_TX_PKT_XT_CHKSTARTOFFSET(x) \
@@ -2190,7 +2241,8 @@ struct cpl_t7_tx_data_iso {
__be32 num_pi_bytes_seglen_offset;
__be32 datasn_offset;
__be32 buffer_offset;
- __be32 reserved3;
+ __be32 pdo_pkd;
+ /* encapsulated CPL_TX_DATA follows here */
};
#define S_CPL_T7_TX_DATA_ISO_OPCODE 24
@@ -2274,6 +2326,12 @@ struct cpl_t7_tx_data_iso {
(((x) >> S_CPL_T7_TX_DATA_ISO_DATASEGLENOFFSET) & \
M_CPL_T7_TX_DATA_ISO_DATASEGLENOFFSET)
+#define S_CPL_TX_DATA_ISO_PDO 0
+#define M_CPL_TX_DATA_ISO_PDO 0xff
+#define V_CPL_TX_DATA_ISO_PDO(x) ((x) << S_CPL_TX_DATA_ISO_PDO)
+#define G_CPL_TX_DATA_ISO_PDO(x) \
+ (((x) >> S_CPL_TX_DATA_ISO_PDO) & M_CPL_TX_DATA_ISO_PDO)
+
struct cpl_iscsi_hdr {
RSS_HDR
union opcode_tid ot;
@@ -2419,6 +2477,74 @@ struct cpl_rx_data_ack_core {
#define V_RX_DACK_CHANGE(x) ((x) << S_RX_DACK_CHANGE)
#define F_RX_DACK_CHANGE V_RX_DACK_CHANGE(1U)
+struct cpl_rx_phys_addr {
+ __be32 RSS[2];
+ __be32 op_to_tid;
+ __be32 pci_rlx_order_to_len;
+ __be64 phys_addr;
+};
+
+#define S_CPL_RX_PHYS_ADDR_OPCODE 24
+#define M_CPL_RX_PHYS_ADDR_OPCODE 0xff
+#define V_CPL_RX_PHYS_ADDR_OPCODE(x) ((x) << S_CPL_RX_PHYS_ADDR_OPCODE)
+#define G_CPL_RX_PHYS_ADDR_OPCODE(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_OPCODE) & M_CPL_RX_PHYS_ADDR_OPCODE)
+
+#define S_CPL_RX_PHYS_ADDR_ISRDMA 23
+#define M_CPL_RX_PHYS_ADDR_ISRDMA 0x1
+#define V_CPL_RX_PHYS_ADDR_ISRDMA(x) ((x) << S_CPL_RX_PHYS_ADDR_ISRDMA)
+#define G_CPL_RX_PHYS_ADDR_ISRDMA(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_ISRDMA) & M_CPL_RX_PHYS_ADDR_ISRDMA)
+#define F_CPL_RX_PHYS_ADDR_ISRDMA V_CPL_RX_PHYS_ADDR_ISRDMA(1U)
+
+#define S_CPL_RX_PHYS_ADDR_TID 0
+#define M_CPL_RX_PHYS_ADDR_TID 0xfffff
+#define V_CPL_RX_PHYS_ADDR_TID(x) ((x) << S_CPL_RX_PHYS_ADDR_TID)
+#define G_CPL_RX_PHYS_ADDR_TID(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_TID) & M_CPL_RX_PHYS_ADDR_TID)
+
+#define S_CPL_RX_PHYS_ADDR_PCIRLXORDER 31
+#define M_CPL_RX_PHYS_ADDR_PCIRLXORDER 0x1
+#define V_CPL_RX_PHYS_ADDR_PCIRLXORDER(x) \
+ ((x) << S_CPL_RX_PHYS_ADDR_PCIRLXORDER)
+#define G_CPL_RX_PHYS_ADDR_PCIRLXORDER(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_PCIRLXORDER) & M_CPL_RX_PHYS_ADDR_PCIRLXORDER)
+#define F_CPL_RX_PHYS_ADDR_PCIRLXORDER V_CPL_RX_PHYS_ADDR_PCIRLXORDER(1U)
+
+#define S_CPL_RX_PHYS_ADDR_PCINOSNOOP 30
+#define M_CPL_RX_PHYS_ADDR_PCINOSNOOP 0x1
+#define V_CPL_RX_PHYS_ADDR_PCINOSNOOP(x) \
+ ((x) << S_CPL_RX_PHYS_ADDR_PCINOSNOOP)
+#define G_CPL_RX_PHYS_ADDR_PCINOSNOOP(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_PCINOSNOOP) & M_CPL_RX_PHYS_ADDR_PCINOSNOOP)
+#define F_CPL_RX_PHYS_ADDR_PCINOSNOOP V_CPL_RX_PHYS_ADDR_PCINOSNOOP(1U)
+
+#define S_CPL_RX_PHYS_ADDR_PCITPHINTEN 29
+#define M_CPL_RX_PHYS_ADDR_PCITPHINTEN 0x1
+#define V_CPL_RX_PHYS_ADDR_PCITPHINTEN(x) \
+ ((x) << S_CPL_RX_PHYS_ADDR_PCITPHINTEN)
+#define G_CPL_RX_PHYS_ADDR_PCITPHINTEN(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_PCITPHINTEN) & M_CPL_RX_PHYS_ADDR_PCITPHINTEN)
+#define F_CPL_RX_PHYS_ADDR_PCITPHINTEN V_CPL_RX_PHYS_ADDR_PCITPHINTEN(1U)
+
+#define S_CPL_RX_PHYS_ADDR_PCITPHINT 27
+#define M_CPL_RX_PHYS_ADDR_PCITPHINT 0x3
+#define V_CPL_RX_PHYS_ADDR_PCITPHINT(x) ((x) << S_CPL_RX_PHYS_ADDR_PCITPHINT)
+#define G_CPL_RX_PHYS_ADDR_PCITPHINT(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_PCITPHINT) & M_CPL_RX_PHYS_ADDR_PCITPHINT)
+
+#define S_CPL_RX_PHYS_ADDR_DCAID 16
+#define M_CPL_RX_PHYS_ADDR_DCAID 0x7ff
+#define V_CPL_RX_PHYS_ADDR_DCAID(x) ((x) << S_CPL_RX_PHYS_ADDR_DCAID)
+#define G_CPL_RX_PHYS_ADDR_DCAID(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_DCAID) & M_CPL_RX_PHYS_ADDR_DCAID)
+
+#define S_CPL_RX_PHYS_ADDR_LEN 0
+#define M_CPL_RX_PHYS_ADDR_LEN 0xffff
+#define V_CPL_RX_PHYS_ADDR_LEN(x) ((x) << S_CPL_RX_PHYS_ADDR_LEN)
+#define G_CPL_RX_PHYS_ADDR_LEN(x) \
+ (((x) >> S_CPL_RX_PHYS_ADDR_LEN) & M_CPL_RX_PHYS_ADDR_LEN)
+
struct cpl_rx_ddp_complete {
RSS_HDR
union opcode_tid ot;
@@ -4059,13 +4185,6 @@ struct cpl_rdma_cqe_ext {
#define G_CPL_RDMA_CQE_EXT_QPID(x) \
(((x) >> S_CPL_RDMA_CQE_EXT_QPID) & M_CPL_RDMA_CQE_EXT_QPID)
-#define S_CPL_RDMA_CQE_EXT_EXTMODE 11
-#define M_CPL_RDMA_CQE_EXT_EXTMODE 0x1
-#define V_CPL_RDMA_CQE_EXT_EXTMODE(x) ((x) << S_CPL_RDMA_CQE_EXT_EXTMODE)
-#define G_CPL_RDMA_CQE_EXT_EXTMODE(x) \
- (((x) >> S_CPL_RDMA_CQE_EXT_EXTMODE) & M_CPL_RDMA_CQE_EXT_EXTMODE)
-#define F_CPL_RDMA_CQE_EXT_EXTMODE V_CPL_RDMA_CQE_EXT_EXTMODE(1U)
-
#define S_CPL_RDMA_CQE_EXT_GENERATION_BIT 10
#define M_CPL_RDMA_CQE_EXT_GENERATION_BIT 0x1
#define V_CPL_RDMA_CQE_EXT_GENERATION_BIT(x) \
@@ -4109,6 +4228,13 @@ struct cpl_rdma_cqe_ext {
#define G_CPL_RDMA_CQE_EXT_WR_TYPE_EXT(x) \
(((x) >> S_CPL_RDMA_CQE_EXT_WR_TYPE_EXT) & M_CPL_RDMA_CQE_EXT_WR_TYPE_EXT)
+#define S_CPL_RDMA_CQE_EXT_EXTMODE 23
+#define M_CPL_RDMA_CQE_EXT_EXTMODE 0x1
+#define V_CPL_RDMA_CQE_EXT_EXTMODE(x) ((x) << S_CPL_RDMA_CQE_EXT_EXTMODE)
+#define G_CPL_RDMA_CQE_EXT_EXTMODE(x) \
+ (((x) >> S_CPL_RDMA_CQE_EXT_EXTMODE) & M_CPL_RDMA_CQE_EXT_EXTMODE)
+#define F_CPL_RDMA_CQE_EXT_EXTMODE V_CPL_RDMA_CQE_EXT_EXTMODE(1U)
+
#define S_CPL_RDMA_CQE_EXT_SRQ 0
#define M_CPL_RDMA_CQE_EXT_SRQ 0xfff
#define V_CPL_RDMA_CQE_EXT_SRQ(x) ((x) << S_CPL_RDMA_CQE_EXT_SRQ)
@@ -4161,14 +4287,6 @@ struct cpl_rdma_cqe_fw_ext {
#define G_CPL_RDMA_CQE_FW_EXT_QPID(x) \
(((x) >> S_CPL_RDMA_CQE_FW_EXT_QPID) & M_CPL_RDMA_CQE_FW_EXT_QPID)
-#define S_CPL_RDMA_CQE_FW_EXT_EXTMODE 11
-#define M_CPL_RDMA_CQE_FW_EXT_EXTMODE 0x1
-#define V_CPL_RDMA_CQE_FW_EXT_EXTMODE(x) \
- ((x) << S_CPL_RDMA_CQE_FW_EXT_EXTMODE)
-#define G_CPL_RDMA_CQE_FW_EXT_EXTMODE(x) \
- (((x) >> S_CPL_RDMA_CQE_FW_EXT_EXTMODE) & M_CPL_RDMA_CQE_FW_EXT_EXTMODE)
-#define F_CPL_RDMA_CQE_FW_EXT_EXTMODE V_CPL_RDMA_CQE_FW_EXT_EXTMODE(1U)
-
#define S_CPL_RDMA_CQE_FW_EXT_GENERATION_BIT 10
#define M_CPL_RDMA_CQE_FW_EXT_GENERATION_BIT 0x1
#define V_CPL_RDMA_CQE_FW_EXT_GENERATION_BIT(x) \
@@ -4215,6 +4333,14 @@ struct cpl_rdma_cqe_fw_ext {
(((x) >> S_CPL_RDMA_CQE_FW_EXT_WR_TYPE_EXT) & \
M_CPL_RDMA_CQE_FW_EXT_WR_TYPE_EXT)
+#define S_CPL_RDMA_CQE_FW_EXT_EXTMODE 23
+#define M_CPL_RDMA_CQE_FW_EXT_EXTMODE 0x1
+#define V_CPL_RDMA_CQE_FW_EXT_EXTMODE(x) \
+ ((x) << S_CPL_RDMA_CQE_FW_EXT_EXTMODE)
+#define G_CPL_RDMA_CQE_FW_EXT_EXTMODE(x) \
+ (((x) >> S_CPL_RDMA_CQE_FW_EXT_EXTMODE) & M_CPL_RDMA_CQE_FW_EXT_EXTMODE)
+#define F_CPL_RDMA_CQE_FW_EXT_EXTMODE V_CPL_RDMA_CQE_FW_EXT_EXTMODE(1U)
+
#define S_CPL_RDMA_CQE_FW_EXT_SRQ 0
#define M_CPL_RDMA_CQE_FW_EXT_SRQ 0xfff
#define V_CPL_RDMA_CQE_FW_EXT_SRQ(x) ((x) << S_CPL_RDMA_CQE_FW_EXT_SRQ)
@@ -4267,14 +4393,6 @@ struct cpl_rdma_cqe_err_ext {
#define G_CPL_RDMA_CQE_ERR_EXT_QPID(x) \
(((x) >> S_CPL_RDMA_CQE_ERR_EXT_QPID) & M_CPL_RDMA_CQE_ERR_EXT_QPID)
-#define S_CPL_RDMA_CQE_ERR_EXT_EXTMODE 11
-#define M_CPL_RDMA_CQE_ERR_EXT_EXTMODE 0x1
-#define V_CPL_RDMA_CQE_ERR_EXT_EXTMODE(x) \
- ((x) << S_CPL_RDMA_CQE_ERR_EXT_EXTMODE)
-#define G_CPL_RDMA_CQE_ERR_EXT_EXTMODE(x) \
- (((x) >> S_CPL_RDMA_CQE_ERR_EXT_EXTMODE) & M_CPL_RDMA_CQE_ERR_EXT_EXTMODE)
-#define F_CPL_RDMA_CQE_ERR_EXT_EXTMODE V_CPL_RDMA_CQE_ERR_EXT_EXTMODE(1U)
-
#define S_CPL_RDMA_CQE_ERR_EXT_GENERATION_BIT 10
#define M_CPL_RDMA_CQE_ERR_EXT_GENERATION_BIT 0x1
#define V_CPL_RDMA_CQE_ERR_EXT_GENERATION_BIT(x) \
@@ -4323,6 +4441,14 @@ struct cpl_rdma_cqe_err_ext {
(((x) >> S_CPL_RDMA_CQE_ERR_EXT_WR_TYPE_EXT) & \
M_CPL_RDMA_CQE_ERR_EXT_WR_TYPE_EXT)
+#define S_CPL_RDMA_CQE_ERR_EXT_EXTMODE 23
+#define M_CPL_RDMA_CQE_ERR_EXT_EXTMODE 0x1
+#define V_CPL_RDMA_CQE_ERR_EXT_EXTMODE(x) \
+ ((x) << S_CPL_RDMA_CQE_ERR_EXT_EXTMODE)
+#define G_CPL_RDMA_CQE_ERR_EXT_EXTMODE(x) \
+ (((x) >> S_CPL_RDMA_CQE_ERR_EXT_EXTMODE) & M_CPL_RDMA_CQE_ERR_EXT_EXTMODE)
+#define F_CPL_RDMA_CQE_ERR_EXT_EXTMODE V_CPL_RDMA_CQE_ERR_EXT_EXTMODE(1U)
+
#define S_CPL_RDMA_CQE_ERR_EXT_SRQ 0
#define M_CPL_RDMA_CQE_ERR_EXT_SRQ 0xfff
#define V_CPL_RDMA_CQE_ERR_EXT_SRQ(x) ((x) << S_CPL_RDMA_CQE_ERR_EXT_SRQ)
@@ -5040,6 +5166,58 @@ struct cpl_tx_tnl_lso {
#define G_CPL_TX_TNL_LSO_SIZE(x) \
(((x) >> S_CPL_TX_TNL_LSO_SIZE) & M_CPL_TX_TNL_LSO_SIZE)
+#define S_CPL_TX_TNL_LSO_BTH_OPCODE 24
+#define M_CPL_TX_TNL_LSO_BTH_OPCODE 0xff
+#define V_CPL_TX_TNL_LSO_BTH_OPCODE(x) ((x) << S_CPL_TX_TNL_LSO_BTH_OPCODE)
+#define G_CPL_TX_TNL_LSO_BTH_OPCODE(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_BTH_OPCODE) & \
+ M_CPL_TX_TNL_LSO_BTH_OPCODE)
+
+#define S_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN 0
+#define M_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN 0xffffff
+#define V_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN(x) \
+ ((x) << S_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN)
+#define G_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN) & \
+ M_CPL_TX_TNL_LSO_TCPSEQOFFSET_PSN)
+
+#define S_CPL_TX_TNL_LSO_MSS_TVER 8
+#define M_CPL_TX_TNL_LSO_MSS_TVER 0xf
+#define V_CPL_TX_TNL_LSO_MSS_TVER(x) ((x) << S_CPL_TX_TNL_LSO_MSS_TVER)
+#define G_CPL_TX_TNL_LSO_MSS_TVER(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_TVER) & M_CPL_TX_TNL_LSO_MSS_TVER)
+
+#define S_CPL_TX_TNL_LSO_MSS_M 7
+#define M_CPL_TX_TNL_LSO_MSS_M 0x1
+#define V_CPL_TX_TNL_LSO_MSS_M(x) ((x) << S_CPL_TX_TNL_LSO_MSS_M)
+#define G_CPL_TX_TNL_LSO_MSS_M(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_M) & M_CPL_TX_TNL_LSO_MSS_M)
+
+#define S_CPL_TX_TNL_LSO_MSS_PMTU 4
+#define M_CPL_TX_TNL_LSO_MSS_PMTU 0x7
+#define V_CPL_TX_TNL_LSO_MSS_PMTU(x) ((x) << S_CPL_TX_TNL_LSO_MSS_PMTU)
+#define G_CPL_TX_TNL_LSO_MSS_PMTU(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_PMTU) & M_CPL_TX_TNL_LSO_MSS_PMTU)
+
+#define S_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR 3
+#define M_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR 0x1
+#define V_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR(x) \
+ ((x) << S_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR)
+#define G_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR) & M_CPL_TX_TNL_LSO_MSS_RR_MSN_INCR)
+
+#define S_CPL_TX_TNL_LSO_MSS_ACKREQ 1
+#define M_CPL_TX_TNL_LSO_MSS_ACKREQ 0x3
+#define V_CPL_TX_TNL_LSO_MSS_ACKREQ(x) ((x) << S_CPL_TX_TNL_LSO_MSS_ACKREQ)
+#define G_CPL_TX_TNL_LSO_MSS_ACKREQ(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_ACKREQ) & M_CPL_TX_TNL_LSO_MSS_ACKREQ)
+
+#define S_CPL_TX_TNL_LSO_MSS_SE 0
+#define M_CPL_TX_TNL_LSO_MSS_SE 0x1
+#define V_CPL_TX_TNL_LSO_MSS_SE(x) ((x) << S_CPL_TX_TNL_LSO_MSS_SE)
+#define G_CPL_TX_TNL_LSO_MSS_SE(x) \
+ (((x) >> S_CPL_TX_TNL_LSO_MSS_SE) & M_CPL_TX_TNL_LSO_MSS_SE)
+
struct cpl_rx_mps_pkt {
__be32 op_to_r1_hi;
__be32 r1_lo_length;
@@ -5839,10 +6017,10 @@ struct cpl_tx_tls_ack {
#define G_CPL_TX_TLS_ACK_OPCODE(x) \
(((x) >> S_CPL_TX_TLS_ACK_OPCODE) & M_CPL_TX_TLS_ACK_OPCODE)
-#define S_T7_CPL_TX_TLS_ACK_RXCHID 22
-#define M_T7_CPL_TX_TLS_ACK_RXCHID 0x3
-#define V_T7_CPL_TX_TLS_ACK_RXCHID(x) ((x) << S_T7_CPL_TX_TLS_ACK_RXCHID)
-#define G_T7_CPL_TX_TLS_ACK_RXCHID(x) \
+#define S_T7_CPL_TX_TLS_ACK_RXCHID 22
+#define M_T7_CPL_TX_TLS_ACK_RXCHID 0x3
+#define V_T7_CPL_TX_TLS_ACK_RXCHID(x) ((x) << S_T7_CPL_TX_TLS_ACK_RXCHID)
+#define G_T7_CPL_TX_TLS_ACK_RXCHID(x) \
(((x) >> S_T7_CPL_TX_TLS_ACK_RXCHID) & M_T7_CPL_TX_TLS_ACK_RXCHID)
#define S_CPL_TX_TLS_ACK_RXCHID 22
@@ -5905,11 +6083,245 @@ struct cpl_tx_tls_ack {
#define G_CPL_TX_TLS_ACK_PLDLEN(x) \
(((x) >> S_CPL_TX_TLS_ACK_PLDLEN) & M_CPL_TX_TLS_ACK_PLDLEN)
+struct cpl_tx_quic_enc {
+ __be32 op_to_hdrlen;
+ __be32 hdrlen_to_pktlen;
+ __be32 r4[2];
+};
+
+#define S_CPL_TX_QUIC_ENC_OPCODE 24
+#define M_CPL_TX_QUIC_ENC_OPCODE 0xff
+#define V_CPL_TX_QUIC_ENC_OPCODE(x) ((x) << S_CPL_TX_QUIC_ENC_OPCODE)
+#define G_CPL_TX_QUIC_ENC_OPCODE(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_OPCODE) & M_CPL_TX_QUIC_ENC_OPCODE)
+
+#define S_CPL_TX_QUIC_ENC_KEYSIZE 22
+#define M_CPL_TX_QUIC_ENC_KEYSIZE 0x3
+#define V_CPL_TX_QUIC_ENC_KEYSIZE(x) ((x) << S_CPL_TX_QUIC_ENC_KEYSIZE)
+#define G_CPL_TX_QUIC_ENC_KEYSIZE(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_KEYSIZE) & M_CPL_TX_QUIC_ENC_KEYSIZE)
+
+#define S_CPL_TX_QUIC_ENC_PKTNUMSIZE 20
+#define M_CPL_TX_QUIC_ENC_PKTNUMSIZE 0x3
+#define V_CPL_TX_QUIC_ENC_PKTNUMSIZE(x) ((x) << S_CPL_TX_QUIC_ENC_PKTNUMSIZE)
+#define G_CPL_TX_QUIC_ENC_PKTNUMSIZE(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_PKTNUMSIZE) & M_CPL_TX_QUIC_ENC_PKTNUMSIZE)
+
+#define S_CPL_TX_QUIC_ENC_HDRTYPE 19
+#define M_CPL_TX_QUIC_ENC_HDRTYPE 0x1
+#define V_CPL_TX_QUIC_ENC_HDRTYPE(x) ((x) << S_CPL_TX_QUIC_ENC_HDRTYPE)
+#define G_CPL_TX_QUIC_ENC_HDRTYPE(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_HDRTYPE) & M_CPL_TX_QUIC_ENC_HDRTYPE)
+#define F_CPL_TX_QUIC_ENC_HDRTYPE V_CPL_TX_QUIC_ENC_HDRTYPE(1U)
+
+#define S_CPL_TX_QUIC_ENC_HDRSTARTOFFSET 4
+#define M_CPL_TX_QUIC_ENC_HDRSTARTOFFSET 0xfff
+#define V_CPL_TX_QUIC_ENC_HDRSTARTOFFSET(x) \
+ ((x) << S_CPL_TX_QUIC_ENC_HDRSTARTOFFSET)
+#define G_CPL_TX_QUIC_ENC_HDRSTARTOFFSET(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_HDRSTARTOFFSET) & \
+ M_CPL_TX_QUIC_ENC_HDRSTARTOFFSET)
+
+#define S_CPL_TX_QUIC_ENC_HDRLENGTH_HI 0
+#define M_CPL_TX_QUIC_ENC_HDRLENGTH_HI 0x3
+#define V_CPL_TX_QUIC_ENC_HDRLENGTH_HI(x) \
+ ((x) << S_CPL_TX_QUIC_ENC_HDRLENGTH_HI)
+#define G_CPL_TX_QUIC_ENC_HDRLENGTH_HI(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_HDRLENGTH_HI) & M_CPL_TX_QUIC_ENC_HDRLENGTH_HI)
+
+#define S_CPL_TX_QUIC_ENC_HDRLENGTH_LO 24
+#define M_CPL_TX_QUIC_ENC_HDRLENGTH_LO 0xff
+#define V_CPL_TX_QUIC_ENC_HDRLENGTH_LO(x) \
+ ((x) << S_CPL_TX_QUIC_ENC_HDRLENGTH_LO)
+#define G_CPL_TX_QUIC_ENC_HDRLENGTH_LO(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_HDRLENGTH_LO) & M_CPL_TX_QUIC_ENC_HDRLENGTH_LO)
+
+#define S_CPL_TX_QUIC_ENC_NUMPKT 16
+#define M_CPL_TX_QUIC_ENC_NUMPKT 0xff
+#define V_CPL_TX_QUIC_ENC_NUMPKT(x) ((x) << S_CPL_TX_QUIC_ENC_NUMPKT)
+#define G_CPL_TX_QUIC_ENC_NUMPKT(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_NUMPKT) & M_CPL_TX_QUIC_ENC_NUMPKT)
+
+#define S_CPL_TX_QUIC_ENC_PKTLEN 0
+#define M_CPL_TX_QUIC_ENC_PKTLEN 0xffff
+#define V_CPL_TX_QUIC_ENC_PKTLEN(x) ((x) << S_CPL_TX_QUIC_ENC_PKTLEN)
+#define G_CPL_TX_QUIC_ENC_PKTLEN(x) \
+ (((x) >> S_CPL_TX_QUIC_ENC_PKTLEN) & M_CPL_TX_QUIC_ENC_PKTLEN)
+
+struct cpl_tls_tx_scmd_fmt {
+ __be32 op_to_num_ivs;
+ __be32 enb_dbgId_to_hdrlen;
+ __be32 seq_num[2];
+};
+
+#define S_CPL_TLS_TX_SCMD_FMT_OPCODE 31
+#define M_CPL_TLS_TX_SCMD_FMT_OPCODE 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_OPCODE(x) ((x) << S_CPL_TLS_TX_SCMD_FMT_OPCODE)
+#define G_CPL_TLS_TX_SCMD_FMT_OPCODE(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_OPCODE) & M_CPL_TLS_TX_SCMD_FMT_OPCODE)
+#define F_CPL_TLS_TX_SCMD_FMT_OPCODE V_CPL_TLS_TX_SCMD_FMT_OPCODE(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL 29
+#define M_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL 0x3
+#define V_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL)
+#define G_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL) & \
+ M_CPL_TLS_TX_SCMD_FMT_SEQNUMBERCTRL)
+
+#define S_CPL_TLS_TX_SCMD_FMT_PROTOVERSION 24
+#define M_CPL_TLS_TX_SCMD_FMT_PROTOVERSION 0xf
+#define V_CPL_TLS_TX_SCMD_FMT_PROTOVERSION(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_PROTOVERSION)
+#define G_CPL_TLS_TX_SCMD_FMT_PROTOVERSION(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_PROTOVERSION) & \
+ M_CPL_TLS_TX_SCMD_FMT_PROTOVERSION)
+
+#define S_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL 23
+#define M_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL)
+#define G_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL) & \
+ M_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL)
+#define F_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL V_CPL_TLS_TX_SCMD_FMT_ENCDECCTRL(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL 22
+#define M_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL)
+#define G_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL) & \
+ M_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL)
+#define F_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL \
+ V_CPL_TLS_TX_SCMD_FMT_CIPHAUTHSEQCTRL(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_CIPHMODE 18
+#define M_CPL_TLS_TX_SCMD_FMT_CIPHMODE 0xf
+#define V_CPL_TLS_TX_SCMD_FMT_CIPHMODE(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_CIPHMODE)
+#define G_CPL_TLS_TX_SCMD_FMT_CIPHMODE(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_CIPHMODE) & M_CPL_TLS_TX_SCMD_FMT_CIPHMODE)
+
+#define S_CPL_TLS_TX_SCMD_FMT_AUTHMODE 14
+#define M_CPL_TLS_TX_SCMD_FMT_AUTHMODE 0xf
+#define V_CPL_TLS_TX_SCMD_FMT_AUTHMODE(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_AUTHMODE)
+#define G_CPL_TLS_TX_SCMD_FMT_AUTHMODE(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_AUTHMODE) & M_CPL_TLS_TX_SCMD_FMT_AUTHMODE)
+
+#define S_CPL_TLS_TX_SCMD_FMT_HMACCTRL 11
+#define M_CPL_TLS_TX_SCMD_FMT_HMACCTRL 0x7
+#define V_CPL_TLS_TX_SCMD_FMT_HMACCTRL(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_HMACCTRL)
+#define G_CPL_TLS_TX_SCMD_FMT_HMACCTRL(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_HMACCTRL) & M_CPL_TLS_TX_SCMD_FMT_HMACCTRL)
+
+#define S_CPL_TLS_TX_SCMD_FMT_IVSIZE 7
+#define M_CPL_TLS_TX_SCMD_FMT_IVSIZE 0xf
+#define V_CPL_TLS_TX_SCMD_FMT_IVSIZE(x) ((x) << S_CPL_TLS_TX_SCMD_FMT_IVSIZE)
+#define G_CPL_TLS_TX_SCMD_FMT_IVSIZE(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_IVSIZE) & M_CPL_TLS_TX_SCMD_FMT_IVSIZE)
+
+#define S_CPL_TLS_TX_SCMD_FMT_NUMIVS 0
+#define M_CPL_TLS_TX_SCMD_FMT_NUMIVS 0x7f
+#define V_CPL_TLS_TX_SCMD_FMT_NUMIVS(x) ((x) << S_CPL_TLS_TX_SCMD_FMT_NUMIVS)
+#define G_CPL_TLS_TX_SCMD_FMT_NUMIVS(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_NUMIVS) & M_CPL_TLS_TX_SCMD_FMT_NUMIVS)
+
+#define S_CPL_TLS_TX_SCMD_FMT_ENBDBGID 31
+#define M_CPL_TLS_TX_SCMD_FMT_ENBDBGID 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_ENBDBGID(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_ENBDBGID)
+#define G_CPL_TLS_TX_SCMD_FMT_ENBDBGID(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_ENBDBGID) & M_CPL_TLS_TX_SCMD_FMT_ENBDBGID)
+#define F_CPL_TLS_TX_SCMD_FMT_ENBDBGID V_CPL_TLS_TX_SCMD_FMT_ENBDBGID(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_IVGENCTRL 30
+#define M_CPL_TLS_TX_SCMD_FMT_IVGENCTRL 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_IVGENCTRL(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_IVGENCTRL)
+#define G_CPL_TLS_TX_SCMD_FMT_IVGENCTRL(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_IVGENCTRL) & \
+ M_CPL_TLS_TX_SCMD_FMT_IVGENCTRL)
+
+#define S_CPL_TLS_TX_SCMD_FMT_MOREFRAGS 20
+#define M_CPL_TLS_TX_SCMD_FMT_MOREFRAGS 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_MOREFRAGS(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_MOREFRAGS)
+#define G_CPL_TLS_TX_SCMD_FMT_MOREFRAGS(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_MOREFRAGS) & \
+ M_CPL_TLS_TX_SCMD_FMT_MOREFRAGS)
+#define F_CPL_TLS_TX_SCMD_FMT_MOREFRAGS V_CPL_TLS_TX_SCMD_FMT_MOREFRAGS(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_LASTFRAGS 19
+#define M_CPL_TLS_TX_SCMD_FMT_LASTFRAGS 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_LASTFRAGS(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_LASTFRAGS)
+#define G_CPL_TLS_TX_SCMD_FMT_LASTFRAGS(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_LASTFRAGS) & \
+ M_CPL_TLS_TX_SCMD_FMT_LASTFRAGS)
+#define F_CPL_TLS_TX_SCMD_FMT_LASTFRAGS V_CPL_TLS_TX_SCMD_FMT_LASTFRAGS(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU 18
+#define M_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU)
+#define G_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU) & \
+ M_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU)
+#define F_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU V_CPL_TLS_TX_SCMD_FMT_TLSCOMPPDU(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY 17
+#define M_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY)
+#define G_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY) & \
+ M_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY)
+#define F_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY \
+ V_CPL_TLS_TX_SCMD_FMT_PAYLOADONLY(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE 16
+#define M_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE)
+#define G_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE) & \
+ M_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE)
+#define F_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE \
+ V_CPL_TLS_TX_SCMD_FMT_TLSFRAGENABLE(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_MACONLY 15
+#define M_CPL_TLS_TX_SCMD_FMT_MACONLY 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_MACONLY(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_MACONLY)
+#define G_CPL_TLS_TX_SCMD_FMT_MACONLY(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_MACONLY) & M_CPL_TLS_TX_SCMD_FMT_MACONLY)
+#define F_CPL_TLS_TX_SCMD_FMT_MACONLY V_CPL_TLS_TX_SCMD_FMT_MACONLY(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_AADIVDROP 14
+#define M_CPL_TLS_TX_SCMD_FMT_AADIVDROP 0x1
+#define V_CPL_TLS_TX_SCMD_FMT_AADIVDROP(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_AADIVDROP)
+#define G_CPL_TLS_TX_SCMD_FMT_AADIVDROP(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_AADIVDROP) & \
+ M_CPL_TLS_TX_SCMD_FMT_AADIVDROP)
+#define F_CPL_TLS_TX_SCMD_FMT_AADIVDROP V_CPL_TLS_TX_SCMD_FMT_AADIVDROP(1U)
+
+#define S_CPL_TLS_TX_SCMD_FMT_HDRLENGTH 0
+#define M_CPL_TLS_TX_SCMD_FMT_HDRLENGTH 0x3fff
+#define V_CPL_TLS_TX_SCMD_FMT_HDRLENGTH(x) \
+ ((x) << S_CPL_TLS_TX_SCMD_FMT_HDRLENGTH)
+#define G_CPL_TLS_TX_SCMD_FMT_HDRLENGTH(x) \
+ (((x) >> S_CPL_TLS_TX_SCMD_FMT_HDRLENGTH) & \
+ M_CPL_TLS_TX_SCMD_FMT_HDRLENGTH)
+
struct cpl_rcb_upd {
__be32 op_to_tid;
__be32 opcode_psn;
__u8 nodata_to_cnprepclr;
- __u8 r0;
+ __u8 rsp_nak_seqclr_pkd;
__be16 wrptr;
__be32 length;
};
@@ -6202,13 +6614,6 @@ struct cpl_roce_cqe {
#define G_CPL_ROCE_CQE_QPID(x) \
(((x) >> S_CPL_ROCE_CQE_QPID) & M_CPL_ROCE_CQE_QPID)
-#define S_CPL_ROCE_CQE_EXTMODE 11
-#define M_CPL_ROCE_CQE_EXTMODE 0x1
-#define V_CPL_ROCE_CQE_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_EXTMODE)
-#define G_CPL_ROCE_CQE_EXTMODE(x) \
- (((x) >> S_CPL_ROCE_CQE_EXTMODE) & M_CPL_ROCE_CQE_EXTMODE)
-#define F_CPL_ROCE_CQE_EXTMODE V_CPL_ROCE_CQE_EXTMODE(1U)
-
#define S_CPL_ROCE_CQE_GENERATION_BIT 10
#define M_CPL_ROCE_CQE_GENERATION_BIT 0x1
#define V_CPL_ROCE_CQE_GENERATION_BIT(x) \
@@ -6249,6 +6654,13 @@ struct cpl_roce_cqe {
#define G_CPL_ROCE_CQE_WR_TYPE_EXT(x) \
(((x) >> S_CPL_ROCE_CQE_WR_TYPE_EXT) & M_CPL_ROCE_CQE_WR_TYPE_EXT)
+#define S_CPL_ROCE_CQE_EXTMODE 23
+#define M_CPL_ROCE_CQE_EXTMODE 0x1
+#define V_CPL_ROCE_CQE_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_EXTMODE)
+#define G_CPL_ROCE_CQE_EXTMODE(x) \
+ (((x) >> S_CPL_ROCE_CQE_EXTMODE) & M_CPL_ROCE_CQE_EXTMODE)
+#define F_CPL_ROCE_CQE_EXTMODE V_CPL_ROCE_CQE_EXTMODE(1U)
+
#define S_CPL_ROCE_CQE_SRQ 0
#define M_CPL_ROCE_CQE_SRQ 0xfff
#define V_CPL_ROCE_CQE_SRQ(x) ((x) << S_CPL_ROCE_CQE_SRQ)
@@ -6304,13 +6716,6 @@ struct cpl_roce_cqe_fw {
#define G_CPL_ROCE_CQE_FW_QPID(x) \
(((x) >> S_CPL_ROCE_CQE_FW_QPID) & M_CPL_ROCE_CQE_FW_QPID)
-#define S_CPL_ROCE_CQE_FW_EXTMODE 11
-#define M_CPL_ROCE_CQE_FW_EXTMODE 0x1
-#define V_CPL_ROCE_CQE_FW_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_FW_EXTMODE)
-#define G_CPL_ROCE_CQE_FW_EXTMODE(x) \
- (((x) >> S_CPL_ROCE_CQE_FW_EXTMODE) & M_CPL_ROCE_CQE_FW_EXTMODE)
-#define F_CPL_ROCE_CQE_FW_EXTMODE V_CPL_ROCE_CQE_FW_EXTMODE(1U)
-
#define S_CPL_ROCE_CQE_FW_GENERATION_BIT 10
#define M_CPL_ROCE_CQE_FW_GENERATION_BIT 0x1
#define V_CPL_ROCE_CQE_FW_GENERATION_BIT(x) \
@@ -6353,6 +6758,14 @@ struct cpl_roce_cqe_fw {
#define G_CPL_ROCE_CQE_FW_WR_TYPE_EXT(x) \
(((x) >> S_CPL_ROCE_CQE_FW_WR_TYPE_EXT) & M_CPL_ROCE_CQE_FW_WR_TYPE_EXT)
+#define S_CPL_ROCE_CQE_FW_EXTMODE 23
+#define M_CPL_ROCE_CQE_FW_EXTMODE 0x1
+#define V_CPL_ROCE_CQE_FW_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_FW_EXTMODE)
+#define G_CPL_ROCE_CQE_FW_EXTMODE(x) \
+ (((x) >> S_CPL_ROCE_CQE_FW_EXTMODE) & M_CPL_ROCE_CQE_FW_EXTMODE)
+#define F_CPL_ROCE_CQE_FW_EXTMODE V_CPL_ROCE_CQE_FW_EXTMODE(1U)
+
+
#define S_CPL_ROCE_CQE_FW_SRQ 0
#define M_CPL_ROCE_CQE_FW_SRQ 0xfff
#define V_CPL_ROCE_CQE_FW_SRQ(x) ((x) << S_CPL_ROCE_CQE_FW_SRQ)
@@ -6360,16 +6773,16 @@ struct cpl_roce_cqe_fw {
(((x) >> S_CPL_ROCE_CQE_FW_SRQ) & M_CPL_ROCE_CQE_FW_SRQ)
struct cpl_roce_cqe_err {
- __be32 op_to_CQID;
- __be32 Tid_FlitCnt;
- __be32 QPID_to_WR_type;
- __be32 Length;
- __be32 TAG;
- __be32 MSN;
- __be32 SE_to_SRQ;
- __be32 RQE;
- __be32 ExtInfoMS[2];
- __be32 ExtInfoLS[2];
+ __be32 op_to_cqid;
+ __be32 tid_flitcnt;
+ __be32 qpid_to_wr_type;
+ __be32 length;
+ __be32 tag;
+ __be32 msn;
+ __be32 se_to_srq;
+ __be32 rqe;
+ __be32 extinfoms[2];
+ __be32 extinfols[2];
};
#define S_CPL_ROCE_CQE_ERR_OPCODE 24
@@ -6408,13 +6821,6 @@ struct cpl_roce_cqe_err {
#define G_CPL_ROCE_CQE_ERR_QPID(x) \
(((x) >> S_CPL_ROCE_CQE_ERR_QPID) & M_CPL_ROCE_CQE_ERR_QPID)
-#define S_CPL_ROCE_CQE_ERR_EXTMODE 11
-#define M_CPL_ROCE_CQE_ERR_EXTMODE 0x1
-#define V_CPL_ROCE_CQE_ERR_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_ERR_EXTMODE)
-#define G_CPL_ROCE_CQE_ERR_EXTMODE(x) \
- (((x) >> S_CPL_ROCE_CQE_ERR_EXTMODE) & M_CPL_ROCE_CQE_ERR_EXTMODE)
-#define F_CPL_ROCE_CQE_ERR_EXTMODE V_CPL_ROCE_CQE_ERR_EXTMODE(1U)
-
#define S_CPL_ROCE_CQE_ERR_GENERATION_BIT 10
#define M_CPL_ROCE_CQE_ERR_GENERATION_BIT 0x1
#define V_CPL_ROCE_CQE_ERR_GENERATION_BIT(x) \
@@ -6458,6 +6864,14 @@ struct cpl_roce_cqe_err {
#define G_CPL_ROCE_CQE_ERR_WR_TYPE_EXT(x) \
(((x) >> S_CPL_ROCE_CQE_ERR_WR_TYPE_EXT) & M_CPL_ROCE_CQE_ERR_WR_TYPE_EXT)
+#define S_CPL_ROCE_CQE_ERR_EXTMODE 23
+#define M_CPL_ROCE_CQE_ERR_EXTMODE 0x1
+#define V_CPL_ROCE_CQE_ERR_EXTMODE(x) ((x) << S_CPL_ROCE_CQE_ERR_EXTMODE)
+#define G_CPL_ROCE_CQE_ERR_EXTMODE(x) \
+ (((x) >> S_CPL_ROCE_CQE_ERR_EXTMODE) & M_CPL_ROCE_CQE_ERR_EXTMODE)
+#define F_CPL_ROCE_CQE_ERR_EXTMODE V_CPL_ROCE_CQE_ERR_EXTMODE(1U)
+
+
#define S_CPL_ROCE_CQE_ERR_SRQ 0
#define M_CPL_ROCE_CQE_ERR_SRQ 0xfff
#define V_CPL_ROCE_CQE_ERR_SRQ(x) ((x) << S_CPL_ROCE_CQE_ERR_SRQ)
diff --git a/sys/dev/cxgbe/common/t4_regs.h b/sys/dev/cxgbe/common/t4_regs.h
index 8f500ec0fbdd..51f150443261 100644
--- a/sys/dev/cxgbe/common/t4_regs.h
+++ b/sys/dev/cxgbe/common/t4_regs.h
@@ -27,11 +27,11 @@
*/
/* This file is automatically generated --- changes will be lost */
-/* Generation Date : Thu Sep 11 05:25:56 PM IST 2025 */
+/* Generation Date : Tue Oct 28 05:23:45 PM IST 2025 */
/* Directory name: t4_reg.txt, Date: Not specified */
/* Directory name: t5_reg.txt, Changeset: 6945:54ba4ba7ee8b */
/* Directory name: t6_reg.txt, Changeset: 4277:9c165d0f4899 */
-/* Directory name: t7_reg.txt, Changeset: 5945:1487219ecb20 */
+/* Directory name: t7_sw_reg.txt, Changeset: 5946:0b60ff298e7d */
#define MYPF_BASE 0x1b000
#define MYPF_REG(reg_addr) (MYPF_BASE + (reg_addr))
@@ -44006,10 +44006,57 @@
#define V_MPS2CRYPTO_RX_INTF_FIFO(x) ((x) << S_MPS2CRYPTO_RX_INTF_FIFO)
#define G_MPS2CRYPTO_RX_INTF_FIFO(x) (((x) >> S_MPS2CRYPTO_RX_INTF_FIFO) & M_MPS2CRYPTO_RX_INTF_FIFO)
-#define S_RX_PRE_PROC_PERR 9
-#define M_RX_PRE_PROC_PERR 0x7ffU
-#define V_RX_PRE_PROC_PERR(x) ((x) << S_RX_PRE_PROC_PERR)
-#define G_RX_PRE_PROC_PERR(x) (((x) >> S_RX_PRE_PROC_PERR) & M_RX_PRE_PROC_PERR)
+#define S_MAC_RX_PPROC_MPS2TP_TF 19
+#define V_MAC_RX_PPROC_MPS2TP_TF(x) ((x) << S_MAC_RX_PPROC_MPS2TP_TF)
+#define F_MAC_RX_PPROC_MPS2TP_TF V_MAC_RX_PPROC_MPS2TP_TF(1U)
+
+#define S_MAC_RX_PPROC_LB_CH3 18
+#define V_MAC_RX_PPROC_LB_CH3(x) ((x) << S_MAC_RX_PPROC_LB_CH3)
+#define F_MAC_RX_PPROC_LB_CH3 V_MAC_RX_PPROC_LB_CH3(1U)
+
+#define S_MAC_RX_PPROC_LB_CH2 17
+#define V_MAC_RX_PPROC_LB_CH2(x) ((x) << S_MAC_RX_PPROC_LB_CH2)
+#define F_MAC_RX_PPROC_LB_CH2 V_MAC_RX_PPROC_LB_CH2(1U)
+
+#define S_MAC_RX_PPROC_LB_CH1 16
+#define V_MAC_RX_PPROC_LB_CH1(x) ((x) << S_MAC_RX_PPROC_LB_CH1)
+#define F_MAC_RX_PPROC_LB_CH1 V_MAC_RX_PPROC_LB_CH1(1U)
+
+#define S_MAC_RX_PPROC_LB_CH0 15
+#define V_MAC_RX_PPROC_LB_CH0(x) ((x) << S_MAC_RX_PPROC_LB_CH0)
+#define F_MAC_RX_PPROC_LB_CH0 V_MAC_RX_PPROC_LB_CH0(1U)
+
+#define S_MAC_RX_PPROC_DWRR_CH0_3 14
+#define V_MAC_RX_PPROC_DWRR_CH0_3(x) ((x) << S_MAC_RX_PPROC_DWRR_CH0_3)
+#define F_MAC_RX_PPROC_DWRR_CH0_3 V_MAC_RX_PPROC_DWRR_CH0_3(1U)
+
+#define S_MAC_RX_FIFO_PERR 13
+#define V_MAC_RX_FIFO_PERR(x) ((x) << S_MAC_RX_FIFO_PERR)
+#define F_MAC_RX_FIFO_PERR V_MAC_RX_FIFO_PERR(1U)
+
+#define S_MAC2MPS_PT3_PERR 12
+#define V_MAC2MPS_PT3_PERR(x) ((x) << S_MAC2MPS_PT3_PERR)
+#define F_MAC2MPS_PT3_PERR V_MAC2MPS_PT3_PERR(1U)
+
+#define S_MAC2MPS_PT2_PERR 11
+#define V_MAC2MPS_PT2_PERR(x) ((x) << S_MAC2MPS_PT2_PERR)
+#define F_MAC2MPS_PT2_PERR V_MAC2MPS_PT2_PERR(1U)
+
+#define S_MAC2MPS_PT1_PERR 10
+#define V_MAC2MPS_PT1_PERR(x) ((x) << S_MAC2MPS_PT1_PERR)
+#define F_MAC2MPS_PT1_PERR V_MAC2MPS_PT1_PERR(1U)
+
+#define S_MAC2MPS_PT0_PERR 9
+#define V_MAC2MPS_PT0_PERR(x) ((x) << S_MAC2MPS_PT0_PERR)
+#define F_MAC2MPS_PT0_PERR V_MAC2MPS_PT0_PERR(1U)
+
+#define S_LPBK_FIFO_PERR 8
+#define V_LPBK_FIFO_PERR(x) ((x) << S_LPBK_FIFO_PERR)
+#define F_LPBK_FIFO_PERR V_LPBK_FIFO_PERR(1U)
+
+#define S_TP2MPS_TF_FIFO_PERR 7
+#define V_TP2MPS_TF_FIFO_PERR(x) ((x) << S_TP2MPS_TF_FIFO_PERR)
+#define F_TP2MPS_TF_FIFO_PERR V_TP2MPS_TF_FIFO_PERR(1U)
#define A_MPS_RX_PAUSE_GEN_TH_1 0x11090
#define A_MPS_RX_PERR_INT_ENABLE2 0x11090
@@ -78258,6 +78305,26 @@
#define G_RX_CDR_LANE_SEL(x) (((x) >> S_RX_CDR_LANE_SEL) & M_RX_CDR_LANE_SEL)
#define A_MAC_DEBUG_PL_IF_1 0x381c4
+#define A_MAC_HSS0_ANALOG_TEST_CTRL 0x381d0
+
+#define S_WP_PMT_IN_I 0
+#define M_WP_PMT_IN_I 0xfU
+#define V_WP_PMT_IN_I(x) ((x) << S_WP_PMT_IN_I)
+#define G_WP_PMT_IN_I(x) (((x) >> S_WP_PMT_IN_I) & M_WP_PMT_IN_I)
+
+#define A_MAC_HSS1_ANALOG_TEST_CTRL 0x381d4
+#define A_MAC_HSS2_ANALOG_TEST_CTRL 0x381d8
+#define A_MAC_HSS3_ANALOG_TEST_CTRL 0x381dc
+#define A_MAC_HSS0_ANALOG_TEST_STATUS 0x381e0
+
+#define S_WP_PMT_OUT_O 0
+#define M_WP_PMT_OUT_O 0xfU
+#define V_WP_PMT_OUT_O(x) ((x) << S_WP_PMT_OUT_O)
+#define G_WP_PMT_OUT_O(x) (((x) >> S_WP_PMT_OUT_O) & M_WP_PMT_OUT_O)
+
+#define A_MAC_HSS1_ANALOG_TEST_STATUS 0x381e4
+#define A_MAC_HSS2_ANALOG_TEST_STATUS 0x381e8
+#define A_MAC_HSS3_ANALOG_TEST_STATUS 0x381ec
#define A_MAC_SIGNAL_DETECT_CTRL 0x381f0
#define S_SIGNAL_DET_LN7 15
@@ -80933,6 +81000,27 @@
#define F_Q1_LOS_0_ASSERT V_Q1_LOS_0_ASSERT(1U)
#define A_MAC_IOS_INTR_CAUSE_QUAD1 0x3a09c
+#define A_MAC_HSS0_PMD_RECEIVE_SIGNAL_DETECT 0x3a93c
+
+#define S_PMD_RECEIVE_SIGNAL_DETECT_1N3 4
+#define V_PMD_RECEIVE_SIGNAL_DETECT_1N3(x) ((x) << S_PMD_RECEIVE_SIGNAL_DETECT_1N3)
+#define F_PMD_RECEIVE_SIGNAL_DETECT_1N3 V_PMD_RECEIVE_SIGNAL_DETECT_1N3(1U)
+
+#define S_PMD_RECEIVE_SIGNAL_DETECT_1N2 3
+#define V_PMD_RECEIVE_SIGNAL_DETECT_1N2(x) ((x) << S_PMD_RECEIVE_SIGNAL_DETECT_1N2)
+#define F_PMD_RECEIVE_SIGNAL_DETECT_1N2 V_PMD_RECEIVE_SIGNAL_DETECT_1N2(1U)
+
+#define S_PMD_RECEIVE_SIGNAL_DETECT_LN1 2
+#define V_PMD_RECEIVE_SIGNAL_DETECT_LN1(x) ((x) << S_PMD_RECEIVE_SIGNAL_DETECT_LN1)
+#define F_PMD_RECEIVE_SIGNAL_DETECT_LN1 V_PMD_RECEIVE_SIGNAL_DETECT_LN1(1U)
+
+#define S_PMD_RECEIVE_SIGNAL_DETECT_1N0 1
+#define V_PMD_RECEIVE_SIGNAL_DETECT_1N0(x) ((x) << S_PMD_RECEIVE_SIGNAL_DETECT_1N0)
+#define F_PMD_RECEIVE_SIGNAL_DETECT_1N0 V_PMD_RECEIVE_SIGNAL_DETECT_1N0(1U)
+
+#define A_MAC_HSS1_PMD_RECEIVE_SIGNAL_DETECT 0x3b93c
+#define A_MAC_HSS2_PMD_RECEIVE_SIGNAL_DETECT 0x3c93c
+#define A_MAC_HSS3_PMD_RECEIVE_SIGNAL_DETECT 0x3d93c
#define A_MAC_MTIP_PCS_1G_0_CONTROL 0x3e000
#define S_SPEED_SEL_1 13
diff --git a/sys/dev/cxgbe/crypto/t7_kern_tls.c b/sys/dev/cxgbe/crypto/t7_kern_tls.c
index 217459126361..d9710b5bd13f 100644
--- a/sys/dev/cxgbe/crypto/t7_kern_tls.c
+++ b/sys/dev/cxgbe/crypto/t7_kern_tls.c
@@ -141,7 +141,8 @@ alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
tlsp->tx_key_addr = -1;
tlsp->ghash_offset = -1;
tlsp->rx_chid = pi->rx_chan;
- tlsp->rx_qid = sc->sge.rxq[pi->vi->first_rxq].iq.abs_id;
+ tlsp->rx_qid = -1;
+ tlsp->txq = NULL;
mbufq_init(&tlsp->pending_mbufs, INT_MAX);
return (tlsp);
@@ -157,7 +158,8 @@ t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
struct vi_info *vi;
struct inpcb *inp;
struct sge_txq *txq;
- int error, iv_size, keyid, mac_first;
+ int error, iv_size, keyid, mac_first, qidx;
+ uint32_t flowid;
tls = params->tls.tls;
@@ -250,11 +252,15 @@ t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
goto failed;
}
- txq = &sc->sge.txq[vi->first_txq];
if (inp->inp_flowtype != M_HASHTYPE_NONE)
- txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) +
- vi->rsrv_noflowq);
- tlsp->txq = txq;
+ flowid = inp->inp_flowid;
+ else
+ flowid = arc4random();
+ qidx = flowid % vi->nrxq + vi->first_rxq;
+ tlsp->rx_qid = sc->sge.rxq[qidx].iq.abs_id;
+ qidx = (flowid % (vi->ntxq - vi->rsrv_noflowq)) + vi->rsrv_noflowq +
+ vi->first_txq;
+ tlsp->txq = txq = &sc->sge.txq[qidx];
INP_RUNLOCK(inp);
error = ktls_setup_keys(tlsp, tls, txq);
diff --git a/sys/dev/cxgbe/firmware/t4fw_interface.h b/sys/dev/cxgbe/firmware/t4fw_interface.h
index 5874f0343b03..b11552dce021 100644
--- a/sys/dev/cxgbe/firmware/t4fw_interface.h
+++ b/sys/dev/cxgbe/firmware/t4fw_interface.h
@@ -8967,9 +8967,10 @@ enum fw_port_type {
FW_PORT_TYPE_SFP28 = 20, /* No, 1, 25G/10G/1G */
FW_PORT_TYPE_KR_SFP28 = 21, /* No, 1, 25G/10G/1G using Backplane */
FW_PORT_TYPE_KR_XLAUI = 22, /* No, 4, 40G/10G/1G, No AN*/
- FW_PORT_TYPE_SFP56 = 26,
- FW_PORT_TYPE_QSFP56 = 27,
- FW_PORT_TYPE_NONE = M_FW_PORT_CMD_PTYPE
+ FW_PORT_TYPE_SFP56 = 26, /* No, 1, 50G/25G */
+ FW_PORT_TYPE_QSFP56 = 27, /* No, 4, 200G/100G/50G/25G */
+ FW_PORT_TYPE_QSFPDD = 34, /* No, 8, 400G/200G/100G/50G */
+ FW_PORT_TYPE_NONE = M_FW_PORT_CMD_PORTTYPE32
};
static inline bool
diff --git a/sys/dev/cxgbe/nvmf/nvmf_che.c b/sys/dev/cxgbe/nvmf/nvmf_che.c
new file mode 100644
index 000000000000..5c2174b8a40b
--- /dev/null
+++ b/sys/dev/cxgbe/nvmf/nvmf_che.c
@@ -0,0 +1,3331 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_inet.h"
+
+#include <sys/param.h>
+#include <sys/libkern.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#ifdef TCP_OFFLOAD
+#include <sys/bitset.h>
+#include <sys/capsicum.h>
+#include <sys/file.h>
+#include <sys/kthread.h>
+#include <sys/ktr.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/nv.h>
+#include <sys/protosw.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp_var.h>
+#include <netinet/toecore.h>
+
+#include <dev/nvmf/nvmf.h>
+#include <dev/nvmf/nvmf_proto.h>
+#include <dev/nvmf/nvmf_tcp.h>
+#include <dev/nvmf/nvmf_transport.h>
+#include <dev/nvmf/nvmf_transport_internal.h>
+
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+
+#include "common/common.h"
+#include "common/t4_regs.h"
+#include "common/t4_tcb.h"
+#include "tom/t4_tom.h"
+
+/* Status code values in CPL_NVMT_CMP. */
+#define CMP_STATUS_ERROR_MASK 0x7f
+#define CMP_STATUS_NO_ERROR 0
+#define CMP_STATUS_HEADER_DIGEST 1
+#define CMP_STATUS_DIRECTION_MISMATCH 2
+#define CMP_STATUS_DIGEST_FLAG_MISMATCH 3
+#define CMP_STATUS_SUCCESS_NOT_LAST 4
+#define CMP_STATUS_BAD_DATA_LENGTH 5
+#define CMP_STATUS_USER_MODE_UNALLOCATED 6
+#define CMP_STATUS_RQT_LIMIT 7
+#define CMP_STATUS_RQT_WRAP 8
+#define CMP_STATUS_RQT_BOUND 9
+#define CMP_STATUS_TPT_LIMIT 16
+#define CMP_STATUS_TPT_INVALID 17
+#define CMP_STATUS_TPT_COLOUR_MISMATCH 18
+#define CMP_STATUS_TPT_MISC 19
+#define CMP_STATUS_TPT_WRAP 20
+#define CMP_STATUS_TPT_BOUND 21
+#define CMP_STATUS_TPT_LAST_PDU_UNALIGNED 22
+#define CMP_STATUS_PBL_LIMIT 24
+#define CMP_STATUS_DATA_DIGEST 25
+#define CMP_STATUS_DDP 0x80
+
+/*
+ * Transfer tags and CIDs with the MSB set are "unallocated" tags that
+ * pass data through to the freelist without using DDP.
+ */
+#define CHE_FL_TAG_MASK 0x8000
+#define CHE_MAX_FL_TAG 0x7fff
+#define CHE_NUM_FL_TAGS (CHE_MAX_FL_TAG + 1)
+
+#define CHE_TAG_IS_FL(ttag) (((ttag) & CHE_FL_TAG_MASK) == CHE_FL_TAG_MASK)
+#define CHE_RAW_FL_TAG(ttag) ((ttag) & ~CHE_FL_TAG_MASK)
+#define CHE_DDP_TAG(stag_idx, color) ((stag_idx) << 4 | (color))
+#define CHE_STAG_COLOR(stag) ((stag) & 0xf)
+#define CHE_STAG_IDX(stag) ((stag) >> 4)
+#define CHE_DDP_MAX_COLOR 0xf
+
+#define CHE_DDP_NO_TAG 0xffff
+
+/*
+ * A bitmap of non-DDP CIDs in use on the host. Since there is no
+ * _BIT_FFC (find first clear), the bitset is inverted so that a clear
+ * bit indicates an in-use CID.
+ */
+BITSET_DEFINE(fl_cid_set, CHE_NUM_FL_TAGS);
+#define FL_CID_INIT(p) __BIT_FILL(CHE_NUM_FL_TAGS, p)
+#define FL_CID_BUSY(n, p) __BIT_CLR(CHE_NUM_FL_TAGS, n, p)
+#define FL_CID_ISACTIVE(n, p) !__BIT_ISSET(CHE_NUM_FL_TAGS, n, p)
+#define FL_CID_FREE(n, p) __BIT_SET(CHE_NUM_FL_TAGS, n, p)
+#define FL_CID_FINDFREE_AT(p, start) __BIT_FFS_AT(CHE_NUM_FL_TAGS, p, start)
+
+/*
+ * The TCP sequence number of both CPL_NVMT_DATA and CPL_NVMT_CMP
+ * mbufs are saved here while the mbuf is in qp->rx_data and qp->rx_pdus.
+ */
+#define nvmf_tcp_seq PH_loc.thirtytwo[0]
+
+/*
+ * The CPL status of CPL_NVMT_CMP mbufs are saved here while the mbuf
+ * is in qp->rx_pdus.
+ */
+#define nvmf_cpl_status PH_loc.eight[4]
+
+struct nvmf_che_capsule;
+struct nvmf_che_qpair;
+
+struct nvmf_che_adapter {
+ struct adapter *sc;
+
+ u_int ddp_threshold;
+ u_int max_transmit_pdu;
+ u_int max_receive_pdu;
+ bool nvmt_data_iqe;
+
+ struct sysctl_ctx_list ctx; /* from uld_activate to deactivate */
+};
+
+struct nvmf_che_command_buffer {
+ struct nvmf_che_qpair *qp;
+
+ struct nvmf_io_request io;
+ size_t data_len;
+ size_t data_xfered;
+ uint32_t data_offset;
+
+ u_int refs;
+ int error;
+
+ bool ddp_ok;
+ uint16_t cid;
+ uint16_t ttag;
+ uint16_t original_cid; /* Host only */
+
+ TAILQ_ENTRY(nvmf_che_command_buffer) link;
+
+ /* Fields used for DDP. */
+ struct fw_ri_tpte tpte;
+ uint64_t *pbl;
+ uint32_t pbl_addr;
+ uint32_t pbl_len;
+
+ /* Controller only */
+ struct nvmf_che_capsule *cc;
+};
+
+struct nvmf_che_command_buffer_list {
+ TAILQ_HEAD(, nvmf_che_command_buffer) head;
+ struct mtx lock;
+};
+
+struct nvmf_che_qpair {
+ struct nvmf_qpair qp;
+
+ struct socket *so;
+ struct toepcb *toep;
+ struct nvmf_che_adapter *nca;
+
+ volatile u_int refs; /* Every allocated capsule holds a reference */
+ uint8_t txpda;
+ uint8_t rxpda;
+ bool header_digests;
+ bool data_digests;
+ uint32_t maxr2t;
+ uint32_t maxh2cdata; /* Controller only */
+ uint32_t max_rx_data;
+ uint32_t max_tx_data;
+ uint32_t max_icd; /* Host only */
+ uint32_t max_ioccsz; /* Controller only */
+ union {
+ uint16_t next_fl_ttag; /* Controller only */
+ uint16_t next_cid; /* Host only */
+ };
+ uint16_t next_ddp_tag;
+ u_int num_fl_ttags; /* Controller only */
+ u_int active_fl_ttags; /* Controller only */
+ u_int num_ddp_tags;
+ u_int active_ddp_tags;
+ bool send_success; /* Controller only */
+ uint8_t ddp_color;
+ uint32_t tpt_offset;
+
+ /* Receive state. */
+ struct thread *rx_thread;
+ struct cv rx_cv;
+ bool rx_shutdown;
+ int rx_error;
+ struct mbufq rx_data; /* Data received via CPL_NVMT_DATA. */
+ struct mbufq rx_pdus; /* PDU headers received via CPL_NVMT_CMP. */
+
+ /* Transmit state. */
+ struct thread *tx_thread;
+ struct cv tx_cv;
+ bool tx_shutdown;
+ STAILQ_HEAD(, nvmf_che_capsule) tx_capsules;
+
+ struct nvmf_che_command_buffer_list tx_buffers;
+ struct nvmf_che_command_buffer_list rx_buffers;
+
+ /*
+ * For the controller, an RX command buffer can be in one of
+ * three locations, all protected by the rx_buffers.lock. If
+ * a receive request is waiting for either an R2T slot for its
+ * command (due to exceeding MAXR2T), or a transfer tag it is
+ * placed on the rx_buffers list. When a request is allocated
+ * an active transfer tag, it moves to either the
+ * open_ddp_tags[] or open_fl_ttags[] array (indexed by the
+ * tag) until it completes.
+ *
+ * For the host, an RX command buffer using DDP is in
+ * open_ddp_tags[], otherwise it is in rx_buffers.
+ */
+ struct nvmf_che_command_buffer **open_ddp_tags;
+ struct nvmf_che_command_buffer **open_fl_ttags; /* Controller only */
+
+ /*
+ * For the host, CIDs submitted by nvmf(4) must be rewritten
+ * to either use DDP or not use DDP. The CID in response
+ * capsules must be restored to their original value. For
+ * DDP, the original CID is stored in the command buffer.
+ * These variables manage non-DDP CIDs.
+ */
+ uint16_t *fl_cids; /* Host only */
+ struct fl_cid_set *fl_cid_set; /* Host only */
+ struct mtx fl_cid_lock; /* Host only */
+};
+
+struct nvmf_che_rxpdu {
+ struct mbuf *m;
+ const struct nvme_tcp_common_pdu_hdr *hdr;
+ uint32_t data_len;
+ bool data_digest_mismatch;
+ bool ddp;
+};
+
+struct nvmf_che_capsule {
+ struct nvmf_capsule nc;
+
+ volatile u_int refs;
+
+ struct nvmf_che_rxpdu rx_pdu;
+
+ uint32_t active_r2ts; /* Controller only */
+#ifdef INVARIANTS
+ uint32_t tx_data_offset; /* Controller only */
+ u_int pending_r2ts; /* Controller only */
+#endif
+
+ STAILQ_ENTRY(nvmf_che_capsule) link;
+};
+
+#define CCAP(nc) ((struct nvmf_che_capsule *)(nc))
+#define CQP(qp) ((struct nvmf_che_qpair *)(qp))
+
+static void che_release_capsule(struct nvmf_che_capsule *cc);
+static void che_free_qpair(struct nvmf_qpair *nq);
+
+SYSCTL_NODE(_kern_nvmf, OID_AUTO, che, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Chelsio TCP offload transport");
+
+static u_int che_max_transmit_pdu = 32 * 1024;
+SYSCTL_UINT(_kern_nvmf_che, OID_AUTO, max_transmit_pdu, CTLFLAG_RWTUN,
+ &che_max_transmit_pdu, 0,
+ "Maximum size of a transmitted PDU");
+
+static u_int che_max_receive_pdu = 32 * 1024;
+SYSCTL_UINT(_kern_nvmf_che, OID_AUTO, max_receive_pdu, CTLFLAG_RWTUN,
+ &che_max_receive_pdu, 0,
+ "Maximum size of a received PDU");
+
+static int use_dsgl = 1;
+SYSCTL_INT(_kern_nvmf_che, OID_AUTO, use_dsgl, CTLFLAG_RWTUN, &use_dsgl, 0,
+ "Use DSGL for PBL/FastReg (default=1)");
+
+static int inline_threshold = 256;
+SYSCTL_INT(_kern_nvmf_che, OID_AUTO, inline_threshold, CTLFLAG_RWTUN,
+ &inline_threshold, 0,
+ "inline vs dsgl threshold (default=256)");
+
+static int ddp_tags_per_qp = 128;
+SYSCTL_INT(_kern_nvmf_che, OID_AUTO, ddp_tags_per_qp, CTLFLAG_RWTUN,
+ &ddp_tags_per_qp, 0,
+ "Number of DDP tags to reserve for each queue pair");
+
+static MALLOC_DEFINE(M_NVMF_CHE, "nvmf_che", "Chelsio NVMe-TCP offload");
+
+/*
+ * PBL regions consist of N full-sized pages. TPT entries support an
+ * initial offset into the first page (FBO) and can handle a partial
+ * length on the last page.
+ */
+static bool
+che_ddp_io_check(struct nvmf_che_qpair *qp, const struct nvmf_io_request *io)
+{
+ const struct memdesc *mem = &io->io_mem;
+ struct bus_dma_segment *ds;
+ int i;
+
+ if (io->io_len < qp->nca->ddp_threshold) {
+ return (false);
+ }
+
+ switch (mem->md_type) {
+ case MEMDESC_VADDR:
+ case MEMDESC_PADDR:
+ case MEMDESC_VMPAGES:
+ return (true);
+ case MEMDESC_VLIST:
+ case MEMDESC_PLIST:
+ /*
+ * Require all but the first segment to start on a
+ * page boundary. Require all but the last segment to
+ * end on a page boundary.
+ */
+ ds = mem->u.md_list;
+ for (i = 0; i < mem->md_nseg; i++, ds++) {
+ if (i != 0 && ds->ds_addr % PAGE_SIZE != 0)
+ return (false);
+ if (i != mem->md_nseg - 1 &&
+ (ds->ds_addr + ds->ds_len) % PAGE_SIZE != 0)
+ return (false);
+ }
+ return (true);
+ default:
+ /*
+ * Other types could be validated with more work, but
+ * they aren't used currently by nvmf(4) or nvmft(4).
+ */
+ return (false);
+ }
+}
+
+static u_int
+che_fbo(struct nvmf_che_command_buffer *cb)
+{
+ struct memdesc *mem = &cb->io.io_mem;
+
+ switch (mem->md_type) {
+ case MEMDESC_VADDR:
+ return ((uintptr_t)mem->u.md_vaddr & PAGE_MASK);
+ case MEMDESC_PADDR:
+ return (mem->u.md_paddr & PAGE_MASK);
+ case MEMDESC_VMPAGES:
+ return (mem->md_offset);
+ case MEMDESC_VLIST:
+ case MEMDESC_PLIST:
+ return (mem->u.md_list[0].ds_addr & PAGE_MASK);
+ default:
+ __assert_unreachable();
+ }
+}
+
+static u_int
+che_npages(struct nvmf_che_command_buffer *cb)
+{
+ return (howmany(che_fbo(cb) + cb->io.io_len, PAGE_SIZE));
+}
+
+static struct nvmf_che_command_buffer *
+che_alloc_command_buffer(struct nvmf_che_qpair *qp,
+ const struct nvmf_io_request *io, uint32_t data_offset, size_t data_len,
+ uint16_t cid)
+{
+ struct nvmf_che_command_buffer *cb;
+
+ cb = malloc(sizeof(*cb), M_NVMF_CHE, M_WAITOK);
+ cb->qp = qp;
+ cb->io = *io;
+ cb->data_offset = data_offset;
+ cb->data_len = data_len;
+ cb->data_xfered = 0;
+ refcount_init(&cb->refs, 1);
+ cb->error = 0;
+ cb->ddp_ok = che_ddp_io_check(qp, io);
+ cb->cid = cid;
+ cb->ttag = 0;
+ cb->original_cid = 0;
+ cb->cc = NULL;
+ cb->pbl = NULL;
+
+ return (cb);
+}
+
+static void
+che_hold_command_buffer(struct nvmf_che_command_buffer *cb)
+{
+ refcount_acquire(&cb->refs);
+}
+
+static void
+che_free_command_buffer(struct nvmf_che_command_buffer *cb)
+{
+ nvmf_complete_io_request(&cb->io, cb->data_xfered, cb->error);
+ if (cb->cc != NULL)
+ che_release_capsule(cb->cc);
+ MPASS(cb->pbl == NULL);
+ free(cb, M_NVMF_CHE);
+}
+
+static void
+che_release_command_buffer(struct nvmf_che_command_buffer *cb)
+{
+ if (refcount_release(&cb->refs))
+ che_free_command_buffer(cb);
+}
+
+static void
+che_add_command_buffer(struct nvmf_che_command_buffer_list *list,
+ struct nvmf_che_command_buffer *cb)
+{
+ mtx_assert(&list->lock, MA_OWNED);
+ TAILQ_INSERT_HEAD(&list->head, cb, link);
+}
+
+static struct nvmf_che_command_buffer *
+che_find_command_buffer(struct nvmf_che_command_buffer_list *list,
+ uint16_t cid)
+{
+ struct nvmf_che_command_buffer *cb;
+
+ mtx_assert(&list->lock, MA_OWNED);
+ TAILQ_FOREACH(cb, &list->head, link) {
+ if (cb->cid == cid)
+ return (cb);
+ }
+ return (NULL);
+}
+
+static void
+che_remove_command_buffer(struct nvmf_che_command_buffer_list *list,
+ struct nvmf_che_command_buffer *cb)
+{
+ mtx_assert(&list->lock, MA_OWNED);
+ TAILQ_REMOVE(&list->head, cb, link);
+}
+
+static void
+che_purge_command_buffer(struct nvmf_che_command_buffer_list *list,
+ uint16_t cid)
+{
+ struct nvmf_che_command_buffer *cb;
+
+ mtx_lock(&list->lock);
+ cb = che_find_command_buffer(list, cid);
+ if (cb != NULL) {
+ che_remove_command_buffer(list, cb);
+ mtx_unlock(&list->lock);
+ che_release_command_buffer(cb);
+ } else
+ mtx_unlock(&list->lock);
+}
+
+static int
+che_write_mem_inline(struct adapter *sc, struct toepcb *toep, uint32_t addr,
+ uint32_t len, void *data, struct mbufq *wrq)
+{
+ struct mbuf *m;
+ char *cp;
+ int copy_len, i, num_wqe, wr_len;
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: addr 0x%x len %u", __func__, addr << 5, len);
+#endif
+ num_wqe = DIV_ROUND_UP(len, T4_MAX_INLINE_SIZE);
+ cp = data;
+ for (i = 0; i < num_wqe; i++) {
+ copy_len = min(len, T4_MAX_INLINE_SIZE);
+ wr_len = T4_WRITE_MEM_INLINE_LEN(copy_len);
+
+ m = alloc_raw_wr_mbuf(wr_len);
+ if (m == NULL)
+ return (ENOMEM);
+ t4_write_mem_inline_wr(sc, mtod(m, void *), wr_len, toep->tid,
+ addr, copy_len, cp, 0);
+ if (cp != NULL)
+ cp += T4_MAX_INLINE_SIZE;
+ addr += T4_MAX_INLINE_SIZE >> 5;
+ len -= T4_MAX_INLINE_SIZE;
+
+ mbufq_enqueue(wrq, m);
+ }
+ return (0);
+}
+
+static int
+che_write_mem_dma_aligned(struct adapter *sc, struct toepcb *toep,
+ uint32_t addr, uint32_t len, void *data, struct mbufq *wrq)
+{
+ struct mbuf *m;
+ vm_offset_t va;
+ u_int todo;
+ int wr_len;
+
+ /* First page. */
+ va = (vm_offset_t)data;
+ todo = min(PAGE_SIZE - (va % PAGE_SIZE), len);
+ wr_len = T4_WRITE_MEM_DMA_LEN;
+ m = alloc_raw_wr_mbuf(wr_len);
+ if (m == NULL)
+ return (ENOMEM);
+ t4_write_mem_dma_wr(sc, mtod(m, void *), wr_len, toep->tid, addr,
+ todo, pmap_kextract(va), 0);
+ mbufq_enqueue(wrq, m);
+ len -= todo;
+ addr += todo >> 5;
+ va += todo;
+
+ while (len > 0) {
+ MPASS(va == trunc_page(va));
+ todo = min(PAGE_SIZE, len);
+ m = alloc_raw_wr_mbuf(wr_len);
+ if (m == NULL)
+ return (ENOMEM);
+ t4_write_mem_dma_wr(sc, mtod(m, void *), wr_len, toep->tid,
+ addr, todo, pmap_kextract(va), 0);
+ mbufq_enqueue(wrq, m);
+ len -= todo;
+ addr += todo >> 5;
+ va += todo;
+ }
+ return (0);
+}
+
+static int
+che_write_adapter_mem(struct nvmf_che_qpair *qp, uint32_t addr, uint32_t len,
+ void *data)
+{
+ struct adapter *sc = qp->nca->sc;
+ struct toepcb *toep = qp->toep;
+ struct socket *so = qp->so;
+ struct inpcb *inp = sotoinpcb(so);
+ struct mbufq mq;
+ int error;
+
+ mbufq_init(&mq, INT_MAX);
+ if (!use_dsgl || len < inline_threshold || data == NULL)
+ error = che_write_mem_inline(sc, toep, addr, len, data, &mq);
+ else
+ error = che_write_mem_dma_aligned(sc, toep, addr, len, data,
+ &mq);
+ if (__predict_false(error != 0))
+ goto error;
+
+ INP_WLOCK(inp);
+ if ((inp->inp_flags & INP_DROPPED) != 0) {
+ INP_WUNLOCK(inp);
+ error = ECONNRESET;
+ goto error;
+ }
+ mbufq_concat(&toep->ulp_pduq, &mq);
+ INP_WUNLOCK(inp);
+ return (0);
+
+error:
+ mbufq_drain(&mq);
+ return (error);
+}
+
+static bool
+che_alloc_pbl(struct nvmf_che_qpair *qp, struct nvmf_che_command_buffer *cb)
+{
+ struct adapter *sc = qp->nca->sc;
+ struct memdesc *mem = &cb->io.io_mem;
+ uint64_t *pbl;
+ uint32_t addr, len;
+ u_int i, npages;
+ int error;
+
+ MPASS(cb->pbl == NULL);
+ MPASS(cb->ddp_ok);
+
+ /* Hardware limit? iWARP only enforces this for T5. */
+ if (cb->io.io_len >= (8 * 1024 * 1024 * 1024ULL))
+ return (false);
+
+ npages = che_npages(cb);
+ len = roundup2(npages, 4) * sizeof(*cb->pbl);
+ addr = t4_pblpool_alloc(sc, len);
+ if (addr == 0)
+ return (false);
+
+ pbl = malloc(len, M_NVMF_CHE, M_NOWAIT | M_ZERO);
+ if (pbl == NULL) {
+ t4_pblpool_free(sc, addr, len);
+ return (false);
+ }
+
+ switch (mem->md_type) {
+ case MEMDESC_VADDR:
+ {
+ vm_offset_t va;
+
+ va = trunc_page((uintptr_t)mem->u.md_vaddr);
+ for (i = 0; i < npages; i++)
+ pbl[i] = htobe64(pmap_kextract(va + i * PAGE_SIZE));
+ break;
+ }
+ case MEMDESC_PADDR:
+ {
+ vm_paddr_t pa;
+
+ pa = trunc_page(mem->u.md_paddr);
+ for (i = 0; i < npages; i++)
+ pbl[i] = htobe64(pa + i * PAGE_SIZE);
+ break;
+ }
+ case MEMDESC_VMPAGES:
+ for (i = 0; i < npages; i++)
+ pbl[i] = htobe64(VM_PAGE_TO_PHYS(mem->u.md_ma[i]));
+ break;
+ case MEMDESC_VLIST:
+ {
+ struct bus_dma_segment *ds;
+ vm_offset_t va;
+ vm_size_t len;
+ u_int j, k;
+
+ i = 0;
+ ds = mem->u.md_list;
+ for (j = 0; j < mem->md_nseg; j++, ds++) {
+ va = trunc_page((uintptr_t)ds->ds_addr);
+ len = ds->ds_len;
+ if (ds->ds_addr % PAGE_SIZE != 0)
+ len += ds->ds_addr % PAGE_SIZE;
+ for (k = 0; k < howmany(len, PAGE_SIZE); k++) {
+ pbl[i] = htobe64(pmap_kextract(va +
+ k * PAGE_SIZE));
+ i++;
+ }
+ }
+ MPASS(i == npages);
+ break;
+ }
+ case MEMDESC_PLIST:
+ {
+ struct bus_dma_segment *ds;
+ vm_paddr_t pa;
+ vm_size_t len;
+ u_int j, k;
+
+ i = 0;
+ ds = mem->u.md_list;
+ for (j = 0; j < mem->md_nseg; j++, ds++) {
+ pa = trunc_page((vm_paddr_t)ds->ds_addr);
+ len = ds->ds_len;
+ if (ds->ds_addr % PAGE_SIZE != 0)
+ len += ds->ds_addr % PAGE_SIZE;
+ for (k = 0; k < howmany(len, PAGE_SIZE); k++) {
+ pbl[i] = htobe64(pa + k * PAGE_SIZE);
+ i++;
+ }
+ }
+ MPASS(i == npages);
+ break;
+ }
+ default:
+ __assert_unreachable();
+ }
+
+ error = che_write_adapter_mem(qp, addr >> 5, len, pbl);
+ if (error != 0) {
+ t4_pblpool_free(sc, addr, len);
+ free(pbl, M_NVMF_CHE);
+ return (false);
+ }
+
+ cb->pbl = pbl;
+ cb->pbl_addr = addr;
+ cb->pbl_len = len;
+
+ return (true);
+}
+
+static void
+che_free_pbl(struct nvmf_che_command_buffer *cb)
+{
+ free(cb->pbl, M_NVMF_CHE);
+ t4_pblpool_free(cb->qp->nca->sc, cb->pbl_addr, cb->pbl_len);
+ cb->pbl = NULL;
+ cb->pbl_addr = 0;
+ cb->pbl_len = 0;
+}
+
+static bool
+che_write_tpt_entry(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb, uint16_t stag)
+{
+ uint32_t tpt_addr;
+ int error;
+
+ cb->tpte.valid_to_pdid = htobe32(F_FW_RI_TPTE_VALID |
+ V_FW_RI_TPTE_STAGKEY(CHE_STAG_COLOR(stag)) |
+ F_FW_RI_TPTE_STAGSTATE |
+ V_FW_RI_TPTE_STAGTYPE(FW_RI_STAG_NSMR) |
+ V_FW_RI_TPTE_PDID(0));
+ cb->tpte.locread_to_qpid = htobe32(
+ V_FW_RI_TPTE_PERM(FW_RI_MEM_ACCESS_REM_WRITE) |
+ V_FW_RI_TPTE_ADDRTYPE(FW_RI_ZERO_BASED_TO) |
+ V_FW_RI_TPTE_PS(PAGE_SIZE) |
+ V_FW_RI_TPTE_QPID(qp->toep->tid));
+#define PBL_OFF(qp, a) ((a) - (qp)->nca->sc->vres.pbl.start)
+ cb->tpte.nosnoop_pbladdr =
+ htobe32(V_FW_RI_TPTE_PBLADDR(PBL_OFF(qp, cb->pbl_addr) >> 3));
+ cb->tpte.len_lo = htobe32(cb->data_len);
+ cb->tpte.va_hi = 0;
+ cb->tpte.va_lo_fbo = htobe32(che_fbo(cb));
+ cb->tpte.dca_mwbcnt_pstag = 0;
+ cb->tpte.len_hi = htobe32(cb->data_offset);
+
+ tpt_addr = qp->tpt_offset + CHE_STAG_IDX(stag) +
+ (qp->nca->sc->vres.stag.start >> 5);
+
+ error = che_write_adapter_mem(qp, tpt_addr, sizeof(cb->tpte),
+ &cb->tpte);
+ return (error == 0);
+}
+
+static void
+che_clear_tpt_entry(struct nvmf_che_qpair *qp, uint16_t stag)
+{
+ uint32_t tpt_addr;
+
+ tpt_addr = qp->tpt_offset + CHE_STAG_IDX(stag) +
+ (qp->nca->sc->vres.stag.start >> 5);
+
+ (void)che_write_adapter_mem(qp, tpt_addr, sizeof(struct fw_ri_tpte),
+ NULL);
+}
+
+static uint16_t
+che_alloc_ddp_stag(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb)
+{
+ uint16_t stag_idx;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+ MPASS(cb->ddp_ok);
+
+ if (qp->active_ddp_tags == qp->num_ddp_tags)
+ return (CHE_DDP_NO_TAG);
+
+ MPASS(qp->num_ddp_tags != 0);
+
+ stag_idx = qp->next_ddp_tag;
+ for (;;) {
+ if (qp->open_ddp_tags[stag_idx] == NULL)
+ break;
+ if (stag_idx == qp->num_ddp_tags - 1) {
+ stag_idx = 0;
+ if (qp->ddp_color == CHE_DDP_MAX_COLOR)
+ qp->ddp_color = 0;
+ else
+ qp->ddp_color++;
+ } else
+ stag_idx++;
+ MPASS(stag_idx != qp->next_ddp_tag);
+ }
+ if (stag_idx == qp->num_ddp_tags - 1)
+ qp->next_ddp_tag = 0;
+ else
+ qp->next_ddp_tag = stag_idx + 1;
+
+ qp->active_ddp_tags++;
+ qp->open_ddp_tags[stag_idx] = cb;
+
+ return (CHE_DDP_TAG(stag_idx, qp->ddp_color));
+}
+
+static void
+che_free_ddp_stag(struct nvmf_che_qpair *qp, struct nvmf_che_command_buffer *cb,
+ uint16_t stag)
+{
+ MPASS(!CHE_TAG_IS_FL(stag));
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ MPASS(qp->open_ddp_tags[CHE_STAG_IDX(stag)] == cb);
+
+ qp->open_ddp_tags[CHE_STAG_IDX(stag)] = NULL;
+ qp->active_ddp_tags--;
+}
+
+static uint16_t
+che_alloc_ddp_tag(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb)
+{
+ uint16_t stag;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ if (!cb->ddp_ok)
+ return (CHE_DDP_NO_TAG);
+
+ stag = che_alloc_ddp_stag(qp, cb);
+ if (stag == CHE_DDP_NO_TAG) {
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_setup_no_stag,
+ 1);
+ return (CHE_DDP_NO_TAG);
+ }
+
+ if (!che_alloc_pbl(qp, cb)) {
+ che_free_ddp_stag(qp, cb, stag);
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_setup_error, 1);
+ return (CHE_DDP_NO_TAG);
+ }
+
+ if (!che_write_tpt_entry(qp, cb, stag)) {
+ che_free_pbl(cb);
+ che_free_ddp_stag(qp, cb, stag);
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_setup_error, 1);
+ return (CHE_DDP_NO_TAG);
+ }
+
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_setup_ok, 1);
+ return (stag);
+}
+
+static void
+che_free_ddp_tag(struct nvmf_che_qpair *qp, struct nvmf_che_command_buffer *cb,
+ uint16_t stag)
+{
+ MPASS(!CHE_TAG_IS_FL(stag));
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ MPASS(qp->open_ddp_tags[CHE_STAG_IDX(stag)] == cb);
+
+ che_clear_tpt_entry(qp, stag);
+ che_free_pbl(cb);
+ che_free_ddp_stag(qp, cb, stag);
+}
+
+static void
+nvmf_che_write_pdu(struct nvmf_che_qpair *qp, struct mbuf *m)
+{
+ struct epoch_tracker et;
+ struct socket *so = qp->so;
+ struct inpcb *inp = sotoinpcb(so);
+ struct toepcb *toep = qp->toep;
+
+ CURVNET_SET(so->so_vnet);
+ NET_EPOCH_ENTER(et);
+ INP_WLOCK(inp);
+ if (__predict_false(inp->inp_flags & INP_DROPPED) ||
+ __predict_false((toep->flags & TPF_ATTACHED) == 0)) {
+ m_freem(m);
+ } else {
+ mbufq_enqueue(&toep->ulp_pduq, m);
+ t4_push_pdus(toep->vi->adapter, toep, 0);
+ }
+ INP_WUNLOCK(inp);
+ NET_EPOCH_EXIT(et);
+ CURVNET_RESTORE();
+}
+
+static void
+nvmf_che_report_error(struct nvmf_che_qpair *qp, uint16_t fes, uint32_t fei,
+ struct mbuf *rx_pdu, u_int hlen)
+{
+ struct nvme_tcp_term_req_hdr *hdr;
+ struct mbuf *m;
+
+ if (hlen != 0) {
+ hlen = min(hlen, NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE);
+ hlen = min(hlen, m_length(rx_pdu, NULL));
+ }
+
+ m = m_get2(sizeof(*hdr) + hlen, M_WAITOK, MT_DATA, M_PKTHDR);
+ m->m_len = sizeof(*hdr) + hlen;
+ m->m_pkthdr.len = m->m_len;
+ hdr = mtod(m, void *);
+ memset(hdr, 0, sizeof(*hdr));
+ hdr->common.pdu_type = qp->qp.nq_controller ?
+ NVME_TCP_PDU_TYPE_C2H_TERM_REQ : NVME_TCP_PDU_TYPE_H2C_TERM_REQ;
+ hdr->common.hlen = sizeof(*hdr);
+ hdr->common.plen = sizeof(*hdr) + hlen;
+ hdr->fes = htole16(fes);
+ le32enc(hdr->fei, fei);
+ if (hlen != 0)
+ m_copydata(rx_pdu, 0, hlen, (caddr_t)(hdr + 1));
+
+ nvmf_che_write_pdu(qp, m);
+}
+
+static int
+nvmf_che_validate_pdu(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_common_pdu_hdr *ch;
+ struct mbuf *m = pdu->m;
+ uint32_t data_len, fei, plen, rx_digest;
+ u_int hlen, cpl_error;
+ int error;
+ uint16_t fes;
+
+ /* Determine how large of a PDU header to return for errors. */
+ ch = pdu->hdr;
+ hlen = ch->hlen;
+ plen = le32toh(ch->plen);
+ if (hlen < sizeof(*ch) || hlen > plen)
+ hlen = sizeof(*ch);
+
+ cpl_error = m->m_pkthdr.nvmf_cpl_status & CMP_STATUS_ERROR_MASK;
+ switch (cpl_error) {
+ case CMP_STATUS_NO_ERROR:
+ break;
+ case CMP_STATUS_HEADER_DIGEST:
+ counter_u64_add(
+ qp->toep->ofld_rxq->rx_nvme_header_digest_errors, 1);
+ printf("NVMe/TCP: Header digest mismatch\n");
+ rx_digest = le32dec(mtodo(m, ch->hlen));
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_HDGST_ERROR, rx_digest, m,
+ hlen);
+ return (EBADMSG);
+ case CMP_STATUS_DIRECTION_MISMATCH:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ printf("NVMe/TCP: Invalid PDU type %u\n", ch->pdu_type);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_common_pdu_hdr, pdu_type), m,
+ hlen);
+ return (EBADMSG);
+ case CMP_STATUS_SUCCESS_NOT_LAST:
+ case CMP_STATUS_DIGEST_FLAG_MISMATCH:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ printf("NVMe/TCP: Invalid PDU header flags %#x\n", ch->flags);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_common_pdu_hdr, flags), m, hlen);
+ return (EBADMSG);
+ case CMP_STATUS_BAD_DATA_LENGTH:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ printf("NVMe/TCP: Invalid PDU length %u\n", plen);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_common_pdu_hdr, plen), m, hlen);
+ return (EBADMSG);
+ case CMP_STATUS_USER_MODE_UNALLOCATED:
+ case CMP_STATUS_RQT_LIMIT:
+ case CMP_STATUS_RQT_WRAP:
+ case CMP_STATUS_RQT_BOUND:
+ device_printf(qp->nca->sc->dev,
+ "received invalid NVMET error %u\n",
+ cpl_error);
+ return (ECONNRESET);
+ case CMP_STATUS_TPT_LIMIT:
+ case CMP_STATUS_TPT_INVALID:
+ case CMP_STATUS_TPT_COLOUR_MISMATCH:
+ case CMP_STATUS_TPT_MISC:
+ case CMP_STATUS_TPT_WRAP:
+ case CMP_STATUS_TPT_BOUND:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ switch (ch->pdu_type) {
+ case NVME_TCP_PDU_TYPE_H2C_DATA:
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_h2c_data_hdr, ttag),
+ pdu->m, pdu->hdr->hlen);
+ return (EBADMSG);
+ case NVME_TCP_PDU_TYPE_C2H_DATA:
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_c2h_data_hdr, cccid), m,
+ hlen);
+ return (EBADMSG);
+ default:
+ device_printf(qp->nca->sc->dev,
+ "received DDP NVMET error %u for PDU %u\n",
+ cpl_error, ch->pdu_type);
+ return (ECONNRESET);
+ }
+ case CMP_STATUS_TPT_LAST_PDU_UNALIGNED:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, m, hlen);
+ return (EBADMSG);
+ case CMP_STATUS_PBL_LIMIT:
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_invalid_headers, 1);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, m,
+ hlen);
+ return (EBADMSG);
+ case CMP_STATUS_DATA_DIGEST:
+ /* Handled below. */
+ break;
+ default:
+ device_printf(qp->nca->sc->dev,
+ "received unknown NVMET error %u\n",
+ cpl_error);
+ return (ECONNRESET);
+ }
+
+ error = nvmf_tcp_validate_pdu_header(ch, qp->qp.nq_controller,
+ qp->header_digests, qp->data_digests, qp->rxpda, &data_len, &fes,
+ &fei);
+ if (error != 0) {
+ if (error != ECONNRESET)
+ nvmf_che_report_error(qp, fes, fei, m, hlen);
+ return (error);
+ }
+
+ /* Check data digest if present. */
+ pdu->data_digest_mismatch = false;
+ if ((ch->flags & NVME_TCP_CH_FLAGS_DDGSTF) != 0) {
+ if (cpl_error == CMP_STATUS_DATA_DIGEST) {
+ printf("NVMe/TCP: Data digest mismatch\n");
+ pdu->data_digest_mismatch = true;
+ counter_u64_add(
+ qp->toep->ofld_rxq->rx_nvme_data_digest_errors, 1);
+ }
+ }
+
+ pdu->data_len = data_len;
+
+ return (0);
+}
+
+static void
+nvmf_che_free_pdu(struct nvmf_che_rxpdu *pdu)
+{
+ m_freem(pdu->m);
+ pdu->m = NULL;
+ pdu->hdr = NULL;
+}
+
+static int
+nvmf_che_handle_term_req(struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_term_req_hdr *hdr;
+
+ hdr = (const void *)pdu->hdr;
+
+ printf("NVMe/TCP: Received termination request: fes %#x fei %#x\n",
+ le16toh(hdr->fes), le32dec(hdr->fei));
+ nvmf_che_free_pdu(pdu);
+ return (ECONNRESET);
+}
+
+static int
+nvmf_che_save_command_capsule(struct nvmf_che_qpair *qp,
+ struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_cmd *cmd;
+ struct nvmf_capsule *nc;
+ struct nvmf_che_capsule *cc;
+
+ cmd = (const void *)pdu->hdr;
+
+ nc = nvmf_allocate_command(&qp->qp, &cmd->ccsqe, M_WAITOK);
+
+ cc = CCAP(nc);
+ cc->rx_pdu = *pdu;
+
+ nvmf_capsule_received(&qp->qp, nc);
+ return (0);
+}
+
+static int
+nvmf_che_save_response_capsule(struct nvmf_che_qpair *qp,
+ struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_rsp *rsp;
+ struct nvme_completion cpl;
+ struct nvmf_capsule *nc;
+ struct nvmf_che_capsule *cc;
+ uint16_t cid;
+
+ rsp = (const void *)pdu->hdr;
+
+ /*
+ * Restore the original CID and ensure any command buffers
+ * associated with this CID have been released. Once the CQE
+ * has been received, no further transfers to the command
+ * buffer for the associated CID can occur.
+ */
+ cpl = rsp->rccqe;
+ cid = le16toh(cpl.cid);
+ if (CHE_TAG_IS_FL(cid)) {
+ cid = CHE_RAW_FL_TAG(cid);
+ mtx_lock(&qp->fl_cid_lock);
+ MPASS(FL_CID_ISACTIVE(cid, qp->fl_cid_set));
+ cpl.cid = qp->fl_cids[cid];
+ FL_CID_FREE(cid, qp->fl_cid_set);
+ mtx_unlock(&qp->fl_cid_lock);
+
+ che_purge_command_buffer(&qp->rx_buffers, rsp->rccqe.cid);
+ che_purge_command_buffer(&qp->tx_buffers, rsp->rccqe.cid);
+ } else {
+ struct nvmf_che_command_buffer *cb;
+
+ mtx_lock(&qp->rx_buffers.lock);
+ cb = qp->open_ddp_tags[CHE_STAG_IDX(cid)];
+ MPASS(cb != NULL);
+ MPASS(cb->cid == rsp->rccqe.cid);
+ cpl.cid = cb->original_cid;
+ che_free_ddp_tag(qp, cb, cid);
+ mtx_unlock(&qp->rx_buffers.lock);
+ che_release_command_buffer(cb);
+ }
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u freed cid 0x%04x for 0x%04x", __func__,
+ qp->toep->tid, le16toh(rsp->rccqe.cid), cpl.cid);
+#endif
+
+ nc = nvmf_allocate_response(&qp->qp, &cpl, M_WAITOK);
+
+ nc->nc_sqhd_valid = true;
+ cc = CCAP(nc);
+ cc->rx_pdu = *pdu;
+
+ nvmf_capsule_received(&qp->qp, nc);
+ return (0);
+}
+
+/*
+ * Construct a PDU that contains an optional data payload. This
+ * includes dealing with the length fields in the common header. The
+ * adapter inserts digests and padding when the PDU is transmitted.
+ */
+static struct mbuf *
+nvmf_che_construct_pdu(struct nvmf_che_qpair *qp, void *hdr, size_t hlen,
+ struct mbuf *data, uint32_t data_len)
+{
+ struct nvme_tcp_common_pdu_hdr *ch;
+ struct mbuf *top;
+ uint32_t pdo, plen;
+ uint8_t ulp_submode;
+
+ plen = hlen;
+ if (qp->header_digests)
+ plen += sizeof(uint32_t);
+ if (data_len != 0) {
+ KASSERT(m_length(data, NULL) == data_len, ("length mismatch"));
+ pdo = roundup(plen, qp->txpda);
+ plen = pdo + data_len;
+ if (qp->data_digests)
+ plen += sizeof(uint32_t);
+ } else {
+ KASSERT(data == NULL, ("payload mbuf with zero length"));
+ pdo = 0;
+ }
+
+ top = m_get2(hlen, M_WAITOK, MT_DATA, M_PKTHDR);
+ top->m_len = hlen;
+ top->m_pkthdr.len = hlen;
+ ch = mtod(top, void *);
+ memcpy(ch, hdr, hlen);
+ ch->hlen = hlen;
+ ulp_submode = 0;
+ if (qp->header_digests) {
+ ch->flags |= NVME_TCP_CH_FLAGS_HDGSTF;
+ ulp_submode |= ULP_CRC_HEADER;
+ }
+ if (qp->data_digests && data_len != 0) {
+ ch->flags |= NVME_TCP_CH_FLAGS_DDGSTF;
+ ulp_submode |= ULP_CRC_DATA;
+ }
+ ch->pdo = pdo;
+ ch->plen = htole32(plen);
+ set_mbuf_ulp_submode(top, ulp_submode);
+
+ if (data_len != 0) {
+ top->m_pkthdr.len += data_len;
+ top->m_next = data;
+ }
+
+ return (top);
+}
+
+/* Allocate the next free freelist transfer tag. */
+static bool
+nvmf_che_allocate_fl_ttag(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb)
+{
+ uint16_t ttag;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ if (qp->active_fl_ttags == qp->num_fl_ttags)
+ return (false);
+
+ ttag = qp->next_fl_ttag;
+ for (;;) {
+ if (qp->open_fl_ttags[ttag] == NULL)
+ break;
+ if (ttag == qp->num_fl_ttags - 1)
+ ttag = 0;
+ else
+ ttag++;
+ MPASS(ttag != qp->next_fl_ttag);
+ }
+ if (ttag == qp->num_fl_ttags - 1)
+ qp->next_fl_ttag = 0;
+ else
+ qp->next_fl_ttag = ttag + 1;
+
+ qp->active_fl_ttags++;
+ qp->open_fl_ttags[ttag] = cb;
+
+ cb->ttag = ttag | CHE_FL_TAG_MASK;
+ return (true);
+}
+
+/* Attempt to allocate a free transfer tag and assign it to cb. */
+static bool
+nvmf_che_allocate_ttag(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb)
+{
+ uint16_t stag;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ stag = che_alloc_ddp_tag(qp, cb);
+ if (stag == CHE_DDP_NO_TAG) {
+ if (!nvmf_che_allocate_fl_ttag(qp, cb))
+ return (false);
+ } else {
+ cb->ttag = stag;
+ }
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u allocated ttag 0x%04x", __func__,
+ qp->toep->tid, cb->ttag);
+#endif
+ cb->cc->active_r2ts++;
+ return (true);
+}
+
+/* Find the next command buffer eligible to schedule for R2T. */
+static struct nvmf_che_command_buffer *
+nvmf_che_next_r2t(struct nvmf_che_qpair *qp)
+{
+ struct nvmf_che_command_buffer *cb;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ TAILQ_FOREACH(cb, &qp->rx_buffers.head, link) {
+ /* NB: maxr2t is 0's based. */
+ if (cb->cc->active_r2ts > qp->maxr2t)
+ continue;
+
+ if (!nvmf_che_allocate_ttag(qp, cb))
+ return (NULL);
+#ifdef INVARIANTS
+ cb->cc->pending_r2ts--;
+#endif
+ TAILQ_REMOVE(&qp->rx_buffers.head, cb, link);
+ return (cb);
+ }
+ return (NULL);
+}
+
+/* NB: cid and is little-endian already. */
+static void
+che_send_r2t(struct nvmf_che_qpair *qp, uint16_t cid, uint16_t ttag,
+ uint32_t data_offset, uint32_t data_len)
+{
+ struct nvme_tcp_r2t_hdr r2t;
+ struct mbuf *m;
+
+ memset(&r2t, 0, sizeof(r2t));
+ r2t.common.pdu_type = NVME_TCP_PDU_TYPE_R2T;
+ r2t.cccid = cid;
+ r2t.ttag = htole16(ttag);
+ r2t.r2to = htole32(data_offset);
+ r2t.r2tl = htole32(data_len);
+
+ m = nvmf_che_construct_pdu(qp, &r2t, sizeof(r2t), NULL, 0);
+ nvmf_che_write_pdu(qp, m);
+}
+
+/*
+ * Release a transfer tag and schedule another R2T.
+ *
+ * NB: This drops the rx_buffers.lock mutex.
+ */
+static void
+nvmf_che_send_next_r2t(struct nvmf_che_qpair *qp,
+ struct nvmf_che_command_buffer *cb)
+{
+ struct nvmf_che_command_buffer *ncb;
+
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u freed ttag 0x%04x", __func__, qp->toep->tid,
+ cb->ttag);
+#endif
+ if (CHE_TAG_IS_FL(cb->ttag)) {
+ uint16_t ttag;
+
+ ttag = CHE_RAW_FL_TAG(cb->ttag);
+ MPASS(qp->open_fl_ttags[ttag] == cb);
+
+ /* Release this transfer tag. */
+ qp->open_fl_ttags[ttag] = NULL;
+ qp->active_fl_ttags--;
+ } else
+ che_free_ddp_tag(qp, cb, cb->ttag);
+
+ cb->cc->active_r2ts--;
+
+ /* Schedule another R2T. */
+ ncb = nvmf_che_next_r2t(qp);
+ mtx_unlock(&qp->rx_buffers.lock);
+ if (ncb != NULL)
+ che_send_r2t(qp, ncb->cid, ncb->ttag, ncb->data_offset,
+ ncb->data_len);
+}
+
+/*
+ * Copy len bytes starting at offset skip from an mbuf chain into an
+ * I/O buffer at destination offset io_offset.
+ */
+static void
+mbuf_copyto_io(struct mbuf *m, u_int skip, u_int len,
+ struct nvmf_io_request *io, u_int io_offset)
+{
+ u_int todo;
+
+ while (m->m_len <= skip) {
+ skip -= m->m_len;
+ m = m->m_next;
+ }
+ while (len != 0) {
+ MPASS((m->m_flags & M_EXTPG) == 0);
+
+ todo = min(m->m_len - skip, len);
+ memdesc_copyback(&io->io_mem, io_offset, todo, mtodo(m, skip));
+ skip = 0;
+ io_offset += todo;
+ len -= todo;
+ m = m->m_next;
+ }
+}
+
+static int
+nvmf_che_handle_h2c_data(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_h2c_data_hdr *h2c;
+ struct nvmf_che_command_buffer *cb;
+ uint32_t data_len, data_offset;
+ uint16_t ttag, fl_ttag;
+
+ h2c = (const void *)pdu->hdr;
+ if (le32toh(h2c->datal) > qp->maxh2cdata) {
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_LIMIT_EXCEEDED, 0,
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ ttag = le16toh(h2c->ttag);
+ if (CHE_TAG_IS_FL(ttag)) {
+ fl_ttag = CHE_RAW_FL_TAG(ttag);
+ if (fl_ttag >= qp->num_fl_ttags) {
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_h2c_data_hdr, ttag),
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ mtx_lock(&qp->rx_buffers.lock);
+ cb = qp->open_fl_ttags[fl_ttag];
+ } else {
+ if (CHE_STAG_IDX(ttag) >= qp->num_ddp_tags) {
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_h2c_data_hdr, ttag),
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ mtx_lock(&qp->rx_buffers.lock);
+ cb = qp->open_ddp_tags[CHE_STAG_IDX(ttag)];
+ }
+
+ if (cb == NULL) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_h2c_data_hdr, ttag), pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+ MPASS(cb->ttag == ttag);
+
+ /* For a data digest mismatch, fail the I/O request. */
+ if (pdu->data_digest_mismatch) {
+ nvmf_che_send_next_r2t(qp, cb);
+ cb->error = EINTEGRITY;
+ che_release_command_buffer(cb);
+ nvmf_che_free_pdu(pdu);
+ return (0);
+ }
+
+ data_len = le32toh(h2c->datal);
+ if (data_len != pdu->data_len) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_h2c_data_hdr, datal), pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ data_offset = le32toh(h2c->datao);
+ if (data_offset < cb->data_offset ||
+ data_offset + data_len > cb->data_offset + cb->data_len) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ if (data_offset != cb->data_offset + cb->data_xfered) {
+ if (CHE_TAG_IS_FL(ttag)) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ } else {
+ uint32_t ddp_bytes;
+
+ /* Account for PDUs silently received via DDP. */
+ ddp_bytes = data_offset -
+ (cb->data_offset + cb->data_xfered);
+ cb->data_xfered += ddp_bytes;
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u previous ddp_bytes %u",
+ __func__, qp->toep->tid, ddp_bytes);
+#endif
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_octets,
+ ddp_bytes);
+ }
+ }
+
+ if ((cb->data_xfered + data_len == cb->data_len) !=
+ ((pdu->hdr->flags & NVME_TCP_H2C_DATA_FLAGS_LAST_PDU) != 0)) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ cb->data_xfered += data_len;
+ data_offset -= cb->data_offset;
+ if (cb->data_xfered == cb->data_len) {
+ nvmf_che_send_next_r2t(qp, cb);
+ } else {
+ che_hold_command_buffer(cb);
+ mtx_unlock(&qp->rx_buffers.lock);
+ }
+
+ if (CHE_TAG_IS_FL(ttag))
+ mbuf_copyto_io(pdu->m->m_next, 0, data_len, &cb->io,
+ data_offset);
+
+ che_release_command_buffer(cb);
+ nvmf_che_free_pdu(pdu);
+ return (0);
+}
+
+static int
+nvmf_che_handle_c2h_data(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_c2h_data_hdr *c2h;
+ struct nvmf_che_command_buffer *cb;
+ uint32_t data_len, data_offset;
+ uint16_t cid, original_cid;
+
+ /*
+ * Unlike freelist command buffers, DDP command buffers are
+ * not released until the response capsule is received to keep
+ * the STAG allocated until the command has completed.
+ */
+ c2h = (const void *)pdu->hdr;
+
+ cid = le16toh(c2h->cccid);
+ if (CHE_TAG_IS_FL(cid)) {
+ mtx_lock(&qp->rx_buffers.lock);
+ cb = che_find_command_buffer(&qp->rx_buffers, c2h->cccid);
+ } else {
+ if (CHE_STAG_IDX(cid) >= qp->num_ddp_tags) {
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_c2h_data_hdr, cccid),
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ mtx_lock(&qp->rx_buffers.lock);
+ cb = qp->open_ddp_tags[CHE_STAG_IDX(cid)];
+ }
+
+ if (cb == NULL) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ /*
+ * XXX: Could be PDU sequence error if cccid is for a
+ * command that doesn't use a command buffer.
+ */
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_c2h_data_hdr, cccid), pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ /* For a data digest mismatch, fail the I/O request. */
+ if (pdu->data_digest_mismatch) {
+ cb->error = EINTEGRITY;
+ if (CHE_TAG_IS_FL(cid)) {
+ che_remove_command_buffer(&qp->rx_buffers, cb);
+ mtx_unlock(&qp->rx_buffers.lock);
+ che_release_command_buffer(cb);
+ } else
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_free_pdu(pdu);
+ return (0);
+ }
+
+ data_len = le32toh(c2h->datal);
+ if (data_len != pdu->data_len) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_c2h_data_hdr, datal), pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ data_offset = le32toh(c2h->datao);
+ if (data_offset < cb->data_offset ||
+ data_offset + data_len > cb->data_offset + cb->data_len) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0,
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ if (data_offset != cb->data_offset + cb->data_xfered) {
+ if (CHE_TAG_IS_FL(cid)) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ } else {
+ uint32_t ddp_bytes;
+
+ /* Account for PDUs silently received via DDP. */
+ ddp_bytes = data_offset -
+ (cb->data_offset + cb->data_xfered);
+ cb->data_xfered += ddp_bytes;
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u previous ddp_bytes %u",
+ __func__, qp->toep->tid, ddp_bytes);
+#endif
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_octets,
+ ddp_bytes);
+ }
+ }
+
+ if ((cb->data_xfered + data_len == cb->data_len) !=
+ ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_LAST_PDU) != 0)) {
+ mtx_unlock(&qp->rx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ cb->data_xfered += data_len;
+ original_cid = cb->original_cid;
+
+ if (CHE_TAG_IS_FL(cid)) {
+ data_offset -= cb->data_offset;
+ if (cb->data_xfered == cb->data_len)
+ che_remove_command_buffer(&qp->rx_buffers, cb);
+ else
+ che_hold_command_buffer(cb);
+ mtx_unlock(&qp->rx_buffers.lock);
+
+ if ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_SUCCESS) != 0) {
+ /*
+ * Free the CID as the command has now been
+ * completed.
+ */
+ cid = CHE_RAW_FL_TAG(cid);
+ mtx_lock(&qp->fl_cid_lock);
+ MPASS(FL_CID_ISACTIVE(cid, qp->fl_cid_set));
+ MPASS(original_cid == qp->fl_cids[cid]);
+ FL_CID_FREE(cid, qp->fl_cid_set);
+ mtx_unlock(&qp->fl_cid_lock);
+ }
+
+ mbuf_copyto_io(pdu->m->m_next, 0, data_len, &cb->io,
+ data_offset);
+
+ che_release_command_buffer(cb);
+ } else {
+ if ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_SUCCESS) != 0) {
+ /*
+ * Free the command buffer and STAG as the
+ * command has now been completed.
+ */
+ che_free_ddp_tag(qp, cb, cid);
+ mtx_unlock(&qp->rx_buffers.lock);
+ che_release_command_buffer(cb);
+ } else
+ mtx_unlock(&qp->rx_buffers.lock);
+ }
+
+ if ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_SUCCESS) != 0) {
+ struct nvme_completion cqe;
+ struct nvmf_capsule *nc;
+
+ memset(&cqe, 0, sizeof(cqe));
+ cqe.cid = original_cid;
+
+ nc = nvmf_allocate_response(&qp->qp, &cqe, M_WAITOK);
+ nc->nc_sqhd_valid = false;
+
+ nvmf_capsule_received(&qp->qp, nc);
+ }
+
+ nvmf_che_free_pdu(pdu);
+ return (0);
+}
+
+/* Called when m_free drops refcount to 0. */
+static void
+nvmf_che_mbuf_done(struct mbuf *m)
+{
+ struct nvmf_che_command_buffer *cb = m->m_ext.ext_arg1;
+
+ che_free_command_buffer(cb);
+}
+
+static struct mbuf *
+nvmf_che_mbuf(void *arg, int how, void *data, size_t len)
+{
+ struct nvmf_che_command_buffer *cb = arg;
+ struct mbuf *m;
+
+ m = m_get(how, MT_DATA);
+ m->m_flags |= M_RDONLY;
+ m_extaddref(m, data, len, &cb->refs, nvmf_che_mbuf_done, cb, NULL);
+ m->m_len = len;
+ return (m);
+}
+
+static void
+nvmf_che_free_mext_pg(struct mbuf *m)
+{
+ struct nvmf_che_command_buffer *cb = m->m_ext.ext_arg1;
+
+ M_ASSERTEXTPG(m);
+ che_release_command_buffer(cb);
+}
+
+static struct mbuf *
+nvmf_che_mext_pg(void *arg, int how)
+{
+ struct nvmf_che_command_buffer *cb = arg;
+ struct mbuf *m;
+
+ m = mb_alloc_ext_pgs(how, nvmf_che_free_mext_pg, M_RDONLY);
+ m->m_ext.ext_arg1 = cb;
+ che_hold_command_buffer(cb);
+ return (m);
+}
+
+/*
+ * Return an mbuf chain for a range of data belonging to a command
+ * buffer.
+ *
+ * The mbuf chain uses M_EXT mbufs which hold references on the
+ * command buffer so that it remains "alive" until the data has been
+ * fully transmitted. If truncate_ok is true, then the mbuf chain
+ * might return a short chain to avoid gratuitously splitting up a
+ * page.
+ */
+static struct mbuf *
+nvmf_che_command_buffer_mbuf(struct nvmf_che_command_buffer *cb,
+ uint32_t data_offset, uint32_t data_len, uint32_t *actual_len,
+ bool can_truncate)
+{
+ struct mbuf *m;
+ size_t len;
+
+ m = memdesc_alloc_ext_mbufs(&cb->io.io_mem, nvmf_che_mbuf,
+ nvmf_che_mext_pg, cb, M_WAITOK, data_offset, data_len, &len,
+ can_truncate);
+ if (actual_len != NULL)
+ *actual_len = len;
+ return (m);
+}
+
+/* NB: cid and ttag and little-endian already. */
+static void
+che_send_h2c_pdu(struct nvmf_che_qpair *qp, uint16_t cid, uint16_t ttag,
+ uint32_t data_offset, struct mbuf *m, size_t len, bool last_pdu)
+{
+ struct nvme_tcp_h2c_data_hdr h2c;
+ struct mbuf *top;
+
+ memset(&h2c, 0, sizeof(h2c));
+ h2c.common.pdu_type = NVME_TCP_PDU_TYPE_H2C_DATA;
+ if (last_pdu)
+ h2c.common.flags |= NVME_TCP_H2C_DATA_FLAGS_LAST_PDU;
+ h2c.cccid = cid;
+ h2c.ttag = ttag;
+ h2c.datao = htole32(data_offset);
+ h2c.datal = htole32(len);
+
+ top = nvmf_che_construct_pdu(qp, &h2c, sizeof(h2c), m, len);
+ nvmf_che_write_pdu(qp, top);
+}
+
+static int
+nvmf_che_handle_r2t(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ const struct nvme_tcp_r2t_hdr *r2t;
+ struct nvmf_che_command_buffer *cb;
+ uint32_t data_len, data_offset;
+
+ r2t = (const void *)pdu->hdr;
+
+ mtx_lock(&qp->tx_buffers.lock);
+ cb = che_find_command_buffer(&qp->tx_buffers, r2t->cccid);
+ if (cb == NULL) {
+ mtx_unlock(&qp->tx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD,
+ offsetof(struct nvme_tcp_r2t_hdr, cccid), pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ data_offset = le32toh(r2t->r2to);
+ if (data_offset != cb->data_xfered) {
+ mtx_unlock(&qp->tx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,
+ pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ /*
+ * XXX: The spec does not specify how to handle R2T tranfers
+ * out of range of the original command.
+ */
+ data_len = le32toh(r2t->r2tl);
+ if (data_offset + data_len > cb->data_len) {
+ mtx_unlock(&qp->tx_buffers.lock);
+ nvmf_che_report_error(qp,
+ NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0,
+ pdu->m, pdu->hdr->hlen);
+ nvmf_che_free_pdu(pdu);
+ return (EBADMSG);
+ }
+
+ cb->data_xfered += data_len;
+ if (cb->data_xfered == cb->data_len)
+ che_remove_command_buffer(&qp->tx_buffers, cb);
+ else
+ che_hold_command_buffer(cb);
+ mtx_unlock(&qp->tx_buffers.lock);
+
+ /*
+ * Queue one or more H2C_DATA PDUs containing the requested
+ * data.
+ */
+ while (data_len > 0) {
+ struct mbuf *m;
+ uint32_t sent, todo;
+
+ todo = min(data_len, qp->max_tx_data);
+ m = nvmf_che_command_buffer_mbuf(cb, data_offset, todo, &sent,
+ todo < data_len);
+ che_send_h2c_pdu(qp, r2t->cccid, r2t->ttag, data_offset, m,
+ sent, sent == data_len);
+
+ data_offset += sent;
+ data_len -= sent;
+ }
+
+ che_release_command_buffer(cb);
+ nvmf_che_free_pdu(pdu);
+ return (0);
+}
+
+static int
+nvmf_che_dispatch_pdu(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ /*
+ * The PDU header should always be contiguous in the mbuf from
+ * CPL_NVMT_CMP.
+ */
+ pdu->hdr = mtod(pdu->m, void *);
+ KASSERT(pdu->m->m_len == pdu->hdr->hlen +
+ ((pdu->hdr->flags & NVME_TCP_CH_FLAGS_HDGSTF) != 0 ?
+ sizeof(uint32_t) : 0),
+ ("%s: mismatched PDU header mbuf length", __func__));
+
+ switch (pdu->hdr->pdu_type) {
+ default:
+ __assert_unreachable();
+ break;
+ case NVME_TCP_PDU_TYPE_H2C_TERM_REQ:
+ case NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
+ return (nvmf_che_handle_term_req(pdu));
+ case NVME_TCP_PDU_TYPE_CAPSULE_CMD:
+ return (nvmf_che_save_command_capsule(qp, pdu));
+ case NVME_TCP_PDU_TYPE_CAPSULE_RESP:
+ return (nvmf_che_save_response_capsule(qp, pdu));
+ case NVME_TCP_PDU_TYPE_H2C_DATA:
+ return (nvmf_che_handle_h2c_data(qp, pdu));
+ case NVME_TCP_PDU_TYPE_C2H_DATA:
+ return (nvmf_che_handle_c2h_data(qp, pdu));
+ case NVME_TCP_PDU_TYPE_R2T:
+ return (nvmf_che_handle_r2t(qp, pdu));
+ }
+}
+
+static int
+nvmf_che_attach_pdu_data(struct nvmf_che_qpair *qp, struct nvmf_che_rxpdu *pdu)
+{
+ struct socket *so = qp->so;
+ struct mbuf *m, *n;
+ uint32_t tcp_seq;
+ size_t len;
+ int error;
+
+ /* Check for DDP data. */
+ if (pdu->ddp) {
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_pdus, 1);
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_ddp_octets,
+ pdu->data_len);
+ return (0);
+ }
+
+ error = 0;
+ len = pdu->data_len;
+ tcp_seq = pdu->m->m_pkthdr.nvmf_tcp_seq;
+ m = pdu->m;
+ SOCKBUF_LOCK(&so->so_rcv);
+ while (len > 0) {
+ n = mbufq_dequeue(&qp->rx_data);
+ KASSERT(n != NULL, ("%s: missing %zu data", __func__, len));
+ if (n == NULL) {
+ error = ENOBUFS;
+ break;
+ }
+
+ KASSERT(n->m_pkthdr.nvmf_tcp_seq == tcp_seq,
+ ("%s: TCP seq mismatch", __func__));
+ KASSERT(n->m_pkthdr.len <= len,
+ ("%s: too much data", __func__));
+ if (n->m_pkthdr.nvmf_tcp_seq != tcp_seq ||
+ n->m_pkthdr.len > len) {
+ m_freem(n);
+ error = ENOBUFS;
+ break;
+ }
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__,
+ qp->toep->tid, n->m_pkthdr.len, n->m_pkthdr.nvmf_tcp_seq);
+#endif
+ pdu->m->m_pkthdr.len += n->m_pkthdr.len;
+ len -= n->m_pkthdr.len;
+ tcp_seq += n->m_pkthdr.len;
+ m_demote_pkthdr(n);
+ m->m_next = n;
+ m = m_last(n);
+ }
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ if (error == 0) {
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_fl_pdus, 1);
+ counter_u64_add(qp->toep->ofld_rxq->rx_nvme_fl_octets,
+ pdu->data_len);
+ }
+ return (error);
+}
+
+static void
+nvmf_che_receive(void *arg)
+{
+ struct nvmf_che_qpair *qp = arg;
+ struct socket *so = qp->so;
+ struct nvmf_che_rxpdu pdu;
+ struct mbuf *m;
+ int error, terror;
+
+ SOCKBUF_LOCK(&so->so_rcv);
+ while (!qp->rx_shutdown) {
+ /* Wait for a PDU. */
+ if (so->so_error != 0 || so->so_rerror != 0) {
+ if (so->so_error != 0)
+ error = so->so_error;
+ else
+ error = so->so_rerror;
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ error:
+ nvmf_qpair_error(&qp->qp, error);
+ SOCKBUF_LOCK(&so->so_rcv);
+ while (!qp->rx_shutdown)
+ cv_wait(&qp->rx_cv, SOCKBUF_MTX(&so->so_rcv));
+ break;
+ }
+
+ m = mbufq_dequeue(&qp->rx_pdus);
+ if (m == NULL) {
+ if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) != 0) {
+ error = 0;
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ goto error;
+ }
+ cv_wait(&qp->rx_cv, SOCKBUF_MTX(&so->so_rcv));
+ continue;
+ }
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ pdu.m = m;
+ pdu.hdr = mtod(m, const void *);
+ pdu.ddp = (m->m_pkthdr.nvmf_cpl_status & CMP_STATUS_DDP) != 0;
+
+ error = nvmf_che_validate_pdu(qp, &pdu);
+ if (error == 0 && pdu.data_len != 0)
+ error = nvmf_che_attach_pdu_data(qp, &pdu);
+ if (error != 0)
+ nvmf_che_free_pdu(&pdu);
+ else
+ error = nvmf_che_dispatch_pdu(qp, &pdu);
+ if (error != 0) {
+ /*
+ * If we received a termination request, close
+ * the connection immediately.
+ */
+ if (error == ECONNRESET)
+ goto error;
+
+ /*
+ * Wait for up to 30 seconds for the socket to
+ * be closed by the other end.
+ */
+ SOCKBUF_LOCK(&so->so_rcv);
+ if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
+ terror = cv_timedwait(&qp->rx_cv,
+ SOCKBUF_MTX(&so->so_rcv), 30 * hz);
+ if (terror == ETIMEDOUT)
+ printf("NVMe/TCP: Timed out after sending terminate request\n");
+ }
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ goto error;
+ }
+
+ SOCKBUF_LOCK(&so->so_rcv);
+ }
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ kthread_exit();
+}
+
+static int
+nvmf_che_soupcall_receive(struct socket *so, void *arg, int waitflag)
+{
+ struct nvmf_che_qpair *qp = arg;
+
+ cv_signal(&qp->rx_cv);
+ return (SU_OK);
+}
+
+static int
+do_nvmt_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+ struct adapter *sc = iq->adapter;
+ struct nvmf_che_adapter *nca = sc->nvme_ulp_softc;
+ const struct cpl_nvmt_data *cpl;
+ u_int tid;
+ struct toepcb *toep;
+ struct nvmf_che_qpair *qp;
+ struct socket *so;
+ struct inpcb *inp;
+ struct tcpcb *tp;
+ int len __diagused;
+
+ if (nca->nvmt_data_iqe) {
+ cpl = (const void *)(rss + 1);
+ } else {
+ cpl = mtod(m, const void *);
+
+ /* strip off CPL header */
+ m_adj(m, sizeof(*cpl));
+ }
+ tid = GET_TID(cpl);
+ toep = lookup_tid(sc, tid);
+
+ KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
+
+ len = m->m_pkthdr.len;
+
+ KASSERT(len == be16toh(cpl->length),
+ ("%s: payload length mismatch", __func__));
+
+ inp = toep->inp;
+ INP_WLOCK(inp);
+ if (inp->inp_flags & INP_DROPPED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
+ __func__, tid, len, inp->inp_flags);
+ INP_WUNLOCK(inp);
+ m_freem(m);
+ return (0);
+ }
+
+ /* Save TCP sequence number. */
+ m->m_pkthdr.nvmf_tcp_seq = be32toh(cpl->seq);
+
+ qp = toep->ulpcb;
+ so = qp->so;
+ SOCKBUF_LOCK(&so->so_rcv);
+ mbufq_enqueue(&qp->rx_data, m);
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ tp = intotcpcb(inp);
+ tp->t_rcvtime = ticks;
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
+ be32toh(cpl->seq));
+#endif
+
+ INP_WUNLOCK(inp);
+ return (0);
+}
+
+static int
+do_nvmt_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_nvmt_cmp *cpl = mtod(m, const void *);
+ u_int tid = GET_TID(cpl);
+ struct toepcb *toep = lookup_tid(sc, tid);
+ struct nvmf_che_qpair *qp = toep->ulpcb;
+ struct socket *so = qp->so;
+ struct inpcb *inp = toep->inp;
+ u_int hlen __diagused;
+ bool empty;
+
+ KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
+ KASSERT(!(toep->flags & TPF_SYNQE),
+ ("%s: toep %p claims to be a synq entry", __func__, toep));
+
+ /* strip off CPL header */
+ m_adj(m, sizeof(*cpl));
+ hlen = m->m_pkthdr.len;
+
+ KASSERT(hlen == be16toh(cpl->length),
+ ("%s: payload length mismatch", __func__));
+
+ INP_WLOCK(inp);
+ if (inp->inp_flags & INP_DROPPED) {
+ CTR(KTR_CXGBE, "%s: tid %u, rx (hlen %u), inp_flags 0x%x",
+ __func__, tid, hlen, inp->inp_flags);
+ INP_WUNLOCK(inp);
+ m_freem(m);
+ return (0);
+ }
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u hlen %u seq %u status %u", __func__, tid,
+ hlen, be32toh(cpl->seq), cpl->status);
+#endif
+
+ /* Save TCP sequence number and CPL status. */
+ m->m_pkthdr.nvmf_tcp_seq = be32toh(cpl->seq);
+ m->m_pkthdr.nvmf_cpl_status = cpl->status;
+
+ SOCKBUF_LOCK(&so->so_rcv);
+ empty = mbufq_len(&qp->rx_pdus) == 0;
+ mbufq_enqueue(&qp->rx_pdus, m);
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ INP_WUNLOCK(inp);
+ if (empty)
+ cv_signal(&qp->rx_cv);
+ return (0);
+}
+
+static uint16_t
+che_alloc_fl_cid(struct nvmf_che_qpair *qp, uint16_t original_cid)
+{
+ uint16_t new_cid;
+
+ mtx_lock(&qp->fl_cid_lock);
+ new_cid = FL_CID_FINDFREE_AT(qp->fl_cid_set, qp->next_cid);
+ if (new_cid == 0) {
+ new_cid = FL_CID_FINDFREE_AT(qp->fl_cid_set, 0);
+ MPASS(new_cid != 0);
+ }
+ new_cid--;
+ FL_CID_BUSY(new_cid, qp->fl_cid_set);
+ if (new_cid == CHE_MAX_FL_TAG)
+ qp->next_cid = 0;
+ else
+ qp->next_cid = new_cid + 1;
+ qp->fl_cids[new_cid] = original_cid;
+ mtx_unlock(&qp->fl_cid_lock);
+
+ return (new_cid | CHE_FL_TAG_MASK);
+}
+
+static uint16_t
+che_alloc_ddp_cid(struct nvmf_che_qpair *qp, struct nvmf_che_command_buffer *cb)
+{
+ mtx_assert(&qp->rx_buffers.lock, MA_OWNED);
+
+ return (che_alloc_ddp_tag(qp, cb));
+}
+
+static struct mbuf *
+che_command_pdu(struct nvmf_che_qpair *qp, struct nvmf_che_capsule *cc)
+{
+ struct nvmf_capsule *nc = &cc->nc;
+ struct nvmf_che_command_buffer *cb;
+ struct nvme_sgl_descriptor *sgl;
+ struct nvme_tcp_cmd cmd;
+ struct mbuf *top, *m;
+ uint16_t cid;
+ bool use_icd;
+
+ use_icd = false;
+ cb = NULL;
+ m = NULL;
+
+ if (nc->nc_data.io_len != 0) {
+ cb = che_alloc_command_buffer(qp, &nc->nc_data, 0,
+ nc->nc_data.io_len, nc->nc_sqe.cid);
+ cb->original_cid = nc->nc_sqe.cid;
+
+ if (nc->nc_send_data && nc->nc_data.io_len <= qp->max_icd) {
+ cid = che_alloc_fl_cid(qp, nc->nc_sqe.cid);
+ use_icd = true;
+ m = nvmf_che_command_buffer_mbuf(cb, 0,
+ nc->nc_data.io_len, NULL, false);
+ cb->data_xfered = nc->nc_data.io_len;
+ che_release_command_buffer(cb);
+ } else if (nc->nc_send_data) {
+ cid = che_alloc_fl_cid(qp, nc->nc_sqe.cid);
+ cb->cid = htole16(cid);
+ mtx_lock(&qp->tx_buffers.lock);
+ che_add_command_buffer(&qp->tx_buffers, cb);
+ mtx_unlock(&qp->tx_buffers.lock);
+ } else {
+ mtx_lock(&qp->rx_buffers.lock);
+ cid = che_alloc_ddp_cid(qp, cb);
+ if (cid == CHE_DDP_NO_TAG) {
+ cid = che_alloc_fl_cid(qp, nc->nc_sqe.cid);
+ che_add_command_buffer(&qp->rx_buffers, cb);
+ }
+ cb->cid = htole16(cid);
+ mtx_unlock(&qp->rx_buffers.lock);
+ }
+ } else
+ cid = che_alloc_fl_cid(qp, nc->nc_sqe.cid);
+
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE, "%s: tid %u allocated cid 0x%04x for 0x%04x", __func__,
+ qp->toep->tid, cid, nc->nc_sqe.cid);
+#endif
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.common.pdu_type = NVME_TCP_PDU_TYPE_CAPSULE_CMD;
+ cmd.ccsqe = nc->nc_sqe;
+ cmd.ccsqe.cid = htole16(cid);
+
+ /* Populate SGL in SQE. */
+ sgl = &cmd.ccsqe.sgl;
+ memset(sgl, 0, sizeof(*sgl));
+ sgl->address = 0;
+ sgl->length = htole32(nc->nc_data.io_len);
+ if (use_icd) {
+ /* Use in-capsule data. */
+ sgl->type = NVME_SGL_TYPE_ICD;
+ } else {
+ /* Use a command buffer. */
+ sgl->type = NVME_SGL_TYPE_COMMAND_BUFFER;
+ }
+
+ top = nvmf_che_construct_pdu(qp, &cmd, sizeof(cmd), m, m != NULL ?
+ nc->nc_data.io_len : 0);
+ return (top);
+}
+
+static struct mbuf *
+che_response_pdu(struct nvmf_che_qpair *qp, struct nvmf_che_capsule *cc)
+{
+ struct nvmf_capsule *nc = &cc->nc;
+ struct nvme_tcp_rsp rsp;
+
+ memset(&rsp, 0, sizeof(rsp));
+ rsp.common.pdu_type = NVME_TCP_PDU_TYPE_CAPSULE_RESP;
+ rsp.rccqe = nc->nc_cqe;
+
+ return (nvmf_che_construct_pdu(qp, &rsp, sizeof(rsp), NULL, 0));
+}
+
+static struct mbuf *
+capsule_to_pdu(struct nvmf_che_qpair *qp, struct nvmf_che_capsule *cc)
+{
+ if (cc->nc.nc_qe_len == sizeof(struct nvme_command))
+ return (che_command_pdu(qp, cc));
+ else
+ return (che_response_pdu(qp, cc));
+}
+
+static void
+nvmf_che_send(void *arg)
+{
+ struct nvmf_che_qpair *qp = arg;
+ struct nvmf_che_capsule *cc;
+ struct socket *so = qp->so;
+ struct mbuf *m;
+ int error;
+
+ m = NULL;
+ SOCKBUF_LOCK(&so->so_snd);
+ while (!qp->tx_shutdown) {
+ if (so->so_error != 0) {
+ error = so->so_error;
+ SOCKBUF_UNLOCK(&so->so_snd);
+ m_freem(m);
+ nvmf_qpair_error(&qp->qp, error);
+ SOCKBUF_LOCK(&so->so_snd);
+ while (!qp->tx_shutdown)
+ cv_wait(&qp->tx_cv, SOCKBUF_MTX(&so->so_snd));
+ break;
+ }
+
+ if (STAILQ_EMPTY(&qp->tx_capsules)) {
+ cv_wait(&qp->tx_cv, SOCKBUF_MTX(&so->so_snd));
+ continue;
+ }
+
+ /* Convert a capsule into a PDU. */
+ cc = STAILQ_FIRST(&qp->tx_capsules);
+ STAILQ_REMOVE_HEAD(&qp->tx_capsules, link);
+ SOCKBUF_UNLOCK(&so->so_snd);
+
+ m = capsule_to_pdu(qp, cc);
+ che_release_capsule(cc);
+
+ nvmf_che_write_pdu(qp, m);
+
+ SOCKBUF_LOCK(&so->so_snd);
+ }
+ SOCKBUF_UNLOCK(&so->so_snd);
+ kthread_exit();
+}
+
+static int
+nvmf_che_setsockopt(struct socket *so, u_int sspace, u_int rspace)
+{
+ struct sockopt opt;
+ int error, one = 1;
+
+ /* Don't lower the buffer sizes, just enforce a minimum. */
+ SOCKBUF_LOCK(&so->so_snd);
+ if (sspace < so->so_snd.sb_hiwat)
+ sspace = so->so_snd.sb_hiwat;
+ SOCKBUF_UNLOCK(&so->so_snd);
+ SOCKBUF_LOCK(&so->so_rcv);
+ if (rspace < so->so_rcv.sb_hiwat)
+ rspace = so->so_rcv.sb_hiwat;
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ error = soreserve(so, sspace, rspace);
+ if (error != 0)
+ return (error);
+ SOCKBUF_LOCK(&so->so_snd);
+ so->so_snd.sb_flags |= SB_AUTOSIZE;
+ SOCKBUF_UNLOCK(&so->so_snd);
+ SOCKBUF_LOCK(&so->so_rcv);
+ so->so_rcv.sb_flags |= SB_AUTOSIZE;
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ /*
+ * Disable Nagle.
+ */
+ bzero(&opt, sizeof(opt));
+ opt.sopt_dir = SOPT_SET;
+ opt.sopt_level = IPPROTO_TCP;
+ opt.sopt_name = TCP_NODELAY;
+ opt.sopt_val = &one;
+ opt.sopt_valsize = sizeof(one);
+ error = sosetopt(so, &opt);
+ if (error != 0)
+ return (error);
+
+ return (0);
+}
+
+static void
+t4_nvme_set_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
+ uint64_t val)
+{
+ struct adapter *sc = td_adapter(toep->td);
+
+ t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0);
+}
+
+static void
+set_ulp_mode_nvme(struct toepcb *toep, u_int ulp_submode, uint8_t rxpda)
+{
+ uint64_t val;
+
+ CTR(KTR_CXGBE, "%s: tid %u, ULP_MODE_NVMET, submode=%#x, rxpda=%u",
+ __func__, toep->tid, ulp_submode, rxpda);
+
+ val = V_TCB_ULP_TYPE(ULP_MODE_NVMET) | V_TCB_ULP_RAW(ulp_submode);
+ t4_nvme_set_tcb_field(toep, W_TCB_ULP_TYPE,
+ V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val);
+
+ val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL);
+ t4_nvme_set_tcb_field(toep, W_TCB_T_FLAGS, val, val);
+
+ val = V_TCB_RSVD((rxpda / 4) - 1);
+ t4_nvme_set_tcb_field(toep, W_TCB_RSVD, V_TCB_RSVD(M_TCB_RSVD), val);
+
+ /* 0 disables CPL_NVMT_CMP_IMM which is not useful in this driver. */
+ val = 0;
+ t4_nvme_set_tcb_field(toep, W_TCB_CMP_IMM_SZ,
+ V_TCB_CMP_IMM_SZ(M_TCB_CMP_IMM_SZ), val);
+}
+
+static u_int
+pdu_max_data_len(const nvlist_t *nvl, u_int max_pdu_len, u_int hlen,
+ uint8_t pda)
+{
+ u_int max_data_len;
+
+ if (nvlist_get_bool(nvl, "header_digests"))
+ hlen += sizeof(uint32_t);
+ hlen = roundup(hlen, pda);
+ max_data_len = max_pdu_len - hlen;
+ if (nvlist_get_bool(nvl, "data_digests"))
+ max_data_len -= sizeof(uint32_t);
+ return (max_data_len);
+}
+
+static struct nvmf_qpair *
+che_allocate_qpair(bool controller, const nvlist_t *nvl)
+{
+ struct nvmf_che_adapter *nca;
+ struct nvmf_che_qpair *qp;
+ struct adapter *sc;
+ struct file *fp;
+ struct socket *so;
+ struct inpcb *inp;
+ struct tcpcb *tp;
+ struct toepcb *toep;
+ cap_rights_t rights;
+ u_int max_tx_pdu_len, num_ddp_tags;
+ int error, ulp_submode;
+
+ if (!nvlist_exists_number(nvl, "fd") ||
+ !nvlist_exists_number(nvl, "rxpda") ||
+ !nvlist_exists_number(nvl, "txpda") ||
+ !nvlist_exists_bool(nvl, "header_digests") ||
+ !nvlist_exists_bool(nvl, "data_digests") ||
+ !nvlist_exists_number(nvl, "maxr2t") ||
+ !nvlist_exists_number(nvl, "maxh2cdata") ||
+ !nvlist_exists_number(nvl, "max_icd"))
+ return (NULL);
+
+ error = fget(curthread, nvlist_get_number(nvl, "fd"),
+ cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
+ if (error != 0)
+ return (NULL);
+ if (fp->f_type != DTYPE_SOCKET) {
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+ so = fp->f_data;
+ if (so->so_type != SOCK_STREAM ||
+ so->so_proto->pr_protocol != IPPROTO_TCP) {
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+
+ sc = find_offload_adapter(so);
+ if (sc == NULL) {
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+ nca = sc->nvme_ulp_softc;
+
+ /*
+ * Controller: Require advertised MAXH2CDATA to be small
+ * enough.
+ */
+ if (controller) {
+ u_int max_rx_data;
+
+ max_rx_data = pdu_max_data_len(nvl, nca->max_receive_pdu,
+ sizeof(struct nvme_tcp_h2c_data_hdr),
+ nvlist_get_number(nvl, "rxpda"));
+ if (nvlist_get_number(nvl, "maxh2cdata") > max_rx_data) {
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+ }
+
+ /*
+ * Host: Require the queue size to be small enough that all of
+ * the command ids allocated by nvmf(4) will fit in the
+ * unallocated range.
+ *
+ * XXX: Alternatively this driver could just queue commands
+ * when an unallocated ID isn't available.
+ */
+ if (!controller) {
+ u_int num_commands;
+
+ num_commands = nvlist_get_number(nvl, "qsize") - 1;
+ if (nvlist_get_bool(nvl, "admin"))
+ num_commands += 8; /* Max AER */
+ if (num_commands > CHE_NUM_FL_TAGS) {
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+ }
+
+ qp = malloc(sizeof(*qp), M_NVMF_CHE, M_WAITOK | M_ZERO);
+ qp->txpda = nvlist_get_number(nvl, "txpda");
+ qp->rxpda = nvlist_get_number(nvl, "rxpda");
+ qp->header_digests = nvlist_get_bool(nvl, "header_digests");
+ qp->data_digests = nvlist_get_bool(nvl, "data_digests");
+ qp->maxr2t = nvlist_get_number(nvl, "maxr2t");
+ if (controller)
+ qp->maxh2cdata = nvlist_get_number(nvl, "maxh2cdata");
+
+ if (controller) {
+ /* NB: maxr2t is 0's based. */
+ qp->num_fl_ttags = MIN(CHE_NUM_FL_TAGS,
+ nvlist_get_number(nvl, "qsize") *
+ ((uint64_t)qp->maxr2t + 1));
+ qp->open_fl_ttags = mallocarray(qp->num_fl_ttags,
+ sizeof(*qp->open_fl_ttags), M_NVMF_CHE, M_WAITOK | M_ZERO);
+ } else {
+ qp->fl_cids = mallocarray(CHE_NUM_FL_TAGS,
+ sizeof(*qp->fl_cids), M_NVMF_CHE, M_WAITOK | M_ZERO);
+ qp->fl_cid_set = malloc(sizeof(*qp->fl_cid_set), M_NVMF_CHE,
+ M_WAITOK);
+ FL_CID_INIT(qp->fl_cid_set);
+ mtx_init(&qp->fl_cid_lock, "nvmf/che fl cids", NULL, MTX_DEF);
+ }
+
+ inp = sotoinpcb(so);
+ INP_WLOCK(inp);
+ tp = intotcpcb(inp);
+ if (inp->inp_flags & INP_DROPPED) {
+ INP_WUNLOCK(inp);
+ free(qp->fl_cid_set, M_NVMF_CHE);
+ free(qp->fl_cids, M_NVMF_CHE);
+ free(qp->open_fl_ttags, M_NVMF_CHE);
+ free(qp, M_NVMF_CHE);
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+
+ MPASS(tp->t_flags & TF_TOE);
+ MPASS(tp->tod != NULL);
+ MPASS(tp->t_toe != NULL);
+ toep = tp->t_toe;
+ MPASS(toep->vi->adapter == sc);
+
+ if (ulp_mode(toep) != ULP_MODE_NONE) {
+ INP_WUNLOCK(inp);
+ free(qp->fl_cid_set, M_NVMF_CHE);
+ free(qp->fl_cids, M_NVMF_CHE);
+ free(qp->open_fl_ttags, M_NVMF_CHE);
+ free(qp, M_NVMF_CHE);
+ fdrop(fp, curthread);
+ return (NULL);
+ }
+
+ /* Claim socket from file descriptor. */
+ fp->f_ops = &badfileops;
+ fp->f_data = NULL;
+
+ qp->so = so;
+ qp->toep = toep;
+ qp->nca = nca;
+ refcount_init(&qp->refs, 1);
+
+ /* NB: C2H and H2C headers are the same size. */
+ qp->max_rx_data = pdu_max_data_len(nvl, nca->max_receive_pdu,
+ sizeof(struct nvme_tcp_c2h_data_hdr), qp->rxpda);
+ qp->max_tx_data = pdu_max_data_len(nvl, nca->max_transmit_pdu,
+ sizeof(struct nvme_tcp_c2h_data_hdr), qp->txpda);
+ if (!controller) {
+ qp->max_tx_data = min(qp->max_tx_data,
+ nvlist_get_number(nvl, "maxh2cdata"));
+ qp->max_icd = min(nvlist_get_number(nvl, "max_icd"),
+ pdu_max_data_len(nvl, nca->max_transmit_pdu,
+ sizeof(struct nvme_tcp_cmd), qp->txpda));
+ } else {
+ /*
+ * IOCCSZ represents the size of a logical command
+ * capsule including the 64 byte SQE and the
+ * in-capsule data. Use pdu_max_data_len to compute
+ * the maximum supported ICD length.
+ */
+ qp->max_ioccsz = rounddown(pdu_max_data_len(nvl,
+ nca->max_receive_pdu, sizeof(struct nvme_tcp_cmd),
+ qp->rxpda), 16) + sizeof(struct nvme_command);
+ }
+
+ ulp_submode = 0;
+ if (qp->header_digests)
+ ulp_submode |= FW_NVMET_ULPSUBMODE_HCRC;
+ if (qp->data_digests)
+ ulp_submode |= FW_NVMET_ULPSUBMODE_DCRC;
+ if (!controller)
+ ulp_submode |= FW_NVMET_ULPSUBMODE_ING_DIR;
+
+ max_tx_pdu_len = sizeof(struct nvme_tcp_h2c_data_hdr);
+ if (qp->header_digests)
+ max_tx_pdu_len += sizeof(uint32_t);
+ max_tx_pdu_len = roundup(max_tx_pdu_len, qp->txpda);
+ max_tx_pdu_len += qp->max_tx_data;
+ if (qp->data_digests)
+ max_tx_pdu_len += sizeof(uint32_t);
+
+ /* TODO: ISO limits */
+
+ if (controller) {
+ /* Use the SUCCESS flag if SQ flow control is disabled. */
+ qp->send_success = !nvlist_get_bool(nvl, "sq_flow_control");
+ }
+
+ toep->params.ulp_mode = ULP_MODE_NVMET;
+ toep->ulpcb = qp;
+
+ send_txdataplen_max_flowc_wr(sc, toep,
+ roundup(/* max_iso_pdus * */ max_tx_pdu_len, tp->t_maxseg));
+ set_ulp_mode_nvme(toep, ulp_submode, qp->rxpda);
+ INP_WUNLOCK(inp);
+
+ fdrop(fp, curthread);
+
+ error = nvmf_che_setsockopt(so, max_tx_pdu_len, nca->max_receive_pdu);
+ if (error != 0) {
+ free(qp->fl_cid_set, M_NVMF_CHE);
+ free(qp->fl_cids, M_NVMF_CHE);
+ free(qp->open_fl_ttags, M_NVMF_CHE);
+ free(qp, M_NVMF_CHE);
+ return (NULL);
+ }
+
+ num_ddp_tags = ddp_tags_per_qp;
+ if (num_ddp_tags > 0) {
+ qp->tpt_offset = t4_stag_alloc(sc, num_ddp_tags);
+ if (qp->tpt_offset != T4_STAG_UNSET) {
+#ifdef VERBOSE_TRACES
+ CTR(KTR_CXGBE,
+ "%s: tid %u using %u tags at offset 0x%x",
+ __func__, toep->tid, num_ddp_tags, qp->tpt_offset);
+#endif
+ qp->num_ddp_tags = num_ddp_tags;
+ qp->open_ddp_tags = mallocarray(qp->num_ddp_tags,
+ sizeof(*qp->open_ddp_tags), M_NVMF_CHE, M_WAITOK |
+ M_ZERO);
+
+ t4_nvme_set_tcb_field(toep, W_TCB_TPT_OFFSET,
+ M_TCB_TPT_OFFSET, V_TCB_TPT_OFFSET(qp->tpt_offset));
+ }
+ }
+
+ TAILQ_INIT(&qp->rx_buffers.head);
+ TAILQ_INIT(&qp->tx_buffers.head);
+ mtx_init(&qp->rx_buffers.lock, "nvmf/che rx buffers", NULL, MTX_DEF);
+ mtx_init(&qp->tx_buffers.lock, "nvmf/che tx buffers", NULL, MTX_DEF);
+
+ cv_init(&qp->rx_cv, "-");
+ cv_init(&qp->tx_cv, "-");
+ mbufq_init(&qp->rx_data, 0);
+ mbufq_init(&qp->rx_pdus, 0);
+ STAILQ_INIT(&qp->tx_capsules);
+
+ /* Register socket upcall for receive to handle remote FIN. */
+ SOCKBUF_LOCK(&so->so_rcv);
+ soupcall_set(so, SO_RCV, nvmf_che_soupcall_receive, qp);
+ SOCKBUF_UNLOCK(&so->so_rcv);
+
+ /* Spin up kthreads. */
+ error = kthread_add(nvmf_che_receive, qp, NULL, &qp->rx_thread, 0, 0,
+ "nvmef che rx");
+ if (error != 0) {
+ che_free_qpair(&qp->qp);
+ return (NULL);
+ }
+ error = kthread_add(nvmf_che_send, qp, NULL, &qp->tx_thread, 0, 0,
+ "nvmef che tx");
+ if (error != 0) {
+ che_free_qpair(&qp->qp);
+ return (NULL);
+ }
+
+ return (&qp->qp);
+}
+
+static void
+che_release_qpair(struct nvmf_che_qpair *qp)
+{
+ if (refcount_release(&qp->refs))
+ free(qp, M_NVMF_CHE);
+}
+
+static void
+che_free_qpair(struct nvmf_qpair *nq)
+{
+ struct nvmf_che_qpair *qp = CQP(nq);
+ struct nvmf_che_command_buffer *ncb, *cb;
+ struct nvmf_che_capsule *ncc, *cc;
+ struct socket *so = qp->so;
+ struct toepcb *toep = qp->toep;
+ struct inpcb *inp = sotoinpcb(so);
+
+ /* Shut down kthreads. */
+ SOCKBUF_LOCK(&so->so_snd);
+ qp->tx_shutdown = true;
+ if (qp->tx_thread != NULL) {
+ cv_signal(&qp->tx_cv);
+ mtx_sleep(qp->tx_thread, SOCKBUF_MTX(&so->so_snd), 0,
+ "nvchetx", 0);
+ }
+ SOCKBUF_UNLOCK(&so->so_snd);
+
+ SOCKBUF_LOCK(&so->so_rcv);
+ qp->rx_shutdown = true;
+ if (qp->rx_thread != NULL) {
+ cv_signal(&qp->rx_cv);
+ mtx_sleep(qp->rx_thread, SOCKBUF_MTX(&so->so_rcv), 0,
+ "nvcherx", 0);
+ }
+ soupcall_clear(so, SO_RCV);
+ SOCKBUF_UNLOCK(&so->so_rcv);
+ mbufq_drain(&qp->rx_data);
+ mbufq_drain(&qp->rx_pdus);
+
+ STAILQ_FOREACH_SAFE(cc, &qp->tx_capsules, link, ncc) {
+ nvmf_abort_capsule_data(&cc->nc, ECONNABORTED);
+ che_release_capsule(cc);
+ }
+
+ cv_destroy(&qp->tx_cv);
+ cv_destroy(&qp->rx_cv);
+
+ if (qp->open_fl_ttags != NULL) {
+ for (u_int i = 0; i < qp->num_fl_ttags; i++) {
+ cb = qp->open_fl_ttags[i];
+ if (cb != NULL) {
+ cb->cc->active_r2ts--;
+ cb->error = ECONNABORTED;
+ che_release_command_buffer(cb);
+ }
+ }
+ free(qp->open_fl_ttags, M_NVMF_CHE);
+ }
+ if (qp->num_ddp_tags != 0) {
+ for (u_int i = 0; i < qp->num_ddp_tags; i++) {
+ cb = qp->open_ddp_tags[i];
+ if (cb != NULL) {
+ if (cb->cc != NULL)
+ cb->cc->active_r2ts--;
+ cb->error = ECONNABORTED;
+ mtx_lock(&qp->rx_buffers.lock);
+ che_free_ddp_tag(qp, cb, cb->ttag);
+ mtx_unlock(&qp->rx_buffers.lock);
+ che_release_command_buffer(cb);
+ }
+ }
+ free(qp->open_ddp_tags, M_NVMF_CHE);
+ }
+
+ mtx_lock(&qp->rx_buffers.lock);
+ TAILQ_FOREACH_SAFE(cb, &qp->rx_buffers.head, link, ncb) {
+ che_remove_command_buffer(&qp->rx_buffers, cb);
+ mtx_unlock(&qp->rx_buffers.lock);
+#ifdef INVARIANTS
+ if (cb->cc != NULL)
+ cb->cc->pending_r2ts--;
+#endif
+ cb->error = ECONNABORTED;
+ che_release_command_buffer(cb);
+ mtx_lock(&qp->rx_buffers.lock);
+ }
+ mtx_destroy(&qp->rx_buffers.lock);
+
+ mtx_lock(&qp->tx_buffers.lock);
+ TAILQ_FOREACH_SAFE(cb, &qp->tx_buffers.head, link, ncb) {
+ che_remove_command_buffer(&qp->tx_buffers, cb);
+ mtx_unlock(&qp->tx_buffers.lock);
+ cb->error = ECONNABORTED;
+ che_release_command_buffer(cb);
+ mtx_lock(&qp->tx_buffers.lock);
+ }
+ mtx_destroy(&qp->tx_buffers.lock);
+
+ if (qp->num_ddp_tags != 0)
+ t4_stag_free(qp->nca->sc, qp->tpt_offset, qp->num_ddp_tags);
+
+ if (!qp->qp.nq_controller) {
+ free(qp->fl_cids, M_NVMF_CHE);
+ free(qp->fl_cid_set, M_NVMF_CHE);
+ mtx_destroy(&qp->fl_cid_lock);
+ }
+
+ INP_WLOCK(inp);
+ toep->ulpcb = NULL;
+ mbufq_drain(&toep->ulp_pduq);
+
+ /*
+ * Grab a reference to use when waiting for the final CPL to
+ * be received. If toep->inp is NULL, then
+ * final_cpl_received() has already been called (e.g. due to
+ * the peer sending a RST).
+ */
+ if (toep->inp != NULL) {
+ toep = hold_toepcb(toep);
+ toep->flags |= TPF_WAITING_FOR_FINAL;
+ } else
+ toep = NULL;
+ INP_WUNLOCK(inp);
+
+ soclose(so);
+
+ /*
+ * Wait for the socket to fully close. This ensures any
+ * pending received data has been received (and in particular,
+ * any data that would be received by DDP has been handled).
+ */
+ if (toep != NULL) {
+ struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep);
+
+ mtx_lock(lock);
+ while ((toep->flags & TPF_WAITING_FOR_FINAL) != 0)
+ mtx_sleep(toep, lock, PSOCK, "conclo2", 0);
+ mtx_unlock(lock);
+ free_toepcb(toep);
+ }
+
+ che_release_qpair(qp);
+}
+
+static uint32_t
+che_max_ioccsz(struct nvmf_qpair *nq)
+{
+ struct nvmf_che_qpair *qp = CQP(nq);
+
+ /*
+ * Limit the command capsule size so that with maximum ICD it
+ * fits within the limit of the largest PDU the adapter can
+ * receive.
+ */
+ return (qp->max_ioccsz);
+}
+
+static uint64_t
+che_max_xfer_size(struct nvmf_qpair *nq)
+{
+ struct nvmf_che_qpair *qp = CQP(nq);
+
+ /*
+ * Limit host transfers to the size of the data payload in the
+ * largest PDU the adapter can receive.
+ */
+ return (qp->max_rx_data);
+}
+
+static struct nvmf_capsule *
+che_allocate_capsule(struct nvmf_qpair *nq, int how)
+{
+ struct nvmf_che_qpair *qp = CQP(nq);
+ struct nvmf_che_capsule *cc;
+
+ cc = malloc(sizeof(*cc), M_NVMF_CHE, how | M_ZERO);
+ if (cc == NULL)
+ return (NULL);
+ refcount_init(&cc->refs, 1);
+ refcount_acquire(&qp->refs);
+ return (&cc->nc);
+}
+
+static void
+che_release_capsule(struct nvmf_che_capsule *cc)
+{
+ struct nvmf_che_qpair *qp = CQP(cc->nc.nc_qpair);
+
+ if (!refcount_release(&cc->refs))
+ return;
+
+ MPASS(cc->active_r2ts == 0);
+ MPASS(cc->pending_r2ts == 0);
+
+ nvmf_che_free_pdu(&cc->rx_pdu);
+ free(cc, M_NVMF_CHE);
+ che_release_qpair(qp);
+}
+
+static void
+che_free_capsule(struct nvmf_capsule *nc)
+{
+ che_release_capsule(CCAP(nc));
+}
+
+static int
+che_transmit_capsule(struct nvmf_capsule *nc)
+{
+ struct nvmf_che_qpair *qp = CQP(nc->nc_qpair);
+ struct nvmf_che_capsule *cc = CCAP(nc);
+ struct socket *so = qp->so;
+
+ refcount_acquire(&cc->refs);
+ SOCKBUF_LOCK(&so->so_snd);
+ STAILQ_INSERT_TAIL(&qp->tx_capsules, cc, link);
+ cv_signal(&qp->tx_cv);
+ SOCKBUF_UNLOCK(&so->so_snd);
+ return (0);
+}
+
+static uint8_t
+che_validate_command_capsule(struct nvmf_capsule *nc)
+{
+ struct nvmf_che_capsule *cc = CCAP(nc);
+ struct nvme_sgl_descriptor *sgl;
+
+ KASSERT(cc->rx_pdu.hdr != NULL, ("capsule wasn't received"));
+
+ sgl = &nc->nc_sqe.sgl;
+ switch (sgl->type) {
+ case NVME_SGL_TYPE_ICD:
+ if (cc->rx_pdu.data_len != le32toh(sgl->length)) {
+ printf("NVMe/TCP: Command Capsule with mismatched ICD length\n");
+ return (NVME_SC_DATA_SGL_LENGTH_INVALID);
+ }
+ break;
+ case NVME_SGL_TYPE_COMMAND_BUFFER:
+ if (cc->rx_pdu.data_len != 0) {
+ printf("NVMe/TCP: Command Buffer SGL with ICD\n");
+ return (NVME_SC_INVALID_FIELD);
+ }
+ break;
+ default:
+ printf("NVMe/TCP: Invalid SGL type in Command Capsule\n");
+ return (NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID);
+ }
+
+ if (sgl->address != 0) {
+ printf("NVMe/TCP: Invalid SGL offset in Command Capsule\n");
+ return (NVME_SC_SGL_OFFSET_INVALID);
+ }
+
+ return (NVME_SC_SUCCESS);
+}
+
+static size_t
+che_capsule_data_len(const struct nvmf_capsule *nc)
+{
+ MPASS(nc->nc_qe_len == sizeof(struct nvme_command));
+ return (le32toh(nc->nc_sqe.sgl.length));
+}
+
+static void
+che_receive_r2t_data(struct nvmf_capsule *nc, uint32_t data_offset,
+ struct nvmf_io_request *io)
+{
+ struct nvmf_che_qpair *qp = CQP(nc->nc_qpair);
+ struct nvmf_che_capsule *cc = CCAP(nc);
+ struct nvmf_che_command_buffer *cb;
+
+ cb = che_alloc_command_buffer(qp, io, data_offset, io->io_len,
+ nc->nc_sqe.cid);
+
+ cb->cc = cc;
+ refcount_acquire(&cc->refs);
+
+ /*
+ * If this command has too many active R2Ts or there are no
+ * available transfer tags, queue the request for later.
+ *
+ * NB: maxr2t is 0's based.
+ */
+ mtx_lock(&qp->rx_buffers.lock);
+ if (cc->active_r2ts > qp->maxr2t ||
+ !nvmf_che_allocate_ttag(qp, cb)) {
+#ifdef INVARIANTS
+ cc->pending_r2ts++;
+#endif
+ TAILQ_INSERT_TAIL(&qp->rx_buffers.head, cb, link);
+ mtx_unlock(&qp->rx_buffers.lock);
+ return;
+ }
+ mtx_unlock(&qp->rx_buffers.lock);
+
+ che_send_r2t(qp, nc->nc_sqe.cid, cb->ttag, data_offset, io->io_len);
+}
+
+static void
+che_receive_icd_data(struct nvmf_capsule *nc, uint32_t data_offset,
+ struct nvmf_io_request *io)
+{
+ struct nvmf_che_capsule *cc = CCAP(nc);
+
+ /*
+ * The header is in rx_pdu.m, the padding is discarded, and
+ * the data starts at rx_pdu.m->m_next.
+ */
+ mbuf_copyto_io(cc->rx_pdu.m->m_next, data_offset, io->io_len, io, 0);
+ nvmf_complete_io_request(io, io->io_len, 0);
+}
+
+static int
+che_receive_controller_data(struct nvmf_capsule *nc, uint32_t data_offset,
+ struct nvmf_io_request *io)
+{
+ struct nvme_sgl_descriptor *sgl;
+ size_t data_len;
+
+ if (nc->nc_qe_len != sizeof(struct nvme_command) ||
+ !nc->nc_qpair->nq_controller)
+ return (EINVAL);
+
+ sgl = &nc->nc_sqe.sgl;
+ data_len = le32toh(sgl->length);
+ if (data_offset + io->io_len > data_len)
+ return (EFBIG);
+
+ if (sgl->type == NVME_SGL_TYPE_ICD)
+ che_receive_icd_data(nc, data_offset, io);
+ else
+ che_receive_r2t_data(nc, data_offset, io);
+ return (0);
+}
+
+/* NB: cid is little-endian already. */
+static void
+che_send_c2h_pdu(struct nvmf_che_qpair *qp, uint16_t cid, uint32_t data_offset,
+ struct mbuf *m, size_t len, bool last_pdu, bool success)
+{
+ struct nvme_tcp_c2h_data_hdr c2h;
+ struct mbuf *top;
+
+ memset(&c2h, 0, sizeof(c2h));
+ c2h.common.pdu_type = NVME_TCP_PDU_TYPE_C2H_DATA;
+ if (last_pdu)
+ c2h.common.flags |= NVME_TCP_C2H_DATA_FLAGS_LAST_PDU;
+ if (success)
+ c2h.common.flags |= NVME_TCP_C2H_DATA_FLAGS_SUCCESS;
+ c2h.cccid = cid;
+ c2h.datao = htole32(data_offset);
+ c2h.datal = htole32(len);
+
+ top = nvmf_che_construct_pdu(qp, &c2h, sizeof(c2h), m, len);
+ nvmf_che_write_pdu(qp, top);
+}
+
+static u_int
+che_send_controller_data(struct nvmf_capsule *nc, uint32_t data_offset,
+ struct mbuf *m, size_t len)
+{
+ struct nvmf_che_qpair *qp = CQP(nc->nc_qpair);
+ struct nvme_sgl_descriptor *sgl;
+ uint32_t data_len;
+ bool last_pdu, last_xfer;
+
+ if (nc->nc_qe_len != sizeof(struct nvme_command) ||
+ !qp->qp.nq_controller) {
+ m_freem(m);
+ return (NVME_SC_INVALID_FIELD);
+ }
+
+ sgl = &nc->nc_sqe.sgl;
+ data_len = le32toh(sgl->length);
+ if (data_offset + len > data_len) {
+ m_freem(m);
+ return (NVME_SC_INVALID_FIELD);
+ }
+ last_xfer = (data_offset + len == data_len);
+
+ if (sgl->type != NVME_SGL_TYPE_COMMAND_BUFFER) {
+ m_freem(m);
+ return (NVME_SC_INVALID_FIELD);
+ }
+
+ KASSERT(data_offset == CCAP(nc)->tx_data_offset,
+ ("%s: starting data_offset %u doesn't match end of previous xfer %u",
+ __func__, data_offset, CCAP(nc)->tx_data_offset));
+
+ /* Queue one or more C2H_DATA PDUs containing the data from 'm'. */
+ while (m != NULL) {
+ struct mbuf *n;
+ uint32_t todo;
+
+ if (m->m_len > qp->max_tx_data) {
+ n = m_split(m, qp->max_tx_data, M_WAITOK);
+ todo = m->m_len;
+ } else {
+ struct mbuf *p;
+
+ todo = m->m_len;
+ p = m;
+ n = p->m_next;
+ while (n != NULL) {
+ if (todo + n->m_len > qp->max_tx_data) {
+ p->m_next = NULL;
+ break;
+ }
+ todo += n->m_len;
+ p = n;
+ n = p->m_next;
+ }
+ MPASS(m_length(m, NULL) == todo);
+ }
+
+ last_pdu = (n == NULL && last_xfer);
+ che_send_c2h_pdu(qp, nc->nc_sqe.cid, data_offset, m, todo,
+ last_pdu, last_pdu && qp->send_success);
+
+ data_offset += todo;
+ data_len -= todo;
+ m = n;
+ }
+ MPASS(data_len == 0);
+
+#ifdef INVARIANTS
+ CCAP(nc)->tx_data_offset = data_offset;
+#endif
+ if (!last_xfer)
+ return (NVMF_MORE);
+ else if (qp->send_success)
+ return (NVMF_SUCCESS_SENT);
+ else
+ return (NVME_SC_SUCCESS);
+}
+
+struct nvmf_transport_ops che_ops = {
+ .allocate_qpair = che_allocate_qpair,
+ .free_qpair = che_free_qpair,
+ .max_ioccsz = che_max_ioccsz,
+ .max_xfer_size = che_max_xfer_size,
+ .allocate_capsule = che_allocate_capsule,
+ .free_capsule = che_free_capsule,
+ .transmit_capsule = che_transmit_capsule,
+ .validate_command_capsule = che_validate_command_capsule,
+ .capsule_data_len = che_capsule_data_len,
+ .receive_controller_data = che_receive_controller_data,
+ .send_controller_data = che_send_controller_data,
+ .trtype = NVMF_TRTYPE_TCP,
+ .priority = 10,
+};
+
+NVMF_TRANSPORT(che, che_ops);
+
+static void
+read_pdu_limits(struct adapter *sc, u_int *max_tx_pdu_len,
+ uint32_t *max_rx_pdu_len)
+{
+ uint32_t tx_len, rx_len, r, v;
+
+ /* Copied from cxgbei, but not sure if this is correct. */
+ rx_len = t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE);
+ tx_len = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
+
+ r = t4_read_reg(sc, A_TP_PARA_REG2);
+ rx_len = min(rx_len, G_MAXRXDATA(r));
+ tx_len = min(tx_len, G_MAXRXDATA(r));
+
+ r = t4_read_reg(sc, A_TP_PARA_REG7);
+ v = min(G_PMMAXXFERLEN0(r), G_PMMAXXFERLEN1(r));
+ rx_len = min(rx_len, v);
+ tx_len = min(tx_len, v);
+
+ /* Cannot be larger than 32KB - 256. */
+ rx_len = min(rx_len, 32512);
+ tx_len = min(tx_len, 32512);
+
+ *max_tx_pdu_len = tx_len;
+ *max_rx_pdu_len = rx_len;
+}
+
+static int
+nvmf_che_init(struct adapter *sc, struct nvmf_che_adapter *nca)
+{
+ struct sysctl_oid *oid;
+ struct sysctl_oid_list *children;
+ uint32_t val;
+
+ read_pdu_limits(sc, &nca->max_transmit_pdu, &nca->max_receive_pdu);
+ if (nca->max_transmit_pdu > che_max_transmit_pdu)
+ nca->max_transmit_pdu = che_max_transmit_pdu;
+ if (nca->max_receive_pdu > che_max_receive_pdu)
+ nca->max_receive_pdu = che_max_receive_pdu;
+ val = t4_read_reg(sc, A_SGE_CONTROL2);
+ nca->nvmt_data_iqe = (val & F_RXCPLMODE_NVMT) != 0;
+
+ sysctl_ctx_init(&nca->ctx);
+ oid = device_get_sysctl_tree(sc->dev); /* dev.che.X */
+ children = SYSCTL_CHILDREN(oid);
+
+ oid = SYSCTL_ADD_NODE(&nca->ctx, children, OID_AUTO, "nvme",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NVMe ULP settings");
+ children = SYSCTL_CHILDREN(oid);
+
+ nca->ddp_threshold = 8192;
+ SYSCTL_ADD_UINT(&nca->ctx, children, OID_AUTO, "ddp_threshold",
+ CTLFLAG_RW, &nca->ddp_threshold, 0, "Rx zero copy threshold");
+
+ SYSCTL_ADD_UINT(&nca->ctx, children, OID_AUTO, "max_transmit_pdu",
+ CTLFLAG_RW, &nca->max_transmit_pdu, 0,
+ "Maximum size of a transmitted PDU");
+
+ SYSCTL_ADD_UINT(&nca->ctx, children, OID_AUTO, "max_receive_pdu",
+ CTLFLAG_RW, &nca->max_receive_pdu, 0,
+ "Maximum size of a received PDU");
+
+ return (0);
+}
+
+static void
+nvmf_che_destroy(struct nvmf_che_adapter *nca)
+{
+ sysctl_ctx_free(&nca->ctx);
+ free(nca, M_CXGBE);
+}
+
+static int
+nvmf_che_activate(struct adapter *sc)
+{
+ struct nvmf_che_adapter *nca;
+ int rc;
+
+ ASSERT_SYNCHRONIZED_OP(sc);
+
+ if (uld_active(sc, ULD_NVME)) {
+ KASSERT(0, ("%s: NVMe offload already enabled on adapter %p",
+ __func__, sc));
+ return (0);
+ }
+
+ if ((sc->nvmecaps & FW_CAPS_CONFIG_NVME_TCP) == 0) {
+ device_printf(sc->dev,
+ "not NVMe offload capable, or capability disabled\n");
+ return (ENOSYS);
+ }
+
+ /* per-adapter softc for NVMe */
+ nca = malloc(sizeof(*nca), M_CXGBE, M_ZERO | M_WAITOK);
+ nca->sc = sc;
+
+ rc = nvmf_che_init(sc, nca);
+ if (rc != 0) {
+ free(nca, M_CXGBE);
+ return (rc);
+ }
+
+ sc->nvme_ulp_softc = nca;
+
+ return (0);
+}
+
+static int
+nvmf_che_deactivate(struct adapter *sc)
+{
+ struct nvmf_che_adapter *nca = sc->nvme_ulp_softc;
+
+ ASSERT_SYNCHRONIZED_OP(sc);
+
+ if (nca != NULL) {
+ nvmf_che_destroy(nca);
+ sc->nvme_ulp_softc = NULL;
+ }
+
+ return (0);
+}
+
+static void
+nvmf_che_activate_all(struct adapter *sc, void *arg __unused)
+{
+ if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t7nvact") != 0)
+ return;
+
+ /* Activate NVMe if any port on this adapter has IFCAP_TOE enabled. */
+ if (sc->offload_map && !uld_active(sc, ULD_NVME))
+ (void) t4_activate_uld(sc, ULD_NVME);
+
+ end_synchronized_op(sc, 0);
+}
+
+static void
+nvmf_che_deactivate_all(struct adapter *sc, void *arg __unused)
+{
+ if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t7nvdea") != 0)
+ return;
+
+ if (uld_active(sc, ULD_NVME))
+ (void) t4_deactivate_uld(sc, ULD_NVME);
+
+ end_synchronized_op(sc, 0);
+}
+
+static struct uld_info nvmf_che_uld_info = {
+ .uld_activate = nvmf_che_activate,
+ .uld_deactivate = nvmf_che_deactivate,
+};
+
+static int
+nvmf_che_mod_load(void)
+{
+ int rc;
+
+ t4_register_cpl_handler(CPL_NVMT_CMP, do_nvmt_cmp);
+ t4_register_cpl_handler(CPL_NVMT_DATA, do_nvmt_data);
+
+ rc = t4_register_uld(&nvmf_che_uld_info, ULD_NVME);
+ if (rc != 0)
+ return (rc);
+
+ t4_iterate(nvmf_che_activate_all, NULL);
+
+ return (rc);
+}
+
+static int
+nvmf_che_mod_unload(void)
+{
+ t4_iterate(nvmf_che_deactivate_all, NULL);
+
+ if (t4_unregister_uld(&nvmf_che_uld_info, ULD_NVME) == EBUSY)
+ return (EBUSY);
+
+ t4_register_cpl_handler(CPL_NVMT_CMP, NULL);
+ t4_register_cpl_handler(CPL_NVMT_DATA, NULL);
+
+ return (0);
+}
+#endif
+
+static int
+nvmf_che_modevent(module_t mod, int cmd, void *arg)
+{
+ int rc;
+
+#ifdef TCP_OFFLOAD
+ switch (cmd) {
+ case MOD_LOAD:
+ rc = nvmf_che_mod_load();
+ break;
+ case MOD_UNLOAD:
+ rc = nvmf_che_mod_unload();
+ break;
+ default:
+ rc = EOPNOTSUPP;
+ break;
+ }
+#else
+ printf("nvmf_che: compiled without TCP_OFFLOAD support.\n");
+ rc = EOPNOTSUPP;
+#endif
+
+ return (rc);
+}
+
+static moduledata_t nvmf_che_mod = {
+ "nvmf_che",
+ nvmf_che_modevent,
+ NULL,
+};
+
+MODULE_VERSION(nvmf_che, 1);
+DECLARE_MODULE(nvmf_che, nvmf_che_mod, SI_SUB_EXEC, SI_ORDER_ANY);
+MODULE_DEPEND(nvmf_che, t4_tom, 1, 1, 1);
+MODULE_DEPEND(nvmf_che, cxgbe, 1, 1, 1);
diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index 91a43785aaca..d63accf86e2a 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -196,7 +196,8 @@ enum {
ULD_TOM = 0,
ULD_IWARP,
ULD_ISCSI,
- ULD_MAX = ULD_ISCSI
+ ULD_NVME,
+ ULD_MAX = ULD_NVME
};
struct adapter;
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 22d2f504c257..15b3fd94fa54 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -57,9 +57,7 @@
#include <net/if_types.h>
#include <net/if_dl.h>
#include <net/if_vlan_var.h>
-#ifdef RSS
#include <net/rss_config.h>
-#endif
#include <netinet/in.h>
#include <netinet/ip.h>
#ifdef KERN_TLS
@@ -611,7 +609,7 @@ static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN,
&t4_switchcaps_allowed, 0, "Default switch capabilities");
-static int t4_nvmecaps_allowed = 0;
+static int t4_nvmecaps_allowed = -1;
SYSCTL_INT(_hw_cxgbe, OID_AUTO, nvmecaps_allowed, CTLFLAG_RDTUN,
&t4_nvmecaps_allowed, 0, "Default NVMe capabilities");
@@ -1327,6 +1325,8 @@ t4_attach(device_t dev)
sc->dev = dev;
sysctl_ctx_init(&sc->ctx);
TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
+ if (TUNABLE_INT_FETCH("hw.cxgbe.iflags", &sc->intr_flags) == 0)
+ sc->intr_flags = IHF_INTR_CLEAR_ON_INIT | IHF_CLR_ALL_UNIGNORED;
if ((pci_get_device(dev) & 0xff00) == 0x5400)
t5_attribute_workaround(dev);
@@ -3652,6 +3652,7 @@ port_mword(struct port_info *pi, uint32_t speed)
case FW_PORT_TYPE_SFP28:
case FW_PORT_TYPE_SFP56:
case FW_PORT_TYPE_QSFP56:
+ case FW_PORT_TYPE_QSFPDD:
/* Pluggable transceiver */
switch (pi->mod_type) {
case FW_PORT_MOD_TYPE_LR:
@@ -3671,6 +3672,8 @@ port_mword(struct port_info *pi, uint32_t speed)
return (IFM_100G_LR4);
case FW_PORT_CAP32_SPEED_200G:
return (IFM_200G_LR4);
+ case FW_PORT_CAP32_SPEED_400G:
+ return (IFM_400G_LR8);
}
break;
case FW_PORT_MOD_TYPE_SR:
@@ -3689,6 +3692,8 @@ port_mword(struct port_info *pi, uint32_t speed)
return (IFM_100G_SR4);
case FW_PORT_CAP32_SPEED_200G:
return (IFM_200G_SR4);
+ case FW_PORT_CAP32_SPEED_400G:
+ return (IFM_400G_SR8);
}
break;
case FW_PORT_MOD_TYPE_ER:
@@ -3712,6 +3717,8 @@ port_mword(struct port_info *pi, uint32_t speed)
return (IFM_100G_CR4);
case FW_PORT_CAP32_SPEED_200G:
return (IFM_200G_CR4_PAM4);
+ case FW_PORT_CAP32_SPEED_400G:
+ return (IFM_400G_CR8);
}
break;
case FW_PORT_MOD_TYPE_LRM:
@@ -3723,10 +3730,12 @@ port_mword(struct port_info *pi, uint32_t speed)
return (IFM_100G_DR);
if (speed == FW_PORT_CAP32_SPEED_200G)
return (IFM_200G_DR4);
+ if (speed == FW_PORT_CAP32_SPEED_400G)
+ return (IFM_400G_DR4);
break;
case FW_PORT_MOD_TYPE_NA:
MPASS(0); /* Not pluggable? */
- /* fall throough */
+ /* fall through */
case FW_PORT_MOD_TYPE_ERROR:
case FW_PORT_MOD_TYPE_UNKNOWN:
case FW_PORT_MOD_TYPE_NOTSUPPORTED:
@@ -3735,6 +3744,10 @@ port_mword(struct port_info *pi, uint32_t speed)
return (IFM_NONE);
}
break;
+ case M_FW_PORT_CMD_PTYPE: /* FW_PORT_TYPE_NONE for old firmware */
+ if (chip_id(pi->adapter) >= CHELSIO_T7)
+ return (IFM_UNKNOWN);
+ /* fall through */
case FW_PORT_TYPE_NONE:
return (IFM_NONE);
}
@@ -3930,8 +3943,6 @@ fatal_error_task(void *arg, int pending)
void
t4_fatal_err(struct adapter *sc, bool fw_error)
{
- const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
-
stop_adapter(sc);
if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR)))
return;
@@ -3944,7 +3955,7 @@ t4_fatal_err(struct adapter *sc, bool fw_error)
* main INT_CAUSE registers here to make sure we haven't missed
* anything interesting.
*/
- t4_slow_intr_handler(sc, verbose);
+ t4_slow_intr_handler(sc, sc->intr_flags);
atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
}
t4_report_fw_error(sc);
@@ -5408,6 +5419,7 @@ apply_cfg_and_initialize(struct adapter *sc, char *cfg_file,
caps.toecaps = 0;
caps.rdmacaps = 0;
caps.iscsicaps = 0;
+ caps.nvmecaps = 0;
}
caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
@@ -5881,61 +5893,63 @@ get_params__post_init(struct adapter *sc)
* that will never be used.
*/
sc->iscsicaps = 0;
+ sc->nvmecaps = 0;
sc->rdmacaps = 0;
}
- if (sc->rdmacaps) {
+ if (sc->nvmecaps || sc->rdmacaps) {
param[0] = FW_PARAM_PFVF(STAG_START);
param[1] = FW_PARAM_PFVF(STAG_END);
- param[2] = FW_PARAM_PFVF(RQ_START);
- param[3] = FW_PARAM_PFVF(RQ_END);
- param[4] = FW_PARAM_PFVF(PBL_START);
- param[5] = FW_PARAM_PFVF(PBL_END);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
+ param[2] = FW_PARAM_PFVF(PBL_START);
+ param[3] = FW_PARAM_PFVF(PBL_END);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
if (rc != 0) {
device_printf(sc->dev,
- "failed to query RDMA parameters(1): %d.\n", rc);
+ "failed to query NVMe/RDMA parameters: %d.\n", rc);
return (rc);
}
sc->vres.stag.start = val[0];
sc->vres.stag.size = val[1] - val[0] + 1;
- sc->vres.rq.start = val[2];
- sc->vres.rq.size = val[3] - val[2] + 1;
- sc->vres.pbl.start = val[4];
- sc->vres.pbl.size = val[5] - val[4] + 1;
-
- param[0] = FW_PARAM_PFVF(SQRQ_START);
- param[1] = FW_PARAM_PFVF(SQRQ_END);
- param[2] = FW_PARAM_PFVF(CQ_START);
- param[3] = FW_PARAM_PFVF(CQ_END);
- param[4] = FW_PARAM_PFVF(OCQ_START);
- param[5] = FW_PARAM_PFVF(OCQ_END);
+ sc->vres.pbl.start = val[2];
+ sc->vres.pbl.size = val[3] - val[2] + 1;
+ }
+ if (sc->rdmacaps) {
+ param[0] = FW_PARAM_PFVF(RQ_START);
+ param[1] = FW_PARAM_PFVF(RQ_END);
+ param[2] = FW_PARAM_PFVF(SQRQ_START);
+ param[3] = FW_PARAM_PFVF(SQRQ_END);
+ param[4] = FW_PARAM_PFVF(CQ_START);
+ param[5] = FW_PARAM_PFVF(CQ_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
- "failed to query RDMA parameters(2): %d.\n", rc);
+ "failed to query RDMA parameters(1): %d.\n", rc);
return (rc);
}
- sc->vres.qp.start = val[0];
- sc->vres.qp.size = val[1] - val[0] + 1;
- sc->vres.cq.start = val[2];
- sc->vres.cq.size = val[3] - val[2] + 1;
- sc->vres.ocq.start = val[4];
- sc->vres.ocq.size = val[5] - val[4] + 1;
-
- param[0] = FW_PARAM_PFVF(SRQ_START);
- param[1] = FW_PARAM_PFVF(SRQ_END);
- param[2] = FW_PARAM_DEV(MAXORDIRD_QP);
- param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
+ sc->vres.rq.start = val[0];
+ sc->vres.rq.size = val[1] - val[0] + 1;
+ sc->vres.qp.start = val[2];
+ sc->vres.qp.size = val[3] - val[2] + 1;
+ sc->vres.cq.start = val[4];
+ sc->vres.cq.size = val[5] - val[4] + 1;
+
+ param[0] = FW_PARAM_PFVF(OCQ_START);
+ param[1] = FW_PARAM_PFVF(OCQ_END);
+ param[2] = FW_PARAM_PFVF(SRQ_START);
+ param[3] = FW_PARAM_PFVF(SRQ_END);
+ param[4] = FW_PARAM_DEV(MAXORDIRD_QP);
+ param[5] = FW_PARAM_DEV(MAXIRD_ADAPTER);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
- "failed to query RDMA parameters(3): %d.\n", rc);
+ "failed to query RDMA parameters(2): %d.\n", rc);
return (rc);
}
- sc->vres.srq.start = val[0];
- sc->vres.srq.size = val[1] - val[0] + 1;
- sc->params.max_ordird_qp = val[2];
- sc->params.max_ird_adapter = val[3];
+ sc->vres.ocq.start = val[0];
+ sc->vres.ocq.size = val[1] - val[0] + 1;
+ sc->vres.srq.start = val[2];
+ sc->vres.srq.size = val[3] - val[2] + 1;
+ sc->params.max_ordird_qp = val[4];
+ sc->params.max_ird_adapter = val[5];
}
if (sc->iscsicaps) {
param[0] = FW_PARAM_PFVF(ISCSI_START);
@@ -7019,7 +7033,6 @@ t4_setup_intr_handlers(struct adapter *sc)
static void
write_global_rss_key(struct adapter *sc)
{
-#ifdef RSS
int i;
uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
@@ -7031,7 +7044,6 @@ write_global_rss_key(struct adapter *sc)
rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
}
t4_write_rss_key(sc, &rss_key[0], -1, 1);
-#endif
}
/*
@@ -7111,7 +7123,6 @@ adapter_full_uninit(struct adapter *sc)
sc->flags &= ~FULL_INIT_DONE;
}
-#ifdef RSS
#define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
@@ -7174,7 +7185,6 @@ hashen_to_hashconfig(int hashen)
return (hashconfig);
}
-#endif
/*
* Idempotent.
@@ -7185,9 +7195,9 @@ vi_full_init(struct vi_info *vi)
struct adapter *sc = vi->adapter;
struct sge_rxq *rxq;
int rc, i, j;
+ int hashconfig = rss_gethashconfig();
#ifdef RSS
int nbuckets = rss_getnumbuckets();
- int hashconfig = rss_gethashconfig();
int extra;
#endif
@@ -7243,9 +7253,9 @@ vi_full_init(struct vi_info *vi)
return (rc);
}
-#ifdef RSS
vi->hashen = hashconfig_to_hashen(hashconfig);
+#ifdef RSS
/*
* We may have had to enable some hashes even though the global config
* wants them disabled. This is a potential problem that must be
@@ -7279,11 +7289,6 @@ vi_full_init(struct vi_info *vi)
CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n");
if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n");
-#else
- vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
- F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
- F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
- F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
#endif
rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0],
0, 0);
@@ -7892,6 +7897,9 @@ t4_sysctls(struct adapter *sc)
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
&sc->debug_flags, 0, "flags to enable runtime debugging");
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "iflags", CTLFLAG_RW,
+ &sc->intr_flags, 0, "flags for the slow interrupt handler");
+
SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
@@ -8988,7 +8996,7 @@ sysctl_requested_fec(SYSCTL_HANDLER_ARGS)
struct adapter *sc = pi->adapter;
struct link_config *lc = &pi->link_cfg;
int rc;
- int8_t old;
+ int8_t old = lc->requested_fec;
if (req->newptr == NULL) {
struct sbuf *sb;
@@ -8997,16 +9005,15 @@ sysctl_requested_fec(SYSCTL_HANDLER_ARGS)
if (sb == NULL)
return (ENOMEM);
- sbuf_printf(sb, "%b", lc->requested_fec, t4_fec_bits);
+ sbuf_printf(sb, "%b", old, t4_fec_bits);
rc = sbuf_finish(sb);
sbuf_delete(sb);
} else {
char s[8];
int n;
- snprintf(s, sizeof(s), "%d",
- lc->requested_fec == FEC_AUTO ? -1 :
- lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
+ snprintf(s, sizeof(s), "%d", old == FEC_AUTO ? -1 :
+ old & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
rc = sysctl_handle_string(oidp, s, sizeof(s), req);
if (rc != 0)
@@ -9023,7 +9030,10 @@ sysctl_requested_fec(SYSCTL_HANDLER_ARGS)
if (rc)
return (rc);
PORT_LOCK(pi);
- old = lc->requested_fec;
+ if (lc->requested_fec != old) {
+ rc = EBUSY;
+ goto done;
+ }
if (n == FEC_AUTO)
lc->requested_fec = FEC_AUTO;
else if (n == 0 || n == FEC_NONE)
@@ -12984,6 +12994,9 @@ clear_stats(struct adapter *sc, u_int port_id)
counter_u64_zero(ofld_txq->tx_iscsi_pdus);
counter_u64_zero(ofld_txq->tx_iscsi_octets);
counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs);
+ counter_u64_zero(ofld_txq->tx_nvme_pdus);
+ counter_u64_zero(ofld_txq->tx_nvme_octets);
+ counter_u64_zero(ofld_txq->tx_nvme_iso_wrs);
counter_u64_zero(ofld_txq->tx_aio_jobs);
counter_u64_zero(ofld_txq->tx_aio_octets);
counter_u64_zero(ofld_txq->tx_toe_tls_records);
@@ -13003,6 +13016,22 @@ clear_stats(struct adapter *sc, u_int port_id)
ofld_rxq->rx_iscsi_ddp_octets = 0;
ofld_rxq->rx_iscsi_fl_pdus = 0;
ofld_rxq->rx_iscsi_fl_octets = 0;
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_ok);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_no_stag);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_ddp_setup_error);
+ counter_u64_zero(ofld_rxq->rx_nvme_ddp_pdus);
+ counter_u64_zero(ofld_rxq->rx_nvme_ddp_octets);
+ counter_u64_zero(ofld_rxq->rx_nvme_fl_pdus);
+ counter_u64_zero(ofld_rxq->rx_nvme_fl_octets);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_invalid_headers);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_header_digest_errors);
+ counter_u64_zero(
+ ofld_rxq->rx_nvme_data_digest_errors);
ofld_rxq->rx_aio_ddp_jobs = 0;
ofld_rxq->rx_aio_ddp_octets = 0;
ofld_rxq->rx_toe_tls_records = 0;
@@ -13409,11 +13438,16 @@ toe_capability(struct vi_info *vi, bool enable)
("%s: TOM activated but flag not set", __func__));
}
- /* Activate iWARP and iSCSI too, if the modules are loaded. */
+ /*
+ * Activate iWARP, iSCSI, and NVMe too, if the modules
+ * are loaded.
+ */
if (!uld_active(sc, ULD_IWARP))
(void) t4_activate_uld(sc, ULD_IWARP);
if (!uld_active(sc, ULD_ISCSI))
(void) t4_activate_uld(sc, ULD_ISCSI);
+ if (!uld_active(sc, ULD_NVME))
+ (void) t4_activate_uld(sc, ULD_NVME);
if (pi->uld_vis++ == 0)
setbit(&sc->offload_map, pi->port_id);
@@ -13694,6 +13728,9 @@ tweak_tunables(void)
FW_CAPS_CONFIG_ISCSI_T10DIF;
}
+ if (t4_nvmecaps_allowed == -1)
+ t4_nvmecaps_allowed = FW_CAPS_CONFIG_NVME_TCP;
+
if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS)
t4_tmr_idx_ofld = TMR_IDX_OFLD;
@@ -13705,6 +13742,9 @@ tweak_tunables(void)
if (t4_iscsicaps_allowed == -1)
t4_iscsicaps_allowed = 0;
+
+ if (t4_nvmecaps_allowed == -1)
+ t4_nvmecaps_allowed = 0;
#endif
#ifdef DEV_NETMAP
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 2f9cb1a4ebb5..e9754ace27c2 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -852,6 +852,11 @@ t4_tweak_chip_settings(struct adapter *sc)
/* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
+ if (sc->nvmecaps != 0) {
+ /* Request DDP status bit for NVMe PDU completions. */
+ m |= F_NVME_TCP_DDP_VAL_EN;
+ v |= F_NVME_TCP_DDP_VAL_EN;
+ }
t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
@@ -1335,7 +1340,6 @@ t4_intr_err(void *arg)
{
struct adapter *sc = arg;
uint32_t v;
- const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
if (atomic_load_int(&sc->error_flags) & ADAP_FATAL_ERR)
return;
@@ -1346,7 +1350,7 @@ t4_intr_err(void *arg)
t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v);
}
- if (t4_slow_intr_handler(sc, verbose))
+ if (t4_slow_intr_handler(sc, sc->intr_flags))
t4_fatal_err(sc, false);
}
@@ -4170,6 +4174,20 @@ alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,
ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
ofld_rxq->rx_iscsi_ddp_setup_error =
counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_no_stag =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_setup_error =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_octets = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_ddp_pdus = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_fl_octets = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_fl_pdus = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_invalid_headers = counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_header_digest_errors =
+ counter_u64_alloc(M_WAITOK);
+ ofld_rxq->rx_nvme_data_digest_errors =
+ counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_alloc = counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_reuse = counter_u64_alloc(M_WAITOK);
ofld_rxq->ddp_buffer_free = counter_u64_alloc(M_WAITOK);
@@ -4207,6 +4225,16 @@ free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED));
counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok);
counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_ok);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_no_stag);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_error);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_octets);
+ counter_u64_free(ofld_rxq->rx_nvme_ddp_pdus);
+ counter_u64_free(ofld_rxq->rx_nvme_fl_octets);
+ counter_u64_free(ofld_rxq->rx_nvme_fl_pdus);
+ counter_u64_free(ofld_rxq->rx_nvme_invalid_headers);
+ counter_u64_free(ofld_rxq->rx_nvme_header_digest_errors);
+ counter_u64_free(ofld_rxq->rx_nvme_data_digest_errors);
counter_u64_free(ofld_rxq->ddp_buffer_alloc);
counter_u64_free(ofld_rxq->ddp_buffer_reuse);
counter_u64_free(ofld_rxq->ddp_buffer_free);
@@ -4218,12 +4246,12 @@ static void
add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
struct sge_ofld_rxq *ofld_rxq)
{
- struct sysctl_oid_list *children;
+ struct sysctl_oid_list *children, *top;
if (ctx == NULL || oid == NULL)
return;
- children = SYSCTL_CHILDREN(oid);
+ top = children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "rx_aio_ddp_jobs",
CTLFLAG_RD, &ofld_rxq->rx_aio_ddp_jobs, 0,
"# of aio_read(2) jobs completed via DDP");
@@ -4280,6 +4308,41 @@ add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors",
CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0,
"# of PDUs with invalid data digests");
+
+ oid = SYSCTL_ADD_NODE(ctx, top, OID_AUTO, "nvme",
+ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE NVMe statistics");
+ children = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_ok,
+ "# of times DDP buffer was setup successfully");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_no_stag",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_no_stag,
+ "# of times STAG was not available for DDP buffer setup");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_error,
+ "# of times DDP buffer setup failed");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_octets",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_octets,
+ "# of octets placed directly");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_pdus",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_pdus,
+ "# of PDUs with data placed directly");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_octets",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_octets,
+ "# of data octets delivered in freelist");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_pdus",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_pdus,
+ "# of PDUs with data delivered in freelist");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "invalid_headers",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_invalid_headers,
+ "# of PDUs with invalid header field");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "header_digest_errors",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_header_digest_errors,
+ "# of PDUs with invalid header digests");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "data_digest_errors",
+ CTLFLAG_RD, &ofld_rxq->rx_nvme_data_digest_errors,
+ "# of PDUs with invalid data digests");
}
#endif
@@ -4957,6 +5020,9 @@ alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)
ofld_txq->tx_iscsi_pdus = counter_u64_alloc(M_WAITOK);
ofld_txq->tx_iscsi_octets = counter_u64_alloc(M_WAITOK);
ofld_txq->tx_iscsi_iso_wrs = counter_u64_alloc(M_WAITOK);
+ ofld_txq->tx_nvme_pdus = counter_u64_alloc(M_WAITOK);
+ ofld_txq->tx_nvme_octets = counter_u64_alloc(M_WAITOK);
+ ofld_txq->tx_nvme_iso_wrs = counter_u64_alloc(M_WAITOK);
ofld_txq->tx_aio_jobs = counter_u64_alloc(M_WAITOK);
ofld_txq->tx_aio_octets = counter_u64_alloc(M_WAITOK);
ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK);
@@ -5000,6 +5066,9 @@ free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)
counter_u64_free(ofld_txq->tx_iscsi_pdus);
counter_u64_free(ofld_txq->tx_iscsi_octets);
counter_u64_free(ofld_txq->tx_iscsi_iso_wrs);
+ counter_u64_free(ofld_txq->tx_nvme_pdus);
+ counter_u64_free(ofld_txq->tx_nvme_octets);
+ counter_u64_free(ofld_txq->tx_nvme_iso_wrs);
counter_u64_free(ofld_txq->tx_aio_jobs);
counter_u64_free(ofld_txq->tx_aio_octets);
counter_u64_free(ofld_txq->tx_toe_tls_records);
@@ -5029,6 +5098,15 @@ add_ofld_txq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_iso_wrs",
CTLFLAG_RD, &ofld_txq->tx_iscsi_iso_wrs,
"# of iSCSI segmentation offload work requests");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_nvme_pdus",
+ CTLFLAG_RD, &ofld_txq->tx_nvme_pdus,
+ "# of NVMe PDUs transmitted");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_nvme_octets",
+ CTLFLAG_RD, &ofld_txq->tx_nvme_octets,
+ "# of payload octets in transmitted NVMe PDUs");
+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_nvme_iso_wrs",
+ CTLFLAG_RD, &ofld_txq->tx_nvme_iso_wrs,
+ "# of NVMe segmentation offload work requests");
SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_aio_jobs",
CTLFLAG_RD, &ofld_txq->tx_aio_jobs,
"# of zero-copy aio_write(2) jobs transmitted");
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index 84e31efa8b58..5c39ae5fa8f3 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -66,6 +66,7 @@
#include <vm/vm_page.h>
#include <dev/iscsi/iscsi_proto.h>
+#include <dev/nvmf/nvmf_proto.h>
#include "common/common.h"
#include "common/t4_msg.h"
@@ -495,6 +496,9 @@ t4_close_conn(struct adapter *sc, struct toepcb *toep)
#define MIN_ISO_TX_CREDITS (howmany(sizeof(struct cpl_tx_data_iso), 16))
#define MIN_TX_CREDITS(iso) \
(MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
+#define MIN_OFLD_TX_V2_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_v2_wr) + 1, 16))
+#define MIN_TX_V2_CREDITS(iso) \
+ (MIN_OFLD_TX_V2_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
_Static_assert(MAX_OFLD_TX_CREDITS <= MAX_OFLD_TX_SDESC_CREDITS,
"MAX_OFLD_TX_SDESC_CREDITS too small");
@@ -542,6 +546,46 @@ max_dsgl_nsegs(int tx_credits, int iso)
return (nseg);
}
+/* Maximum amount of immediate data we could stuff in a WR */
+static inline int
+max_imm_payload_v2(int tx_credits, int iso)
+{
+ const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0;
+
+ KASSERT(tx_credits >= 0 &&
+ tx_credits <= MAX_OFLD_TX_CREDITS,
+ ("%s: %d credits", __func__, tx_credits));
+
+ if (tx_credits < MIN_TX_V2_CREDITS(iso))
+ return (0);
+
+ return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_v2_wr) -
+ iso_cpl_size);
+}
+
+/* Maximum number of SGL entries we could stuff in a WR */
+static inline int
+max_dsgl_nsegs_v2(int tx_credits, int iso, int imm_payload)
+{
+ int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
+ int sge_pair_credits = tx_credits - MIN_TX_V2_CREDITS(iso);
+
+ KASSERT(tx_credits >= 0 &&
+ tx_credits <= MAX_OFLD_TX_CREDITS,
+ ("%s: %d credits", __func__, tx_credits));
+
+ if (tx_credits < MIN_TX_V2_CREDITS(iso) ||
+ sge_pair_credits <= howmany(imm_payload, 16))
+ return (0);
+ sge_pair_credits -= howmany(imm_payload, 16);
+
+ nseg += 2 * (sge_pair_credits * 16 / 24);
+ if ((sge_pair_credits * 16) % 24 == 16)
+ nseg++;
+
+ return (nseg);
+}
+
static inline void
write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
@@ -569,6 +613,35 @@ write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
}
}
+static inline void
+write_tx_v2_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
+ unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
+ int ulp_submode)
+{
+ struct fw_ofld_tx_data_v2_wr *txwr = dst;
+ uint32_t flags;
+
+ memset(txwr, 0, sizeof(*txwr));
+ txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) |
+ V_FW_WR_IMMDLEN(immdlen));
+ txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
+ V_FW_WR_LEN16(credits));
+ txwr->plen = htobe32(plen);
+ flags = V_TX_ULP_MODE(ULP_MODE_NVMET) | V_TX_ULP_SUBMODE(ulp_submode) |
+ V_TX_URG(0) | V_TX_SHOVE(shove);
+
+ if (toep->params.tx_align > 0) {
+ if (plen < 2 * toep->params.emss)
+ flags |= F_FW_OFLD_TX_DATA_WR_LSODISABLE;
+ else
+ flags |= F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
+ (toep->params.nagle == 0 ? 0 :
+ F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE);
+ }
+
+ txwr->lsodisable_to_flags = htobe32(flags);
+}
+
/*
* Generate a DSGL from a starting mbuf. The total number of segments and the
* maximum segments in any one mbuf are provided.
@@ -982,8 +1055,8 @@ rqdrop_locked(struct mbufq *q, int plen)
#define ULP_ISO G_TX_ULP_SUBMODE(F_FW_ISCSI_TX_DATA_WR_ULPSUBMODE_ISO)
static void
-write_tx_data_iso(void *dst, u_int ulp_submode, uint8_t flags, uint16_t mss,
- int len, int npdu)
+write_iscsi_tx_data_iso(void *dst, u_int ulp_submode, uint8_t flags,
+ uint16_t mss, int len, int npdu)
{
struct cpl_tx_data_iso *cpl;
unsigned int burst_size;
@@ -1147,7 +1220,7 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
adjusted_plen, credits, shove, ulp_submode | ULP_ISO);
cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1);
MPASS(plen == sndptr->m_pkthdr.len);
- write_tx_data_iso(cpl_iso, ulp_submode,
+ write_iscsi_tx_data_iso(cpl_iso, ulp_submode,
mbuf_iscsi_iso_flags(sndptr), iso_mss, plen, npdu);
p = cpl_iso + 1;
} else {
@@ -1183,21 +1256,269 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
return (wr);
}
+static void
+write_nvme_tx_data_iso(void *dst, u_int ulp_submode, u_int iso_type,
+ uint16_t mss, int len, int npdu, int pdo)
+{
+ struct cpl_t7_tx_data_iso *cpl;
+ unsigned int burst_size;
+
+ /*
+ * TODO: Need to figure out how the LAST_PDU and SUCCESS flags
+ * are handled.
+ *
+ * - Does len need padding bytes? (If so, does padding need
+ * to be in DSGL input?)
+ *
+ * - burst always 0?
+ */
+ burst_size = 0;
+
+ cpl = (struct cpl_t7_tx_data_iso *)dst;
+ cpl->op_to_scsi = htonl(V_CPL_T7_TX_DATA_ISO_OPCODE(CPL_TX_DATA_ISO) |
+ V_CPL_T7_TX_DATA_ISO_FIRST(1) |
+ V_CPL_T7_TX_DATA_ISO_LAST(1) |
+ V_CPL_T7_TX_DATA_ISO_CPLHDRLEN(0) |
+ V_CPL_T7_TX_DATA_ISO_HDRCRC(!!(ulp_submode & ULP_CRC_HEADER)) |
+ V_CPL_T7_TX_DATA_ISO_PLDCRC(!!(ulp_submode & ULP_CRC_DATA)) |
+ V_CPL_T7_TX_DATA_ISO_IMMEDIATE(0) |
+ V_CPL_T7_TX_DATA_ISO_SCSI(iso_type));
+
+ cpl->nvme_tcp_pkd = F_CPL_T7_TX_DATA_ISO_NVME_TCP;
+ cpl->ahs = 0;
+ cpl->mpdu = htons(DIV_ROUND_UP(mss, 4));
+ cpl->burst = htonl(DIV_ROUND_UP(burst_size, 4));
+ cpl->size = htonl(len);
+ cpl->num_pi_bytes_seglen_offset = htonl(0);
+ cpl->datasn_offset = htonl(0);
+ cpl->buffer_offset = htonl(0);
+ cpl->pdo_pkd = pdo;
+}
+
+static struct wrqe *
+write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
+{
+ struct mbuf *m;
+ const struct nvme_tcp_common_pdu_hdr *hdr;
+ struct fw_v2_nvmet_tx_data_wr *txwr;
+ struct cpl_tx_data_iso *cpl_iso;
+ void *p;
+ struct wrqe *wr;
+ u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
+ u_int adjusted_plen, imm_data, ulp_submode;
+ struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
+ int tx_credits, shove, npdu, wr_len;
+ uint16_t iso_mss;
+ bool iso, nomap_mbuf_seen;
+
+ M_ASSERTPKTHDR(sndptr);
+
+ tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
+ if (mbuf_raw_wr(sndptr)) {
+ plen = sndptr->m_pkthdr.len;
+ KASSERT(plen <= SGE_MAX_WR_LEN,
+ ("raw WR len %u is greater than max WR len", plen));
+ if (plen > tx_credits * 16)
+ return (NULL);
+
+ wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq);
+ if (__predict_false(wr == NULL))
+ return (NULL);
+
+ m_copydata(sndptr, 0, plen, wrtod(wr));
+ return (wr);
+ }
+
+ /*
+ * The first mbuf is the PDU header that is always sent as
+ * immediate data.
+ */
+ imm_data = sndptr->m_len;
+
+ iso = mbuf_iscsi_iso(sndptr);
+ max_imm = max_imm_payload_v2(tx_credits, iso);
+
+ /*
+ * Not enough credits for the PDU header.
+ */
+ if (imm_data > max_imm)
+ return (NULL);
+
+ max_nsegs = max_dsgl_nsegs_v2(tx_credits, iso, imm_data);
+ iso_mss = mbuf_iscsi_iso_mss(sndptr);
+
+ plen = imm_data;
+ nsegs = 0;
+ max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
+ nomap_mbuf_seen = false;
+ for (m = sndptr->m_next; m != NULL; m = m->m_next) {
+ int n;
+
+ if (m->m_flags & M_EXTPG)
+ n = sglist_count_mbuf_epg(m, mtod(m, vm_offset_t),
+ m->m_len);
+ else
+ n = sglist_count(mtod(m, void *), m->m_len);
+
+ nsegs += n;
+ plen += m->m_len;
+
+ /*
+ * This mbuf would send us _over_ the nsegs limit.
+ * Suspend tx because the PDU can't be sent out.
+ */
+ if ((nomap_mbuf_seen || plen > max_imm) && nsegs > max_nsegs)
+ return (NULL);
+
+ if (m->m_flags & M_EXTPG)
+ nomap_mbuf_seen = true;
+ if (max_nsegs_1mbuf < n)
+ max_nsegs_1mbuf = n;
+ }
+
+ if (__predict_false(toep->flags & TPF_FIN_SENT))
+ panic("%s: excess tx.", __func__);
+
+ /*
+ * We have a PDU to send. All of it goes out in one WR so 'm'
+ * is NULL. A PDU's length is always a multiple of 4.
+ */
+ MPASS(m == NULL);
+ MPASS((plen & 3) == 0);
+ MPASS(sndptr->m_pkthdr.len == plen);
+
+ shove = !(tp->t_flags & TF_MORETOCOME);
+
+ /*
+ * plen doesn't include header digests, padding, and data
+ * digests which are generated and inserted in the right
+ * places by the TOE, but they do occupy TCP sequence space
+ * and need to be accounted for.
+ *
+ * To determine the overhead, check the PDU header in sndptr.
+ * Note that only certain PDU types can use digests and
+ * padding, and PDO accounts for all but the data digests for
+ * those PDUs.
+ */
+ MPASS((sndptr->m_flags & M_EXTPG) == 0);
+ ulp_submode = mbuf_ulp_submode(sndptr);
+ hdr = mtod(sndptr, const void *);
+ switch (hdr->pdu_type) {
+ case NVME_TCP_PDU_TYPE_H2C_TERM_REQ:
+ case NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
+ MPASS(ulp_submode == 0);
+ MPASS(!iso);
+ break;
+ case NVME_TCP_PDU_TYPE_CAPSULE_RESP:
+ case NVME_TCP_PDU_TYPE_R2T:
+ MPASS((ulp_submode & ULP_CRC_DATA) == 0);
+ /* FALLTHROUGH */
+ case NVME_TCP_PDU_TYPE_CAPSULE_CMD:
+ MPASS(!iso);
+ break;
+ case NVME_TCP_PDU_TYPE_H2C_DATA:
+ case NVME_TCP_PDU_TYPE_C2H_DATA:
+ if (le32toh(hdr->plen) + ((ulp_submode & ULP_CRC_DATA) != 0 ?
+ sizeof(uint32_t) : 0) == plen)
+ MPASS(!iso);
+ break;
+ default:
+ __assert_unreachable();
+ }
+
+ if (iso) {
+ npdu = howmany(plen - hdr->hlen, iso_mss);
+ adjusted_plen = hdr->pdo * npdu + (plen - hdr->hlen);
+ if ((ulp_submode & ULP_CRC_DATA) != 0)
+ adjusted_plen += npdu * sizeof(uint32_t);
+ } else {
+ npdu = 1;
+ adjusted_plen = le32toh(hdr->plen);
+ }
+ wr_len = sizeof(*txwr);
+ if (iso)
+ wr_len += sizeof(struct cpl_tx_data_iso);
+ if (plen <= max_imm && !nomap_mbuf_seen) {
+ /* Immediate data tx for full PDU */
+ imm_data = plen;
+ wr_len += plen;
+ nsegs = 0;
+ } else {
+ /* DSGL tx for PDU data */
+ wr_len += roundup2(imm_data, 16);
+ wr_len += sizeof(struct ulptx_sgl) +
+ ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
+ }
+
+ wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
+ if (wr == NULL) {
+ /* XXX: how will we recover from this? */
+ return (NULL);
+ }
+ txwr = wrtod(wr);
+ credits = howmany(wr->wr_len, 16);
+
+ if (iso) {
+ write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR,
+ imm_data + sizeof(struct cpl_tx_data_iso),
+ adjusted_plen, credits, shove, ulp_submode | ULP_ISO);
+ cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1);
+ MPASS(plen == sndptr->m_pkthdr.len);
+ write_nvme_tx_data_iso(cpl_iso, ulp_submode,
+ (hdr->pdu_type & 0x1) == 0 ? 1 : 2, iso_mss, plen, npdu,
+ hdr->pdo);
+ p = cpl_iso + 1;
+ } else {
+ write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR, imm_data,
+ adjusted_plen, credits, shove, ulp_submode);
+ p = txwr + 1;
+ }
+
+ /* PDU header (and immediate data payload). */
+ m_copydata(sndptr, 0, imm_data, p);
+ if (nsegs != 0) {
+ p = roundup2((char *)p + imm_data, 16);
+ write_tx_sgl(p, sndptr->m_next, NULL, nsegs, max_nsegs_1mbuf);
+ if (wr_len & 0xf) {
+ uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len);
+ *pad = 0;
+ }
+ }
+
+ KASSERT(toep->tx_credits >= credits,
+ ("%s: not enough credits: credits %u "
+ "toep->tx_credits %u tx_credits %u nsegs %u "
+ "max_nsegs %u iso %d", __func__, credits,
+ toep->tx_credits, tx_credits, nsegs, max_nsegs, iso));
+
+ tp->snd_nxt += adjusted_plen;
+ tp->snd_max += adjusted_plen;
+
+ counter_u64_add(toep->ofld_txq->tx_nvme_pdus, npdu);
+ counter_u64_add(toep->ofld_txq->tx_nvme_octets, plen);
+ if (iso)
+ counter_u64_add(toep->ofld_txq->tx_nvme_iso_wrs, 1);
+
+ return (wr);
+}
+
void
t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
{
struct mbuf *sndptr, *m;
struct fw_wr_hdr *wrhdr;
struct wrqe *wr;
- u_int plen, credits;
+ u_int plen, credits, mode;
struct inpcb *inp = toep->inp;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
struct mbufq *pduq = &toep->ulp_pduq;
INP_WLOCK_ASSERT(inp);
+ mode = ulp_mode(toep);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
- KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
+ KASSERT(mode == ULP_MODE_ISCSI || mode == ULP_MODE_NVMET,
("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
@@ -1230,7 +1551,7 @@ t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
if (sbu > 0) {
/*
* The data transmitted before the
- * tid's ULP mode changed to ISCSI is
+ * tid's ULP mode changed to ISCSI/NVMET is
* still in so_snd. Incoming credits
* should account for so_snd first.
*/
@@ -1243,7 +1564,10 @@ t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
}
while ((sndptr = mbufq_first(pduq)) != NULL) {
- wr = write_iscsi_mbuf_wr(toep, sndptr);
+ if (mode == ULP_MODE_ISCSI)
+ wr = write_iscsi_mbuf_wr(toep, sndptr);
+ else
+ wr = write_nvme_mbuf_wr(toep, sndptr);
if (wr == NULL) {
toep->flags |= TPF_TX_SUSPENDED;
return;
@@ -1302,7 +1626,8 @@ static inline void
t4_push_data(struct adapter *sc, struct toepcb *toep, int drop)
{
- if (ulp_mode(toep) == ULP_MODE_ISCSI)
+ if (ulp_mode(toep) == ULP_MODE_ISCSI ||
+ ulp_mode(toep) == ULP_MODE_NVMET)
t4_push_pdus(sc, toep, drop);
else if (toep->flags & TPF_KTLS)
t4_push_ktls(sc, toep, drop);
@@ -1462,7 +1787,8 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
socantrcvmore(so);
if (ulp_mode(toep) == ULP_MODE_RDMA ||
- (ulp_mode(toep) == ULP_MODE_ISCSI && chip_id(sc) >= CHELSIO_T6)) {
+ (ulp_mode(toep) == ULP_MODE_ISCSI && chip_id(sc) >= CHELSIO_T6) ||
+ ulp_mode(toep) == ULP_MODE_NVMET) {
/*
* There might be data received via DDP before the FIN
* not reported to the driver. Just assume the
@@ -2008,7 +2334,8 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
SOCKBUF_LOCK(sb);
sbu = sbused(sb);
- if (ulp_mode(toep) == ULP_MODE_ISCSI) {
+ if (ulp_mode(toep) == ULP_MODE_ISCSI ||
+ ulp_mode(toep) == ULP_MODE_NVMET) {
if (__predict_false(sbu > 0)) {
/*
* The data transmitted before the
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 53a945f8b4cc..8dfffd465345 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -1990,8 +1990,10 @@ t4_tom_deactivate(struct adapter *sc)
if (td == NULL)
return (0); /* XXX. KASSERT? */
- if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI))
- return (EBUSY); /* both iWARP and iSCSI rely on the TOE. */
+ /* These ULDs rely on the TOE. */
+ if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI) ||
+ uld_active(sc, ULD_NVME))
+ return (EBUSY);
if (sc->offload_map != 0) {
for_each_port(sc, i) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-23 19:44:17 +0000