aboutsummaryrefslogtreecommitdiff
path: root/sys/contrib/dev/iwlwifi/pcie/rx.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/contrib/dev/iwlwifi/pcie/rx.c')
-rw-r--r--sys/contrib/dev/iwlwifi/pcie/rx.c170
1 files changed, 123 insertions, 47 deletions
diff --git a/sys/contrib/dev/iwlwifi/pcie/rx.c b/sys/contrib/dev/iwlwifi/pcie/rx.c
index 2f306309578c..7499ba3093af 100644
--- a/sys/contrib/dev/iwlwifi/pcie/rx.c
+++ b/sys/contrib/dev/iwlwifi/pcie/rx.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * Copyright (C) 2003-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2023 Intel Corporation
* Copyright (C) 2013-2015 Intel Mobile Communications GmbH
* Copyright (C) 2016-2017 Intel Deutschland GmbH
*/
@@ -190,11 +190,14 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
}
rxq->write_actual = round_down(rxq->write, 8);
- if (trans->trans_cfg->mq_rx_supported)
+ if (!trans->trans_cfg->mq_rx_supported)
+ iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
+ else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+ iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual |
+ HBUS_TARG_WRPTR_RX_Q(rxq->id));
+ else
iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id),
rxq->write_actual);
- else
- iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
}
static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans)
@@ -231,8 +234,12 @@ static void iwl_pcie_restock_bd(struct iwl_trans *trans,
bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid);
}
+#if defined(__linux__)
+ IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
+#elif defined(__FreeBSD__)
IWL_DEBUG_PCI_RW(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
(u32)rxb->vid, rxq->id, rxq->write);
+#endif
}
/*
@@ -652,23 +659,30 @@ void iwl_pcie_rx_allocator_work(struct work_struct *data)
iwl_pcie_rx_allocator(trans_pcie->trans);
}
-static int iwl_pcie_free_bd_size(struct iwl_trans *trans, bool use_rx_td)
+static int iwl_pcie_free_bd_size(struct iwl_trans *trans)
{
- struct iwl_rx_transfer_desc *rx_td;
+ if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+ return sizeof(struct iwl_rx_transfer_desc);
- if (use_rx_td)
- return sizeof(*rx_td);
- else
- return trans->trans_cfg->mq_rx_supported ? sizeof(__le64) :
- sizeof(__le32);
+ return trans->trans_cfg->mq_rx_supported ?
+ sizeof(__le64) : sizeof(__le32);
+}
+
+static int iwl_pcie_used_bd_size(struct iwl_trans *trans)
+{
+ if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
+ return sizeof(struct iwl_rx_completion_desc_bz);
+
+ if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
+ return sizeof(struct iwl_rx_completion_desc);
+
+ return sizeof(__le32);
}
static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans,
struct iwl_rxq *rxq)
{
- bool use_rx_td = (trans->trans_cfg->device_family >=
- IWL_DEVICE_FAMILY_AX210);
- int free_size = iwl_pcie_free_bd_size(trans, use_rx_td);
+ int free_size = iwl_pcie_free_bd_size(trans);
if (rxq->bd)
dma_free_coherent(trans->dev,
@@ -682,24 +696,32 @@ static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans,
if (rxq->used_bd)
dma_free_coherent(trans->dev,
- (use_rx_td ? sizeof(*rxq->cd) :
- sizeof(__le32)) * rxq->queue_size,
+ iwl_pcie_used_bd_size(trans) *
+ rxq->queue_size,
rxq->used_bd, rxq->used_bd_dma);
rxq->used_bd_dma = 0;
rxq->used_bd = NULL;
}
+static size_t iwl_pcie_rb_stts_size(struct iwl_trans *trans)
+{
+ bool use_rx_td = (trans->trans_cfg->device_family >=
+ IWL_DEVICE_FAMILY_AX210);
+
+ if (use_rx_td)
+ return sizeof(__le16);
+
+ return sizeof(struct iwl_rb_status);
+}
+
static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
struct iwl_rxq *rxq)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ size_t rb_stts_size = iwl_pcie_rb_stts_size(trans);
struct device *dev = trans->dev;
int i;
int free_size;
- bool use_rx_td = (trans->trans_cfg->device_family >=
- IWL_DEVICE_FAMILY_AX210);
- size_t rb_stts_size = use_rx_td ? sizeof(__le16) :
- sizeof(struct iwl_rb_status);
spin_lock_init(&rxq->lock);
if (trans->trans_cfg->mq_rx_supported)
@@ -707,7 +729,7 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
else
rxq->queue_size = RX_QUEUE_SIZE;
- free_size = iwl_pcie_free_bd_size(trans, use_rx_td);
+ free_size = iwl_pcie_free_bd_size(trans);
/*
* Allocate the circular buffer of Read Buffer Descriptors
@@ -720,14 +742,15 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans,
if (trans->trans_cfg->mq_rx_supported) {
rxq->used_bd = dma_alloc_coherent(dev,
- (use_rx_td ? sizeof(*rxq->cd) : sizeof(__le32)) * rxq->queue_size,
+ iwl_pcie_used_bd_size(trans) *
+ rxq->queue_size,
&rxq->used_bd_dma,
GFP_KERNEL);
if (!rxq->used_bd)
goto err;
}
- rxq->rb_stts = (void *)((u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size);
+ rxq->rb_stts = (u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size;
rxq->rb_stts_dma =
trans_pcie->base_rb_stts_dma + rxq->id * rb_stts_size;
@@ -746,11 +769,9 @@ err:
static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ size_t rb_stts_size = iwl_pcie_rb_stts_size(trans);
struct iwl_rb_allocator *rba = &trans_pcie->rba;
int i, ret;
- size_t rb_stts_size = trans->trans_cfg->device_family >=
- IWL_DEVICE_FAMILY_AX210 ?
- sizeof(__le16) : sizeof(struct iwl_rb_status);
if (WARN_ON(trans_pcie->rxq))
return -EINVAL;
@@ -1042,6 +1063,22 @@ static int iwl_pcie_napi_poll_msix(struct napi_struct *napi, int budget)
return ret;
}
+void iwl_pcie_rx_napi_sync(struct iwl_trans *trans)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ int i;
+
+ if (unlikely(!trans_pcie->rxq))
+ return;
+
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+ if (rxq && rxq->napi.poll)
+ napi_synchronize(&rxq->napi);
+ }
+}
+
static int _iwl_pcie_rx_init(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -1099,7 +1136,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
poll = iwl_pcie_napi_poll_msix;
netif_napi_add(&trans_pcie->napi_dev, &rxq->napi,
- poll, NAPI_POLL_WEIGHT);
+ poll);
napi_enable(&rxq->napi);
}
@@ -1166,11 +1203,9 @@ int iwl_pcie_gen2_rx_init(struct iwl_trans *trans)
void iwl_pcie_rx_free(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ size_t rb_stts_size = iwl_pcie_rb_stts_size(trans);
struct iwl_rb_allocator *rba = &trans_pcie->rba;
int i;
- size_t rb_stts_size = trans->trans_cfg->device_family >=
- IWL_DEVICE_FAMILY_AX210 ?
- sizeof(__le16) : sizeof(struct iwl_rb_status);
/*
* if rxq is NULL, it means that nothing has been allocated,
@@ -1307,9 +1342,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
"Q %d: cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n",
rxq->id, offset,
iwl_get_cmd_string(trans,
- iwl_cmd_id(pkt->hdr.cmd,
- pkt->hdr.group_id,
- 0)),
+ WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)),
pkt->hdr.group_id, pkt->hdr.cmd,
le16_to_cpu(pkt->hdr.sequence));
@@ -1319,7 +1352,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN);
/* check that what the device tells us made sense */
- if (offset > max_len)
+ if (len < sizeof(*pkt) || offset > max_len)
break;
trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len);
@@ -1419,6 +1452,7 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans,
u16 vid;
BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc) != 32);
+ BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc_bz) != 4);
if (!trans->trans_cfg->mq_rx_supported) {
rxb = rxq->queue[i];
@@ -1426,11 +1460,20 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans,
return rxb;
}
- if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
- vid = le16_to_cpu(rxq->cd[i].rbid);
- *join = rxq->cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED;
+ if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+ struct iwl_rx_completion_desc_bz *cd = rxq->used_bd;
+
+ vid = le16_to_cpu(cd[i].rbid);
+ *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED;
+ } else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+ struct iwl_rx_completion_desc *cd = rxq->used_bd;
+
+ vid = le16_to_cpu(cd[i].rbid);
+ *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED;
} else {
- vid = le32_to_cpu(rxq->bd_32[i]) & 0x0FFF; /* 12-bit VID */
+ __le32 *cd = rxq->used_bd;
+
+ vid = le32_to_cpu(cd[i]) & 0x0FFF; /* 12-bit VID */
}
if (!vid || vid > RX_POOL_SIZE(trans_pcie->num_rx_bufs))
@@ -1601,18 +1644,22 @@ irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id)
struct msix_entry *entry = dev_id;
struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry);
struct iwl_trans *trans = trans_pcie->trans;
- struct iwl_rxq *rxq = &trans_pcie->rxq[entry->entry];
+ struct iwl_rxq *rxq;
trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0);
if (WARN_ON(entry->entry >= trans->num_rx_queues))
return IRQ_NONE;
- if (WARN_ONCE(!rxq,
- "[%d] Got MSI-X interrupt before we have Rx queues",
- entry->entry))
+ if (!trans_pcie->rxq) {
+ if (net_ratelimit())
+ IWL_ERR(trans,
+ "[%d] Got MSI-X interrupt before we have Rx queues\n",
+ entry->entry);
return IRQ_NONE;
+ }
+ rxq = &trans_pcie->rxq[entry->entry];
lock_map_acquire(&trans->sync_cmd_lockdep_map);
IWL_DEBUG_ISR(trans, "[%d] Got interrupt\n", entry->entry);
@@ -1837,7 +1884,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
return IRQ_NONE;
}
- if (unlikely(inta == 0xFFFFFFFF || (inta & 0xFFFFFFF0) == 0xa5a5a5a0)) {
+ if (unlikely(inta == 0xFFFFFFFF || iwl_trans_is_hw_error_value(inta))) {
/*
* Hardware disappeared. It might have
* already raised an interrupt.
@@ -1958,7 +2005,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
CSR_INT, CSR_INT_BIT_RX_PERIODIC);
}
/* Sending RX interrupt require many steps to be done in the
- * the device:
+ * device:
* 1- write interrupt to current index in ICT table.
* 2- dma RX frame.
* 3- update RX shared data to indicate last write index.
@@ -2002,6 +2049,11 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
/* Wake up uCode load routine, now that load is complete */
trans_pcie->ucode_write_complete = true;
wake_up(&trans_pcie->ucode_write_waitq);
+ /* Wake up IMR write routine, now that write to SRAM is complete */
+ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
+ trans_pcie->imr_status = IMR_D2S_COMPLETED;
+ wake_up(&trans_pcie->ucode_write_waitq);
+ }
}
if (inta & ~handled) {
@@ -2217,7 +2269,17 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
}
/* This "Tx" DMA channel is used only for loading uCode */
- if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) {
+ if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM &&
+ trans_pcie->imr_status == IMR_D2S_REQUESTED) {
+ IWL_DEBUG_ISR(trans, "IMR Complete interrupt\n");
+ isr_stats->tx++;
+
+ /* Wake up IMR routine once write to SRAM is complete */
+ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
+ trans_pcie->imr_status = IMR_D2S_COMPLETED;
+ wake_up(&trans_pcie->ucode_write_waitq);
+ }
+ } else if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) {
IWL_DEBUG_ISR(trans, "uCode load interrupt\n");
isr_stats->tx++;
/*
@@ -2226,6 +2288,12 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
*/
trans_pcie->ucode_write_complete = true;
wake_up(&trans_pcie->ucode_write_waitq);
+
+ /* Wake up IMR routine once write to SRAM is complete */
+ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
+ trans_pcie->imr_status = IMR_D2S_COMPLETED;
+ wake_up(&trans_pcie->ucode_write_waitq);
+ }
}
if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
@@ -2240,7 +2308,10 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
inta_fh);
isr_stats->sw++;
/* during FW reset flow report errors from there */
- if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
+ if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
+ trans_pcie->imr_status = IMR_D2S_ERROR;
+ wake_up(&trans_pcie->imr_waitq);
+ } else if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
trans_pcie->fw_reset_state = FW_RESET_ERROR;
wake_up(&trans_pcie->fw_reset_waitq);
} else {
@@ -2274,7 +2345,12 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
}
}
- if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP) {
+ /*
+ * In some rare cases when the HW is in a bad state, we may
+ * get this interrupt too early, when prph_info is still NULL.
+ * So make sure that it's not NULL to prevent crashing.
+ */
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) {
u32 sleep_notif =
le32_to_cpu(trans_pcie->prph_info->sleep_notif);
if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND ||