diff options
Diffstat (limited to 'sys/contrib/dev/iwlwifi/pcie/rx.c')
-rw-r--r-- | sys/contrib/dev/iwlwifi/pcie/rx.c | 170 |
1 files changed, 123 insertions, 47 deletions
diff --git a/sys/contrib/dev/iwlwifi/pcie/rx.c b/sys/contrib/dev/iwlwifi/pcie/rx.c index 2f306309578c..7499ba3093af 100644 --- a/sys/contrib/dev/iwlwifi/pcie/rx.c +++ b/sys/contrib/dev/iwlwifi/pcie/rx.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2003-2014, 2018-2021 Intel Corporation + * Copyright (C) 2003-2014, 2018-2023 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -190,11 +190,14 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans, } rxq->write_actual = round_down(rxq->write, 8); - if (trans->trans_cfg->mq_rx_supported) + if (!trans->trans_cfg->mq_rx_supported) + iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual); + else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) + iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual | + HBUS_TARG_WRPTR_RX_Q(rxq->id)); + else iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id), rxq->write_actual); - else - iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual); } static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans) @@ -231,8 +234,12 @@ static void iwl_pcie_restock_bd(struct iwl_trans *trans, bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid); } +#if defined(__linux__) + IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n", +#elif defined(__FreeBSD__) IWL_DEBUG_PCI_RW(trans, "Assigned virtual RB ID %u to queue %d index %d\n", (u32)rxb->vid, rxq->id, rxq->write); +#endif } /* @@ -652,23 +659,30 @@ void iwl_pcie_rx_allocator_work(struct work_struct *data) iwl_pcie_rx_allocator(trans_pcie->trans); } -static int iwl_pcie_free_bd_size(struct iwl_trans *trans, bool use_rx_td) +static int iwl_pcie_free_bd_size(struct iwl_trans *trans) { - struct iwl_rx_transfer_desc *rx_td; + if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) + return sizeof(struct iwl_rx_transfer_desc); - if (use_rx_td) - return sizeof(*rx_td); - else - return trans->trans_cfg->mq_rx_supported ? sizeof(__le64) : - sizeof(__le32); + return trans->trans_cfg->mq_rx_supported ? + sizeof(__le64) : sizeof(__le32); +} + +static int iwl_pcie_used_bd_size(struct iwl_trans *trans) +{ + if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) + return sizeof(struct iwl_rx_completion_desc_bz); + + if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) + return sizeof(struct iwl_rx_completion_desc); + + return sizeof(__le32); } static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans, struct iwl_rxq *rxq) { - bool use_rx_td = (trans->trans_cfg->device_family >= - IWL_DEVICE_FAMILY_AX210); - int free_size = iwl_pcie_free_bd_size(trans, use_rx_td); + int free_size = iwl_pcie_free_bd_size(trans); if (rxq->bd) dma_free_coherent(trans->dev, @@ -682,24 +696,32 @@ static void iwl_pcie_free_rxq_dma(struct iwl_trans *trans, if (rxq->used_bd) dma_free_coherent(trans->dev, - (use_rx_td ? sizeof(*rxq->cd) : - sizeof(__le32)) * rxq->queue_size, + iwl_pcie_used_bd_size(trans) * + rxq->queue_size, rxq->used_bd, rxq->used_bd_dma); rxq->used_bd_dma = 0; rxq->used_bd = NULL; } +static size_t iwl_pcie_rb_stts_size(struct iwl_trans *trans) +{ + bool use_rx_td = (trans->trans_cfg->device_family >= + IWL_DEVICE_FAMILY_AX210); + + if (use_rx_td) + return sizeof(__le16); + + return sizeof(struct iwl_rb_status); +} + static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); struct device *dev = trans->dev; int i; int free_size; - bool use_rx_td = (trans->trans_cfg->device_family >= - IWL_DEVICE_FAMILY_AX210); - size_t rb_stts_size = use_rx_td ? sizeof(__le16) : - sizeof(struct iwl_rb_status); spin_lock_init(&rxq->lock); if (trans->trans_cfg->mq_rx_supported) @@ -707,7 +729,7 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans, else rxq->queue_size = RX_QUEUE_SIZE; - free_size = iwl_pcie_free_bd_size(trans, use_rx_td); + free_size = iwl_pcie_free_bd_size(trans); /* * Allocate the circular buffer of Read Buffer Descriptors @@ -720,14 +742,15 @@ static int iwl_pcie_alloc_rxq_dma(struct iwl_trans *trans, if (trans->trans_cfg->mq_rx_supported) { rxq->used_bd = dma_alloc_coherent(dev, - (use_rx_td ? sizeof(*rxq->cd) : sizeof(__le32)) * rxq->queue_size, + iwl_pcie_used_bd_size(trans) * + rxq->queue_size, &rxq->used_bd_dma, GFP_KERNEL); if (!rxq->used_bd) goto err; } - rxq->rb_stts = (void *)((u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size); + rxq->rb_stts = (u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size; rxq->rb_stts_dma = trans_pcie->base_rb_stts_dma + rxq->id * rb_stts_size; @@ -746,11 +769,9 @@ err: static int iwl_pcie_rx_alloc(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; int i, ret; - size_t rb_stts_size = trans->trans_cfg->device_family >= - IWL_DEVICE_FAMILY_AX210 ? - sizeof(__le16) : sizeof(struct iwl_rb_status); if (WARN_ON(trans_pcie->rxq)) return -EINVAL; @@ -1042,6 +1063,22 @@ static int iwl_pcie_napi_poll_msix(struct napi_struct *napi, int budget) return ret; } +void iwl_pcie_rx_napi_sync(struct iwl_trans *trans) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + int i; + + if (unlikely(!trans_pcie->rxq)) + return; + + for (i = 0; i < trans->num_rx_queues; i++) { + struct iwl_rxq *rxq = &trans_pcie->rxq[i]; + + if (rxq && rxq->napi.poll) + napi_synchronize(&rxq->napi); + } +} + static int _iwl_pcie_rx_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -1099,7 +1136,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans) poll = iwl_pcie_napi_poll_msix; netif_napi_add(&trans_pcie->napi_dev, &rxq->napi, - poll, NAPI_POLL_WEIGHT); + poll); napi_enable(&rxq->napi); } @@ -1166,11 +1203,9 @@ int iwl_pcie_gen2_rx_init(struct iwl_trans *trans) void iwl_pcie_rx_free(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + size_t rb_stts_size = iwl_pcie_rb_stts_size(trans); struct iwl_rb_allocator *rba = &trans_pcie->rba; int i; - size_t rb_stts_size = trans->trans_cfg->device_family >= - IWL_DEVICE_FAMILY_AX210 ? - sizeof(__le16) : sizeof(struct iwl_rb_status); /* * if rxq is NULL, it means that nothing has been allocated, @@ -1307,9 +1342,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, "Q %d: cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n", rxq->id, offset, iwl_get_cmd_string(trans, - iwl_cmd_id(pkt->hdr.cmd, - pkt->hdr.group_id, - 0)), + WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)), pkt->hdr.group_id, pkt->hdr.cmd, le16_to_cpu(pkt->hdr.sequence)); @@ -1319,7 +1352,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN); /* check that what the device tells us made sense */ - if (offset > max_len) + if (len < sizeof(*pkt) || offset > max_len) break; trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len); @@ -1419,6 +1452,7 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans, u16 vid; BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc) != 32); + BUILD_BUG_ON(sizeof(struct iwl_rx_completion_desc_bz) != 4); if (!trans->trans_cfg->mq_rx_supported) { rxb = rxq->queue[i]; @@ -1426,11 +1460,20 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans, return rxb; } - if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { - vid = le16_to_cpu(rxq->cd[i].rbid); - *join = rxq->cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; + if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { + struct iwl_rx_completion_desc_bz *cd = rxq->used_bd; + + vid = le16_to_cpu(cd[i].rbid); + *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; + } else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { + struct iwl_rx_completion_desc *cd = rxq->used_bd; + + vid = le16_to_cpu(cd[i].rbid); + *join = cd[i].flags & IWL_RX_CD_FLAGS_FRAGMENTED; } else { - vid = le32_to_cpu(rxq->bd_32[i]) & 0x0FFF; /* 12-bit VID */ + __le32 *cd = rxq->used_bd; + + vid = le32_to_cpu(cd[i]) & 0x0FFF; /* 12-bit VID */ } if (!vid || vid > RX_POOL_SIZE(trans_pcie->num_rx_bufs)) @@ -1601,18 +1644,22 @@ irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id) struct msix_entry *entry = dev_id; struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); struct iwl_trans *trans = trans_pcie->trans; - struct iwl_rxq *rxq = &trans_pcie->rxq[entry->entry]; + struct iwl_rxq *rxq; trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0); if (WARN_ON(entry->entry >= trans->num_rx_queues)) return IRQ_NONE; - if (WARN_ONCE(!rxq, - "[%d] Got MSI-X interrupt before we have Rx queues", - entry->entry)) + if (!trans_pcie->rxq) { + if (net_ratelimit()) + IWL_ERR(trans, + "[%d] Got MSI-X interrupt before we have Rx queues\n", + entry->entry); return IRQ_NONE; + } + rxq = &trans_pcie->rxq[entry->entry]; lock_map_acquire(&trans->sync_cmd_lockdep_map); IWL_DEBUG_ISR(trans, "[%d] Got interrupt\n", entry->entry); @@ -1837,7 +1884,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) return IRQ_NONE; } - if (unlikely(inta == 0xFFFFFFFF || (inta & 0xFFFFFFF0) == 0xa5a5a5a0)) { + if (unlikely(inta == 0xFFFFFFFF || iwl_trans_is_hw_error_value(inta))) { /* * Hardware disappeared. It might have * already raised an interrupt. @@ -1958,7 +2005,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) CSR_INT, CSR_INT_BIT_RX_PERIODIC); } /* Sending RX interrupt require many steps to be done in the - * the device: + * device: * 1- write interrupt to current index in ICT table. * 2- dma RX frame. * 3- update RX shared data to indicate last write index. @@ -2002,6 +2049,11 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) /* Wake up uCode load routine, now that load is complete */ trans_pcie->ucode_write_complete = true; wake_up(&trans_pcie->ucode_write_waitq); + /* Wake up IMR write routine, now that write to SRAM is complete */ + if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { + trans_pcie->imr_status = IMR_D2S_COMPLETED; + wake_up(&trans_pcie->ucode_write_waitq); + } } if (inta & ~handled) { @@ -2217,7 +2269,17 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) } /* This "Tx" DMA channel is used only for loading uCode */ - if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) { + if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM && + trans_pcie->imr_status == IMR_D2S_REQUESTED) { + IWL_DEBUG_ISR(trans, "IMR Complete interrupt\n"); + isr_stats->tx++; + + /* Wake up IMR routine once write to SRAM is complete */ + if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { + trans_pcie->imr_status = IMR_D2S_COMPLETED; + wake_up(&trans_pcie->ucode_write_waitq); + } + } else if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) { IWL_DEBUG_ISR(trans, "uCode load interrupt\n"); isr_stats->tx++; /* @@ -2226,6 +2288,12 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) */ trans_pcie->ucode_write_complete = true; wake_up(&trans_pcie->ucode_write_waitq); + + /* Wake up IMR routine once write to SRAM is complete */ + if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { + trans_pcie->imr_status = IMR_D2S_COMPLETED; + wake_up(&trans_pcie->ucode_write_waitq); + } } if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) @@ -2240,7 +2308,10 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) inta_fh); isr_stats->sw++; /* during FW reset flow report errors from there */ - if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { + if (trans_pcie->imr_status == IMR_D2S_REQUESTED) { + trans_pcie->imr_status = IMR_D2S_ERROR; + wake_up(&trans_pcie->imr_waitq); + } else if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) { trans_pcie->fw_reset_state = FW_RESET_ERROR; wake_up(&trans_pcie->fw_reset_waitq); } else { @@ -2274,7 +2345,12 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) } } - if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP) { + /* + * In some rare cases when the HW is in a bad state, we may + * get this interrupt too early, when prph_info is still NULL. + * So make sure that it's not NULL to prevent crashing. + */ + if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) { u32 sleep_notif = le32_to_cpu(trans_pcie->prph_info->sleep_notif); if (sleep_notif == IWL_D3_SLEEP_STATUS_SUSPEND || |