diff options
author | Alan Cox <alc@FreeBSD.org> | 2022-07-31 19:28:30 +0000 |
---|---|---|
committer | Alan Cox <alc@FreeBSD.org> | 2022-08-06 18:05:58 +0000 |
commit | 7f46deccbed74436b62f8fd02655ff4ad89f1023 (patch) | |
tree | a9059230eaff4cd94c4bff60040b91c4bf211108 | |
parent | 35ce7d40f9861059caa460296d1663ff172de931 (diff) | |
download | src-7f46deccbed74436b62f8fd02655ff4ad89f1023.tar.gz src-7f46deccbed74436b62f8fd02655ff4ad89f1023.zip |
x86/iommu: Reduce the number of queued invalidation interrupts
Restructure dmar_qi_task() so as to reduce the number of invalidation
completion interrupts. Specifically, because processing completed
invalidations in dmar_qi_task() can take quite some time, don't reenable
completion interrupts until processing has completed a first time. Then,
check a second time after reenabling completion interrupts, so that
any invalidations that complete just before interrupts are reenabled
do not linger until a future invalidation might raise an interrupt.
(Recent changes have made checking for completed invalidations cheap; no
locking is required.)
Reviewed by: kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D36054
-rw-r--r-- | sys/x86/iommu/intel_qi.c | 45 |
1 files changed, 29 insertions, 16 deletions
diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c index baaf5b472a2c..8a8e656083e3 100644 --- a/sys/x86/iommu/intel_qi.c +++ b/sys/x86/iommu/intel_qi.c @@ -412,13 +412,33 @@ dmar_qi_intr(void *arg) } static void +dmar_qi_drain_tlb_flush(struct dmar_unit *unit) +{ + struct iommu_map_entry *entry, *head; + + for (head = unit->tlb_flush_head;; head = entry) { + entry = (struct iommu_map_entry *) + atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next); + if (entry == NULL || + !dmar_qi_seq_processed(unit, &entry->gseq)) + break; + unit->tlb_flush_head = entry; + iommu_gas_free_entry(head); + if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) + iommu_gas_free_region(entry); + else + iommu_gas_free_space(entry); + } +} + +static void dmar_qi_task(void *arg, int pending __unused) { struct dmar_unit *unit; - struct iommu_map_entry *entry, *head; uint32_t ics; unit = arg; + dmar_qi_drain_tlb_flush(unit); /* * Request an interrupt on the completion of the next invalidation @@ -428,23 +448,16 @@ dmar_qi_task(void *arg, int pending __unused) if ((ics & DMAR_ICS_IWC) != 0) { ics = DMAR_ICS_IWC; dmar_write4(unit, DMAR_ICS_REG, ics); - } - for (;;) { - head = unit->tlb_flush_head; - entry = (struct iommu_map_entry *) - atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next); - if (entry == NULL) - break; - if (!dmar_qi_seq_processed(unit, &entry->gseq)) - break; - unit->tlb_flush_head = entry; - iommu_gas_free_entry(head); - if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) - iommu_gas_free_region(entry); - else - iommu_gas_free_space(entry); + /* + * Drain a second time in case the DMAR processes an entry + * after the first call and before clearing DMAR_ICS_IWC. + * Otherwise, such entries will linger until a later entry + * that requests an interrupt is processed. + */ + dmar_qi_drain_tlb_flush(unit); } + if (unit->inv_seq_waiters > 0) { /* * Acquire the DMAR lock so that wakeup() is called only after |