AMD IOMMU: fix per-device IOMMU bypass when IR is enabled

When interrupt remapping (IR) is enabled, the device table entry
(DTE) for a given device will likely be initialized by
amdiommu_ir_find() during MSI configuration.  This function directly
calls amdiommu_get_ctx_for_dev() with id_mapped=false, which means that
any attempt to disable DMA remapping for the device (e.g. by setting
hw.busdma.pciD.B.S.F='bounce' in the loader tunables) will effectively
be ignored as the paging mode field in the DTE will not be set to
0 as required for identity mapping.  This will ultimately produce
an unusable device, because busdma will later observe the bounce
configuration through iommu_instantiate_ctx() and will employ the
non-translated 'bounce' busdma methods for the device, while the DTE
remains configured to enable translation.

Fix this by tweaking iommu_instantiate_ctx() to always return the
relevant per-device context object even if translation is disabled,
and adopt it in amdiommu_ir_find() instead of directly calling
amdiommu_get_ctx_for_dev().

Reviewed by:	kib
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D53209

2 files changed, 9 insertions, 7 deletions

diff --git a/sys/dev/iommu/busdma_iommu.c b/sys/dev/iommu/busdma_iommu.c
index 668ccf056463..82f73d469585 100644
--- a/sys/dev/iommu/busdma_iommu.c
+++ b/sys/dev/iommu/busdma_iommu.c
@@ -295,7 +295,6 @@ iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
 		} else {
 			iommu_free_ctx_locked(unit, ctx);
 		}
-		ctx = NULL;
 	}
 	return (ctx);
 }
@@ -303,6 +302,7 @@ iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
 struct iommu_ctx *
 iommu_get_dev_ctx(device_t dev)
 {
+	struct iommu_ctx *ctx;
 	struct iommu_unit *unit;
 
 	unit = iommu_find(dev, bootverbose);
@@ -313,7 +313,10 @@ iommu_get_dev_ctx(device_t dev)
 		return (NULL);
 
 	iommu_unit_pre_instantiate_ctx(unit);
-	return (iommu_instantiate_ctx(unit, dev, false));
+	ctx = iommu_instantiate_ctx(unit, dev, false);
+	if (ctx != NULL && (ctx->flags & IOMMU_CTX_DISABLED) != 0)
+		ctx = NULL;
+	return (ctx);
 }
 
 bus_dma_tag_t
diff --git a/sys/x86/iommu/amd_intrmap.c b/sys/x86/iommu/amd_intrmap.c
index f8900fe0561f..cce4f57ca323 100644
--- a/sys/x86/iommu/amd_intrmap.c
+++ b/sys/x86/iommu/amd_intrmap.c
@@ -223,9 +223,9 @@ static struct amdiommu_ctx *
 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu)
 {
 	devclass_t src_class;
-	device_t requester;
 	struct amdiommu_unit *unit;
 	struct amdiommu_ctx *ctx;
+	struct iommu_ctx *ioctx;
 	uint32_t edte;
 	uint16_t rid;
 	uint8_t dte;
@@ -255,10 +255,9 @@ amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu)
 		error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte,
 		    bootverbose);
 		if (error == 0) {
-			error = iommu_get_requester(src, &requester, &rid);
-			MPASS(error == 0);
-			ctx = amdiommu_get_ctx_for_dev(unit, src,
-			    rid, 0, false /* XXXKIB */, false, dte, edte);
+			ioctx = iommu_instantiate_ctx(AMD2IOMMU(unit), src, false);
+			if (ioctx != NULL)
+				ctx = IOCTX2CTX(ioctx);
 		}
 	}
 	if (ridp != NULL)