Introduce IOMMU support for arm64 platform.

This adds an arm64 iommu interface and a driver for Arm System Memory
Management Unit version 3.2 (ARM SMMU v3.2) specified in ARM IHI 0070C
document.

Hardware overview is provided in the header of smmu.c file.

The support is disabled by default. To enable add 'options IOMMU' to your
kernel configuration file.

The support was developed on Arm Neoverse N1 System Development Platform
(ARM N1SDP), kindly provided by ARM Ltd.

Currently, PCI-based devices and ACPI platforms are supported only.
The support was tested on IOMMU-enabled Marvell SATA controller,
Realtek Ethernet controller and a TI xHCI USB controller with a low to
medium load only.

Many thanks to Konstantin Belousov for help forming the generic IOMMU
framework that is vital for this project; to Andrew Turner for adding
IOMMU support to MSI interrupt code; to Mark Johnston for help with SMMU
page management; to John Baldwin for explaining various IOMMU bits.

Reviewed by:	mmel
Relnotes:	yes
Sponsored by:	DARPA / AFRL
Sponsored by:	Innovate UK (Digital Security by Design programme)
Differential Revision:	https://reviews.freebsd.org/D24618

10 files changed, 3530 insertions, 0 deletions

diff --git a/sys/arm64/include/iommu.h b/sys/arm64/include/iommu.h
new file mode 100644
index 000000000000..95b5c274ff1e
--- /dev/null
+++ b/sys/arm64/include/iommu.h
@@ -0,0 +1,11 @@
+/*-
+ * This file is in the public domain.
+ */
+/* $FreeBSD$ */
+
+#ifndef	_MACHINE_IOMMU_H_
+#define	_MACHINE_IOMMU_H_
+
+#include <arm64/iommu/iommu.h>
+
+#endif /* !_MACHINE_IOMMU_H_ */
diff --git a/sys/arm64/iommu/iommu.c b/sys/arm64/iommu/iommu.c
new file mode 100644
index 000000000000..0fad03c7cd8e
--- /dev/null
+++ b/sys/arm64/iommu/iommu.c
@@ -0,0 +1,397 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Portions of this work was supported by Innovate UK project 105694,
+ * "Digital Security by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+#include <vm/vm.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <machine/bus.h>
+#include <dev/iommu/busdma_iommu.h>
+#include <machine/vmparam.h>
+
+#include "iommu.h"
+#include "iommu_if.h"
+
+static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
+
+#define	IOMMU_LIST_LOCK()		mtx_lock(&iommu_mtx)
+#define	IOMMU_LIST_UNLOCK()		mtx_unlock(&iommu_mtx)
+#define	IOMMU_LIST_ASSERT_LOCKED()	mtx_assert(&iommu_mtx, MA_OWNED)
+
+#define dprintf(fmt, ...)
+
+static struct mtx iommu_mtx;
+
+struct iommu_entry {
+	struct iommu_unit *iommu;
+	LIST_ENTRY(iommu_entry) next;
+};
+static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
+
+static int
+iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
+    iommu_gaddr_t size, int flags)
+{
+	struct iommu_unit *iommu;
+	int error;
+
+	iommu = iodom->iommu;
+
+	error = IOMMU_UNMAP(iommu->dev, iodom, base, size);
+
+	return (error);
+}
+
+static int
+iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
+    iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
+{
+	struct iommu_unit *iommu;
+	vm_prot_t prot;
+	vm_offset_t va;
+	int error;
+
+	dprintf("%s: base %lx, size %lx\n", __func__, base, size);
+
+	prot = 0;
+	if (eflags & IOMMU_MAP_ENTRY_READ)
+		prot |= VM_PROT_READ;
+	if (eflags & IOMMU_MAP_ENTRY_WRITE)
+		prot |= VM_PROT_WRITE;
+
+	va = base;
+
+	iommu = iodom->iommu;
+
+	error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot);
+
+	return (0);
+}
+
+static const struct iommu_domain_map_ops domain_map_ops = {
+	.map = iommu_domain_map_buf,
+	.unmap = iommu_domain_unmap_buf,
+};
+
+static struct iommu_domain *
+iommu_domain_alloc(struct iommu_unit *iommu)
+{
+	struct iommu_domain *iodom;
+
+	iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu);
+	if (iodom == NULL)
+		return (NULL);
+
+	iommu_domain_init(iommu, iodom, &domain_map_ops);
+	iodom->end = VM_MAXUSER_ADDRESS;
+	iodom->iommu = iommu;
+	iommu_gas_init_domain(iodom);
+
+	return (iodom);
+}
+
+static int
+iommu_domain_free(struct iommu_domain *iodom)
+{
+	struct iommu_unit *iommu;
+
+	iommu = iodom->iommu;
+
+	IOMMU_LOCK(iommu);
+
+	if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
+		IOMMU_DOMAIN_LOCK(iodom);
+		iommu_gas_fini_domain(iodom);
+		IOMMU_DOMAIN_UNLOCK(iodom);
+	}
+
+	iommu_domain_fini(iodom);
+
+	IOMMU_DOMAIN_FREE(iommu->dev, iodom);
+	IOMMU_UNLOCK(iommu);
+
+	return (0);
+}
+
+static void
+iommu_tag_init(struct bus_dma_tag_iommu *t)
+{
+	bus_addr_t maxaddr;
+
+	maxaddr = BUS_SPACE_MAXADDR;
+
+	t->common.ref_count = 0;
+	t->common.impl = &bus_dma_iommu_impl;
+	t->common.alignment = 1;
+	t->common.boundary = 0;
+	t->common.lowaddr = maxaddr;
+	t->common.highaddr = maxaddr;
+	t->common.maxsize = maxaddr;
+	t->common.nsegments = BUS_SPACE_UNRESTRICTED;
+	t->common.maxsegsz = maxaddr;
+}
+
+static struct iommu_ctx *
+iommu_ctx_alloc(device_t dev, struct iommu_domain *iodom, bool disabled)
+{
+	struct iommu_unit *iommu;
+	struct iommu_ctx *ioctx;
+
+	iommu = iodom->iommu;
+
+	ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, dev, disabled);
+	if (ioctx == NULL)
+		return (NULL);
+
+	/*
+	 * iommu can also be used for non-PCI based devices.
+	 * This should be reimplemented as new newbus method with
+	 * pci_get_rid() as a default for PCI device class.
+	 */
+	ioctx->rid = pci_get_rid(dev);
+
+	return (ioctx);
+}
+
+struct iommu_ctx *
+iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
+    uint16_t rid, bool disabled, bool rmrr)
+{
+	struct iommu_ctx *ioctx;
+	struct iommu_domain *iodom;
+	struct bus_dma_tag_iommu *tag;
+
+	IOMMU_LOCK(iommu);
+	ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester);
+	if (ioctx) {
+		IOMMU_UNLOCK(iommu);
+		return (ioctx);
+	}
+	IOMMU_UNLOCK(iommu);
+
+	/*
+	 * In our current configuration we have a domain per each ctx.
+	 * So allocate a domain first.
+	 */
+	iodom = iommu_domain_alloc(iommu);
+	if (iodom == NULL)
+		return (NULL);
+
+	ioctx = iommu_ctx_alloc(requester, iodom, disabled);
+	if (ioctx == NULL) {
+		iommu_domain_free(iodom);
+		return (NULL);
+	}
+
+	tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu),
+	    M_IOMMU, M_WAITOK | M_ZERO);
+	tag->owner = requester;
+	tag->ctx = ioctx;
+	tag->ctx->domain = iodom;
+
+	iommu_tag_init(tag);
+
+	ioctx->domain = iodom;
+
+	return (ioctx);
+}
+
+void
+iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx)
+{
+	struct bus_dma_tag_iommu *tag;
+
+	IOMMU_ASSERT_LOCKED(iommu);
+
+	tag = ioctx->tag;
+
+	IOMMU_CTX_FREE(iommu->dev, ioctx);
+
+	free(tag, M_IOMMU);
+}
+
+void
+iommu_free_ctx(struct iommu_ctx *ioctx)
+{
+	struct iommu_unit *iommu;
+	struct iommu_domain *iodom;
+	int error;
+
+	iodom = ioctx->domain;
+	iommu = iodom->iommu;
+
+	IOMMU_LOCK(iommu);
+	iommu_free_ctx_locked(iommu, ioctx);
+	IOMMU_UNLOCK(iommu);
+
+	/* Since we have a domain per each ctx, remove the domain too. */
+	error = iommu_domain_free(iodom);
+	if (error)
+		device_printf(iommu->dev, "Could not free a domain\n");
+}
+
+static void
+iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
+{
+	struct iommu_domain *iodom;
+
+	iodom = entry->domain;
+
+	IOMMU_DOMAIN_LOCK(iodom);
+	iommu_gas_free_space(iodom, entry);
+	IOMMU_DOMAIN_UNLOCK(iodom);
+
+	if (free)
+		iommu_gas_free_entry(iodom, entry);
+	else
+		entry->flags = 0;
+}
+
+void
+iommu_domain_unload(struct iommu_domain *iodom,
+    struct iommu_map_entries_tailq *entries, bool cansleep)
+{
+	struct iommu_map_entry *entry, *entry1;
+	int error;
+
+	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
+		KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
+		    ("not mapped entry %p %p", iodom, entry));
+		error = iodom->ops->unmap(iodom, entry->start, entry->end -
+		    entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
+		KASSERT(error == 0, ("unmap %p error %d", iodom, error));
+		TAILQ_REMOVE(entries, entry, dmamap_link);
+		iommu_domain_free_entry(entry, true);
+        }
+
+	if (TAILQ_EMPTY(entries))
+		return;
+
+	panic("entries map is not empty");
+}
+
+int
+iommu_register(struct iommu_unit *iommu)
+{
+	struct iommu_entry *entry;
+
+	mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF);
+
+	entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO);
+	entry->iommu = iommu;
+
+	IOMMU_LIST_LOCK();
+	LIST_INSERT_HEAD(&iommu_list, entry, next);
+	IOMMU_LIST_UNLOCK();
+
+	iommu_init_busdma(iommu);
+
+	return (0);
+}
+
+int
+iommu_unregister(struct iommu_unit *iommu)
+{
+	struct iommu_entry *entry, *tmp;
+
+	IOMMU_LIST_LOCK();
+	LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) {
+		if (entry->iommu == iommu) {
+			LIST_REMOVE(entry, next);
+			free(entry, M_IOMMU);
+		}
+	}
+	IOMMU_LIST_UNLOCK();
+
+	iommu_fini_busdma(iommu);
+
+	mtx_destroy(&iommu->lock);
+
+	return (0);
+}
+
+struct iommu_unit *
+iommu_find(device_t dev, bool verbose)
+{
+	struct iommu_entry *entry;
+	struct iommu_unit *iommu;
+	int error;
+
+	IOMMU_LIST_LOCK();
+	LIST_FOREACH(entry, &iommu_list, next) {
+		iommu = entry->iommu;
+		error = IOMMU_FIND(iommu->dev, dev);
+		if (error == 0) {
+			IOMMU_LIST_UNLOCK();
+			return (entry->iommu);
+		}
+	}
+	IOMMU_LIST_UNLOCK();
+
+	return (NULL);
+}
+
+void
+iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free)
+{
+
+	dprintf("%s\n", __func__);
+
+	iommu_domain_free_entry(entry, free);
+}
+
+static void
+iommu_init(void)
+{
+
+	mtx_init(&iommu_mtx, "IOMMU", NULL, MTX_DEF);
+}
+
+SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);
diff --git a/sys/arm64/iommu/iommu.h b/sys/arm64/iommu/iommu.h
new file mode 100644
index 000000000000..2071173070df
--- /dev/null
+++ b/sys/arm64/iommu/iommu.h
@@ -0,0 +1,44 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_ARM64_IOMMU_IOMMU_H_
+#define	_ARM64_IOMMU_IOMMU_H_
+
+#define	IOMMU_PAGE_SIZE		4096
+#define	IOMMU_PAGE_MASK		(IOMMU_PAGE_SIZE - 1)
+
+int iommu_unregister(struct iommu_unit *unit);
+int iommu_register(struct iommu_unit *unit);
+
+#endif /* _ARM64_IOMMU_IOMMU_H_ */
diff --git a/sys/arm64/iommu/iommu_if.m b/sys/arm64/iommu/iommu_if.m
new file mode 100644
index 000000000000..ef8c3e7b57b8
--- /dev/null
+++ b/sys/arm64/iommu/iommu_if.m
@@ -0,0 +1,125 @@
+#-
+# SPDX-License-Identifier: BSD-2-Clause
+#:
+# Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+#
+# This software was developed by SRI International and the University of
+# Cambridge Computer Laboratory (Department of Computer Science and
+# Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+# DARPA SSITH research programme.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+#include <sys/types.h>
+#include <sys/taskqueue.h>
+#include <sys/bus.h>
+#include <sys/sysctl.h>
+#include <sys/tree.h>
+#include <vm/vm.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+INTERFACE iommu;
+
+#
+# Check if the iommu controller dev is responsible to serve traffic
+# for a given child.
+#
+METHOD int find {
+	device_t		dev;
+	device_t		child;
+};
+
+#
+# Map a virtual address VA to a physical address PA.
+#
+METHOD int map {
+	device_t		dev;
+	struct iommu_domain	*iodom;
+	vm_offset_t		va;
+	vm_page_t		*ma;
+	bus_size_t		size;
+	vm_prot_t		prot;
+};
+
+#
+# Unmap a virtual address VA.
+#
+METHOD int unmap {
+	device_t		dev;
+	struct iommu_domain	*iodom;
+	vm_offset_t		va;
+	bus_size_t		size;
+};
+
+#
+# Allocate an IOMMU domain.
+#
+METHOD struct iommu_domain * domain_alloc {
+	device_t		dev;
+	struct iommu_unit	*iommu;
+};
+
+#
+# Release all the resources held by IOMMU domain.
+#
+METHOD void domain_free {
+	device_t		dev;
+	struct iommu_domain	*iodom;
+};
+
+#
+# Find a domain allocated for a dev.
+#
+METHOD struct iommu_domain * domain_lookup {
+	device_t		dev;
+};
+
+#
+# Find an allocated context for a device.
+#
+METHOD struct iommu_ctx * ctx_lookup {
+	device_t		dev;
+	device_t		child;
+};
+
+#
+# Allocate a new iommu context.
+#
+METHOD struct iommu_ctx * ctx_alloc {
+	device_t		dev;
+	struct iommu_domain	*iodom;
+	device_t		child;
+	bool			disabled;
+};
+
+#
+# Free the iommu context.
+#
+METHOD void ctx_free {
+	device_t		dev;
+	struct iommu_ctx	*ioctx;
+};
diff --git a/sys/arm64/iommu/smmu.c b/sys/arm64/iommu/smmu.c
new file mode 100644
index 000000000000..9df6bcfc4351
--- /dev/null
+++ b/sys/arm64/iommu/smmu.c
@@ -0,0 +1,1937 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Hardware overview.
+ *
+ * An incoming transaction from a peripheral device has an address, size,
+ * attributes and StreamID.
+ *
+ * In case of PCI-based devices, StreamID is a PCI rid.
+ *
+ * The StreamID is used to select a Stream Table Entry (STE) in a Stream table,
+ * which contains per-device configuration.
+ *
+ * Stream table is a linear or 2-level walk table (this driver supports both).
+ * Note that a linear table could occupy 1GB or more of memory depending on
+ * sid_bits value.
+ *
+ * STE is used to locate a Context Descriptor, which is a struct in memory
+ * that describes stages of translation, translation table type, pointer to
+ * level 0 of page tables, ASID, etc.
+ *
+ * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2):
+ *  o S1 is used for the host machine traffic translation
+ *  o S2 is for a hypervisor
+ *
+ * This driver enables S1 stage with standard AArch64 page tables.
+ *
+ * Note that SMMU does not share TLB with a main CPU.
+ * Command queue is used by this driver to Invalidate SMMU TLB, STE cache.
+ *
+ * An arm64 SoC could have more than one SMMU instance.
+ * ACPI IORT table describes which SMMU unit is assigned for a particular
+ * peripheral device.
+ *
+ * Queues.
+ *
+ * Register interface and Memory-based circular buffer queues are used
+ * to inferface SMMU.
+ *
+ * These are a Command queue for commands to send to the SMMU and an Event
+ * queue for event/fault reports from the SMMU. Optionally PRI queue is
+ * designed for PCIe page requests reception.
+ *
+ * Note that not every hardware supports PRI services. For instance they were
+ * not found in Neoverse N1 SDP machine.
+ * (This drivers does not implement PRI queue.)
+ *
+ * All SMMU queues are arranged as circular buffers in memory. They are used
+ * in a producer-consumer fashion so that an output queue contains data
+ * produced by the SMMU and consumed by software.
+ * An input queue contains data produced by software, consumed by the SMMU.
+ *
+ * Interrupts.
+ *
+ * Interrupts are not required by this driver for normal operation.
+ * The standard wired interrupt is only triggered when an event comes from
+ * the SMMU, which is only in a case of errors (e.g. translation fault).
+ */
+
+#include "opt_platform.h"
+#include "opt_acpi.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/rman.h>
+#include <sys/lock.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#if DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+#endif
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include "iommu.h"
+#include "iommu_if.h"
+
+#include "smmureg.h"
+#include "smmuvar.h"
+
+#define	STRTAB_L1_SZ_SHIFT	20
+#define	STRTAB_SPLIT		8
+
+#define	STRTAB_L1_DESC_L2PTR_M	(0x3fffffffffff << 6)
+#define	STRTAB_L1_DESC_DWORDS	1
+
+#define	STRTAB_STE_DWORDS	8
+
+#define	CMDQ_ENTRY_DWORDS	2
+#define	EVTQ_ENTRY_DWORDS	4
+#define	PRIQ_ENTRY_DWORDS	2
+
+#define	CD_DWORDS		8
+
+#define	Q_WRP(q, p)		((p) & (1 << (q)->size_log2))
+#define	Q_IDX(q, p)		((p) & ((1 << (q)->size_log2) - 1))
+#define	Q_OVF(p)		((p) & (1 << 31)) /* Event queue overflowed */
+
+#define	SMMU_Q_ALIGN		(64 * 1024)
+
+static struct resource_spec smmu_spec[] = {
+	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
+	{ SYS_RES_IRQ, 0, RF_ACTIVE },
+	{ SYS_RES_IRQ, 1, RF_ACTIVE },
+	{ SYS_RES_IRQ, 2, RF_ACTIVE },
+	RESOURCE_SPEC_END
+};
+
+MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR);
+
+#define	dprintf(fmt, ...)
+
+struct smmu_event {
+	int ident;
+	char *str;
+	char *msg;
+};
+
+static struct smmu_event events[] = {
+	{ 0x01, "F_UUT",
+		"Unsupported Upstream Transaction."},
+	{ 0x02, "C_BAD_STREAMID",
+		"Transaction StreamID out of range."},
+	{ 0x03, "F_STE_FETCH",
+		"Fetch of STE caused external abort."},
+	{ 0x04, "C_BAD_STE",
+		"Used STE invalid."},
+	{ 0x05, "F_BAD_ATS_TREQ",
+		"Address Translation Request disallowed for a StreamID "
+		"and a PCIe ATS Translation Request received."},
+	{ 0x06, "F_STREAM_DISABLED",
+		"The STE of a transaction marks non-substream transactions "
+		"disabled."},
+	{ 0x07, "F_TRANSL_FORBIDDEN",
+		"An incoming PCIe transaction is marked Translated but "
+		"SMMU bypass is disallowed for this StreamID."},
+	{ 0x08, "C_BAD_SUBSTREAMID",
+		"Incoming SubstreamID present, but configuration is invalid."},
+	{ 0x09, "F_CD_FETCH",
+		"Fetch of CD caused external abort."},
+	{ 0x0a, "C_BAD_CD",
+		"Fetched CD invalid."},
+	{ 0x0b, "F_WALK_EABT",
+		"An external abort occurred fetching (or updating) "
+		"a translation table descriptor."},
+	{ 0x10, "F_TRANSLATION",
+		"Translation fault."},
+	{ 0x11, "F_ADDR_SIZE",
+		"Address Size fault."},
+	{ 0x12, "F_ACCESS",
+		"Access flag fault due to AF == 0 in a page or block TTD."},
+	{ 0x13, "F_PERMISSION",
+		"Permission fault occurred on page access."},
+	{ 0x20, "F_TLB_CONFLICT",
+		"A TLB conflict occurred because of the transaction."},
+	{ 0x21, "F_CFG_CONFLICT",
+		"A configuration cache conflict occurred due to "
+		"the transaction."},
+	{ 0x24, "E_PAGE_REQUEST",
+		"Speculative page request hint."},
+	{ 0x25, "F_VMS_FETCH",
+		"Fetch of VMS caused external abort."},
+	{ 0, NULL, NULL },
+};
+
+static int
+smmu_q_has_space(struct smmu_queue *q)
+{
+
+	/*
+	 * See 6.3.27 SMMU_CMDQ_PROD
+	 *
+	 * There is space in the queue for additional commands if:
+	 *  SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR ||
+	 *  SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP
+	 */
+
+	if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||
+	    Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
+		return (1);
+
+	return (0);
+}
+
+static int
+smmu_q_empty(struct smmu_queue *q)
+{
+
+	if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&
+	    Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
+		return (1);
+
+	return (0);
+}
+
+static int __unused
+smmu_q_consumed(struct smmu_queue *q, uint32_t prod)
+{
+
+	if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&
+	    (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))
+		return (1);
+
+	if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&
+	    (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))
+		return (1);
+
+	return (0);
+}
+
+static uint32_t
+smmu_q_inc_cons(struct smmu_queue *q)
+{
+	uint32_t cons;
+	uint32_t val;
+
+	cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;
+	val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));
+
+	return (val);
+}
+
+static uint32_t
+smmu_q_inc_prod(struct smmu_queue *q)
+{
+	uint32_t prod;
+	uint32_t val;
+
+	prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;
+	val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));
+
+	return (val);
+}
+
+static int
+smmu_write_ack(struct smmu_softc *sc, uint32_t reg,
+    uint32_t reg_ack, uint32_t val)
+{
+	uint32_t v;
+	int timeout;
+
+	timeout = 100000;
+
+	bus_write_4(sc->res[0], reg, val);
+
+	do {
+		v = bus_read_4(sc->res[0], reg_ack);
+		if (v == val)
+			break;
+	} while (timeout--);
+
+	if (timeout <= 0) {
+		device_printf(sc->dev, "Failed to write reg.\n");
+		return (-1);
+	}
+
+	return (0);
+}
+
+static inline int
+ilog2(long x)
+{
+
+	KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x));
+
+	return (flsl(x) - 1);
+}
+
+static int
+smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q,
+    uint32_t prod_off, uint32_t cons_off, uint32_t dwords)
+{
+	int sz;
+
+	sz = (1 << q->size_log2) * dwords * 8;
+
+	/* Set up the command circular buffer */
+	q->vaddr = contigmalloc(sz, M_SMMU,
+	    M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);
+	if (q->vaddr == NULL) {
+		device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
+		return (-1);
+	}
+
+	q->prod_off = prod_off;
+	q->cons_off = cons_off;
+	q->paddr = vtophys(q->vaddr);
+
+	q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;
+	q->base |= q->paddr & Q_BASE_ADDR_M;
+	q->base |= q->size_log2 << Q_LOG2SIZE_S;
+
+	return (0);
+}
+
+static int
+smmu_init_queues(struct smmu_softc *sc)
+{
+	int err;
+
+	/* Command queue. */
+	err = smmu_init_queue(sc, &sc->cmdq,
+	    SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS);
+	if (err)
+		return (ENXIO);
+
+	/* Event queue. */
+	err = smmu_init_queue(sc, &sc->evtq,
+	    SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS);
+	if (err)
+		return (ENXIO);
+
+	if (!(sc->features & SMMU_FEATURE_PRI))
+		return (0);
+
+	/* PRI queue. */
+	err = smmu_init_queue(sc, &sc->priq,
+	    SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS);
+	if (err)
+		return (ENXIO);
+
+	return (0);
+}
+
+/*
+ * Dump 2LVL or linear STE.
+ */
+static void
+smmu_dump_ste(struct smmu_softc *sc, int sid)
+{
+	struct smmu_strtab *strtab;
+	struct l1_desc *l1_desc;
+	uint64_t *ste, *l1;
+	int i;
+
+	strtab = &sc->strtab;
+
+	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+		i = sid >> STRTAB_SPLIT;
+		l1 = (void *)((uint64_t)strtab->vaddr +
+		    STRTAB_L1_DESC_DWORDS * 8 * i);
+		device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);
+
+		l1_desc = &strtab->l1[i];
+		ste = l1_desc->va;
+		if (ste == NULL) /* L2 is not initialized */
+			return;
+	} else {
+		ste = (void *)((uint64_t)strtab->vaddr +
+		    sid * (STRTAB_STE_DWORDS << 3));
+	}
+
+	/* Dump L2 or linear STE. */
+	for (i = 0; i < STRTAB_STE_DWORDS; i++)
+		device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);
+}
+
+static void __unused
+smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd)
+{
+	uint64_t *vaddr;
+	int i;
+
+	device_printf(sc->dev, "%s\n", __func__);
+
+	vaddr = cd->vaddr;
+	for (i = 0; i < CD_DWORDS; i++)
+		device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);
+}
+
+static void
+smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt)
+{
+	struct smmu_queue *evtq;
+	void *entry_addr;
+
+	evtq = &sc->evtq;
+
+	evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);
+	entry_addr = (void *)((uint64_t)evtq->vaddr +
+	    evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);
+	memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8);
+	evtq->lc.cons = smmu_q_inc_cons(evtq);
+	bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);
+}
+
+static void
+smmu_print_event(struct smmu_softc *sc, uint32_t *evt)
+{
+	struct smmu_event *ev;
+	uintptr_t input_addr;
+	uint8_t event_id;
+	device_t dev;
+	int sid;
+	int i;
+
+	dev = sc->dev;
+
+	ev = NULL;
+	event_id = evt[0] & 0xff;
+	for (i = 0; events[i].ident != 0; i++) {
+		if (events[i].ident == event_id) {
+			ev = &events[i];
+			break;
+		}
+	}
+
+	sid = evt[1];
+	input_addr = evt[5];
+	input_addr <<= 32;
+	input_addr |= evt[4];
+
+	if (smmu_quirks_check(dev, sid, event_id, input_addr)) {
+		/* The event is known. Don't print anything. */
+		return;
+	}
+
+	if (ev) {
+		device_printf(sc->dev,
+		    "Event %s (%s) received.\n", ev->str, ev->msg);
+	} else
+		device_printf(sc->dev, "Event 0x%x received\n", event_id);
+
+	device_printf(sc->dev, "SID %x, Input Address: %jx\n",
+	    sid, input_addr);
+
+	for (i = 0; i < 8; i++)
+		device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);
+
+	smmu_dump_ste(sc, sid);
+}
+
+static void
+make_cmd(struct smmu_softc *sc, uint64_t *cmd,
+    struct smmu_cmdq_entry *entry)
+{
+
+	memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8);
+	cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;
+
+	switch (entry->opcode) {
+	case CMD_TLBI_NH_VA:
+		cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
+		cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;
+		if (entry->tlbi.leaf) {
+			/*
+			 * Leaf flag means that only cached entries
+			 * for the last level of translation table walk
+			 * are required to be invalidated.
+			 */
+			cmd[1] |= TLBI_1_LEAF;
+		}
+		break;
+	case CMD_TLBI_NH_ASID:
+		cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
+		break;
+	case CMD_TLBI_NSNH_ALL:
+	case CMD_TLBI_NH_ALL:
+	case CMD_TLBI_EL2_ALL:
+		break;
+	case CMD_CFGI_CD:
+		cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);
+		/* FALLTROUGH */
+	case CMD_CFGI_STE:
+		cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);
+		cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);
+		break;
+	case CMD_CFGI_STE_RANGE:
+		cmd[1] = (31 << CFGI_1_STE_RANGE_S);
+		break;
+	case CMD_SYNC:
+		cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB;
+		if (entry->sync.msiaddr) {
+			cmd[0] |= SYNC_0_CS_SIG_IRQ;
+			cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);
+		} else
+			cmd[0] |= SYNC_0_CS_SIG_SEV;
+		break;
+	case CMD_PREFETCH_CONFIG:
+		cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);
+		break;
+	};
+}
+
+static void
+smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry)
+{
+	uint64_t cmd[CMDQ_ENTRY_DWORDS];
+	struct smmu_queue *cmdq;
+	void *entry_addr;
+
+	cmdq = &sc->cmdq;
+
+	make_cmd(sc, cmd, entry);
+
+	SMMU_LOCK(sc);
+
+	/* Ensure that a space is available. */
+	do {
+		cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);
+	} while (smmu_q_has_space(cmdq) == 0);
+
+	/* Write the command to the current prod entry. */
+	entry_addr = (void *)((uint64_t)cmdq->vaddr +
+	    Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);
+	memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8);
+
+	/* Increment prod index. */
+	cmdq->lc.prod = smmu_q_inc_prod(cmdq);
+	bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);
+
+	SMMU_UNLOCK(sc);
+}
+
+static void __unused
+smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q)
+{
+
+	while (1) {
+		q->lc.val = bus_read_8(sc->res[0], q->prod_off);
+		if (smmu_q_empty(q))
+			break;
+		cpu_spinwait();
+	}
+}
+
+static int
+smmu_sync(struct smmu_softc *sc)
+{
+	struct smmu_cmdq_entry cmd;
+	struct smmu_queue *q;
+	uint32_t *base;
+	int timeout;
+	int prod;
+
+	q = &sc->cmdq;
+	prod = q->lc.prod;
+
+	/* Enqueue sync command. */
+	cmd.opcode = CMD_SYNC;
+	cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+
+	/* Wait for the sync completion. */
+	base = (void *)((uint64_t)q->vaddr +
+	    Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8);
+
+	/*
+	 * It takes around 200 loops (6 instructions each)
+	 * on Neoverse N1 to complete the sync.
+	 */
+	timeout = 10000;
+
+	do {
+		if (*base == 0) {
+			/* MSI write completed. */
+			break;
+		}
+		cpu_spinwait();
+	} while (timeout--);
+
+	if (timeout < 0)
+		device_printf(sc->dev, "Failed to sync\n");
+
+	return (0);
+}
+
+static int
+smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf)
+{
+	struct smmu_cmdq_entry cmd;
+
+	cmd.opcode = CMD_CFGI_CD;
+	cmd.cfgi.sid = sid;
+	cmd.cfgi.ssid = ssid;
+	cmd.cfgi.leaf = leaf;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+
+	return (0);
+}
+
+static void
+smmu_invalidate_all_sid(struct smmu_softc *sc)
+{
+	struct smmu_cmdq_entry cmd;
+
+	/* Invalidate cached config */
+	cmd.opcode = CMD_CFGI_STE_RANGE;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+	smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_all(struct smmu_softc *sc)
+{
+	struct smmu_cmdq_entry cmd;
+
+	/* Invalidate entire TLB */
+	cmd.opcode = CMD_TLBI_NSNH_ALL;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+	smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid)
+{
+	struct smmu_cmdq_entry cmd;
+
+	/* Invalidate TLB for an ASID. */
+	cmd.opcode = CMD_TLBI_NH_ASID;
+	cmd.tlbi.asid = asid;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+	smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid)
+{
+	struct smmu_cmdq_entry cmd;
+
+	/* Invalidate specific range */
+	cmd.opcode = CMD_TLBI_NH_VA;
+	cmd.tlbi.asid = asid;
+	cmd.tlbi.vmid = 0;
+	cmd.tlbi.leaf = true; /* We change only L3. */
+	cmd.tlbi.addr = va;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+}
+
+static void
+smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid)
+{
+	struct smmu_cmdq_entry cmd;
+
+	/* Invalidate cached config */
+	cmd.opcode = CMD_CFGI_STE;
+	cmd.cfgi.sid = sid;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+	smmu_sync(sc);
+}
+
+static void
+smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid)
+{
+	struct smmu_cmdq_entry cmd;
+
+	cmd.opcode = CMD_PREFETCH_CONFIG;
+	cmd.prefetch.sid = sid;
+	smmu_cmdq_enqueue_cmd(sc, &cmd);
+	smmu_sync(sc);
+}
+
+/*
+ * Init STE in bypass mode. Traffic is not translated for the sid.
+ */
+static void
+smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste)
+{
+	uint64_t val;
+
+	val = STE0_VALID | STE0_CONFIG_BYPASS;
+
+	ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS;
+	ste[2] = 0;
+	ste[3] = 0;
+	ste[4] = 0;
+	ste[5] = 0;
+	ste[6] = 0;
+	ste[7] = 0;
+
+	smmu_invalidate_sid(sc, sid);
+	ste[0] = val;
+	dsb(sy);
+	smmu_invalidate_sid(sc, sid);
+
+	smmu_prefetch_sid(sc, sid);
+}
+
+/*
+ * Enable Stage1 (S1) translation for the sid.
+ */
+static int
+smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd,
+    uint32_t sid, uint64_t *ste)
+{
+	uint64_t val;
+
+	val = STE0_VALID;
+
+	/* S1 */
+	ste[1] = STE1_EATS_FULLATS	|
+		 STE1_S1CSH_IS		|
+		 STE1_S1CIR_WBRA	|
+		 STE1_S1COR_WBRA	|
+		 STE1_STRW_NS_EL1;
+	ste[2] = 0;
+	ste[3] = 0;
+	ste[4] = 0;
+	ste[5] = 0;
+	ste[6] = 0;
+	ste[7] = 0;
+
+	if (sc->features & SMMU_FEATURE_STALL &&
+	    ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))
+		ste[1] |= STE1_S1STALLD;
+
+	/* Configure STE */
+	val |= (cd->paddr & STE0_S1CONTEXTPTR_M);
+	val |= STE0_CONFIG_S1_TRANS;
+
+	smmu_invalidate_sid(sc, sid);
+
+	/* The STE[0] has to be written in a single blast, last of all. */
+	ste[0] = val;
+	dsb(sy);
+
+	smmu_invalidate_sid(sc, sid);
+	smmu_sync_cd(sc, sid, 0, true);
+	smmu_invalidate_sid(sc, sid);
+
+	/* The sid will be used soon most likely. */
+	smmu_prefetch_sid(sc, sid);
+
+	return (0);
+}
+
+static int
+smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass)
+{
+	struct smmu_strtab *strtab;
+	struct l1_desc *l1_desc;
+	uint64_t *addr;
+
+	strtab = &sc->strtab;
+
+	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+		l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+		addr = l1_desc->va;
+		addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
+	} else {
+		addr = (void *)((uint64_t)strtab->vaddr +
+		    STRTAB_STE_DWORDS * 8 * sid);
+	};
+
+	if (bypass)
+		smmu_init_ste_bypass(sc, sid, addr);
+	else
+		smmu_init_ste_s1(sc, cd, sid, addr);
+
+	smmu_sync(sc);
+
+	return (0);
+}
+
+static int
+smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain)
+{
+	vm_paddr_t paddr;
+	uint64_t *ptr;
+	uint64_t val;
+	vm_size_t size;
+	struct smmu_cd *cd;
+	pmap_t p;
+
+	size = 1 * (CD_DWORDS << 3);
+
+	p = &domain->p;
+	cd = domain->cd = malloc(sizeof(struct smmu_cd),
+	    M_SMMU, M_WAITOK | M_ZERO);
+
+	cd->vaddr = contigmalloc(size, M_SMMU,
+	    M_WAITOK | M_ZERO,	/* flags */
+	    0,			/* low */
+	    (1ul << 40) - 1,	/* high */
+	    size,		/* alignment */
+	    0);			/* boundary */
+	if (cd->vaddr == NULL) {
+		device_printf(sc->dev, "Failed to allocate CD\n");
+		return (ENXIO);
+	}
+
+	cd->size = size;
+	cd->paddr = vtophys(cd->vaddr);
+
+	ptr = cd->vaddr;
+
+	val = CD0_VALID;
+	val |= CD0_AA64;
+	val |= CD0_R;
+	val |= CD0_A;
+	val |= CD0_ASET;
+	val |= (uint64_t)domain->asid << CD0_ASID_S;
+	val |= CD0_TG0_4KB;
+	val |= CD0_EPD1; /* Disable TT1 */
+	val |= ((64 - sc->ias) << CD0_T0SZ_S);
+	val |= CD0_IPS_48BITS;
+
+	paddr = p->pm_l0_paddr & CD1_TTB0_M;
+	KASSERT(paddr == p->pm_l0_paddr, ("bad allocation 1"));
+
+	ptr[1] = paddr;
+	ptr[2] = 0;
+	ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE)	|
+		 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE)	|
+		 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK)	|
+		 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH);
+
+	/* Install the CD. */
+	ptr[0] = val;
+
+	return (0);
+}
+
+static int
+smmu_init_strtab_linear(struct smmu_softc *sc)
+{
+	struct smmu_strtab *strtab;
+	vm_paddr_t base;
+	uint32_t size;
+	uint64_t reg;
+
+	strtab = &sc->strtab;
+	strtab->num_l1_entries = (1 << sc->sid_bits);
+
+	size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);
+
+	if (bootverbose)
+		device_printf(sc->dev,
+		    "%s: linear strtab size %d, num_l1_entries %d\n",
+		    __func__, size, strtab->num_l1_entries);
+
+	strtab->vaddr = contigmalloc(size, M_SMMU,
+	    M_WAITOK | M_ZERO,	/* flags */
+	    0,			/* low */
+	    (1ul << 48) - 1,	/* high */
+	    size,		/* alignment */
+	    0);			/* boundary */
+	if (strtab->vaddr == NULL) {
+		device_printf(sc->dev, "failed to allocate strtab\n");
+		return (ENXIO);
+	}
+
+	reg = STRTAB_BASE_CFG_FMT_LINEAR;
+	reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;
+	strtab->base_cfg = (uint32_t)reg;
+
+	base = vtophys(strtab->vaddr);
+
+	reg = base & STRTAB_BASE_ADDR_M;
+	KASSERT(reg == base, ("bad allocation 2"));
+	reg |= STRTAB_BASE_RA;
+	strtab->base = reg;
+
+	return (0);
+}
+
+static int
+smmu_init_strtab_2lvl(struct smmu_softc *sc)
+{
+	struct smmu_strtab *strtab;
+	vm_paddr_t base;
+	uint64_t reg_base;
+	uint32_t l1size;
+	uint32_t size;
+	uint32_t reg;
+	int sz;
+
+	strtab = &sc->strtab;
+
+	size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
+	size = min(size, sc->sid_bits - STRTAB_SPLIT);
+	strtab->num_l1_entries = (1 << size);
+	size += STRTAB_SPLIT;
+
+	l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);
+
+	if (bootverbose)
+		device_printf(sc->dev,
+		    "%s: size %d, l1 entries %d, l1size %d\n",
+		    __func__, size, strtab->num_l1_entries, l1size);
+
+	strtab->vaddr = contigmalloc(l1size, M_SMMU,
+	    M_WAITOK | M_ZERO,	/* flags */
+	    0,			/* low */
+	    (1ul << 48) - 1,	/* high */
+	    l1size,		/* alignment */
+	    0);			/* boundary */
+	if (strtab->vaddr == NULL) {
+		device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");
+		return (ENOMEM);
+	}
+
+	sz = strtab->num_l1_entries * sizeof(struct l1_desc);
+
+	strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);
+	if (strtab->l1 == NULL) {
+		contigfree(strtab->vaddr, l1size, M_SMMU);
+		return (ENOMEM);
+	}
+
+	reg = STRTAB_BASE_CFG_FMT_2LVL;
+	reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S;
+	reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S;
+	strtab->base_cfg = (uint32_t)reg;
+
+	base = vtophys(strtab->vaddr);
+
+	reg_base = base & STRTAB_BASE_ADDR_M;
+	KASSERT(reg_base == base, ("bad allocation 3"));
+	reg_base |= STRTAB_BASE_RA;
+	strtab->base = reg_base;
+
+	return (0);
+}
+
+static int
+smmu_init_strtab(struct smmu_softc *sc)
+{
+	int error;
+
+	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)
+		error = smmu_init_strtab_2lvl(sc);
+	else
+		error = smmu_init_strtab_linear(sc);
+
+	return (error);
+}
+
+static int
+smmu_init_l1_entry(struct smmu_softc *sc, int sid)
+{
+	struct smmu_strtab *strtab;
+	struct l1_desc *l1_desc;
+	uint64_t *addr;
+	uint64_t val;
+	size_t size;
+	int i;
+
+	strtab = &sc->strtab;
+	l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+
+	size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
+
+	l1_desc->span = STRTAB_SPLIT + 1;
+	l1_desc->size = size;
+	l1_desc->va = contigmalloc(size, M_SMMU,
+	    M_WAITOK | M_ZERO,	/* flags */
+	    0,			/* low */
+	    (1ul << 48) - 1,	/* high */
+	    size,		/* alignment */
+	    0);			/* boundary */
+	if (l1_desc->va == NULL) {
+		device_printf(sc->dev, "failed to allocate l2 entry\n");
+		return (ENXIO);
+	}
+
+	l1_desc->pa = vtophys(l1_desc->va);
+
+	i = sid >> STRTAB_SPLIT;
+	addr = (void *)((uint64_t)strtab->vaddr +
+	    STRTAB_L1_DESC_DWORDS * 8 * i);
+
+	/* Install the L1 entry. */
+	val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;
+	KASSERT(val == l1_desc->pa, ("bad allocation 4"));
+	val |= l1_desc->span;
+	*addr = val;
+
+	return (0);
+}
+
+static void
+smmu_deinit_l1_entry(struct smmu_softc *sc, int sid)
+{
+	struct smmu_strtab *strtab;
+	struct l1_desc *l1_desc;
+	uint64_t *addr;
+	int i;
+
+	strtab = &sc->strtab;
+
+	i = sid >> STRTAB_SPLIT;
+	addr = (void *)((uint64_t)strtab->vaddr +
+	    STRTAB_L1_DESC_DWORDS * 8 * i);
+	*addr = 0;
+
+	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+		l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+		contigfree(l1_desc->va, l1_desc->size, M_SMMU);
+	}
+}
+
+static int
+smmu_disable(struct smmu_softc *sc)
+{
+	uint32_t reg;
+	int error;
+
+	/* Disable SMMU */
+	reg = bus_read_4(sc->res[0], SMMU_CR0);
+	reg &= ~CR0_SMMUEN;
+	error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+	if (error)
+		device_printf(sc->dev, "Could not disable SMMU.\n");
+
+	return (0);
+}
+
+static int
+smmu_event_intr(void *arg)
+{
+	uint32_t evt[EVTQ_ENTRY_DWORDS * 2];
+	struct smmu_softc *sc;
+
+	sc = arg;
+
+	do {
+		smmu_evtq_dequeue(sc, evt);
+		smmu_print_event(sc, evt);
+	} while (!smmu_q_empty(&sc->evtq));
+
+	return (FILTER_HANDLED);
+}
+
+static int __unused
+smmu_sync_intr(void *arg)
+{
+	struct smmu_softc *sc;
+
+	sc = arg;
+
+	device_printf(sc->dev, "%s\n", __func__);
+
+	return (FILTER_HANDLED);
+}
+
+static int
+smmu_gerr_intr(void *arg)
+{
+	struct smmu_softc *sc;
+
+	sc = arg;
+
+	device_printf(sc->dev, "SMMU Global Error\n");
+
+	return (FILTER_HANDLED);
+}
+
+static int
+smmu_enable_interrupts(struct smmu_softc *sc)
+{
+	uint32_t reg;
+	int error;
+
+	/* Disable MSI. */
+	bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);
+	bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);
+	bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);
+
+	bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);
+	bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);
+	bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);
+
+	if (sc->features & CR0_PRIQEN) {
+		bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);
+		bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);
+		bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);
+	}
+
+	/* Disable any interrupts. */
+	error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0);
+	if (error) {
+		device_printf(sc->dev, "Could not disable interrupts.\n");
+		return (ENXIO);
+	}
+
+	/* Enable interrupts. */
+	reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
+	if (sc->features & SMMU_FEATURE_PRI)
+		reg |= IRQ_CTRL_PRIQ_IRQEN;
+
+	error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg);
+	if (error) {
+		device_printf(sc->dev, "Could not enable interrupts.\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+#if DEV_ACPI
+static void
+smmu_configure_intr(struct smmu_softc *sc, struct resource *res)
+{
+	struct intr_map_data_acpi *ad;
+	struct intr_map_data *data;
+
+	data = rman_get_virtual(res);
+	KASSERT(data != NULL, ("data is NULL"));
+
+	if (data->type == INTR_MAP_DATA_ACPI) {
+		ad = (struct intr_map_data_acpi *)data;
+		ad->trig = INTR_TRIGGER_EDGE;
+		ad->pol = INTR_POLARITY_HIGH;
+	}
+}
+#endif
+
+static int
+smmu_setup_interrupts(struct smmu_softc *sc)
+{
+	device_t dev;
+	int error;
+
+	dev = sc->dev;
+
+#if DEV_ACPI
+	/*
+	 * Configure SMMU interrupts as EDGE triggered manually
+	 * as ACPI tables carries no information for that.
+	 */
+	smmu_configure_intr(sc, sc->res[1]);
+	smmu_configure_intr(sc, sc->res[2]);
+	smmu_configure_intr(sc, sc->res[3]);
+#endif
+
+	error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
+	    smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);
+	if (error) {
+		device_printf(dev, "Couldn't setup Event interrupt handler\n");
+		return (ENXIO);
+	}
+
+	error = bus_setup_intr(dev, sc->res[3], INTR_TYPE_MISC,
+	    smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);
+	if (error) {
+		device_printf(dev, "Couldn't setup Gerr interrupt handler\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static int
+smmu_reset(struct smmu_softc *sc)
+{
+	struct smmu_cmdq_entry cmd;
+	struct smmu_strtab *strtab;
+	int error;
+	int reg;
+
+	reg = bus_read_4(sc->res[0], SMMU_CR0);
+
+	if (reg & CR0_SMMUEN)
+		device_printf(sc->dev,
+		    "%s: Warning: SMMU is enabled\n", __func__);
+
+	error = smmu_disable(sc);
+	if (error)
+		device_printf(sc->dev,
+		    "%s: Could not disable SMMU.\n", __func__);
+
+	if (smmu_enable_interrupts(sc) != 0) {
+		device_printf(sc->dev, "Could not enable interrupts.\n");
+		return (ENXIO);
+	}
+
+	reg = CR1_TABLE_SH_IS	|
+	      CR1_TABLE_OC_WBC	|
+	      CR1_TABLE_IC_WBC	|
+	      CR1_QUEUE_SH_IS	|
+	      CR1_QUEUE_OC_WBC	|
+	      CR1_QUEUE_IC_WBC;
+	bus_write_4(sc->res[0], SMMU_CR1, reg);
+
+	reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
+	bus_write_4(sc->res[0], SMMU_CR2, reg);
+
+	/* Stream table. */
+	strtab = &sc->strtab;
+	bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);
+	bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);
+
+	/* Command queue. */
+	bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);
+	bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);
+	bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);
+
+	reg = CR0_CMDQEN;
+	error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+	if (error) {
+		device_printf(sc->dev, "Could not enable command queue\n");
+		return (ENXIO);
+	}
+
+	/* Invalidate cached configuration. */
+	smmu_invalidate_all_sid(sc);
+
+	if (sc->features & SMMU_FEATURE_HYP) {
+		cmd.opcode = CMD_TLBI_EL2_ALL;
+		smmu_cmdq_enqueue_cmd(sc, &cmd);
+	};
+
+	/* Invalidate TLB. */
+	smmu_tlbi_all(sc);
+
+	/* Event queue */
+	bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);
+	bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);
+	bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);
+
+	reg |= CR0_EVENTQEN;
+	error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+	if (error) {
+		device_printf(sc->dev, "Could not enable event queue\n");
+		return (ENXIO);
+	}
+
+	if (sc->features & SMMU_FEATURE_PRI) {
+		/* PRI queue */
+		bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);
+		bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);
+		bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);
+
+		reg |= CR0_PRIQEN;
+		error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+		if (error) {
+			device_printf(sc->dev, "Could not enable PRI queue\n");
+			return (ENXIO);
+		}
+	}
+
+	if (sc->features & SMMU_FEATURE_ATS) {
+		reg |= CR0_ATSCHK;
+		error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+		if (error) {
+			device_printf(sc->dev, "Could not enable ATS check.\n");
+			return (ENXIO);
+		}
+	}
+
+	reg |= CR0_SMMUEN;
+	error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+	if (error) {
+		device_printf(sc->dev, "Could not enable SMMU.\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static int
+smmu_check_features(struct smmu_softc *sc)
+{
+	uint32_t reg;
+	uint32_t val;
+
+	sc->features = 0;
+
+	reg = bus_read_4(sc->res[0], SMMU_IDR0);
+
+	if (reg & IDR0_ST_LVL_2) {
+		if (bootverbose)
+			device_printf(sc->dev,
+			    "2-level stream table supported.\n");
+		sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;
+	}
+
+	if (reg & IDR0_CD2L) {
+		if (bootverbose)
+			device_printf(sc->dev,
+			    "2-level CD table supported.\n");
+		sc->features |= SMMU_FEATURE_2_LVL_CD;
+	}
+
+	switch (reg & IDR0_TTENDIAN_M) {
+	case IDR0_TTENDIAN_MIXED:
+		if (bootverbose)
+			device_printf(sc->dev, "Mixed endianess supported.\n");
+		sc->features |= SMMU_FEATURE_TT_LE;
+		sc->features |= SMMU_FEATURE_TT_BE;
+		break;
+	case IDR0_TTENDIAN_LITTLE:
+		if (bootverbose)
+			device_printf(sc->dev,
+			    "Little endian supported only.\n");
+		sc->features |= SMMU_FEATURE_TT_LE;
+		break;
+	case IDR0_TTENDIAN_BIG:
+		if (bootverbose)
+			device_printf(sc->dev, "Big endian supported only.\n");
+		sc->features |= SMMU_FEATURE_TT_BE;
+		break;
+	default:
+		device_printf(sc->dev, "Unsupported endianness.\n");
+		return (ENXIO);
+	}
+
+	if (reg & IDR0_SEV)
+		sc->features |= SMMU_FEATURE_SEV;
+
+	if (reg & IDR0_MSI) {
+		if (bootverbose)
+			device_printf(sc->dev, "MSI feature present.\n");
+		sc->features |= SMMU_FEATURE_MSI;
+	}
+
+	if (reg & IDR0_HYP) {
+		if (bootverbose)
+			device_printf(sc->dev, "HYP feature present.\n");
+		sc->features |= SMMU_FEATURE_HYP;
+	}
+
+	if (reg & IDR0_ATS)
+		sc->features |= SMMU_FEATURE_ATS;
+
+	if (reg & IDR0_PRI)
+		sc->features |= SMMU_FEATURE_PRI;
+
+	switch (reg & IDR0_STALL_MODEL_M) {
+	case IDR0_STALL_MODEL_FORCE:
+		/* Stall is forced. */
+		sc->features |= SMMU_FEATURE_STALL_FORCE;
+		/* FALLTHROUGH */
+	case IDR0_STALL_MODEL_STALL:
+		sc->features |= SMMU_FEATURE_STALL;
+		break;
+	}
+
+	/* Grab translation stages supported. */
+	if (reg & IDR0_S1P) {
+		if (bootverbose)
+			device_printf(sc->dev,
+			    "Stage 1 translation supported.\n");
+		sc->features |= SMMU_FEATURE_S1P;
+	}
+	if (reg & IDR0_S2P) {
+		if (bootverbose)
+			device_printf(sc->dev,
+			    "Stage 2 translation supported.\n");
+		sc->features |= SMMU_FEATURE_S2P;
+	}
+
+	switch (reg & IDR0_TTF_M) {
+	case IDR0_TTF_ALL:
+	case IDR0_TTF_AA64:
+		sc->ias = 40;
+		break;
+	default:
+		device_printf(sc->dev, "No AArch64 table format support.\n");
+		return (ENXIO);
+	}
+
+	if (reg & IDR0_ASID16)
+		sc->asid_bits = 16;
+	else
+		sc->asid_bits = 8;
+
+	if (bootverbose)
+		device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);
+
+	if (reg & IDR0_VMID16)
+		sc->vmid_bits = 16;
+	else
+		sc->vmid_bits = 8;
+
+	reg = bus_read_4(sc->res[0], SMMU_IDR1);
+
+	if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
+		device_printf(sc->dev,
+		    "Embedded implementations not supported by this driver.\n");
+		return (ENXIO);
+	}
+
+	val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S;
+	sc->cmdq.size_log2 = val;
+	if (bootverbose)
+		device_printf(sc->dev, "CMD queue bits %d\n", val);
+
+	val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S;
+	sc->evtq.size_log2 = val;
+	if (bootverbose)
+		device_printf(sc->dev, "EVENT queue bits %d\n", val);
+
+	if (sc->features & SMMU_FEATURE_PRI) {
+		val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S;
+		sc->priq.size_log2 = val;
+		if (bootverbose)
+			device_printf(sc->dev, "PRI queue bits %d\n", val);
+	}
+
+	sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;
+	sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;
+
+	if (sc->sid_bits <= STRTAB_SPLIT)
+		sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;
+
+	if (bootverbose) {
+		device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);
+		device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);
+	}
+
+	/* IDR3 */
+	reg = bus_read_4(sc->res[0], SMMU_IDR3);
+	if (reg & IDR3_RIL)
+		sc->features |= SMMU_FEATURE_RANGE_INV;
+
+	/* IDR5 */
+	reg = bus_read_4(sc->res[0], SMMU_IDR5);
+
+	switch (reg & IDR5_OAS_M) {
+	case IDR5_OAS_32:
+		sc->oas = 32;
+		break;
+	case IDR5_OAS_36:
+		sc->oas = 36;
+		break;
+	case IDR5_OAS_40:
+		sc->oas = 40;
+		break;
+	case IDR5_OAS_42:
+		sc->oas = 42;
+		break;
+	case IDR5_OAS_44:
+		sc->oas = 44;
+		break;
+	case IDR5_OAS_48:
+		sc->oas = 48;
+		break;
+	case IDR5_OAS_52:
+		sc->oas = 52;
+		break;
+	}
+
+	sc->pgsizes = 0;
+	if (reg & IDR5_GRAN64K)
+		sc->pgsizes |= 64 * 1024;
+	if (reg & IDR5_GRAN16K)
+		sc->pgsizes |= 16 * 1024;
+	if (reg & IDR5_GRAN4K)
+		sc->pgsizes |= 4 * 1024;
+
+	if ((reg & IDR5_VAX_M) == IDR5_VAX_52)
+		sc->features |= SMMU_FEATURE_VAX;
+
+	return (0);
+}
+
+static void
+smmu_init_asids(struct smmu_softc *sc)
+{
+
+	sc->asid_set_size = (1 << sc->asid_bits);
+	sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);
+	mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);
+}
+
+static int
+smmu_asid_alloc(struct smmu_softc *sc, int *new_asid)
+{
+
+	mtx_lock_spin(&sc->asid_set_mutex);
+	bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);
+	if (*new_asid == -1) {
+		mtx_unlock_spin(&sc->asid_set_mutex);
+		return (ENOMEM);
+	}
+	bit_set(sc->asid_set, *new_asid);
+	mtx_unlock_spin(&sc->asid_set_mutex);
+
+	return (0);
+}
+
+static void
+smmu_asid_free(struct smmu_softc *sc, int asid)
+{
+
+	mtx_lock_spin(&sc->asid_set_mutex);
+	bit_clear(sc->asid_set, asid);
+	mtx_unlock_spin(&sc->asid_set_mutex);
+}
+
+/*
+ * Device interface.
+ */
+int
+smmu_attach(device_t dev)
+{
+	struct smmu_softc *sc;
+	int error;
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);
+
+	error = bus_alloc_resources(dev, smmu_spec, sc->res);
+	if (error) {
+		device_printf(dev, "Couldn't allocate resources.\n");
+		return (ENXIO);
+	}
+
+	error = smmu_setup_interrupts(sc);
+	if (error) {
+		bus_release_resources(dev, smmu_spec, sc->res);
+		return (ENXIO);
+	}
+
+	error = smmu_check_features(sc);
+	if (error) {
+		device_printf(dev, "Some features are required "
+		    "but not supported by hardware.\n");
+		return (ENXIO);
+	}
+
+	smmu_init_asids(sc);
+
+	error = smmu_init_queues(sc);
+	if (error) {
+		device_printf(dev, "Couldn't allocate queues.\n");
+		return (ENXIO);
+	}
+
+	error = smmu_init_strtab(sc);
+	if (error) {
+		device_printf(dev, "Couldn't allocate strtab.\n");
+		return (ENXIO);
+	}
+
+	error = smmu_reset(sc);
+	if (error) {
+		device_printf(dev, "Couldn't reset SMMU.\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+int
+smmu_detach(device_t dev)
+{
+	struct smmu_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	bus_release_resources(dev, smmu_spec, sc->res);
+
+	return (0);
+}
+
+static int
+smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+	struct smmu_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	device_printf(sc->dev, "%s\n", __func__);
+
+	return (ENOENT);
+}
+
+static int
+smmu_unmap(device_t dev, struct iommu_domain *iodom,
+    vm_offset_t va, bus_size_t size)
+{
+	struct smmu_domain *domain;
+	struct smmu_softc *sc;
+	int err;
+	int i;
+
+	sc = device_get_softc(dev);
+
+	domain = (struct smmu_domain *)iodom;
+
+	err = 0;
+
+	dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		if (pmap_sremove(&domain->p, va) == 0) {
+			/* pmap entry removed, invalidate TLB. */
+			smmu_tlbi_va(sc, va, domain->asid);
+		} else {
+			err = ENOENT;
+			break;
+		}
+		va += PAGE_SIZE;
+	}
+
+	smmu_sync(sc);
+
+	return (err);
+}
+
+static int
+smmu_map(device_t dev, struct iommu_domain *iodom,
+    vm_offset_t va, vm_page_t *ma, vm_size_t size,
+    vm_prot_t prot)
+{
+	struct smmu_domain *domain;
+	struct smmu_softc *sc;
+	vm_paddr_t pa;
+	int error;
+	int i;
+
+	sc = device_get_softc(dev);
+
+	domain = (struct smmu_domain *)iodom;
+
+	dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,
+	    domain->asid);
+
+	for (i = 0; size > 0; size -= PAGE_SIZE) {
+		pa = VM_PAGE_TO_PHYS(ma[i++]);
+		error = pmap_senter(&domain->p, va, pa, prot, 0);
+		if (error)
+			return (error);
+		smmu_tlbi_va(sc, va, domain->asid);
+		va += PAGE_SIZE;
+	}
+
+	smmu_sync(sc);
+
+	return (0);
+}
+
+static struct iommu_domain *
+smmu_domain_alloc(device_t dev, struct iommu_unit *iommu)
+{
+	struct smmu_domain *domain;
+	struct smmu_unit *unit;
+	struct smmu_softc *sc;
+	int error;
+	int new_asid;
+
+	sc = device_get_softc(dev);
+
+	unit = (struct smmu_unit *)iommu;
+
+	domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO);
+
+	error = smmu_asid_alloc(sc, &new_asid);
+	if (error) {
+		free(domain, M_SMMU);
+		device_printf(sc->dev,
+		    "Could not allocate ASID for a new domain.\n");
+		return (NULL);
+	}
+
+	domain->asid = (uint16_t)new_asid;
+
+	pmap_pinit(&domain->p);
+	PMAP_LOCK_INIT(&domain->p);
+
+	error = smmu_init_cd(sc, domain);
+	if (error) {
+		free(domain, M_SMMU);
+		device_printf(sc->dev, "Could not initialize CD\n");
+		return (NULL);
+	}
+
+	smmu_tlbi_asid(sc, domain->asid);
+
+	LIST_INIT(&domain->ctx_list);
+
+	IOMMU_LOCK(iommu);
+	LIST_INSERT_HEAD(&unit->domain_list, domain, next);
+	IOMMU_UNLOCK(iommu);
+
+	return (&domain->iodom);
+}
+
+static void
+smmu_domain_free(device_t dev, struct iommu_domain *iodom)
+{
+	struct smmu_domain *domain;
+	struct smmu_softc *sc;
+	struct smmu_cd *cd;
+
+	sc = device_get_softc(dev);
+
+	domain = (struct smmu_domain *)iodom;
+
+	LIST_REMOVE(domain, next);
+
+	cd = domain->cd;
+
+	pmap_sremove_pages(&domain->p);
+	pmap_release(&domain->p);
+
+	smmu_tlbi_asid(sc, domain->asid);
+	smmu_asid_free(sc, domain->asid);
+
+	contigfree(cd->vaddr, cd->size, M_SMMU);
+	free(cd, M_SMMU);
+
+	free(domain, M_SMMU);
+}
+
+static int
+smmu_set_buswide(device_t dev, struct smmu_domain *domain,
+    struct smmu_ctx *ctx)
+{
+	struct smmu_softc *sc;
+	int i;
+
+	sc = device_get_softc(dev);
+
+	for (i = 0; i < PCI_SLOTMAX; i++)
+		smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass);
+
+	return (0);
+}
+
+static struct iommu_ctx *
+smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child,
+    bool disabled)
+{
+	struct smmu_domain *domain;
+	struct smmu_softc *sc;
+	struct smmu_ctx *ctx;
+	uint16_t rid;
+	u_int xref, sid;
+	int seg;
+	int err;
+
+	sc = device_get_softc(dev);
+	domain = (struct smmu_domain *)iodom;
+
+	seg = pci_get_domain(child);
+	rid = pci_get_rid(child);
+	err = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
+	if (err)
+		return (NULL);
+
+	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+		err = smmu_init_l1_entry(sc, sid);
+		if (err)
+			return (NULL);
+	}
+
+	ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO);
+	ctx->vendor = pci_get_vendor(child);
+	ctx->device = pci_get_device(child);
+	ctx->dev = child;
+	ctx->sid = sid;
+	ctx->domain = domain;
+	if (disabled)
+		ctx->bypass = true;
+
+	/*
+	 * Neoverse N1 SDP:
+	 * 0x800 xhci
+	 * 0x700 re
+	 * 0x600 sata
+	 */
+
+	smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);
+
+	if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))
+		smmu_set_buswide(dev, domain, ctx);
+
+	IOMMU_DOMAIN_LOCK(iodom);
+	LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
+	IOMMU_DOMAIN_UNLOCK(iodom);
+
+	return (&ctx->ioctx);
+}
+
+static void
+smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx)
+{
+	struct smmu_softc *sc;
+	struct smmu_ctx *ctx;
+
+	IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
+
+	sc = device_get_softc(dev);
+	ctx = (struct smmu_ctx *)ioctx;
+
+	smmu_deinit_l1_entry(sc, ctx->sid);
+
+	LIST_REMOVE(ctx, next);
+
+	free(ctx, M_SMMU);
+}
+
+struct smmu_ctx *
+smmu_ctx_lookup_by_sid(device_t dev, u_int sid)
+{
+	struct smmu_softc *sc;
+	struct smmu_domain *domain;
+	struct smmu_unit *unit;
+	struct smmu_ctx *ctx;
+
+	sc = device_get_softc(dev);
+
+	unit = &sc->unit;
+
+	LIST_FOREACH(domain, &unit->domain_list, next) {
+		LIST_FOREACH(ctx, &domain->ctx_list, next) {
+			if (ctx->sid == sid)
+				return (ctx);
+		}
+	}
+
+	return (NULL);
+}
+
+static struct iommu_ctx *
+smmu_ctx_lookup(device_t dev, device_t child)
+{
+	struct iommu_unit *iommu;
+	struct smmu_softc *sc;
+	struct smmu_domain *domain;
+	struct smmu_unit *unit;
+	struct smmu_ctx *ctx;
+
+	sc = device_get_softc(dev);
+
+	unit = &sc->unit;
+	iommu = &unit->iommu;
+
+	IOMMU_ASSERT_LOCKED(iommu);
+
+	LIST_FOREACH(domain, &unit->domain_list, next) {
+		IOMMU_DOMAIN_LOCK(&domain->iodom);
+		LIST_FOREACH(ctx, &domain->ctx_list, next) {
+			if (ctx->dev == child) {
+				IOMMU_DOMAIN_UNLOCK(&domain->iodom);
+				return (&ctx->ioctx);
+			}
+		}
+		IOMMU_DOMAIN_UNLOCK(&domain->iodom);
+	}
+
+	return (NULL);
+}
+
+static int
+smmu_find(device_t dev, device_t child)
+{
+	struct smmu_softc *sc;
+	u_int xref, sid;
+	uint16_t rid;
+	int error;
+	int seg;
+
+	sc = device_get_softc(dev);
+
+	rid = pci_get_rid(child);
+	seg = pci_get_domain(child);
+
+	/*
+	 * Find an xref of an IOMMU controller that serves traffic for dev.
+	 */
+#ifdef DEV_ACPI
+	error = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
+	if (error) {
+		/* Could not find reference to an SMMU device. */
+		return (ENOENT);
+	}
+#else
+	/* TODO: add FDT support. */
+	return (ENXIO);
+#endif
+
+	/* Check if xref is ours. */
+	if (xref != sc->xref)
+		return (EFAULT);
+
+	return (0);
+}
+
+static device_method_t smmu_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_detach,	smmu_detach),
+
+	/* SMMU interface */
+	DEVMETHOD(iommu_find,		smmu_find),
+	DEVMETHOD(iommu_map,		smmu_map),
+	DEVMETHOD(iommu_unmap,		smmu_unmap),
+	DEVMETHOD(iommu_domain_alloc,	smmu_domain_alloc),
+	DEVMETHOD(iommu_domain_free,	smmu_domain_free),
+	DEVMETHOD(iommu_ctx_alloc,	smmu_ctx_alloc),
+	DEVMETHOD(iommu_ctx_free,	smmu_ctx_free),
+	DEVMETHOD(iommu_ctx_lookup,	smmu_ctx_lookup),
+
+	/* Bus interface */
+	DEVMETHOD(bus_read_ivar,	smmu_read_ivar),
+
+	/* End */
+	DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc));
diff --git a/sys/arm64/iommu/smmu_acpi.c b/sys/arm64/iommu/smmu_acpi.c
new file mode 100644
index 000000000000..d7a2c0444fdf
--- /dev/null
+++ b/sys/arm64/iommu/smmu_acpi.c
@@ -0,0 +1,255 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_acpi.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/bitstring.h>
+#include <sys/kernel.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include <arm64/iommu/iommu.h>
+
+#include "smmuvar.h"
+
+#define	MEMORY_RESOURCE_SIZE	0x20000
+#define	MAX_SMMU		8
+
+struct smmu_acpi_devinfo {
+	struct resource_list	di_rl;
+};
+
+struct iort_table_data {
+	device_t parent;
+	device_t dev;
+	ACPI_IORT_SMMU_V3 *smmu[MAX_SMMU];
+	int count;
+};
+
+static void
+iort_handler(ACPI_SUBTABLE_HEADER *entry, void *arg)
+{
+	struct iort_table_data *iort_data;
+	ACPI_IORT_NODE *node;
+	int i;
+
+	iort_data = (struct iort_table_data *)arg;
+	i = iort_data->count;
+
+	switch(entry->Type) {
+	case ACPI_IORT_NODE_SMMU_V3:
+		if (i == MAX_SMMU) {
+			printf("SMMUv3 found, but no space available.\n");
+			break;
+		}
+
+		if (iort_data->smmu[i] != NULL) {
+			if (bootverbose)
+				device_printf(iort_data->parent,
+				    "smmu: Already have an SMMU table");
+			break;
+		}
+		node = (ACPI_IORT_NODE *)entry;
+		iort_data->smmu[i] = (ACPI_IORT_SMMU_V3 *)node->NodeData;
+		iort_data->count++;
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+smmu_acpi_identify(driver_t *driver, device_t parent)
+{
+	struct iort_table_data iort_data;
+	ACPI_TABLE_IORT *iort;
+	vm_paddr_t iort_pa;
+	uintptr_t priv;
+	device_t dev;
+	int i;
+
+	iort_pa = acpi_find_table(ACPI_SIG_IORT);
+	if (iort_pa == 0)
+		return;
+
+	iort = acpi_map_table(iort_pa, ACPI_SIG_IORT);
+	if (iort == NULL) {
+		device_printf(parent, "smmu: Unable to map the IORT\n");
+		return;
+	}
+
+	iort_data.parent = parent;
+	for (i = 0; i < MAX_SMMU; i++)
+		iort_data.smmu[i] = NULL;
+	iort_data.count = 0;
+
+	acpi_walk_subtables(iort + 1, (char *)iort + iort->Header.Length,
+	    iort_handler, &iort_data);
+	if (iort_data.count == 0) {
+		device_printf(parent, "No SMMU found.\n");
+		goto out;
+	}
+
+	for (i = 0; i < iort_data.count; i++) {
+		dev = BUS_ADD_CHILD(parent,
+		    BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, "smmu", -1);
+		if (dev == NULL) {
+			device_printf(parent, "add smmu child failed\n");
+			goto out;
+		}
+
+		/* Add the IORT data */
+		BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 0,
+		    iort_data.smmu[i]->EventGsiv, 1);
+		BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 1,
+		    iort_data.smmu[i]->SyncGsiv, 1);
+		BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 2,
+		    iort_data.smmu[i]->GerrGsiv, 1);
+		BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0,
+		    iort_data.smmu[i]->BaseAddress, MEMORY_RESOURCE_SIZE);
+
+		priv = iort_data.smmu[i]->Flags;
+		priv <<= 32;
+		priv |= iort_data.smmu[i]->Model;
+
+		acpi_set_private(dev, (void *)priv);
+	}
+
+	iort_data.dev = dev;
+
+out:
+	acpi_unmap_table(iort);
+}
+
+static int
+smmu_acpi_probe(device_t dev)
+{
+
+	switch((uintptr_t)acpi_get_private(dev) & 0xffffffff) {
+	case ACPI_IORT_SMMU_V3_GENERIC:
+		/* Generic SMMUv3 */
+		break;
+	default:
+		return (ENXIO);
+	}
+
+	device_set_desc(dev, SMMU_DEVSTR);
+
+	return (BUS_PROBE_NOWILDCARD);
+}
+
+static int
+smmu_acpi_attach(device_t dev)
+{
+	struct smmu_softc *sc;
+	struct smmu_unit *unit;
+	struct iommu_unit *iommu;
+	uintptr_t priv;
+	int err;
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	priv = (uintptr_t)acpi_get_private(dev);
+	if ((priv >> 32) & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
+		sc->features |= SMMU_FEATURE_COHERENCY;
+
+	if (bootverbose)
+		device_printf(sc->dev, "%s: features %x\n",
+		    __func__, sc->features);
+
+	err = smmu_attach(dev);
+	if (err != 0)
+		goto error;
+
+	unit = &sc->unit;
+	unit->dev = dev;
+
+	iommu = &unit->iommu;
+	iommu->dev = dev;
+
+	LIST_INIT(&unit->domain_list);
+
+	/* Use memory start address as an xref. */
+	sc->xref = bus_get_resource_start(dev, SYS_RES_MEMORY, 0);
+
+	err = iommu_register(iommu);
+	if (err) {
+		device_printf(dev, "Failed to register SMMU.\n");
+		return (ENXIO);
+	}
+
+	return (0);
+
+error:
+	if (bootverbose) {
+		device_printf(dev,
+		    "Failed to attach. Error %d\n", err);
+	}
+	/* Failure so free resources. */
+	smmu_detach(dev);
+
+	return (err);
+}
+
+static device_method_t smmu_acpi_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_identify,		smmu_acpi_identify),
+	DEVMETHOD(device_probe,			smmu_acpi_probe),
+	DEVMETHOD(device_attach,		smmu_acpi_attach),
+
+	/* End */
+	DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(smmu, smmu_acpi_driver, smmu_acpi_methods,
+    sizeof(struct smmu_softc), smmu_driver);
+
+static devclass_t smmu_acpi_devclass;
+
+EARLY_DRIVER_MODULE(smmu, acpi, smmu_acpi_driver, smmu_acpi_devclass,
+    0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
diff --git a/sys/arm64/iommu/smmu_quirks.c b/sys/arm64/iommu/smmu_quirks.c
new file mode 100644
index 000000000000..bc7237c0730c
--- /dev/null
+++ b/sys/arm64/iommu/smmu_quirks.c
@@ -0,0 +1,88 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bitstring.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <sys/sysctl.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include "smmuvar.h"
+
+struct smmu_quirk_entry {
+	uint16_t vendor;
+	uint16_t device;
+	const char *name;
+	uint8_t event_id;
+	uintptr_t input_address;
+};
+
+/* List of events that are known and will be silenced. */
+static const struct smmu_quirk_entry smmu_quirk_table[] = {
+	{ 0x10ec, 0x8168, "RealTek 8168/8111", 0x10 /* F_TRANSLATION */, 0x0 },
+	{ 0, 0, NULL, 0, 0 },
+};
+
+bool
+smmu_quirks_check(device_t dev, u_int sid, uint8_t event_id,
+    uintptr_t input_addr)
+{
+	const struct smmu_quirk_entry *q;
+	struct smmu_ctx *ctx;
+	int i;
+
+	ctx = smmu_ctx_lookup_by_sid(dev, sid);
+	if (!ctx)
+		return (false);
+
+	for (i = 0; smmu_quirk_table[i].vendor != 0; i++) {
+		q = &smmu_quirk_table[i];
+		if (ctx->vendor == q->vendor &&
+		    ctx->device == q->device &&
+		    input_addr == q->input_address &&
+		    event_id == q->event_id)
+			return (true);
+	}
+
+	return (false);
+}
diff --git a/sys/arm64/iommu/smmureg.h b/sys/arm64/iommu/smmureg.h
new file mode 100644
index 000000000000..416b9f44dfce
--- /dev/null
+++ b/sys/arm64/iommu/smmureg.h
@@ -0,0 +1,478 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_ARM64_IOMMU_SMMUREG_H_
+#define	_ARM64_IOMMU_SMMUREG_H_
+
+#define	SMMU_IDR0		0x000
+#define	 IDR0_ST_LVL_S		27
+#define	 IDR0_ST_LVL_M		(0x3 << IDR0_ST_LVL_S)
+#define	 IDR0_ST_LVL_LINEAR	(0x0 << IDR0_ST_LVL_S) /* Linear Stream table*/
+#define	 IDR0_ST_LVL_2		(0x1 << IDR0_ST_LVL_S) /* 2-level Stream Table*/
+#define	 IDR0_ST_TERM_MODEL	(1 << 26) /* Terminate model behavior */
+#define	 IDR0_STALL_MODEL_S	24 /*  Stall model support */
+#define	 IDR0_STALL_MODEL_M	(0x3 << IDR0_STALL_MODEL_S)
+#define	 IDR0_STALL_MODEL_STALL	(0x0 << IDR0_STALL_MODEL_S) /* Stall and Term*/
+#define	 IDR0_STALL_MODEL_FORCE	(0x2 << IDR0_STALL_MODEL_S) /* Stall is forced*/
+#define	 IDR0_TTENDIAN_S	21 /* Endianness for translation table walks.*/
+#define	 IDR0_TTENDIAN_M	(0x3 << IDR0_TTENDIAN_S)
+#define	 IDR0_TTENDIAN_MIXED	(0x0 << IDR0_TTENDIAN_S)
+#define	 IDR0_TTENDIAN_LITTLE	(0x2 << IDR0_TTENDIAN_S)
+#define	 IDR0_TTENDIAN_BIG	(0x3 << IDR0_TTENDIAN_S)
+#define	 IDR0_VATOS		(1 << 20) / * Virtual ATOS page interface */
+#define	 IDR0_CD2L		(1 << 19) /* 2-level Context descriptor table*/
+#define	 IDR0_VMID16		(1 << 18) /* 16-bit VMID supported */
+#define	 IDR0_VMW		(1 << 17) /* VMID wildcard-matching */
+#define	 IDR0_PRI		(1 << 16) /* Page Request Interface supported*/
+#define	 IDR0_ATOS		(1 << 15) /* Address Translation Operations */
+#define	 IDR0_SEV		(1 << 14) /* WFE wake-up events */
+#define	 IDR0_MSI		(1 << 13) /* Message Signalled Interrupts */
+#define	 IDR0_ASID16		(1 << 12) /* 16-bit ASID supported */
+#define	 IDR0_NS1ATS		(1 << 11) /* Split-stage ATS not supported */
+#define	 IDR0_ATS		(1 << 10) /* PCIe ATS supported by SMMU */
+#define	 IDR0_HYP		(1 << 9) /* Hypervisor stage 1 contexts */
+#define	 IDR0_DORMHINT		(1 << 8) /* Dormant hint supported */
+#define	 IDR0_HTTU_S		6 /* H/W transl. table A-flag and Dirty state */
+#define	 IDR0_HTTU_M		(0x3 << IDR0_HTTU_S)
+#define	 IDR0_HTTU_A		(0x1 << IDR0_HTTU_S) /* Access flag (A-flag) */
+#define	 IDR0_HTTU_AD		(0x2 << IDR0_HTTU_S) /* A-flag and Dirty State*/
+#define	 IDR0_BTM		(1 << 5) /* Broadcast TLB Maintenance */
+#define	 IDR0_COHACC		(1 << 4) /* Coherent access to translations*/
+#define	 IDR0_TTF_S		2 /* Translation Table Formats supported */
+#define	 IDR0_TTF_M		(0x3 << IDR0_TTF_S)
+#define	 IDR0_TTF_AA32		(0x1 << IDR0_TTF_S) /* AArch32 (LPAE) */
+#define	 IDR0_TTF_AA64		(0x2 << IDR0_TTF_S) /* AArch64 */
+#define	 IDR0_TTF_ALL		(0x3 << IDR0_TTF_S) /* AArch32 and AArch64 */
+#define	 IDR0_S1P		(1 << 1) /* Stage1 translation supported. */
+#define	 IDR0_S2P		(1 << 0) /* Stage2 translation supported. */
+#define	SMMU_IDR1		0x004
+#define	 IDR1_TABLES_PRESET	(1 << 30) /* Table base addresses fixed. */
+#define	 IDR1_QUEUES_PRESET	(1 << 29) /* Queue base addresses fixed. */
+#define	 IDR1_REL		(1 << 28) /* Relative base pointers */
+#define	 IDR1_ATTR_TYPES_OVR	(1 << 27) /* Incoming attrs can be overridden*/
+#define	 IDR1_ATTR_PERMS_OVR	(1 << 26) /* Incoming attrs can be overridden*/
+#define	 IDR1_CMDQS_S		21 /* Maximum number of Command queue entries*/
+#define	 IDR1_CMDQS_M		(0x1f << IDR1_CMDQS_S)
+#define	 IDR1_EVENTQS_S		16 /* Maximum number of Event queue entries */
+#define	 IDR1_EVENTQS_M		(0x1f << IDR1_EVENTQS_S)
+#define	 IDR1_PRIQS_S		11 /* Maximum number of PRI queue entries */
+#define	 IDR1_PRIQS_M		(0x1f << IDR1_PRIQS_S)
+#define	 IDR1_SSIDSIZE_S	6 /* Max bits of SubstreamID */
+#define	 IDR1_SSIDSIZE_M	(0x1f << IDR1_SSIDSIZE_S)
+#define	 IDR1_SIDSIZE_S		0 /* Max bits of StreamID */
+#define	 IDR1_SIDSIZE_M		(0x3f << IDR1_SIDSIZE_S)
+#define	SMMU_IDR2		0x008
+#define	SMMU_IDR3		0x00C
+#define	 IDR3_RIL		(1 << 10) /* Range-based Invalidations. */
+#define	SMMU_IDR4		0x010
+#define	SMMU_IDR5		0x014
+#define	 IDR5_STALL_MAX_S	16 /* Max outstanding stalled transactions */
+#define	 IDR5_STALL_MAX_M	(0xffff << IDR5_STALL_MAX_S)
+#define	 IDR5_VAX_S		10 /* Virtual Address eXtend */
+#define	 IDR5_VAX_M		(0x3 << IDR5_VAX_S)
+#define	 IDR5_VAX_48		(0 << IDR5_VAX_S)
+#define	 IDR5_VAX_52		(1 << IDR5_VAX_S)
+#define	 IDR5_GRAN64K		(1 << 6) /* 64KB translation granule */
+#define	 IDR5_GRAN16K		(1 << 5) /* 16KB translation granule */
+#define	 IDR5_GRAN4K		(1 << 4) /* 4KB translation granule */
+#define	 IDR5_OAS_S		0 /* Output Address Size */
+#define	 IDR5_OAS_M		(0x7 << IDR5_OAS_S)
+#define	 IDR5_OAS_32		(0x0 << IDR5_OAS_S)
+#define	 IDR5_OAS_36		(0x1 << IDR5_OAS_S)
+#define	 IDR5_OAS_40		(0x2 << IDR5_OAS_S)
+#define	 IDR5_OAS_42		(0x3 << IDR5_OAS_S)
+#define	 IDR5_OAS_44		(0x4 << IDR5_OAS_S)
+#define	 IDR5_OAS_48		(0x5 << IDR5_OAS_S)
+#define	 IDR5_OAS_52		(0x6 << IDR5_OAS_S) /* Reserved in SMMU v3.0 */
+#define	SMMU_IIDR		0x018
+#define	SMMU_AIDR		0x01C
+#define	SMMU_CR0		0x020
+#define	 CR0_VMW_S		6 /* VMID Wildcard */
+#define	 CR0_VMW_M		(0x7 << CR0_VMW_S)
+#define	 CR0_ATSCHK		(1 << 4) /* ATS behavior: Safe mode */
+#define	 CR0_CMDQEN		(1 << 3) /* Enable Command queue processing */
+#define	 CR0_EVENTQEN		(1 << 2) /* Enable Event queue writes */
+#define	 CR0_PRIQEN		(1 << 1) /* Enable PRI queue writes */
+#define	 CR0_SMMUEN		(1 << 0) /* Non-secure SMMU enable */
+#define	SMMU_CR0ACK		0x024
+#define	SMMU_CR1		0x028
+#define	 CR1_TABLE_SH_S		10 /* Table access Shareability. */
+#define	 CR1_TABLE_SH_M		(0x3 << CR1_TABLE_SH_S)
+#define	 CR1_TABLE_SH_NS	(0x0 << CR1_TABLE_SH_S)
+#define	 CR1_TABLE_SH_OS	(0x2 << CR1_TABLE_SH_S)
+#define	 CR1_TABLE_SH_IS	(0x3 << CR1_TABLE_SH_S)
+#define	 CR1_TABLE_OC_S		8 /* Table access Outer Cacheability. */
+#define	 CR1_TABLE_OC_M		(0x3 << CR1_TABLE_OC_S)
+#define	 CR1_TABLE_OC_NC	(0x0 << CR1_TABLE_OC_S)
+#define	 CR1_TABLE_OC_WBC	(0x1 << CR1_TABLE_OC_S)
+#define	 CR1_TABLE_OC_WTC	(0x2 << CR1_TABLE_OC_S)
+#define	 CR1_TABLE_IC_S		6 /* Table access Inner Cacheability. */
+#define	 CR1_TABLE_IC_M		(0x3 << CR1_TABLE_IC_S)
+#define	 CR1_TABLE_IC_NC	(0x0 << CR1_TABLE_IC_S)
+#define	 CR1_TABLE_IC_WBC	(0x1 << CR1_TABLE_IC_S)
+#define	 CR1_TABLE_IC_WTC	(0x2 << CR1_TABLE_IC_S)
+#define	 CR1_QUEUE_SH_S		4 /* Queue access Shareability. */
+#define	 CR1_QUEUE_SH_M		(0x3 << CR1_QUEUE_SH_S)
+#define	 CR1_QUEUE_SH_NS	(0x0 << CR1_QUEUE_SH_S)
+#define	 CR1_QUEUE_SH_OS	(0x2 << CR1_QUEUE_SH_S)
+#define	 CR1_QUEUE_SH_IS	(0x3 << CR1_QUEUE_SH_S)
+#define	 CR1_QUEUE_OC_S		2 /* Queue access Outer Cacheability. */
+#define	 CR1_QUEUE_OC_M		(0x3 << CR1_QUEUE_OC_S)
+#define	 CR1_QUEUE_OC_NC	(0x0 << CR1_QUEUE_OC_S)
+#define	 CR1_QUEUE_OC_WBC	(0x1 << CR1_QUEUE_OC_S)
+#define	 CR1_QUEUE_OC_WTC	(0x2 << CR1_QUEUE_OC_S)
+#define	 CR1_QUEUE_IC_S		0 /* Queue access Inner Cacheability. */
+#define	 CR1_QUEUE_IC_M		(0x3 << CR1_QUEUE_IC_S)
+#define	 CR1_QUEUE_IC_NC	(0x0 << CR1_QUEUE_IC_S)
+#define	 CR1_QUEUE_IC_WBC	(0x1 << CR1_QUEUE_IC_S)
+#define	 CR1_QUEUE_IC_WTC	(0x2 << CR1_QUEUE_IC_S)
+#define	SMMU_CR2		0x02C
+#define	 CR2_PTM		(1 << 2) /* Private TLB Maintenance. */
+#define	 CR2_RECINVSID		(1 << 1) /* Record invalid SID. */
+#define	 CR2_E2H		(1 << 0) /* Enable EL2-E2H translation regime */
+#define	SMMU_STATUSR		0x040
+#define	SMMU_GBPA		0x044
+#define	SMMU_AGBPA		0x048
+#define	SMMU_IRQ_CTRL		0x050
+#define	 IRQ_CTRL_EVENTQ_IRQEN	(1 << 2) /* NS Event queue interrupts enabled.*/
+#define	 IRQ_CTRL_PRIQ_IRQEN	(1 << 1) /* PRI queue interrupts are enabled.*/
+#define	 IRQ_CTRL_GERROR_IRQEN	(1 << 0) /* Global errors int are enabled. */
+#define	SMMU_IRQ_CTRLACK	0x054
+#define	SMMU_GERROR		0x060
+#define	SMMU_GERRORN		0x064
+#define	SMMU_GERROR_IRQ_CFG0	0x068
+#define	SMMU_GERROR_IRQ_CFG1	0x070
+#define	SMMU_GERROR_IRQ_CFG2	0x074
+#define	SMMU_STRTAB_BASE	0x080
+#define	 STRTAB_BASE_RA		(1UL << 62) /* Read-Allocate. */
+#define	 STRTAB_BASE_ADDR_S	6 /* Physical address of Stream table base */
+#define	 STRTAB_BASE_ADDR_M	(0x3fffffffffffUL << STRTAB_BASE_ADDR_S)
+#define	SMMU_STRTAB_BASE_CFG	0x088
+#define	 STRTAB_BASE_CFG_FMT_S		16 /* Format of Stream table. */
+#define	 STRTAB_BASE_CFG_FMT_M		(0x3 << STRTAB_BASE_CFG_FMT_S)
+#define	 STRTAB_BASE_CFG_FMT_LINEAR	(0x0 << STRTAB_BASE_CFG_FMT_S)
+#define	 STRTAB_BASE_CFG_FMT_2LVL	(0x1 << STRTAB_BASE_CFG_FMT_S)
+#define	 STRTAB_BASE_CFG_SPLIT_S	6 /* SID split point for 2lvl table. */
+#define	 STRTAB_BASE_CFG_SPLIT_M	(0x1f << STRTAB_BASE_CFG_SPLIT_S)
+#define	 STRTAB_BASE_CFG_SPLIT_4KB	(6 << STRTAB_BASE_CFG_SPLIT_S)
+#define	 STRTAB_BASE_CFG_SPLIT_16KB	(8 << STRTAB_BASE_CFG_SPLIT_S)
+#define	 STRTAB_BASE_CFG_SPLIT_64KB	(10 << STRTAB_BASE_CFG_SPLIT_S)
+#define	 STRTAB_BASE_CFG_LOG2SIZE_S	0 /* Table size as log2(entries) */
+#define	 STRTAB_BASE_CFG_LOG2SIZE_M	(0x3f << STRTAB_BASE_CFG_LOG2SIZE_S)
+#define	SMMU_CMDQ_BASE		0x090
+#define	 CMDQ_BASE_RA		(1UL << 62) /* Read-Allocate. */
+#define	 Q_BASE_ADDR_S		5 /* PA of queue base */
+#define	 Q_BASE_ADDR_M		(0x7fffffffffff << Q_BASE_ADDR_S)
+#define	 Q_LOG2SIZE_S		0 /* Queue size as log2(entries) */
+#define	 Q_LOG2SIZE_M		(0x1f << Q_LOG2SIZE_S)
+#define	SMMU_CMDQ_PROD		0x098
+#define	SMMU_CMDQ_CONS		0x09C
+#define	 CMDQ_CONS_ERR_S	24
+#define	 CMDQ_CONS_ERR_M	(0x7f << CMDQ_CONS_ERR_S)
+#define	SMMU_EVENTQ_BASE	0x0A0
+#define	 EVENTQ_BASE_WA		(1UL << 62) /* Write-Allocate. */
+#define	SMMU_EVENTQ_PROD	0x100A8
+#define	SMMU_EVENTQ_CONS	0x100AC
+#define	SMMU_EVENTQ_IRQ_CFG0	0x0B0
+#define	SMMU_EVENTQ_IRQ_CFG1	0x0B8
+#define	SMMU_EVENTQ_IRQ_CFG2	0x0BC
+#define	SMMU_PRIQ_BASE		0x0C0
+#define	 PRIQ_BASE_WA		(1UL < 62) /* Write-Allocate. */
+#define	SMMU_PRIQ_PROD		0x100C8
+#define	SMMU_PRIQ_CONS		0x100CC
+#define	SMMU_PRIQ_IRQ_CFG0	0x0D0
+#define	SMMU_PRIQ_IRQ_CFG1	0x0D8
+#define	SMMU_PRIQ_IRQ_CFG2	0x0DC
+#define	SMMU_GATOS_CTRL		0x100
+#define	SMMU_GATOS_SID		0x108
+#define	SMMU_GATOS_ADDR		0x110
+#define	SMMU_GATOS_PAR		0x118
+#define	SMMU_VATOS_SEL		0x180
+#define	SMMU_S_IDR0		0x8000
+#define	SMMU_S_IDR1		0x8004
+#define	SMMU_S_IDR2		0x8008
+#define	SMMU_S_IDR3		0x800C
+#define	SMMU_S_IDR4		0x8010
+#define	SMMU_S_CR0		0x8020
+#define	SMMU_S_CR0ACK		0x8024
+#define	SMMU_S_CR1		0x8028
+#define	SMMU_S_CR2		0x802C
+#define	SMMU_S_INIT		0x803C
+#define	SMMU_S_GBPA		0x8044
+#define	SMMU_S_AGBPA		0x8048
+#define	SMMU_S_IRQ_CTRL		0x8050
+#define	SMMU_S_IRQ_CTRLACK	0x8054
+#define	SMMU_S_GERROR		0x8060
+#define	SMMU_S_GERRORN		0x8064
+#define	SMMU_S_GERROR_IRQ_CFG0	0x8068
+#define	SMMU_S_GERROR_IRQ_CFG1	0x8070
+#define	SMMU_S_GERROR_IRQ_CFG2	0x8074
+#define	SMMU_S_STRTAB_BASE	0x8080
+#define	SMMU_S_STRTAB_BASE_CFG	0x8088
+#define	SMMU_S_CMDQ_BASE	0x8090
+#define	SMMU_S_CMDQ_PROD	0x8098
+#define	SMMU_S_CMDQ_CONS	0x809C
+#define	SMMU_S_EVENTQ_BASE	0x80A0
+#define	SMMU_S_EVENTQ_PROD	0x80A8
+#define	SMMU_S_EVENTQ_CONS	0x80AC
+#define	SMMU_S_EVENTQ_IRQ_CFG0	0x80B0
+#define	SMMU_S_EVENTQ_IRQ_CFG1	0x80B8
+#define	SMMU_S_EVENTQ_IRQ_CFG2	0x80BC
+#define	SMMU_S_GATOS_CTRL	0x8100
+#define	SMMU_S_GATOS_SID	0x8108
+#define	SMMU_S_GATOS_ADDR	0x8110
+#define	SMMU_S_GATOS_PAR	0x8118
+
+#define	CMD_QUEUE_OPCODE_S	0
+#define	CMD_QUEUE_OPCODE_M	(0xff << CMD_QUEUE_OPCODE_S)
+
+#define	CMD_PREFETCH_CONFIG	0x01
+#define	 PREFETCH_0_SID_S	32
+#define	CMD_PREFETCH_ADDR	0x02
+#define	CMD_CFGI_STE		0x03
+#define	 CFGI_0_STE_SID_S	32
+#define	CMD_CFGI_STE_RANGE	0x04
+#define	 CFGI_1_STE_RANGE_S	0
+#define	CMD_CFGI_CD		0x05
+#define	 CFGI_0_SSID_S		12
+#define	 CFGI_1_LEAF_S		0
+#define	CMD_CFGI_CD_ALL		0x06
+#define	CMD_TLBI_NH_ALL		0x10
+#define	CMD_TLBI_NH_ASID	0x11
+#define	CMD_TLBI_NH_VA		0x12
+#define	 TLBI_0_ASID_S		48
+#define	 TLBI_1_LEAF		(1 << 0)
+#define	 TLBI_1_ADDR_S		12
+#define	 TLBI_1_ADDR_M		(0xfffffffffffff << TLBI_1_ADDR_S)
+#define	CMD_TLBI_NH_VAA		0x13
+#define	CMD_TLBI_EL3_ALL	0x18
+#define	CMD_TLBI_EL3_VA		0x1A
+#define	CMD_TLBI_EL2_ALL	0x20
+#define	CMD_TLBI_EL2_ASID	0x21
+#define	CMD_TLBI_EL2_VA		0x22
+#define	CMD_TLBI_EL2_VAA	0x23
+#define	CMD_TLBI_S12_VMALL	0x28
+#define	CMD_TLBI_S2_IPA		0x2A
+#define	CMD_TLBI_NSNH_ALL	0x30
+#define	CMD_ATC_INV		0x40
+#define	CMD_PRI_RESP		0x41
+#define	CMD_RESUME		0x44
+#define	CMD_STALL_TERM		0x45
+#define	CMD_SYNC		0x46
+#define	 SYNC_0_CS_S		12 /* The ComplSignal */
+#define	 SYNC_0_CS_M		(0x3 << SYNC_0_CS_S)
+#define	 SYNC_0_CS_SIG_NONE	(0x0 << SYNC_0_CS_S)
+#define	 SYNC_0_CS_SIG_IRQ	(0x1 << SYNC_0_CS_S)
+#define	 SYNC_0_CS_SIG_SEV	(0x2 << SYNC_0_CS_S)
+#define	 SYNC_0_MSH_S		22 /* Shareability attribute for MSI write */
+#define	 SYNC_0_MSH_M		(0x3 << SYNC_0_MSH_S)
+#define	 SYNC_0_MSH_NS		(0x0 << SYNC_0_MSH_S) /* Non-shareable */
+#define	 SYNC_0_MSH_OS		(0x2 << SYNC_0_MSH_S) /* Outer Shareable */
+#define	 SYNC_0_MSH_IS		(0x3 << SYNC_0_MSH_S) /* Inner Shareable */
+#define	 SYNC_0_MSIATTR_S	24 /* Write attribute for MSI */
+#define	 SYNC_0_MSIATTR_M	(0xf << SYNC_0_MSIATTR_S)
+#define	 SYNC_0_MSIATTR_OIWB	(0xf << SYNC_0_MSIATTR_S)
+#define	 SYNC_0_MSIDATA_S	32
+#define	 SYNC_1_MSIADDRESS_S	2
+#define	 SYNC_1_MSIADDRESS_M	(0x3ffffffffffff << SYNC_1_MSIADDRESS_S)
+#define	STE0_VALID		(1 << 0) /* Structure contents are valid. */
+#define	STE0_CONFIG_S		1
+#define	STE0_CONFIG_M		(0x7 << STE0_CONFIG_S)
+#define	STE0_CONFIG_ABORT	(0x0 << STE0_CONFIG_S)
+#define	STE0_CONFIG_BYPASS	(0x4 << STE0_CONFIG_S)
+#define	STE0_CONFIG_S1_TRANS	(0x5 << STE0_CONFIG_S)
+#define	STE0_CONFIG_S2_TRANS	(0x6 << STE0_CONFIG_S)
+#define	STE0_CONFIG_ALL_TRANS	(0x7 << STE0_CONFIG_S)
+#define	STE0_S1FMT_S		4
+#define	STE0_S1FMT_M		(0x3 << STE0_S1FMT_S)
+#define	STE0_S1FMT_LINEAR	(0x0 << STE0_S1FMT_S)
+#define	STE0_S1FMT_4KB_L2	(0x1 << STE0_S1FMT_S)
+#define	STE0_S1FMT_64KB_L2	(0x2 << STE0_S1FMT_S)
+#define	STE0_S1CONTEXTPTR_S	6
+#define	STE0_S1CONTEXTPTR_M	(0x3fffffffffff << STE0_S1CONTEXTPTR_S)
+#define	STE0_S1CDMAX_S		59
+#define	STE0_S1CDMAX_M		(0x1f << STE0_S1CDMAX_S)
+
+#define	STE1_S1DSS_S		0
+#define	STE1_S1DSS_M		(0x3 << STE1_S1DSS_S)
+#define	STE1_S1DSS_TERMINATE	(0x0 << STE1_S1DSS_S)
+#define	STE1_S1DSS_BYPASS	(0x1 << STE1_S1DSS_S)
+#define	STE1_S1DSS_SUBSTREAM0	(0x2 << STE1_S1DSS_S)
+#define	STE1_S1CIR_S		2
+#define	STE1_S1CIR_M		(0x3 << STE1_S1CIR_S)
+#define	STE1_S1CIR_NC		(0x0 << STE1_S1CIR_S)
+#define	STE1_S1CIR_WBRA		(0x1 << STE1_S1CIR_S)
+#define	STE1_S1CIR_WT		(0x2 << STE1_S1CIR_S)
+#define	STE1_S1CIR_WB		(0x3 << STE1_S1CIR_S)
+#define	STE1_S1COR_S		4
+#define	STE1_S1COR_M		(0x3 << STE1_S1COR_S)
+#define	STE1_S1COR_NC		(0x0 << STE1_S1COR_S)
+#define	STE1_S1COR_WBRA		(0x1 << STE1_S1COR_S)
+#define	STE1_S1COR_WT		(0x2 << STE1_S1COR_S)
+#define	STE1_S1COR_WB		(0x3 << STE1_S1COR_S)
+#define	STE1_S1CSH_S		6
+#define	STE1_S1CSH_NS		(0x0 << STE1_S1CSH_S)
+#define	STE1_S1CSH_OS		(0x2 << STE1_S1CSH_S)
+#define	STE1_S1CSH_IS		(0x3 << STE1_S1CSH_S)
+#define	STE1_S2HWU59		(1 << 8)
+#define	STE1_S2HWU60		(1 << 9)
+#define	STE1_S2HWU61		(1 << 10)
+#define	STE1_S2HWU62		(1 << 11)
+#define	STE1_DRE		(1 << 12) /* Destructive Read Enable. */
+#define	STE1_CONT_S		13 /* Contiguous Hint */
+#define	STE1_CONT_M		(0xf << STE1_CONT_S)
+#define	STE1_DCP		(1 << 17) /* Directed Cache Prefetch. */
+#define	STE1_PPAR		(1 << 18) /* PRI Page request Auto Responses */
+#define	STE1_MEV		(1 << 19) /* Merge Events */
+#define	STE1_S1STALLD		(1 << 27) /* Stage 1 Stall Disable */
+#define	STE1_EATS_S		28 /* Enable PCIe ATS translation and traffic */
+#define	STE1_EATS_M		(0x3 << STE1_EATS_S)
+#define	STE1_EATS_ABORT		(0x0 << STE1_EATS_S)
+#define	STE1_EATS_FULLATS	(0x1 << STE1_EATS_S) /* Full ATS */
+#define	STE1_EATS_S1		(0x2 << STE1_EATS_S) /* Split-stage ATS */
+#define	STE1_STRW_S		30 /* StreamWorld control */
+#define	STE1_STRW_M		(0x3 << STE1_STRW_S)
+#define	STE1_STRW_NS_EL1	(0x0 << STE1_STRW_S)
+#define	STE1_STRW_NS_EL2	(0x2 << STE1_STRW_S)
+#define	STE1_MEMATTR_S		32
+#define	STE1_MTCFG		(1 << 36)
+#define	STE1_ALLOCCFG_S		37
+#define	STE1_SHCFG_S		44
+#define	STE1_SHCFG_M		(0x3UL << STE1_SHCFG_S)
+#define	STE1_SHCFG_NS		(0x0UL << STE1_SHCFG_S)
+#define	STE1_SHCFG_INCOMING	(0x1UL << STE1_SHCFG_S)
+#define	STE1_SHCFG_OS		(0x2UL << STE1_SHCFG_S)
+#define	STE1_SHCFG_IS		(0x3UL << STE1_SHCFG_S)
+#define	STE1_NSCFG_S		46
+#define	STE1_NSCFG_M		(0x3UL << STE1_NSCFG_S)
+#define	STE1_NSCFG_SECURE	(0x2UL << STE1_NSCFG_S)
+#define	STE1_NSCFG_NONSECURE	(0x3UL << STE1_NSCFG_S)
+#define	STE1_PRIVCFG_S		48
+#define	STE1_INSTCFG_S		50
+
+#define	STE2_S2VMID_S		0
+#define	STE2_S2VMID_M		(0xffff << STE2_S2VMID_S)
+#define	STE2_S2T0SZ_S		32 /* Size of IPA input region */
+#define	STE2_S2T0SZ_M		(0x3f << STE2_S2T0SZ_S)
+#define	STE2_S2SL0_S		38 /* Starting level of stage 2 tt walk */
+#define	STE2_S2SL0_M		(0x3 << STE2_S2SL0_S)
+#define	STE2_S2IR0_S		40
+#define	STE2_S2IR0_M		(0x3 << STE2_S2IR0_S)
+#define	STE2_S2OR0_S		42
+#define	STE2_S2OR0_M		(0x3 << STE2_S2OR0_S)
+#define	STE2_S2SH0_S		44
+#define	STE2_S2SH0_M		(0x3 << STE2_S2SH0_S)
+#define	STE2_S2TG_S		46
+#define	STE2_S2TG_M		(0x3 << STE2_S2TG_S)
+#define	STE2_S2PS_S		48 /* Physical address Size */
+#define	STE2_S2PS_M		(0x7 << STE2_S2PS_S)
+#define	STE2_S2AA64		(1 << 51) /* Stage 2 tt is AArch64 */
+#define	STE2_S2ENDI		(1 << 52) /* Stage 2 tt endianness */
+#define	STE2_S2AFFD		(1 << 53) /* Stage 2 Access Flag Fault Disable*/
+#define	STE2_S2PTW		(1 << 54) /* Protected Table Walk */
+#define	STE2_S2S		(1 << 57)
+#define	STE2_S2R		(1 << 58)
+
+#define	STE3_S2TTB_S		4 /* Address of Translation Table base */
+#define	STE3_S2TTB_M		(0xffffffffffff << STE3_S2TTB_S)
+
+#define	CD0_T0SZ_S		0 /* VA region size covered by TT0. */
+#define	CD0_T0SZ_M		(0x3f << CD0_T0SZ_S)
+#define	CD0_TG0_S		6 /* TT0 Translation Granule size */
+#define	CD0_TG0_M		(0x3 << CD0_TG0_S)
+#define	CD0_TG0_4KB		(0x0 << CD0_TG0_S)
+#define	CD0_TG0_64KB		(0x1 << CD0_TG0_S)
+#define	CD0_TG0_16KB		(0x2 << CD0_TG0_S)
+#define	CD0_IR0_S		8 /* Inner region Cacheability for TT0 access*/
+#define	CD0_IR0_M		(0x3 << CD0_IR0_S)
+#define	CD0_IR0_NC		(0x0 << CD0_IR0_S)
+#define	CD0_IR0_WBC_RWA		(0x1 << CD0_IR0_S)
+#define	CD0_IR0_WTC_RA		(0x2 << CD0_IR0_S)
+#define	CD0_IR0_WBC_RA		(0x3 << CD0_IR0_S)
+#define	CD0_OR0_S		10 /* Outer region Cacheability for TT0 access*/
+#define	CD0_OR0_M		(0x3 << CD0_OR0_S)
+#define	CD0_OR0_NC		(0x0 << CD0_OR0_S)
+#define	CD0_OR0_WBC_RWA		(0x1 << CD0_OR0_S)
+#define	CD0_OR0_WTC_RA		(0x2 << CD0_OR0_S)
+#define	CD0_OR0_WBC_RA		(0x3 << CD0_OR0_S)
+#define	CD0_SH0_S		12 /* Shareability for TT0 access */
+#define	CD0_SH0_M		(0x3 << CD0_SH0_S)
+#define	CD0_SH0_NS		(0x0 << CD0_SH0_S)
+#define	CD0_SH0_OS		(0x2 << CD0_SH0_S) /* Outer Shareable */
+#define	CD0_SH0_IS		(0x3 << CD0_SH0_S) /* Inner Shareable */
+#define	CD0_EPD0		(1 << 14) /* TT0 walk disable */
+#define	CD0_ENDI		(1 << 15) /* Big Endian */
+#define	CD0_T1SZ_S		16 /* VA region size covered by TT1 */
+#define	CD0_T1SZ_M		(0x3f << CD0_T1SZ_S)
+#define	CD0_TG1_S		22 /* TT1 Translation Granule size */
+#define	CD0_TG1_M		(0x3 << CD0_TG1_S)
+#define	CD0_TG1_4KB		(0x2 << CD0_TG1_S)
+#define	CD0_TG1_64KB		(0x3 << CD0_TG1_S)
+#define	CD0_TG1_16KB		(0x1 << CD0_TG1_S)
+#define	CD0_IR1_S		24 /* Inner region Cacheability for TT1 access*/
+#define	CD0_IR1_M		(0x3 << CD0_IR1_S)
+#define	CD0_OR1_S		26
+#define	CD0_OR1_M		(0x3 << CD0_OR1_S)
+#define	CD0_SH1_S		28
+#define	CD0_SH1_M		(0x3 << CD0_SH1_S)
+#define	CD0_EPD1		(1UL << 30) /* TT1 tt walk disable*/
+#define	CD0_VALID		(1UL << 31) /* CD Valid. */
+#define	CD0_IPS_S		32 /* Intermediate Physical Size */
+#define	CD0_IPS_M		(0x7UL << CD0_IPS_S)
+#define	CD0_IPS_32BITS		(0x0UL << CD0_IPS_S)
+#define	CD0_IPS_36BITS		(0x1UL << CD0_IPS_S)
+#define	CD0_IPS_40BITS		(0x2UL << CD0_IPS_S)
+#define	CD0_IPS_42BITS		(0x3UL << CD0_IPS_S)
+#define	CD0_IPS_44BITS		(0x4UL << CD0_IPS_S)
+#define	CD0_IPS_48BITS		(0x5UL << CD0_IPS_S)
+#define	CD0_IPS_52BITS		(0x6UL << CD0_IPS_S) /* SMMUv3.1 only */
+#define	CD0_AFFD		(1UL << 35) /* Access Flag Fault Disable */
+#define	CD0_WXN			(1UL << 36) /* Write eXecute Never */
+#define	CD0_UWXN		(1UL << 37) /* Unprivileged Write eXecut Never*/
+#define	CD0_TBI0		(1UL << 38) /* Top Byte Ignore for TTB0 */
+#define	CD0_TBI1		(1UL << 39) /* Top Byte Ignore for TTB1 */
+#define	CD0_PAN			(1UL << 40) /* Privileged Access Never */
+#define	CD0_AA64		(1UL << 41) /* TTB{0,1} is AArch64-format TT */
+#define	CD0_HD			(1UL << 42)
+#define	CD0_HA			(1UL << 43)
+#define	CD0_S			(1UL << 44)
+#define	CD0_R			(1UL << 45)
+#define	CD0_A			(1UL << 46)
+#define	CD0_ASET		(1UL << 47) /* ASID Set. */
+#define	CD0_ASID_S		48 /* Address Space Identifier */
+#define	CD0_ASID_M		(0xffff << CD0_ASID_S)
+#define	CD1_TTB0_S		4 /* Address of TT0 base. */
+#define	CD1_TTB0_M		(0xffffffffffff << CD1_TTB0_S)
+
+#endif /* _ARM64_IOMMU_SMMUREG_H_ */
diff --git a/sys/arm64/iommu/smmuvar.h b/sys/arm64/iommu/smmuvar.h
new file mode 100644
index 000000000000..3a1e5a269c01
--- /dev/null
+++ b/sys/arm64/iommu/smmuvar.h
@@ -0,0 +1,188 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_ARM64_IOMMU_SMMUVAR_H_
+#define	_ARM64_IOMMU_SMMUVAR_H_
+
+#define	SMMU_DEVSTR		"ARM System Memory Management Unit"
+#define	SMMU_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
+#define	SMMU_UNLOCK(_sc)	mtx_unlock(&(_sc)->sc_mtx)
+
+DECLARE_CLASS(smmu_driver);
+
+struct smmu_unit {
+	struct iommu_unit		iommu;
+	LIST_HEAD(, smmu_domain)	domain_list;
+	LIST_ENTRY(smmu_unit)		next;
+	device_t			dev;
+	intptr_t			xref;
+};
+
+struct smmu_domain {
+	struct iommu_domain		iodom;
+	LIST_HEAD(, smmu_ctx)		ctx_list;
+	LIST_ENTRY(smmu_domain)	next;
+	u_int entries_cnt;
+	struct smmu_cd			*cd;
+	struct pmap			p;
+	uint16_t			asid;
+};
+
+struct smmu_ctx {
+	struct iommu_ctx		ioctx;
+	struct smmu_domain		*domain;
+	LIST_ENTRY(smmu_ctx)		next;
+	device_t			dev;
+	bool				bypass;
+	int				sid;
+	uint16_t			vendor;
+	uint16_t			device;
+};
+
+struct smmu_queue_local_copy {
+	union {
+		uint64_t val;
+		struct {
+			uint32_t prod;
+			uint32_t cons;
+		};
+	};
+};
+
+struct smmu_cd {
+	vm_paddr_t paddr;
+	vm_size_t size;
+	void *vaddr;
+};
+
+struct smmu_queue {
+	struct smmu_queue_local_copy lc;
+	vm_paddr_t paddr;
+	void *vaddr;
+	uint32_t prod_off;
+	uint32_t cons_off;
+	int size_log2;
+	uint64_t base;
+};
+
+struct smmu_cmdq_entry {
+	uint8_t opcode;
+	union {
+		struct {
+			uint16_t asid;
+			uint16_t vmid;
+			vm_offset_t addr;
+			bool leaf;
+		} tlbi;
+		struct {
+			uint32_t sid;
+			uint32_t ssid;
+			bool leaf;
+		} cfgi;
+		struct {
+			uint32_t sid;
+		} prefetch;
+		struct {
+			uint64_t msiaddr;
+		} sync;
+	};
+};
+
+struct l1_desc {
+	uint8_t		span;
+	size_t		size;
+	void		*va;
+	vm_paddr_t	pa;
+};
+
+struct smmu_strtab {
+	void		*vaddr;
+	uint64_t	base;
+	uint32_t	base_cfg;
+	uint32_t	num_l1_entries;
+	struct l1_desc	*l1;
+};
+
+struct smmu_softc {
+	device_t		dev;
+	struct resource		*res[4];
+	void			*intr_cookie[3];
+	uint32_t		ias; /* Intermediate Physical Address */
+	uint32_t		oas; /* Physical Address */
+	uint32_t		asid_bits;
+	uint32_t		vmid_bits;
+	uint32_t		sid_bits;
+	uint32_t		ssid_bits;
+	uint32_t		pgsizes;
+	uint32_t		features;
+#define	SMMU_FEATURE_2_LVL_STREAM_TABLE		(1 << 0)
+#define	SMMU_FEATURE_2_LVL_CD			(1 << 1)
+#define	SMMU_FEATURE_TT_LE			(1 << 2)
+#define	SMMU_FEATURE_TT_BE			(1 << 3)
+#define	SMMU_FEATURE_SEV			(1 << 4)
+#define	SMMU_FEATURE_MSI			(1 << 5)
+#define	SMMU_FEATURE_HYP			(1 << 6)
+#define	SMMU_FEATURE_ATS			(1 << 7)
+#define	SMMU_FEATURE_PRI			(1 << 8)
+#define	SMMU_FEATURE_STALL_FORCE		(1 << 9)
+#define	SMMU_FEATURE_STALL			(1 << 10)
+#define	SMMU_FEATURE_S1P			(1 << 11)
+#define	SMMU_FEATURE_S2P			(1 << 12)
+#define	SMMU_FEATURE_VAX			(1 << 13)
+#define	SMMU_FEATURE_COHERENCY			(1 << 14)
+#define	SMMU_FEATURE_RANGE_INV			(1 << 15)
+	struct smmu_queue cmdq;
+	struct smmu_queue evtq;
+	struct smmu_queue priq;
+	struct smmu_strtab strtab;
+	int				sync;
+	struct mtx			sc_mtx;
+	bitstr_t			*asid_set;
+	int				asid_set_size;
+	struct mtx			asid_set_mutex;
+	struct smmu_unit		unit;
+	uintptr_t			xref;
+};
+
+MALLOC_DECLARE(M_SMMU);
+
+/* Device methods */
+int smmu_attach(device_t dev);
+int smmu_detach(device_t dev);
+
+struct smmu_ctx *smmu_ctx_lookup_by_sid(device_t dev, u_int sid);
+bool smmu_quirks_check(device_t dev, u_int sid, uint8_t event_id,
+    uintptr_t input_addr);
+
+#endif /* _ARM64_IOMMU_SMMUVAR_H_ */
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
index 6a112168533e..87e0326f683d 100644
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -211,6 +211,11 @@ arm64/coresight/coresight_tmc_fdt.c		optional fdt
 arm64/intel/firmware.c				optional soc_intel_stratix10
 arm64/intel/stratix10-soc-fpga-mgr.c		optional soc_intel_stratix10
 arm64/intel/stratix10-svc.c			optional soc_intel_stratix10
+arm64/iommu/iommu.c				optional iommu
+arm64/iommu/iommu_if.m				optional iommu
+arm64/iommu/smmu.c				optional iommu
+arm64/iommu/smmu_acpi.c				optional acpi iommu
+arm64/iommu/smmu_quirks.c			optional iommu
 arm64/qoriq/ls1046_gpio.c			optional ls1046_gpio gpio fdt SOC_NXP_LS
 arm64/qoriq/clk/ls1046a_clkgen.c		optional clk SOC_NXP_LS
 arm64/qoriq/clk/qoriq_clk_pll.c			optional clk SOC_NXP_LS
@@ -314,6 +319,8 @@ dev/iicbus/sy8106a.c		optional	sy8106a fdt
 dev/iicbus/twsi/mv_twsi.c	optional	twsi fdt
 dev/iicbus/twsi/a10_twsi.c	optional	twsi fdt
 dev/iicbus/twsi/twsi.c		optional	twsi fdt
+dev/iommu/busdma_iommu.c	optional	iommu
+dev/iommu/iommu_gas.c		optional	iommu
 dev/mbox/mbox_if.m		optional	soc_brcm_bcm2837
 dev/mmc/host/dwmmc.c		optional	dwmmc fdt
 dev/mmc/host/dwmmc_altera.c	optional	dwmmc dwmmc_altera fdt