bhyve: Make most I/O port handling specific to amd64

- The qemu_fwcfg interface, as implemented, is I/O port-based, but QEMU
  implements an MMIO interface that we'll eventually want to port for
  arm64.
- Retain support for I/O space PCI BARs, simply treat them like MMIO
  BARs for most purposes, similar to what the arm64 kernel does.  Such
  BARs are created by virtio devices.

Reviewed by:	corvink, jhb
MFC after:	1 week
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D40741

(cherry picked from commit 31cf78c9217b8298816115474973f4f0568962cf)

8 files changed, 73 insertions, 11 deletions

diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 1bb3c9a1b053..7c0750fae7d7 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -30,7 +30,6 @@ SRCS=	\
 	ctl_util.c		\
 	gdb.c			\
 	hda_codec.c		\
-	inout.c			\
 	iov.c			\
 	mem.c			\
 	mevent.c		\
diff --git a/usr.sbin/bhyve/amd64/Makefile.inc b/usr.sbin/bhyve/amd64/Makefile.inc
index 435327155fc3..96aaecafae4c 100644
--- a/usr.sbin/bhyve/amd64/Makefile.inc
+++ b/usr.sbin/bhyve/amd64/Makefile.inc
@@ -2,6 +2,7 @@ SRCS+=	\
 	atkbdc.c	\
 	e820.c		\
 	fwctl.c		\
+	inout.c		\
 	ioapic.c	\
 	kernemu_dev.c	\
 	mptbl.c		\
diff --git a/usr.sbin/bhyve/inout.c b/usr.sbin/bhyve/amd64/inout.c
index 898dc9aa1ea5..898dc9aa1ea5 100644
--- a/usr.sbin/bhyve/inout.c
+++ b/usr.sbin/bhyve/amd64/inout.c
diff --git a/usr.sbin/bhyve/inout.h b/usr.sbin/bhyve/amd64/inout.h
index 0d6131c9e886..0d6131c9e886 100644
--- a/usr.sbin/bhyve/inout.h
+++ b/usr.sbin/bhyve/amd64/inout.h
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index b8e59d3b0fc0..2da4e198344a 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -78,7 +78,9 @@
 #endif
 #include "bootrom.h"
 #include "config.h"
-#include "inout.h"
+#ifdef __amd64__
+#include "amd64/inout.h"
+#endif
 #include "debug.h"
 #ifdef __amd64__
 #include "amd64/e820.h"
@@ -1032,8 +1034,8 @@ main(int argc, char *argv[])
 #endif
 
 	init_mem(guest_ncpus);
-	init_inout();
 #ifdef __amd64__
+	init_inout();
 	kernemu_dev_init();
 #endif
 	init_bootrom(ctx);
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 6b2e46ce917d..e91b4d0a1e20 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -51,8 +51,8 @@
 #include "bhyverun.h"
 #include "config.h"
 #include "debug.h"
-#include "inout.h"
 #ifdef __amd64__
+#include "amd64/inout.h"
 #include "amd64/ioapic.h"
 #endif
 #include "mem.h"
@@ -502,6 +502,7 @@ pci_msix_pba_bar(struct pci_devinst *pi)
 		return (-1);
 }
 
+#ifdef __amd64__
 static int
 pci_emul_io_handler(struct vmctx *ctx __unused, int in, int port,
     int bytes, uint32_t *eax, void *arg)
@@ -530,6 +531,31 @@ pci_emul_io_handler(struct vmctx *ctx __unused, int in, int port,
 	}
 	return (-1);
 }
+#else
+static int
+pci_emul_iomem_handler(struct vcpu *vcpu __unused, int dir,
+    uint64_t addr, int size, uint64_t *val, void *arg1, long arg2)
+{
+	struct pci_devinst *pdi = arg1;
+	struct pci_devemu *pe = pdi->pi_d;
+	uint64_t offset;
+	int bidx = (int)arg2;
+
+	assert(bidx <= PCI_BARMAX);
+	assert(pdi->pi_bar[bidx].type == PCIBAR_IO);
+	assert(addr >= pdi->pi_bar[bidx].addr &&
+	       addr + size <= pdi->pi_bar[bidx].addr + pdi->pi_bar[bidx].size);
+	assert(size == 1 || size == 2 || size == 4);
+
+	offset = addr - pdi->pi_bar[bidx].addr;
+	if (dir == MEM_F_READ)
+		*val = (*pe->pe_barread)(pdi, bidx, offset, size);
+	else
+		(*pe->pe_barwrite)(pdi, bidx, offset, size, *val);
+
+	return (0);
+}
+#endif /* !__amd64__ */
 
 static int
 pci_emul_mem_handler(struct vcpu *vcpu __unused, int dir,
@@ -538,7 +564,7 @@ pci_emul_mem_handler(struct vcpu *vcpu __unused, int dir,
 	struct pci_devinst *pdi = arg1;
 	struct pci_devemu *pe = pdi->pi_d;
 	uint64_t offset;
-	int bidx = (int) arg2;
+	int bidx = (int)arg2;
 
 	assert(bidx <= PCI_BARMAX);
 	assert(pdi->pi_bar[bidx].type == PCIBAR_MEM32 ||
@@ -601,12 +627,16 @@ modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 {
 	struct pci_devemu *pe;
 	int error;
-	struct inout_port iop;
-	struct mem_range mr;
+	enum pcibar_type type;
 
 	pe = pi->pi_d;
-	switch (pi->pi_bar[idx].type) {
+	type = pi->pi_bar[idx].type;
+	switch (type) {
 	case PCIBAR_IO:
+	{
+#ifdef __amd64__
+		struct inout_port iop;
+
 		bzero(&iop, sizeof(struct inout_port));
 		iop.name = pi->pi_name;
 		iop.port = pi->pi_bar[idx].addr;
@@ -618,9 +648,29 @@ modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 			error = register_inout(&iop);
 		} else
 			error = unregister_inout(&iop);
+#else
+		struct mem_range mr;
+
+		bzero(&mr, sizeof(struct mem_range));
+		mr.name = pi->pi_name;
+		mr.base = pi->pi_bar[idx].addr;
+		mr.size = pi->pi_bar[idx].size;
+		if (registration) {
+			mr.flags = MEM_F_RW;
+			mr.handler = pci_emul_iomem_handler;
+			mr.arg1 = pi;
+			mr.arg2 = idx;
+			error = register_mem(&mr);
+		} else
+			error = unregister_mem(&mr);
+#endif
 		break;
+	}
 	case PCIBAR_MEM32:
 	case PCIBAR_MEM64:
+	{
+		struct mem_range mr;
+
 		bzero(&mr, sizeof(struct mem_range));
 		mr.name = pi->pi_name;
 		mr.base = pi->pi_bar[idx].addr;
@@ -634,6 +684,7 @@ modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
 		} else
 			error = unregister_mem(&mr);
 		break;
+	}
 	case PCIBAR_ROM:
 		error = 0;
 		break;
@@ -2346,6 +2397,7 @@ pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes,
 	}
 }
 
+#ifdef __amd64__
 static int cfgenable, cfgbus, cfgslot, cfgfunc, cfgoff;
 
 static int
@@ -2401,6 +2453,7 @@ INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+0, IOPORT_F_INOUT, pci_emul_cfgdata);
 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+1, IOPORT_F_INOUT, pci_emul_cfgdata);
 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+2, IOPORT_F_INOUT, pci_emul_cfgdata);
 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+3, IOPORT_F_INOUT, pci_emul_cfgdata);
+#endif
 
 #ifdef BHYVE_SNAPSHOT
 /*
diff --git a/usr.sbin/bhyve/pctestdev.c b/usr.sbin/bhyve/pctestdev.c
index b6fc3c336a93..539016a51876 100644
--- a/usr.sbin/bhyve/pctestdev.c
+++ b/usr.sbin/bhyve/pctestdev.c
@@ -43,7 +43,9 @@
 #include <vmmapi.h>
 
 #include "debug.h"
-#include "inout.h"
+#ifdef __amd64__
+#include "amd64/inout.h"
+#endif
 #include "mem.h"
 #include "pctestdev.h"
 
diff --git a/usr.sbin/bhyve/qemu_fwcfg.c b/usr.sbin/bhyve/qemu_fwcfg.c
index 73a401ad8a81..830cee730dbd 100644
--- a/usr.sbin/bhyve/qemu_fwcfg.c
+++ b/usr.sbin/bhyve/qemu_fwcfg.c
@@ -22,8 +22,8 @@
 
 #include "acpi_device.h"
 #include "bhyverun.h"
-#include "inout.h"
 #ifdef __amd64__
+#include "amd64/inout.h"
 #include "amd64/pci_lpc.h"
 #endif
 #include "qemu_fwcfg.h"
@@ -114,6 +114,7 @@ struct qemu_fwcfg_user_file {
 static STAILQ_HEAD(qemu_fwcfg_user_file_list,
     qemu_fwcfg_user_file) user_files = STAILQ_HEAD_INITIALIZER(user_files);
 
+#ifdef __amd64__
 static int
 qemu_fwcfg_selector_port_handler(struct vmctx *const ctx __unused, const int in,
     const int port __unused, const int bytes, uint32_t *const eax,
@@ -181,6 +182,7 @@ qemu_fwcfg_data_port_handler(struct vmctx *const ctx __unused, const int in,
 
 	return (0);
 }
+#endif
 
 static int
 qemu_fwcfg_add_item(const uint16_t architecture, const uint16_t index,
@@ -295,6 +297,7 @@ qemu_fwcfg_add_item_signature(void)
 	    (uint8_t *)fwcfg_signature));
 }
 
+#ifdef __amd64__
 static int
 qemu_fwcfg_register_port(const char *const name, const int port, const int size,
     const int flags, const inout_func_t handler)
@@ -310,6 +313,7 @@ qemu_fwcfg_register_port(const char *const name, const int port, const int size,
 
 	return (register_inout(&iop));
 }
+#endif
 
 int
 qemu_fwcfg_add_file(const char *name, const uint32_t size, void *const data)
@@ -461,7 +465,7 @@ qemu_fwcfg_init(struct vmctx *const ctx)
 			goto done;
 		}
 
-		/* add handlers for fwcfg ports */
+#ifdef __amd64__
 		if ((error = qemu_fwcfg_register_port("qemu_fwcfg_selector",
 		    QEMU_FWCFG_SELECTOR_PORT_NUMBER,
 		    QEMU_FWCFG_SELECTOR_PORT_SIZE,
@@ -481,6 +485,7 @@ qemu_fwcfg_init(struct vmctx *const ctx)
 			    __func__, QEMU_FWCFG_DATA_PORT_NUMBER);
 			goto done;
 		}
+#endif
 	}
 
 	/* add common fwcfg items */