bhyve: Move legacy PCI interrupt handling under amd64/

Specifically, move IO-APIC, LPC and PIRQ routing code under amd64/.

Use ifdefs to conditionally compile related code in other files.  In
particular, legacy PCI interrupt handling is now compiled only on amd64.
This is not too invasive, but suggestions for a more modular approach
would be appreciated.

I am not sure why qemu fwcfg handling is tied to LPC, and I suspect it
should be decoupled.  In this commit I just apply an ifdef hammer, but
we will eventually want fwcfg on arm64 as well.

No functional change intended.

Reviewed by:	corvink, jhb
MFC after:	1 week
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D40739

13 files changed, 77 insertions, 28 deletions

diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 0ec6ad73e2a2..5aaf0d4ea2a0 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -31,7 +31,6 @@ SRCS=	\
 	gdb.c			\
 	hda_codec.c		\
 	inout.c			\
-	ioapic.c		\
 	iov.c			\
 	mem.c			\
 	mevent.c		\
@@ -40,8 +39,6 @@ SRCS=	\
 	pci_emul.c		\
 	pci_fbuf.c		\
 	pci_hostbridge.c	\
-	pci_irq.c		\
-	pci_lpc.c		\
 	pci_nvme.c		\
 	pci_passthru.c		\
 	pci_virtio_9p.c		\
diff --git a/usr.sbin/bhyve/amd64/Makefile.inc b/usr.sbin/bhyve/amd64/Makefile.inc
index 862aeddb16e0..76813891b0a0 100644
--- a/usr.sbin/bhyve/amd64/Makefile.inc
+++ b/usr.sbin/bhyve/amd64/Makefile.inc
@@ -2,12 +2,15 @@ SRCS+=	\
 	atkbdc.c	\
 	e820.c		\
 	fwctl.c		\
+	ioapic.c	\
 	kernemu_dev.c	\
 	mptbl.c		\
 	pci_ahci.c	\
 	pci_e82545.c	\
 	pci_gvt-d.c	\
 	pci_hda.c	\
+	pci_irq.c	\
+	pci_lpc.c	\
 	pci_uart.c	\
 	pci_xhci.c	\
 	pctestdev.c	\
diff --git a/usr.sbin/bhyve/ioapic.c b/usr.sbin/bhyve/amd64/ioapic.c
index 89bb891fdaf0..89bb891fdaf0 100644
--- a/usr.sbin/bhyve/ioapic.c
+++ b/usr.sbin/bhyve/amd64/ioapic.c
diff --git a/usr.sbin/bhyve/ioapic.h b/usr.sbin/bhyve/amd64/ioapic.h
index 73e5c833eac4..73e5c833eac4 100644
--- a/usr.sbin/bhyve/ioapic.h
+++ b/usr.sbin/bhyve/amd64/ioapic.h
diff --git a/usr.sbin/bhyve/pci_irq.c b/usr.sbin/bhyve/amd64/pci_irq.c
index 12c2d94925ad..12c2d94925ad 100644
--- a/usr.sbin/bhyve/pci_irq.c
+++ b/usr.sbin/bhyve/amd64/pci_irq.c
diff --git a/usr.sbin/bhyve/pci_irq.h b/usr.sbin/bhyve/amd64/pci_irq.h
index 50ea2eec00a9..50ea2eec00a9 100644
--- a/usr.sbin/bhyve/pci_irq.h
+++ b/usr.sbin/bhyve/amd64/pci_irq.h
diff --git a/usr.sbin/bhyve/pci_lpc.c b/usr.sbin/bhyve/amd64/pci_lpc.c
index 5c2a2a7965b0..5c2a2a7965b0 100644
--- a/usr.sbin/bhyve/pci_lpc.c
+++ b/usr.sbin/bhyve/amd64/pci_lpc.c
diff --git a/usr.sbin/bhyve/pci_lpc.h b/usr.sbin/bhyve/amd64/pci_lpc.h
index 2dca8f7bec24..2dca8f7bec24 100644
--- a/usr.sbin/bhyve/pci_lpc.h
+++ b/usr.sbin/bhyve/amd64/pci_lpc.h
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index c27cb427f1e2..ba63a6405538 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -85,8 +85,8 @@
 #include "amd64/fwctl.h"
 #endif
 #include "gdb.h"
-#include "ioapic.h"
 #ifdef __amd64__
+#include "amd64/ioapic.h"
 #include "amd64/kernemu_dev.h"
 #endif
 #include "mem.h"
@@ -95,8 +95,10 @@
 #include "amd64/mptbl.h"
 #endif
 #include "pci_emul.h"
-#include "pci_irq.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_irq.h"
+#include "amd64/pci_lpc.h"
+#endif
 #include "qemu_fwcfg.h"
 #include "smbiostbl.h"
 #ifdef BHYVE_SNAPSHOT
@@ -634,8 +636,10 @@ do_open(const char *vmname)
 
 	reinit = romboot = false;
 
+#ifdef __amd64__
 	if (lpc_bootrom())
 		romboot = true;
+#endif
 
 	error = vm_create(vmname);
 	if (error) {
@@ -855,6 +859,7 @@ main(int argc, char *argv[])
 		case 'K':
 			set_config_value("keyboard.layout", optarg);
 			break;
+#ifdef __amd64__
 		case 'l':
 			if (strncmp(optarg, "help", strlen(optarg)) == 0) {
 				lpc_print_supported_devices();
@@ -864,6 +869,7 @@ main(int argc, char *argv[])
 				    "configuration '%s'", optarg);
 			}
 			break;
+#endif
 #ifdef BHYVE_SNAPSHOT
 		case 'r':
 			restore_file = optarg;
@@ -1037,9 +1043,9 @@ main(int argc, char *argv[])
 	init_bootrom(ctx);
 #ifdef __amd64__
 	atkbdc_init(ctx);
-#endif
 	pci_irq_init(ctx);
 	ioapic_init(ctx);
+#endif
 
 #ifdef __amd64__
 	rtc_init(ctx);
@@ -1085,6 +1091,7 @@ main(int argc, char *argv[])
 
 	init_gdb(ctx);
 
+#ifdef __amd64__
 	if (lpc_bootrom()) {
 		if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
 			fprintf(stderr, "ROM boot failed: unrestricted guest "
@@ -1094,6 +1101,7 @@ main(int argc, char *argv[])
 		error = vcpu_reset(bsp);
 		assert(error == 0);
 	}
+#endif
 
 	/*
 	 * Add all vCPUs.
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 5fb25dbfe9c7..6b2e46ce917d 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -52,11 +52,15 @@
 #include "config.h"
 #include "debug.h"
 #include "inout.h"
-#include "ioapic.h"
+#ifdef __amd64__
+#include "amd64/ioapic.h"
+#endif
 #include "mem.h"
 #include "pci_emul.h"
-#include "pci_irq.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_irq.h"
+#include "amd64/pci_lpc.h"
+#endif
 #include "pci_passthru.h"
 #include "qemu_fwcfg.h"
 
@@ -143,9 +147,12 @@ SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE);
 #define	PCI_EMUL_MEMLIMIT32	PCI_EMUL_ECFG_BASE
 #define PCI_EMUL_MEMSIZE64	(32*GB)
 
-static struct pci_devemu *pci_emul_finddev(const char *name);
+#ifdef __amd64__
 static void pci_lintr_route(struct pci_devinst *pi);
 static void pci_lintr_update(struct pci_devinst *pi);
+#endif
+
+static struct pci_devemu *pci_emul_finddev(const char *name);
 static void pci_cfgrw(int in, int bus, int slot, int func, int coff,
     int bytes, uint32_t *val);
 
@@ -1061,11 +1068,13 @@ pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int bus, int slot,
 	pdi->pi_bus = bus;
 	pdi->pi_slot = slot;
 	pdi->pi_func = func;
+#ifdef __amd64__
 	pthread_mutex_init(&pdi->pi_lintr.lock, NULL);
 	pdi->pi_lintr.pin = 0;
 	pdi->pi_lintr.state = IDLE;
 	pdi->pi_lintr.pirq_pin = 0;
 	pdi->pi_lintr.ioapic_irq = 0;
+#endif
 	pdi->pi_d = pde;
 	snprintf(pdi->pi_name, PI_NAMESZ, "%s@pci.%d.%d.%d", pde->pe_emu, bus,
 	    slot, func);
@@ -1203,7 +1212,9 @@ msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
 
 		pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE;
 		pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK;
+#ifdef __amd64__
 		pci_lintr_update(pi);
+#endif
 	}
 
 	CFGWRITE(pi, offset, val, bytes);
@@ -1245,7 +1256,9 @@ msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
 	} else {
 		pi->pi_msi.maxmsgnum = 0;
 	}
+#ifdef __amd64__
 	pci_lintr_update(pi);
+#endif
 }
 
 static void
@@ -1538,6 +1551,7 @@ init_pci(struct vmctx *ctx)
 		bi->memlimit64 = pci_emul_membase64;
 	}
 
+#ifdef __amd64__
 	/*
 	 * PCI backends are initialized before routing INTx interrupts
 	 * so that LPC devices are able to reserve ISA IRQs before
@@ -1558,6 +1572,7 @@ init_pci(struct vmctx *ctx)
 		}
 	}
 	lpc_pirq_routed();
+#endif
 
 	if ((error = init_bootorder()) != 0) {
 		warnx("%s: Unable to init bootorder", __func__);
@@ -1601,6 +1616,7 @@ init_pci(struct vmctx *ctx)
 	return (0);
 }
 
+#ifdef __amd64__
 static void
 pci_apic_prt_entry(int bus __unused, int slot, int pin, int pirq_pin __unused,
     int ioapic_irq, void *arg __unused)
@@ -1633,6 +1649,7 @@ pci_pirq_prt_entry(int bus __unused, int slot, int pin, int pirq_pin,
 	dsdt_line("  },");
 	free(name);
 }
+#endif
 
 /*
  * A bhyve virtual machine has a flat PCI hierarchy with a root port
@@ -1644,7 +1661,7 @@ pci_bus_write_dsdt(int bus)
 	struct businfo *bi;
 	struct slotinfo *si;
 	struct pci_devinst *pi;
-	int count, func, slot;
+	int func, slot;
 
 	/*
 	 * If there are no devices on this 'bus' then just return.
@@ -1747,8 +1764,8 @@ pci_bus_write_dsdt(int bus)
 	dsdt_line("        ,, , AddressRangeMemory, TypeStatic)");
 	dsdt_line("    })");
 
-	count = pci_count_lintr(bus);
-	if (count != 0) {
+#ifdef __amd64__
+	if (pci_count_lintr(bus) != 0) {
 		dsdt_indent(2);
 		dsdt_line("Name (PPRT, Package ()");
 		dsdt_line("{");
@@ -1771,6 +1788,7 @@ pci_bus_write_dsdt(int bus)
 		dsdt_line("}");
 		dsdt_unindent(2);
 	}
+#endif
 
 	dsdt_indent(2);
 	for (slot = 0; slot < MAXSLOTS; slot++) {
@@ -1866,6 +1884,7 @@ pci_generate_msi(struct pci_devinst *pi, int index)
 	}
 }
 
+#ifdef __amd64__
 static bool
 pci_lintr_permitted(struct pci_devinst *pi)
 {
@@ -2026,6 +2045,7 @@ pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg)
 		}
 	}
 }
+#endif /* __amd64__ */
 
 /*
  * Return 1 if the emulated device in 'slot' is a multi-function device.
@@ -2130,11 +2150,13 @@ pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old)
 		}
 	}
 
+#ifdef __amd64__
 	/*
 	 * If INTx has been unmasked and is pending, assert the
 	 * interrupt.
 	 */
 	pci_lintr_update(pi);
+#endif
 }
 
 static void
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index 0fd438151093..c367169113f8 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -115,12 +115,6 @@ struct msix_table_entry {
 #define MAX_MSIX_TABLE_ENTRIES	2048
 #define	PBA_SIZE(msgnum)	(roundup2((msgnum), 64) / 8)
 
-enum lintr_stat {
-	IDLE,
-	ASSERTED,
-	PENDING
-};
-
 struct pci_devinst {
 	struct pci_devemu *pi_d;
 	struct vmctx *pi_vmctx;
@@ -130,13 +124,19 @@ struct pci_devinst {
 	int	  pi_prevcap;
 	int	  pi_capend;
 
+#ifdef __amd64__
 	struct {
 		int8_t    	pin;
-		enum lintr_stat	state;
+		enum {
+			IDLE,
+			ASSERTED,
+			PENDING,
+		} state;
 		int		pirq_pin;
 		int	  	ioapic_irq;
 		pthread_mutex_t	lock;
 	} pi_lintr;
+#endif
 
 	struct {
 		int		enabled;
@@ -221,8 +221,15 @@ struct pciecap {
 } __packed;
 static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed");
 
+#ifdef __amd64__
 typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin,
     int ioapic_irq, void *arg);
+void	pci_lintr_assert(struct pci_devinst *pi);
+void	pci_lintr_deassert(struct pci_devinst *pi);
+void	pci_lintr_request(struct pci_devinst *pi);
+int	pci_count_lintr(int bus);
+void	pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
+#endif
 
 int	init_pci(struct vmctx *ctx);
 void	pci_callback(void);
@@ -241,9 +248,6 @@ void	pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes,
 void	pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old);
 void	pci_generate_msi(struct pci_devinst *pi, int msgnum);
 void	pci_generate_msix(struct pci_devinst *pi, int msgnum);
-void	pci_lintr_assert(struct pci_devinst *pi);
-void	pci_lintr_deassert(struct pci_devinst *pi);
-void	pci_lintr_request(struct pci_devinst *pi);
 int	pci_msi_enabled(struct pci_devinst *pi);
 int	pci_msix_enabled(struct pci_devinst *pi);
 int	pci_msix_table_bar(struct pci_devinst *pi);
@@ -257,11 +261,10 @@ int	pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum);
 int	pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size,
 			     uint64_t value);
 uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size);
-int	pci_count_lintr(int bus);
-void	pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
 void	pci_write_dsdt(void);
 uint64_t pci_ecfg_base(void);
 int	pci_bus_configured(int bus);
+
 #ifdef BHYVE_SNAPSHOT
 struct pci_devinst *pci_next(const struct pci_devinst *cursor);
 int	pci_snapshot(struct vm_snapshot_meta *meta);
diff --git a/usr.sbin/bhyve/qemu_fwcfg.c b/usr.sbin/bhyve/qemu_fwcfg.c
index e845c70950b1..73a401ad8a81 100644
--- a/usr.sbin/bhyve/qemu_fwcfg.c
+++ b/usr.sbin/bhyve/qemu_fwcfg.c
@@ -23,7 +23,9 @@
 #include "acpi_device.h"
 #include "bhyverun.h"
 #include "inout.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_lpc.h"
+#endif
 #include "qemu_fwcfg.h"
 
 #define QEMU_FWCFG_ACPI_DEVICE_NAME "FWCF"
@@ -423,6 +425,18 @@ int
 qemu_fwcfg_init(struct vmctx *const ctx)
 {
 	int error;
+	bool fwcfg_enabled;
+
+	/*
+	 * The fwcfg implementation currently only provides an I/O port
+	 * interface and thus is amd64-specific for now.  An MMIO interface is
+	 * required for other platforms.
+	 */
+#ifdef __amd64__
+	fwcfg_enabled = strcmp(lpc_fwcfg(), "qemu") == 0;
+#else
+	fwcfg_enabled = false;
+#endif
 
 	/*
 	 * Bhyve supports fwctl (bhyve) and fwcfg (qemu) as firmware interfaces.
@@ -430,7 +444,7 @@ qemu_fwcfg_init(struct vmctx *const ctx)
 	 * interfaces at the same time to the guest. Therefore, only create acpi
 	 * tables and register io ports for fwcfg, if it's used.
 	 */
-	if (strcmp(lpc_fwcfg(), "qemu") == 0) {
+	if (fwcfg_enabled) {
 		error = acpi_device_create(&fwcfg_sc.acpi_dev, &fwcfg_sc, ctx,
 		    &qemu_fwcfg_acpi_device_emul);
 		if (error) {
diff --git a/usr.sbin/bhyve/virtio.h b/usr.sbin/bhyve/virtio.h
index 2b72b862ab21..4c6c8004b2d1 100644
--- a/usr.sbin/bhyve/virtio.h
+++ b/usr.sbin/bhyve/virtio.h
@@ -358,7 +358,9 @@ vi_interrupt(struct virtio_softc *vs, uint8_t isr, uint16_t msix_idx)
 		VS_LOCK(vs);
 		vs->vs_isr |= isr;
 		pci_generate_msi(vs->vs_pi, 0);
+#ifdef __amd64__
 		pci_lintr_assert(vs->vs_pi);
+#endif
 		VS_UNLOCK(vs);
 	}
 }