pci: Add an ioctl to perform I/O to BARs

This is useful for bhyve, which otherwise has to use /dev/io to handle
accesses to I/O port BARs when PCI passthrough is in use.

Reviewed by:	imp, kib
Discussed with:	jhb
Sponsored by:	The FreeBSD Foundation

(cherry picked from commit 7e14be0b0717105f4b3b8c62df82a1e883d8ebb6)

3 files changed, 147 insertions, 1 deletions

diff --git a/share/man/man4/pci.4 b/share/man/man4/pci.4
index 28a456d18179..3c2c08afe466 100644
--- a/share/man/man4/pci.4
+++ b/share/man/man4/pci.4
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 27, 2021
+.Dd August 13, 2021
 .Dt PCI 4
 .Os
 .Sh NAME
@@ -430,6 +430,40 @@ even on reads.
 of mapping.
 Currently attempt to mmap an inactive BAR results in error.
 .El
+.It PCIOCBARIO
+This
+.Xr ioctl 2
+command allows users to read from and write to BARs.
+The I/O request parameters are passed in a
+.Va struct pci_bar_ioreq
+structure, which has the following fields:
+.Bl -tag
+.It Vt struct pcisel pbi_sel
+Describes the device to operate on.
+.It Vt int pbi_op
+The operation to perform.
+Currently supported values are
+.Dv PCIBARIO_READ
+and
+.Dv PCIBARIO_WRITE .
+.It Vt uint32_t pbi_bar
+The index of the BAR on which to operate.
+.It Vt uint32_t pbi_offset
+The offset into the BAR at which to operate.
+.It Vt uint32_t pbi_width
+The size, in bytes, of the I/O operation.
+1-byte, 2-byte, 4-byte and 8-byte perations are supported.
+.It Vt uint32_t pbi_value
+For reads, the value is returned in this field.
+For writes, the caller specifies the value to be written in this field.
+.Pp
+Note that this operation maps and unmaps the corresponding resource and
+so is relatively expensive for memory BARs.
+The
+.Va PCIOCBARMMAP
+.Xr ioctl 2
+can be used to create a persistent userspace mapping for such BARs instead.
+.El
 .El
 .Sh LOADER TUNABLES
 Tunables can be set at the
diff --git a/sys/dev/pci/pci_user.c b/sys/dev/pci/pci_user.c
index 640a10abfeca..544cb83ece69 100644
--- a/sys/dev/pci/pci_user.c
+++ b/sys/dev/pci/pci_user.c
@@ -920,6 +920,92 @@ out:
 }
 
 static int
+pci_bar_io(device_t pcidev, struct pci_bar_ioreq *pbi)
+{
+	struct pci_map *pm;
+	struct resource *res;
+	uint32_t offset, width;
+	int bar, error, type;
+
+	if (pbi->pbi_op != PCIBARIO_READ &&
+	    pbi->pbi_op != PCIBARIO_WRITE)
+		return (EINVAL);
+
+	bar = PCIR_BAR(pbi->pbi_bar);
+	pm = pci_find_bar(pcidev, bar);
+	if (pm == NULL)
+		return (EINVAL);
+
+	offset = pbi->pbi_offset;
+	width = pbi->pbi_width;
+
+	if (offset + width < offset ||
+	    ((pci_addr_t)1 << pm->pm_size) < offset + width)
+		return (EINVAL);
+
+	type = PCI_BAR_MEM(pm->pm_value) ? SYS_RES_MEMORY : SYS_RES_IOPORT;
+
+	/*
+	 * This will fail if a driver has allocated the resource.  This could be
+	 * worked around by detecting that case and using bus_map_resource() to
+	 * populate the handle, but so far this is not needed.
+	 */
+	res = bus_alloc_resource_any(pcidev, type, &bar, RF_ACTIVE);
+	if (res == NULL)
+		return (ENOENT);
+
+	error = 0;
+	switch (pbi->pbi_op) {
+	case PCIBARIO_READ:
+		switch (pbi->pbi_width) {
+		case 1:
+			pbi->pbi_value = bus_read_1(res, offset);
+			break;
+		case 2:
+			pbi->pbi_value = bus_read_2(res, offset);
+			break;
+		case 4:
+			pbi->pbi_value = bus_read_4(res, offset);
+			break;
+#ifndef __i386__
+		case 8:
+			pbi->pbi_value = bus_read_8(res, offset);
+			break;
+#endif
+		default:
+			error = EINVAL;
+			break;
+		}
+		break;
+	case PCIBARIO_WRITE:
+		switch (pbi->pbi_width) {
+		case 1:
+			bus_write_1(res, offset, pbi->pbi_value);
+			break;
+		case 2:
+			bus_write_2(res, offset, pbi->pbi_value);
+			break;
+		case 4:
+			bus_write_4(res, offset, pbi->pbi_value);
+			break;
+#ifndef __i386__
+		case 8:
+			bus_write_8(res, offset, pbi->pbi_value);
+			break;
+#endif
+		default:
+			error = EINVAL;
+			break;
+		}
+		break;
+	}
+
+	bus_release_resource(pcidev, type, bar, res);
+
+	return (error);
+}
+
+static int
 pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
 {
 	device_t pcidev;
@@ -928,6 +1014,7 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t
 	struct pci_conf_io *cio = NULL;
 	struct pci_devinfo *dinfo;
 	struct pci_io *io;
+	struct pci_bar_ioreq *pbi;
 	struct pci_bar_io *bio;
 	struct pci_list_vpd_io *lvio;
 	struct pci_match_conf *pattern_buf;
@@ -1303,6 +1390,19 @@ getconfexit:
 		error = pcidev == NULL ? ENODEV : pci_bar_mmap(pcidev, pbm);
 		break;
 
+	case PCIOCBARIO:
+		pbi = (struct pci_bar_ioreq *)data;
+
+		pcidev = pci_find_dbsf(pbi->pbi_sel.pc_domain,
+		    pbi->pbi_sel.pc_bus, pbi->pbi_sel.pc_dev,
+		    pbi->pbi_sel.pc_func);
+		if (pcidev == NULL) {
+			error = ENODEV;
+			break;
+		}
+		error = pci_bar_io(pcidev, pbi);
+		break;
+
 	default:
 		error = ENOTTY;
 		break;
diff --git a/sys/sys/pciio.h b/sys/sys/pciio.h
index 50e9116d63d4..16635a884ca3 100644
--- a/sys/sys/pciio.h
+++ b/sys/sys/pciio.h
@@ -151,6 +151,17 @@ struct pci_bar_mmap {
 	int		pbm_memattr;
 };
 
+struct pci_bar_ioreq {
+	struct pcisel	pbi_sel;	/* device to operate on */
+#define	PCIBARIO_READ		0x1
+#define	PCIBARIO_WRITE		0x2
+	int		pbi_op;
+	uint32_t	pbi_bar;
+	uint32_t	pbi_offset;
+	uint32_t	pbi_width;
+	uint32_t	pbi_value;
+};
+
 #define	PCIIO_BAR_MMAP_FIXED	0x01
 #define	PCIIO_BAR_MMAP_EXCL	0x02
 #define	PCIIO_BAR_MMAP_RW	0x04
@@ -163,5 +174,6 @@ struct pci_bar_mmap {
 #define	PCIOCGETBAR	_IOWR('p', 6, struct pci_bar_io)
 #define	PCIOCLISTVPD	_IOWR('p', 7, struct pci_list_vpd_io)
 #define	PCIOCBARMMAP	_IOWR('p', 8, struct pci_bar_mmap)
+#define	PCIOCBARIO	_IOWR('p', 9, struct pci_bar_ioreq)
 
 #endif /* !_SYS_PCIIO_H_ */