sysutils/bhyve+: New Port: bhyve with unofficial extensions

A patched version of BHyVe, the BSD Hypervisor from the base system
that includes features and fixes that are not yet officially there.

WWW: https://github.com/pgj/freebsd-bhyve-plus-port

18 files changed, 1426 insertions, 0 deletions

diff --git a/sysutils/Makefile b/sysutils/Makefile
index 66f118d826cf..1fcf8d5a8940 100644
--- a/sysutils/Makefile
+++ b/sysutils/Makefile
@@ -119,6 +119,7 @@
     SUBDIR += bfs
     SUBDIR += bhyve-firmware
     SUBDIR += bhyve-rc
+    SUBDIR += bhyve+
     SUBDIR += biosfont
     SUBDIR += bkpupsd
     SUBDIR += bksh
diff --git a/sysutils/bhyve+/Makefile b/sysutils/bhyve+/Makefile
new file mode 100644
index 000000000000..266129c8bf7c
--- /dev/null
+++ b/sysutils/bhyve+/Makefile
@@ -0,0 +1,90 @@
+PORTNAME=	bhyve+
+PORTVERSION=	0.1.0
+CATEGORIES=	sysutils
+
+MAINTAINER=	pali.gabor@gmail.com
+COMMENT=	BHyVe with unofficial extensions
+
+LICENSE=	BSD2CLAUSE
+
+ONLY_FOR_ARCHS=	amd64
+USES=		kmod uidfix
+
+.include <bsd.port.options.mk>
+
+SRCS=		${SRC_BASE}/sys/modules/vmm \
+		${SRC_BASE}/sys/amd64/vmm \
+		${SRC_BASE}/lib/libvmmapi \
+		${SRC_BASE}/usr.sbin/bhyve \
+		${SRC_BASE}/usr.sbin/bhyvectl \
+		${SRC_BASE}/usr.sbin/bhyveload \
+		${SRC_BASE}/share/mk
+
+.for _src in ${SRCS}
+.  if !exists(${_src}) && empty(IGNORE)
+IGNORE+=	requires bhyve and vmm source files in ${_src}
+.  endif
+.endfor
+
+USE_GITHUB=	yes
+GH_ACCOUNT=	conclusiveeng
+GH_PROJECT=	lib9p
+GH_TAGNAME=	7ef466772108856ee3e093735033232a6f1a3f04
+
+USE_LDCONFIG=	yes
+
+.if 1100000 <= ${OSVERSION} && ${OSVERSION} < 1200000
+EXTRA_PATCHES=	${PATCHDIR}/freebsd-11
+.elif 1200000 <= ${OSVERSION} && ${OSVERSION} < 1300000
+EXTRA_PATCHES=	${PATCHDIR}/freebsd-12
+.elif 1300000 <= ${OSVERSION}
+EXTRA_PATCHES=	${PATCHDIR}/freebsd-13
+.else
+.  if empty(IGNORE)
+IGNORE+=	cannot build: unsupported system
+.  endif
+.endif
+
+WRKSRC=		${WRKDIR}
+
+_LINKS=		sys/cam/ctl \
+		sys/dev/e1000 \
+		sys/dev/mii \
+		sys/dev/usb/controller \
+		sys/amd64/include \
+		sys/x86/include \
+		stand/userboot
+
+post-extract:
+	@${MKDIR} ${WRKSRC}/contrib
+	@cd ${WRKSRC} && ${LN} -s ../${GH_PROJECT}-${GH_TAGNAME} contrib/lib9p
+.for _link in ${_LINKS}
+	@${MKDIR} $$(dirname ${WRKSRC}/${_link})
+	@cd ${WRKSRC} && ${LN} -s ${SRC_BASE}/${_link} ${_link}
+.endfor
+.for _src in ${SRCS}
+	@cd ${_src} && ${COPYTREE_SHARE} . ${WRKDIR}/${_src:S/${SRC_BASE}//1}
+.endfor
+	@cd ${FILESDIR} && ${COPYTREE_SHARE} kernconf ${WRKSRC}
+
+do-build:
+	@cd ${WRKSRC}/contrib/lib9p && \
+		${MAKE} SRCTOP=${WRKSRC} SYSDIR=${SRC_BASE}/sys LIB9PDIR=
+.for _src in ${SRCS}
+.  if exists(${_src}/Makefile)
+	@cd ${WRKSRC}/${_src:S/${SRC_BASE}//1} && \
+		${MAKE} SRCTOP=${WRKSRC} SYSDIR=${SRC_BASE}/sys DEBUG_FLAGS=-g \
+		MK_CTF=yes KERNBUILDDIR=${WRKSRC}/kernconf
+.  endif
+.endfor
+
+do-install:
+	${INSTALL_LIB} ${WRKSRC}/contrib/lib9p/lib9p.so.1.1 ${STAGEDIR}${PREFIX}/lib
+	${INSTALL_LIB} ${WRKSRC}/lib/libvmmapi/libvmmapi.so.5.1 ${STAGEDIR}${PREFIX}/lib
+.for _bin in bhyve bhyveload bhyvectl
+	${INSTALL_PROGRAM} ${WRKSRC}/usr.sbin/${_bin}/${_bin} ${STAGEDIR}${PREFIX}/sbin
+	${INSTALL_MAN} ${WRKSRC}/usr.sbin/${_bin}/${_bin}.8.gz ${STAGEDIR}${PREFIX}/man/man8
+.endfor
+	${INSTALL_KLD} ${WRKSRC}/sys/modules/vmm/vmm.ko ${STAGEDIR}${KMODDIR}
+
+.include <bsd.port.mk>
diff --git a/sysutils/bhyve+/distinfo b/sysutils/bhyve+/distinfo
new file mode 100644
index 000000000000..2c17bd4508cf
--- /dev/null
+++ b/sysutils/bhyve+/distinfo
@@ -0,0 +1,3 @@
+TIMESTAMP = 1630740589
+SHA256 (conclusiveeng-lib9p-0.1.0-7ef466772108856ee3e093735033232a6f1a3f04_GH0.tar.gz) = 309c8dfc8cc6ff0ab72af6fa36fb0b7dd26c90bfeac0e11ef39344bf64cc2147
+SIZE (conclusiveeng-lib9p-0.1.0-7ef466772108856ee3e093735033232a6f1a3f04_GH0.tar.gz) = 145598
diff --git a/sysutils/bhyve+/files/freebsd-11/patch-bhyve-shelp b/sysutils/bhyve+/files/freebsd-11/patch-bhyve-shelp
new file mode 100644
index 000000000000..8f2bc12fa573
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-11/patch-bhyve-shelp
@@ -0,0 +1,79 @@
+--- usr.sbin/bhyve/pci_emul.c.orig	2021-08-21 23:52:31 UTC
++++ usr.sbin/bhyve/pci_emul.c
+@@ -237,6 +237,17 @@ done:
+ 	return (error);
+ }
+ 
++void
++pci_print_supported_devices()
++{
++	struct pci_devemu **pdpp, *pdp;
++
++	SET_FOREACH(pdpp, pci_devemu_set) {
++		pdp = *pdpp;
++		printf("%s\n", pdp->pe_emu);
++	}
++}
++
+ static int
+ pci_valid_pba_offset(struct pci_devinst *pi, uint64_t offset)
+ {
+--- usr.sbin/bhyve/pci_emul.h.orig	2021-08-21 23:54:47 UTC
++++ usr.sbin/bhyve/pci_emul.h
+@@ -235,6 +235,7 @@ int	pci_msix_table_bar(struct pci_devins
+ int	pci_msix_pba_bar(struct pci_devinst *pi);
+ int	pci_msi_maxmsgnum(struct pci_devinst *pi);
+ int	pci_parse_slot(char *opt);
++void	pci_print_supported_devices();
+ void	pci_populate_msicap(struct msicap *cap, int msgs, int nextptr);
+ int	pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum);
+ int	pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size,
+diff --git a/bhyve.8.orig b/bhyve.8
+index 32392c9..0d4b16f 100644
+--- usr.sbin/bhyve/bhyve.8.orig
++++ usr.sbin/bhyve/bhyve.8
+@@ -47,7 +47,11 @@
+ .Op Fl l Ar lpcdev Ns Op , Ns Ar conf
+ .Op Fl m Ar memsize Ns Op Ar K|k|M|m|G|g|T|t
+ .Op Fl p Ar vcpu:hostcpu
+-.Op Fl s Ar slot,emulation Ns Op , Ns Ar conf
++.Oo Fl s
++.Sm off
++.Cm help | Ar slot,emulation Ns Op , Ns Ar conf
++.Sm on
++.Oc
+ .Op Fl U Ar uuid
+ .Ar vmname
+ .Sh DESCRIPTION
+@@ -157,7 +161,7 @@ to
+ .Em hostcpu .
+ .It Fl P
+ Force the guest virtual CPU to exit when a PAUSE instruction is detected.
+-.It Fl s Ar slot,emulation Ns Op , Ns Ar conf
++.It Fl s Op Ar help|slot,emulation Ns Op , Ns Ar conf
+ Configure a virtual PCI slot and function.
+ .Pp
+ .Nm
+@@ -166,6 +170,8 @@ slots on the bus.
+ There are 32 available slots, with the option of providing up to 8 functions
+ per slot.
+ .Bl -tag -width 10n
++.It Ar help
++print a list of supported PCI devices.
+ .It Ar slot
+ .Ar pcislot[:function]
+ .Ar bus:pcislot:function
+--- usr.sbin/bhyve/bhyverun.c.orig	2021-08-21 23:52:19 UTC
++++ usr.sbin/bhyve/bhyverun.c
+@@ -1058,7 +1058,10 @@ main(int argc, char *argv[])
+ 			}
+ 			break;
+ 		case 's':
+-			if (pci_parse_slot(optarg) != 0)
++			if (strncmp(optarg, "help", strlen(optarg)) == 0) {
++				pci_print_supported_devices();
++				exit(0);
++			} else if (pci_parse_slot(optarg) != 0)
+ 				exit(4);
+ 			else
+ 				break;
diff --git a/sysutils/bhyve+/files/freebsd-11/patch-virtio-9p b/sysutils/bhyve+/files/freebsd-11/patch-virtio-9p
new file mode 100644
index 000000000000..1f98ebb69ddf
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-11/patch-virtio-9p
@@ -0,0 +1,385 @@
+--- usr.sbin/bhyve/Makefile.orig	2021-08-20 16:08:36 UTC
++++ usr.sbin/bhyve/Makefile
+@@ -3,6 +3,7 @@
+ #
+ 
+ .include <src.opts.mk>
++CFLAGS+=-I${SRCTOP}/contrib/lib9p
+ CFLAGS+=-I${SRCTOP}/sys
+ 
+ PROG=	bhyve
+@@ -37,6 +38,7 @@ SRCS=	\
+ 	pci_irq.c		\
+ 	pci_lpc.c		\
+ 	pci_passthru.c		\
++	pci_virtio_9p.c         \
+ 	pci_virtio_block.c	\
+ 	pci_virtio_console.c	\
+ 	pci_virtio_net.c	\
+@@ -63,7 +65,8 @@ SRCS=	\
+ .PATH:  ${BHYVE_SYSDIR}/sys/amd64/vmm
+ SRCS+=	vmm_instruction_emul.c
+ 
+-LIBADD=	vmmapi md pthread z
++LIBADD=	md pthread z sbuf casper cap_pwd cap_grp
++LDADD+= ../../contrib/lib9p/lib9p.so.1.1 ../../lib/libvmmapi/libvmmapi.so.5.1
+ 
+ .if ${MK_INET_SUPPORT} != "no"
+ CFLAGS+=-DINET
+--- usr.sbin/bhyve/pci_virtio_9p.c.orig	2021-08-20 16:11:13 UTC
++++ usr.sbin/bhyve/pci_virtio_9p.c
+@@ -0,0 +1,344 @@
++/*-
++ * Copyright (c) 2015 iXsystems Inc.
++ * Copyright (c) 2017-2018 Jakub Klama <jceel@FreeBSD.org>
++ * All rights reserved.
++ *
++ * Redistribution and use in source and binary forms, with or without
++ * modification, are permitted provided that the following conditions
++ * are met:
++ * 1. Redistributions of source code must retain the above copyright
++ *    notice, this list of conditions and the following disclaimer
++ *    in this position and unchanged.
++ * 2. Redistributions in binary form must reproduce the above copyright
++ *    notice, this list of conditions and the following disclaimer in the
++ *    documentation and/or other materials provided with the distribution.
++ *
++ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
++ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
++ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
++ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
++ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
++ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
++ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
++ * SUCH DAMAGE.
++ */
++
++/*
++ * VirtIO filesystem passthrough using 9p protocol.
++ */
++
++#include <sys/cdefs.h>
++__FBSDID("$FreeBSD$");
++
++#include <sys/param.h>
++#include <sys/linker_set.h>
++#include <sys/uio.h>
++#include <sys/capsicum.h>
++
++#include <errno.h>
++#include <fcntl.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <string.h>
++#include <unistd.h>
++#include <assert.h>
++#include <pthread.h>
++
++#include <lib9p.h>
++#include <backend/fs.h>
++
++#include "bhyverun.h"
++#include "pci_emul.h"
++#include "virtio.h"
++
++#define	VT9P_MAX_IOV	128
++#define VT9P_RINGSZ	256
++#define	VT9P_MAXTAGSZ	256
++#define	VT9P_CONFIGSPACESZ	(VT9P_MAXTAGSZ + sizeof(uint16_t))
++
++static int pci_vt9p_debug;
++#define DPRINTF(params) if (pci_vt9p_debug) printf params
++#define WPRINTF(params) printf params
++
++/*
++ * Per-device softc
++ */
++struct pci_vt9p_softc {
++	struct virtio_softc      vsc_vs;
++	struct vqueue_info       vsc_vq;
++	pthread_mutex_t          vsc_mtx;
++	uint64_t                 vsc_cfg;
++	uint64_t                 vsc_features;
++	char *                   vsc_rootpath;
++	struct pci_vt9p_config * vsc_config;
++	struct l9p_backend *     vsc_fs_backend;
++	struct l9p_server *      vsc_server;
++        struct l9p_connection *  vsc_conn;
++};
++
++struct pci_vt9p_request {
++	struct pci_vt9p_softc *	vsr_sc;
++	struct iovec *		vsr_iov;
++	size_t			vsr_niov;
++	size_t			vsr_respidx;
++	size_t			vsr_iolen;
++	uint16_t		vsr_idx;
++};
++
++struct pci_vt9p_config {
++	uint16_t tag_len;
++	char tag[0];
++} __attribute__((packed));
++
++static int pci_vt9p_send(struct l9p_request *, const struct iovec *,
++    const size_t, const size_t, void *);
++static void pci_vt9p_drop(struct l9p_request *, const struct iovec *, size_t,
++    void *);
++static void pci_vt9p_reset(void *);
++static void pci_vt9p_notify(void *, struct vqueue_info *);
++static int pci_vt9p_cfgread(void *, int, int, uint32_t *);
++static void pci_vt9p_neg_features(void *, uint64_t);
++
++static struct virtio_consts vt9p_vi_consts = {
++	"vt9p",			/* our name */
++	1,			/* we support 1 virtqueue */
++	VT9P_CONFIGSPACESZ,	/* config reg size */
++	pci_vt9p_reset,		/* reset */
++	pci_vt9p_notify,	/* device-wide qnotify */
++	pci_vt9p_cfgread,	/* read virtio config */
++	NULL,			/* write virtio config */
++	pci_vt9p_neg_features,	/* apply negotiated features */
++	(1 << 0),		/* our capabilities */
++};
++
++
++static void
++pci_vt9p_reset(void *vsc)
++{
++	struct pci_vt9p_softc *sc;
++
++	sc = vsc;
++
++	DPRINTF(("vt9p: device reset requested !\n"));
++	vi_reset_dev(&sc->vsc_vs);
++}
++
++static void
++pci_vt9p_neg_features(void *vsc, uint64_t negotiated_features)
++{
++	struct pci_vt9p_softc *sc = vsc;
++
++	sc->vsc_features = negotiated_features;
++}
++
++static int
++pci_vt9p_cfgread(void *vsc, int offset, int size, uint32_t *retval)
++{
++	struct pci_vt9p_softc *sc = vsc;
++	void *ptr;
++
++	ptr = (uint8_t *)sc->vsc_config + offset;
++	memcpy(retval, ptr, size);
++	return (0);
++}
++
++static int
++pci_vt9p_get_buffer(struct l9p_request *req, struct iovec *iov, size_t *niov,
++    void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	size_t n = preq->vsr_niov - preq->vsr_respidx;
++	
++	memcpy(iov, preq->vsr_iov + preq->vsr_respidx,
++	    n * sizeof(struct iovec));
++	*niov = n;
++	return (0);
++}
++
++static int
++pci_vt9p_send(struct l9p_request *req, const struct iovec *iov,
++    const size_t niov, const size_t iolen, void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	struct pci_vt9p_softc *sc = preq->vsr_sc;
++
++	preq->vsr_iolen = iolen;
++
++	pthread_mutex_lock(&sc->vsc_mtx);
++	vq_relchain(&sc->vsc_vq, preq->vsr_idx, preq->vsr_iolen);
++	vq_endchains(&sc->vsc_vq, 1);
++	pthread_mutex_unlock(&sc->vsc_mtx);
++	free(preq);
++	return (0);
++}
++
++static void
++pci_vt9p_drop(struct l9p_request *req, const struct iovec *iov, size_t niov,
++    void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	struct pci_vt9p_softc *sc = preq->vsr_sc;
++
++	pthread_mutex_lock(&sc->vsc_mtx);
++	vq_relchain(&sc->vsc_vq, preq->vsr_idx, 0);
++	vq_endchains(&sc->vsc_vq, 1);
++	pthread_mutex_unlock(&sc->vsc_mtx);
++	free(preq);
++}
++
++static void
++pci_vt9p_notify(void *vsc, struct vqueue_info *vq)
++{
++	struct iovec iov[VT9P_MAX_IOV];
++	struct pci_vt9p_softc *sc;
++	struct pci_vt9p_request *preq;
++	uint16_t idx, n, i;
++	uint16_t flags[VT9P_MAX_IOV];
++
++	sc = vsc;
++
++	while (vq_has_descs(vq)) {
++		n = vq_getchain(vq, &idx, iov, VT9P_MAX_IOV, flags);
++		preq = calloc(1, sizeof(struct pci_vt9p_request));
++		preq->vsr_sc = sc;
++		preq->vsr_idx = idx;
++		preq->vsr_iov = iov;
++		preq->vsr_niov = n;
++		preq->vsr_respidx = 0;
++
++		/* Count readable descriptors */
++		for (i = 0; i < n; i++) {
++			if (flags[i] & VRING_DESC_F_WRITE)
++				break;
++
++			preq->vsr_respidx++;
++		}
++
++		for (int i = 0; i < n; i++) {
++			DPRINTF(("vt9p: vt9p_notify(): desc%d base=%p, "
++			    "len=%zu, flags=0x%04x\r\n", i, iov[i].iov_base,
++			    iov[i].iov_len, flags[i]));
++		}
++
++		l9p_connection_recv(sc->vsc_conn, iov, preq->vsr_respidx, preq);
++	}
++}
++
++
++static int
++pci_vt9p_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
++{
++	struct pci_vt9p_softc *sc;
++	char *opt;
++	char *sharename = NULL;
++	char *rootpath = NULL;
++	int rootfd;
++	bool ro = false;
++	cap_rights_t rootcap;
++
++	if (opts == NULL) {
++		printf("virtio-9p: share name and path required\n");
++		return (1);
++	}
++
++	while ((opt = strsep(&opts, ",")) != NULL) {
++		if (strchr(opt, '=') != NULL) {
++			if (sharename != NULL) {
++				printf("virtio-9p: more than one share name given\n");
++				return (1);
++			}
++
++			sharename = strsep(&opt, "=");
++			rootpath = opt;
++			continue;
++		}
++
++		if (strcmp(opt, "ro") == 0) {
++			DPRINTF(("read-only mount requested\r\n"));
++			ro = true;
++			continue;
++		}
++
++		printf("virtio-9p: invalid option '%s'\n", opt);
++		return (1);
++	}
++
++	if (strlen(sharename) > VT9P_MAXTAGSZ) {
++		printf("virtio-9p: share name too long\n");
++		return (1);
++	}
++
++	rootfd = open(rootpath, O_DIRECTORY);
++	if (rootfd < 0)
++		return (-1);
++
++	sc = calloc(1, sizeof(struct pci_vt9p_softc));
++	sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config) +
++	    VT9P_MAXTAGSZ);
++
++	pthread_mutex_init(&sc->vsc_mtx, NULL);
++
++	cap_rights_init(&rootcap,
++	    CAP_LOOKUP, CAP_ACL_CHECK, CAP_ACL_DELETE, CAP_ACL_GET,
++	    CAP_ACL_SET, CAP_READ, CAP_WRITE, CAP_SEEK, CAP_FSTAT,
++	    CAP_CREATE, CAP_FCHMODAT, CAP_FCHOWNAT, CAP_FTRUNCATE,
++	    CAP_LINKAT_SOURCE, CAP_LINKAT_TARGET, CAP_MKDIRAT, CAP_MKNODAT,
++	    CAP_PREAD, CAP_PWRITE, CAP_RENAMEAT_SOURCE, CAP_RENAMEAT_TARGET,
++	    CAP_SEEK, CAP_SYMLINKAT, CAP_UNLINKAT, CAP_EXTATTR_DELETE,
++	    CAP_EXTATTR_GET, CAP_EXTATTR_LIST, CAP_EXTATTR_SET,
++	    CAP_FUTIMES, CAP_FSTATFS, CAP_FSYNC, CAP_FPATHCONF);
++
++	if (cap_rights_limit(rootfd, &rootcap) != 0)
++		return (1);
++
++	sc->vsc_config->tag_len = (uint16_t)strlen(sharename);
++	memcpy(sc->vsc_config->tag, sharename, sc->vsc_config->tag_len);
++	
++	if (l9p_backend_fs_init(&sc->vsc_fs_backend, rootfd, ro) != 0) {
++		errno = ENXIO;
++		return (1);
++	}
++
++	if (l9p_server_init(&sc->vsc_server, sc->vsc_fs_backend) != 0) {
++		errno = ENXIO;
++		return (1);
++	}
++
++	if (l9p_connection_init(sc->vsc_server, &sc->vsc_conn) != 0) {
++		errno = EIO;
++		return (1);
++	}
++
++	sc->vsc_conn->lc_msize = L9P_MAX_IOV * PAGE_SIZE;
++	sc->vsc_conn->lc_lt.lt_get_response_buffer = pci_vt9p_get_buffer;
++	sc->vsc_conn->lc_lt.lt_send_response = pci_vt9p_send;
++	sc->vsc_conn->lc_lt.lt_drop_response = pci_vt9p_drop;
++
++	vi_softc_linkup(&sc->vsc_vs, &vt9p_vi_consts, sc, pi, &sc->vsc_vq);
++	sc->vsc_vs.vs_mtx = &sc->vsc_mtx;
++	sc->vsc_vq.vq_qsize = VT9P_RINGSZ;
++
++	/* initialize config space */
++	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_9P);
++	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
++	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
++	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_9P);
++	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
++
++	if (vi_intr_init(&sc->vsc_vs, 1, fbsdrun_virtio_msix()))
++		return (1);
++	vi_set_io_bar(&sc->vsc_vs, 0);
++
++	return (0);
++}
++
++struct pci_devemu pci_de_v9p = {
++	.pe_emu =	"virtio-9p",
++	.pe_init =	pci_vt9p_init,
++	.pe_barwrite =	vi_pci_write,
++	.pe_barread =	vi_pci_read
++};
++PCI_EMUL_SET(pci_de_v9p);
+--- usr.sbin/bhyve/virtio.h.orig	2021-08-20 16:20:49 UTC
++++ usr.sbin/bhyve/virtio.h
+@@ -215,6 +215,7 @@ struct vring_used {
+ #define	VIRTIO_DEV_BLOCK	0x1001
+ #define	VIRTIO_DEV_RANDOM	0x1005
+ #define	VIRTIO_DEV_CONSOLE	0x1003
++#define        VIRTIO_DEV_9P           0x1009
+ 
+ /*
+  * PCI config space constants.
diff --git a/sysutils/bhyve+/files/freebsd-11/patch-vtd-fix b/sysutils/bhyve+/files/freebsd-11/patch-vtd-fix
new file mode 100644
index 000000000000..a5f2ce648941
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-11/patch-vtd-fix
@@ -0,0 +1,211 @@
+--- sys/amd64/vmm/intel/vtd.c
++++ sys/amd64/vmm/intel/vtd.c
+@@ -51,6 +51,8 @@ __FBSDID("$FreeBSD$");
+  * Architecture Spec, September 2008.
+  */
+ 
++#define VTD_DRHD_INCLUDE_PCI_ALL(Flags)  (((Flags) >> 0) & 0x1)
++
+ /* Section 10.4 "Register Descriptions" */
+ struct vtdmap {
+ 	volatile uint32_t	version;
+@@ -116,10 +118,11 @@ struct domain {
+ static SLIST_HEAD(, domain) domhead;
+ 
+ #define	DRHD_MAX_UNITS	8
+-static int		drhd_num;
+-static struct vtdmap	*vtdmaps[DRHD_MAX_UNITS];
+-static int		max_domains;
+-typedef int		(*drhd_ident_func_t)(void);
++static ACPI_DMAR_HARDWARE_UNIT	*drhds[DRHD_MAX_UNITS];
++static int			drhd_num;
++static struct vtdmap		*vtdmaps[DRHD_MAX_UNITS];
++static int			max_domains;
++typedef int			(*drhd_ident_func_t)(void);
+ 
+ static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
+ static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
+@@ -175,6 +178,69 @@ domain_id(void)
+ 	return (id);
+ }
+ 
++static struct vtdmap *
++vtd_device_scope(uint16_t rid)
++{
++	int i, remaining, pathremaining;
++	char *end, *pathend;
++	struct vtdmap *vtdmap;
++	ACPI_DMAR_HARDWARE_UNIT *drhd;
++	ACPI_DMAR_DEVICE_SCOPE *device_scope;
++	ACPI_DMAR_PCI_PATH *path;
++
++	for (i = 0; i < drhd_num; i++) {
++		drhd = drhds[i];
++
++		if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {
++			/*
++			 * From Intel VT-d arch spec, version 3.0:
++			 * If a DRHD structure with INCLUDE_PCI_ALL flag Set is reported
++			 * for a Segment, it must be enumerated by BIOS after all other
++			 * DRHD structures for the same Segment.
++			 */
++			vtdmap = vtdmaps[i];
++			return(vtdmap);
++		}
++
++		end = (char *)drhd + drhd->Header.Length;
++		remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT);
++		while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) {
++			device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining);
++			remaining -= device_scope->Length;
++
++			switch (device_scope->EntryType){
++				/* 0x01 and 0x02 are PCI device entries */
++				case 0x01:
++				case 0x02:
++					break;
++				default:
++					continue;
++			}
++
++			if (PCI_RID2BUS(rid) != device_scope->Bus)
++				continue;
++
++			pathend = (char *)device_scope + device_scope->Length;
++			pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE);
++			while (pathremaining >= sizeof(ACPI_DMAR_PCI_PATH)) {
++				path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining);
++				pathremaining -= sizeof(ACPI_DMAR_PCI_PATH);
++
++				if (PCI_RID2SLOT(rid) != path->Device)
++					continue;
++				if (PCI_RID2FUNC(rid) != path->Function)
++					continue;
++
++				vtdmap = vtdmaps[i];
++				return (vtdmap);
++			}
++		}
++	}
++
++	/* No matching scope */
++	return (NULL);
++}
++
+ static void
+ vtd_wbflush(struct vtdmap *vtdmap)
+ {
+@@ -240,7 +306,7 @@ vtd_translation_disable(struct vtdmap *vtdmap)
+ static int
+ vtd_init(void)
+ {
+-	int i, units, remaining;
++	int i, units, remaining, tmp;
+ 	struct vtdmap *vtdmap;
+ 	vm_paddr_t ctx_paddr;
+ 	char *end, envname[32];
+@@ -291,8 +357,9 @@ vtd_init(void)
+ 			break;
+ 
+ 		drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr;
+-		vtdmaps[units++] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);
+-		if (units >= DRHD_MAX_UNITS)
++		drhds[units] = drhd;
++		vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);
++		if (++units >= DRHD_MAX_UNITS)
+ 			break;
+ 		remaining -= hdr->Length;
+ 	}
+@@ -302,12 +369,18 @@ vtd_init(void)
+ 
+ skip_dmar:
+ 	drhd_num = units;
+-	vtdmap = vtdmaps[0];
+ 
+-	if (VTD_CAP_CM(vtdmap->cap) != 0)
+-		panic("vtd_init: invalid caching mode");
++	max_domains = 64 * 1024; /* maximum valid value */
++	for (i = 0; i < drhd_num; i++){
++		vtdmap = vtdmaps[i];
++
++		if (VTD_CAP_CM(vtdmap->cap) != 0)
++			panic("vtd_init: invalid caching mode");
+ 
+-	max_domains = vtd_max_domains(vtdmap);
++		/* take most compatible (minimum) value */
++		if ((tmp = vtd_max_domains(vtdmap)) < max_domains)
++			max_domains = tmp;
++	}
+ 
+ 	/*
+ 	 * Set up the root-table to point to the context-entry tables
+@@ -373,7 +446,6 @@ vtd_add_device(void *arg, uint16_t rid)
+ 	struct vtdmap *vtdmap;
+ 	uint8_t bus;
+ 
+-	vtdmap = vtdmaps[0];
+ 	bus = PCI_RID2BUS(rid);
+ 	ctxp = ctx_tables[bus];
+ 	pt_paddr = vtophys(dom->ptp);
+@@ -385,6 +457,10 @@ vtd_add_device(void *arg, uint16_t rid)
+ 		      (uint16_t)(ctxp[idx + 1] >> 8));
+ 	}
+ 
++	if ((vtdmap = vtd_device_scope(rid)) == NULL)
++		panic("vtd_add_device: device %x is not in scope for "
++		      "any DMA remapping unit", rid);
++
+ 	/*
+ 	 * Order is important. The 'present' bit is set only after all fields
+ 	 * of the context pointer are initialized.
+@@ -568,8 +644,6 @@ vtd_create_domain(vm_paddr_t maxaddr)
+ 	if (drhd_num <= 0)
+ 		panic("vtd_create_domain: no dma remapping hardware available");
+ 
+-	vtdmap = vtdmaps[0];
+-
+ 	/*
+ 	 * Calculate AGAW.
+ 	 * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec.
+@@ -594,7 +668,14 @@ vtd_create_domain(vm_paddr_t maxaddr)
+ 	pt_levels = 2;
+ 	sagaw = 30;
+ 	addrwidth = 0;
+-	tmp = VTD_CAP_SAGAW(vtdmap->cap);
++
++	tmp = ~0;
++	for (i = 0; i < drhd_num; i++) {
++		vtdmap = vtdmaps[i];
++		/* take most compatible value */
++		tmp &= VTD_CAP_SAGAW(vtdmap->cap);
++	}
++
+ 	for (i = 0; i < 5; i++) {
+ 		if ((tmp & (1 << i)) != 0 && sagaw >= agaw)
+ 			break;
+@@ -606,8 +687,8 @@ vtd_create_domain(vm_paddr_t maxaddr)
+ 	}
+ 
+ 	if (i >= 5) {
+-		panic("vtd_create_domain: SAGAW 0x%lx does not support AGAW %d",
+-		      VTD_CAP_SAGAW(vtdmap->cap), agaw);
++		panic("vtd_create_domain: SAGAW 0x%x does not support AGAW %d",
++		      tmp, agaw);
+ 	}
+ 
+ 	dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK);
+@@ -634,7 +715,12 @@ vtd_create_domain(vm_paddr_t maxaddr)
+ 	 * There is not any code to deal with the demotion at the moment
+ 	 * so we disable superpage mappings altogether.
+ 	 */
+-	dom->spsmask = VTD_CAP_SPS(vtdmap->cap);
++	dom->spsmask = ~0;
++	for (i = 0; i < drhd_num; i++) {
++		vtdmap = vtdmaps[i];
++		/* take most compatible value */
++		dom->spsmask &= VTD_CAP_SPS(vtdmap->cap);
++	}
+ #endif
+ 
+ 	SLIST_INSERT_HEAD(&domhead, dom, next);
+
diff --git a/sysutils/bhyve+/files/freebsd-12/patch-virtio-9p b/sysutils/bhyve+/files/freebsd-12/patch-virtio-9p
new file mode 100644
index 000000000000..6799fd986342
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-12/patch-virtio-9p
@@ -0,0 +1,416 @@
+--- usr.sbin/bhyve/bhyve.8.orig	2021-08-19 22:38:50 UTC
++++ usr.sbin/bhyve/bhyve.8
+@@ -242,6 +242,8 @@ Virtio network interface.
+ Virtio block storage interface.
+ .It Li virtio-scsi
+ Virtio SCSI interface.
++.It Li virtio-9p
++Virtio 9p (VirtFS) interface.
+ .It Li virtio-rnd
+ Virtio RNG interface.
+ .It Li virtio-console
+@@ -327,6 +329,19 @@ are:
+ .It Li iid= Ns Ar IID
+ Initiator ID to use when sending requests to specified CTL port.
+ The default value is 0.
++.El
++.Pp
++9P devices:
++.Bl -tag -width 10n
++.It Pa sharename=/path/to/share[,9p-device-options]
++.El
++.Pp
++The
++.Ar 9p-device-options
++are:
++.Bl -tag -width 10n
++.It Li ro
++Expose the share in read-only mode.
+ .El
+ .Pp
+ TTY devices:
+--- usr.sbin/bhyve/Makefile.orig	2021-08-19 23:00:57 UTC
++++ usr.sbin/bhyve/Makefile
+@@ -3,6 +3,7 @@
+ #
+ 
+ .include <src.opts.mk>
++CFLAGS+=-I${SRCTOP}/contrib/lib9p
+ CFLAGS+=-I${SRCTOP}/sys
+ .PATH:  ${SRCTOP}/sys/cam/ctl
+ 
+@@ -43,6 +44,7 @@ SRCS=	\
+ 	pci_lpc.c		\
+ 	pci_nvme.c		\
+ 	pci_passthru.c		\
++	pci_virtio_9p.c         \
+ 	pci_virtio_block.c	\
+ 	pci_virtio_console.c	\
+ 	pci_virtio_net.c	\
+@@ -71,7 +73,8 @@ SRCS=	\
+ .PATH:  ${BHYVE_SYSDIR}/sys/amd64/vmm
+ SRCS+=	vmm_instruction_emul.c
+ 
+-LIBADD=	vmmapi md pthread z util sbuf cam
++LIBADD=	md pthread z util sbuf cam casper cap_pwd cap_grp
++LDADD=  ../../contrib/lib9p/lib9p.so.1.1 ../../lib/libvmmapi/libvmmapi.so.5.1
+ 
+ .if ${MK_INET_SUPPORT} != "no"
+ CFLAGS+=-DINET
+--- usr.sbin/bhyve/pci_virtio_9p.c.orig	2021-08-19 22:44:10 UTC
++++ usr.sbin/bhyve/pci_virtio_9p.c
+@@ -0,0 +1,344 @@
++/*-
++ * Copyright (c) 2015 iXsystems Inc.
++ * Copyright (c) 2017-2018 Jakub Klama <jceel@FreeBSD.org>
++ * All rights reserved.
++ *
++ * Redistribution and use in source and binary forms, with or without
++ * modification, are permitted provided that the following conditions
++ * are met:
++ * 1. Redistributions of source code must retain the above copyright
++ *    notice, this list of conditions and the following disclaimer
++ *    in this position and unchanged.
++ * 2. Redistributions in binary form must reproduce the above copyright
++ *    notice, this list of conditions and the following disclaimer in the
++ *    documentation and/or other materials provided with the distribution.
++ *
++ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
++ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
++ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
++ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
++ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
++ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
++ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
++ * SUCH DAMAGE.
++ */
++
++/*
++ * VirtIO filesystem passthrough using 9p protocol.
++ */
++
++#include <sys/cdefs.h>
++__FBSDID("$FreeBSD$");
++
++#include <sys/param.h>
++#include <sys/linker_set.h>
++#include <sys/uio.h>
++#include <sys/capsicum.h>
++
++#include <errno.h>
++#include <fcntl.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <string.h>
++#include <unistd.h>
++#include <assert.h>
++#include <pthread.h>
++
++#include <lib9p.h>
++#include <backend/fs.h>
++
++#include "bhyverun.h"
++#include "pci_emul.h"
++#include "virtio.h"
++
++#define	VT9P_MAX_IOV	128
++#define VT9P_RINGSZ	256
++#define	VT9P_MAXTAGSZ	256
++#define	VT9P_CONFIGSPACESZ	(VT9P_MAXTAGSZ + sizeof(uint16_t))
++
++static int pci_vt9p_debug;
++#define DPRINTF(params) if (pci_vt9p_debug) printf params
++#define WPRINTF(params) printf params
++
++/*
++ * Per-device softc
++ */
++struct pci_vt9p_softc {
++	struct virtio_softc      vsc_vs;
++	struct vqueue_info       vsc_vq;
++	pthread_mutex_t          vsc_mtx;
++	uint64_t                 vsc_cfg;
++	uint64_t                 vsc_features;
++	char *                   vsc_rootpath;
++	struct pci_vt9p_config * vsc_config;
++	struct l9p_backend *     vsc_fs_backend;
++	struct l9p_server *      vsc_server;
++        struct l9p_connection *  vsc_conn;
++};
++
++struct pci_vt9p_request {
++	struct pci_vt9p_softc *	vsr_sc;
++	struct iovec *		vsr_iov;
++	size_t			vsr_niov;
++	size_t			vsr_respidx;
++	size_t			vsr_iolen;
++	uint16_t		vsr_idx;
++};
++
++struct pci_vt9p_config {
++	uint16_t tag_len;
++	char tag[0];
++} __attribute__((packed));
++
++static int pci_vt9p_send(struct l9p_request *, const struct iovec *,
++    const size_t, const size_t, void *);
++static void pci_vt9p_drop(struct l9p_request *, const struct iovec *, size_t,
++    void *);
++static void pci_vt9p_reset(void *);
++static void pci_vt9p_notify(void *, struct vqueue_info *);
++static int pci_vt9p_cfgread(void *, int, int, uint32_t *);
++static void pci_vt9p_neg_features(void *, uint64_t);
++
++static struct virtio_consts vt9p_vi_consts = {
++	"vt9p",			/* our name */
++	1,			/* we support 1 virtqueue */
++	VT9P_CONFIGSPACESZ,	/* config reg size */
++	pci_vt9p_reset,		/* reset */
++	pci_vt9p_notify,	/* device-wide qnotify */
++	pci_vt9p_cfgread,	/* read virtio config */
++	NULL,			/* write virtio config */
++	pci_vt9p_neg_features,	/* apply negotiated features */
++	(1 << 0),		/* our capabilities */
++};
++
++
++static void
++pci_vt9p_reset(void *vsc)
++{
++	struct pci_vt9p_softc *sc;
++
++	sc = vsc;
++
++	DPRINTF(("vt9p: device reset requested !\n"));
++	vi_reset_dev(&sc->vsc_vs);
++}
++
++static void
++pci_vt9p_neg_features(void *vsc, uint64_t negotiated_features)
++{
++	struct pci_vt9p_softc *sc = vsc;
++
++	sc->vsc_features = negotiated_features;
++}
++
++static int
++pci_vt9p_cfgread(void *vsc, int offset, int size, uint32_t *retval)
++{
++	struct pci_vt9p_softc *sc = vsc;
++	void *ptr;
++
++	ptr = (uint8_t *)sc->vsc_config + offset;
++	memcpy(retval, ptr, size);
++	return (0);
++}
++
++static int
++pci_vt9p_get_buffer(struct l9p_request *req, struct iovec *iov, size_t *niov,
++    void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	size_t n = preq->vsr_niov - preq->vsr_respidx;
++	
++	memcpy(iov, preq->vsr_iov + preq->vsr_respidx,
++	    n * sizeof(struct iovec));
++	*niov = n;
++	return (0);
++}
++
++static int
++pci_vt9p_send(struct l9p_request *req, const struct iovec *iov,
++    const size_t niov, const size_t iolen, void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	struct pci_vt9p_softc *sc = preq->vsr_sc;
++
++	preq->vsr_iolen = iolen;
++
++	pthread_mutex_lock(&sc->vsc_mtx);
++	vq_relchain(&sc->vsc_vq, preq->vsr_idx, preq->vsr_iolen);
++	vq_endchains(&sc->vsc_vq, 1);
++	pthread_mutex_unlock(&sc->vsc_mtx);
++	free(preq);
++	return (0);
++}
++
++static void
++pci_vt9p_drop(struct l9p_request *req, const struct iovec *iov, size_t niov,
++    void *arg)
++{
++	struct pci_vt9p_request *preq = req->lr_aux;
++	struct pci_vt9p_softc *sc = preq->vsr_sc;
++
++	pthread_mutex_lock(&sc->vsc_mtx);
++	vq_relchain(&sc->vsc_vq, preq->vsr_idx, 0);
++	vq_endchains(&sc->vsc_vq, 1);
++	pthread_mutex_unlock(&sc->vsc_mtx);
++	free(preq);
++}
++
++static void
++pci_vt9p_notify(void *vsc, struct vqueue_info *vq)
++{
++	struct iovec iov[VT9P_MAX_IOV];
++	struct pci_vt9p_softc *sc;
++	struct pci_vt9p_request *preq;
++	uint16_t idx, n, i;
++	uint16_t flags[VT9P_MAX_IOV];
++
++	sc = vsc;
++
++	while (vq_has_descs(vq)) {
++		n = vq_getchain(vq, &idx, iov, VT9P_MAX_IOV, flags);
++		preq = calloc(1, sizeof(struct pci_vt9p_request));
++		preq->vsr_sc = sc;
++		preq->vsr_idx = idx;
++		preq->vsr_iov = iov;
++		preq->vsr_niov = n;
++		preq->vsr_respidx = 0;
++
++		/* Count readable descriptors */
++		for (i = 0; i < n; i++) {
++			if (flags[i] & VRING_DESC_F_WRITE)
++				break;
++
++			preq->vsr_respidx++;
++		}
++
++		for (int i = 0; i < n; i++) {
++			DPRINTF(("vt9p: vt9p_notify(): desc%d base=%p, "
++			    "len=%zu, flags=0x%04x\r\n", i, iov[i].iov_base,
++			    iov[i].iov_len, flags[i]));
++		}
++
++		l9p_connection_recv(sc->vsc_conn, iov, preq->vsr_respidx, preq);
++	}
++}
++
++
++static int
++pci_vt9p_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
++{
++	struct pci_vt9p_softc *sc;
++	char *opt;
++	char *sharename = NULL;
++	char *rootpath = NULL;
++	int rootfd;
++	bool ro = false;
++	cap_rights_t rootcap;
++
++	if (opts == NULL) {
++		printf("virtio-9p: share name and path required\n");
++		return (1);
++	}
++
++	while ((opt = strsep(&opts, ",")) != NULL) {
++		if (strchr(opt, '=') != NULL) {
++			if (sharename != NULL) {
++				printf("virtio-9p: more than one share name given\n");
++				return (1);
++			}
++
++			sharename = strsep(&opt, "=");
++			rootpath = opt;
++			continue;
++		}
++
++		if (strcmp(opt, "ro") == 0) {
++			DPRINTF(("read-only mount requested\r\n"));
++			ro = true;
++			continue;
++		}
++
++		printf("virtio-9p: invalid option '%s'\n", opt);
++		return (1);
++	}
++
++	if (strlen(sharename) > VT9P_MAXTAGSZ) {
++		printf("virtio-9p: share name too long\n");
++		return (1);
++	}
++
++	rootfd = open(rootpath, O_DIRECTORY);
++	if (rootfd < 0)
++		return (-1);
++
++	sc = calloc(1, sizeof(struct pci_vt9p_softc));
++	sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config) +
++	    VT9P_MAXTAGSZ);
++
++	pthread_mutex_init(&sc->vsc_mtx, NULL);
++
++	cap_rights_init(&rootcap,
++	    CAP_LOOKUP, CAP_ACL_CHECK, CAP_ACL_DELETE, CAP_ACL_GET,
++	    CAP_ACL_SET, CAP_READ, CAP_WRITE, CAP_SEEK, CAP_FSTAT,
++	    CAP_CREATE, CAP_FCHMODAT, CAP_FCHOWNAT, CAP_FTRUNCATE,
++	    CAP_LINKAT_SOURCE, CAP_LINKAT_TARGET, CAP_MKDIRAT, CAP_MKNODAT,
++	    CAP_PREAD, CAP_PWRITE, CAP_RENAMEAT_SOURCE, CAP_RENAMEAT_TARGET,
++	    CAP_SEEK, CAP_SYMLINKAT, CAP_UNLINKAT, CAP_EXTATTR_DELETE,
++	    CAP_EXTATTR_GET, CAP_EXTATTR_LIST, CAP_EXTATTR_SET,
++	    CAP_FUTIMES, CAP_FSTATFS, CAP_FSYNC, CAP_FPATHCONF);
++
++	if (cap_rights_limit(rootfd, &rootcap) != 0)
++		return (1);
++
++	sc->vsc_config->tag_len = (uint16_t)strlen(sharename);
++	memcpy(sc->vsc_config->tag, sharename, sc->vsc_config->tag_len);
++	
++	if (l9p_backend_fs_init(&sc->vsc_fs_backend, rootfd, ro) != 0) {
++		errno = ENXIO;
++		return (1);
++	}
++
++	if (l9p_server_init(&sc->vsc_server, sc->vsc_fs_backend) != 0) {
++		errno = ENXIO;
++		return (1);
++	}
++
++	if (l9p_connection_init(sc->vsc_server, &sc->vsc_conn) != 0) {
++		errno = EIO;
++		return (1);
++	}
++
++	sc->vsc_conn->lc_msize = L9P_MAX_IOV * PAGE_SIZE;
++	sc->vsc_conn->lc_lt.lt_get_response_buffer = pci_vt9p_get_buffer;
++	sc->vsc_conn->lc_lt.lt_send_response = pci_vt9p_send;
++	sc->vsc_conn->lc_lt.lt_drop_response = pci_vt9p_drop;
++
++	vi_softc_linkup(&sc->vsc_vs, &vt9p_vi_consts, sc, pi, &sc->vsc_vq);
++	sc->vsc_vs.vs_mtx = &sc->vsc_mtx;
++	sc->vsc_vq.vq_qsize = VT9P_RINGSZ;
++
++	/* initialize config space */
++	pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_9P);
++	pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
++	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
++	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_9P);
++	pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
++
++	if (vi_intr_init(&sc->vsc_vs, 1, fbsdrun_virtio_msix()))
++		return (1);
++	vi_set_io_bar(&sc->vsc_vs, 0);
++
++	return (0);
++}
++
++struct pci_devemu pci_de_v9p = {
++	.pe_emu =	"virtio-9p",
++	.pe_init =	pci_vt9p_init,
++	.pe_barwrite =	vi_pci_write,
++	.pe_barread =	vi_pci_read
++};
++PCI_EMUL_SET(pci_de_v9p);
+--- usr.sbin/bhyve/virtio.h.orig	2021-08-19 22:41:21 UTC
++++ usr.sbin/bhyve/virtio.h
+@@ -216,6 +216,7 @@ struct vring_used {
+ #define	VIRTIO_DEV_CONSOLE	0x1003
+ #define	VIRTIO_DEV_RANDOM	0x1005
+ #define	VIRTIO_DEV_SCSI		0x1008
++#define        VIRTIO_DEV_9P           0x1009
+
+ /*
+  * PCI config space constants.
diff --git a/sysutils/bhyve+/files/freebsd-13/patch-custom-libs b/sysutils/bhyve+/files/freebsd-13/patch-custom-libs
new file mode 100644
index 000000000000..aa5772dc0511
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-13/patch-custom-libs
@@ -0,0 +1,21 @@
+--- usr.sbin/bhyve/Makefile.orig	2021-09-04 02:15:16.364660000 +0200
++++ usr.sbin/bhyve/Makefile	2021-09-04 02:21:26.014446000 +0200
+@@ -3,7 +3,7 @@
+ #
+ 
+ .include <src.opts.mk>
+-CFLAGS+=-I${.CURDIR}/../../contrib/lib9p
++CFLAGS+=-I${SRCTOP}/contrib/lib9p
+ CFLAGS+=-I${SRCTOP}/sys
+ .PATH:  ${SRCTOP}/sys/cam/ctl
+ 
+@@ -83,7 +83,8 @@
+ .PATH:  ${BHYVE_SYSDIR}/sys/amd64/vmm
+ SRCS+=	vmm_instruction_emul.c
+ 
+-LIBADD=	vmmapi md pthread z util sbuf cam 9p
++LIBADD=	md pthread z util sbuf cam
++LDADD=	../../contrib/lib9p/lib9p.so.1.1 ../../lib/libvmmapi/libvmmapi.so.5.1
+ 
+ .if ${MK_CASPER} != "no"
+ LIBADD+=	casper
diff --git a/sysutils/bhyve+/files/freebsd-13/patch-lib9p b/sysutils/bhyve+/files/freebsd-13/patch-lib9p
new file mode 100644
index 000000000000..97f2fd5b00a7
--- /dev/null
+++ b/sysutils/bhyve+/files/freebsd-13/patch-lib9p
@@ -0,0 +1,12 @@
+--- share/mk/src.libnames.mk.orig	2021-09-04 10:14:17.407288000 +0200
++++ share/mk/src.libnames.mk	2021-09-04 10:15:27.832153000 +0200
+@@ -245,6 +245,9 @@
+ # 2nd+ order consumers.  Auto-generating this would be better.
+ _DP_80211=	sbuf bsdxml
+ _DP_9p=		sbuf
++.if ${MK_CASPER} != "no"
++_DP_9p+=       casper cap_pwd cap_grp
++.endif
+ _DP_archive=	z bz2 lzma bsdxml zstd
+ _DP_zstd=	pthread
+ .if ${MK_BLACKLIST} != "no"
diff --git a/sysutils/bhyve+/files/kernconf/opt_acpi.h b/sysutils/bhyve+/files/kernconf/opt_acpi.h
new file mode 100644
index 000000000000..2a4ebae95886
--- /dev/null
+++ b/sysutils/bhyve+/files/kernconf/opt_acpi.h
@@ -0,0 +1,2 @@
+#define ACPI_DMAR 1
+#define DEV_ACPI 1
diff --git a/sysutils/bhyve+/files/kernconf/opt_bhyve_snapshot.h b/sysutils/bhyve+/files/kernconf/opt_bhyve_snapshot.h
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/sysutils/bhyve+/files/kernconf/opt_bhyve_snapshot.h
diff --git a/sysutils/bhyve+/files/kernconf/opt_ddb.h b/sysutils/bhyve+/files/kernconf/opt_ddb.h
new file mode 100644
index 000000000000..79d95ae3ad8d
--- /dev/null
+++ b/sysutils/bhyve+/files/kernconf/opt_ddb.h
@@ -0,0 +1 @@
+#define DDB_CTF 1
diff --git a/sysutils/bhyve+/files/kernconf/opt_global.h b/sysutils/bhyve+/files/kernconf/opt_global.h
new file mode 100644
index 000000000000..3c736873d19c
--- /dev/null
+++ b/sysutils/bhyve+/files/kernconf/opt_global.h
@@ -0,0 +1,3 @@
+#define KDB 1
+#define SMP 1
+#define KDTRACE_HOOKS 1
diff --git a/sysutils/bhyve+/files/patch-lib9p b/sysutils/bhyve+/files/patch-lib9p
new file mode 100644
index 000000000000..f48b6a8efa91
--- /dev/null
+++ b/sysutils/bhyve+/files/patch-lib9p
@@ -0,0 +1,24 @@
+--- contrib/lib9p/Makefile.orig	2021-01-08 21:21:39.000000000 UTC
++++ contrib/lib9p/Makefile
+@@ -5,7 +5,7 @@
+ .include <src.opts.mk>
+ 
+ LIB=		9p
+-SHLIB_MAJOR=	1
++SHLIB_MAJOR=	1.1
+ SRCS=		pack.c \
+ 		connection.c \
+ 		request.c log.c \
+@@ -23,11 +23,9 @@
+ LIBADD=		sbuf
+ 
+ .if ${MK_CASPER} != "no"
+-LIBADD+=	libcasper libcap_pwd libcap_grp
++LIBADD+=	casper cap_pwd cap_grp
+ CFLAGS+=	-DWITH_CASPER
+ .endif
+-
+-SUBDIR=		example
+ 
+ cscope: .PHONY
+ 	cd ${.CURDIR}; cscope -buq $$(find . -name '*.[ch]' -print)
diff --git a/sysutils/bhyve+/files/patch-libvmmapi b/sysutils/bhyve+/files/patch-libvmmapi
new file mode 100644
index 000000000000..16f14a57bf9a
--- /dev/null
+++ b/sysutils/bhyve+/files/patch-libvmmapi
@@ -0,0 +1,33 @@
+--- lib/libvmmapi/Makefile.orig	2021-09-04 01:05:23.110816000 UTC
++++ lib/libvmmapi/Makefile
+@@ -6,6 +6,7 @@
+ INCS=	vmmapi.h
+ 
+ WARNS?=	2
++SHLIB_MAJOR?= 5.1
+ 
+ LIBADD=	util
+
+--- usr.sbin/bhyvectl/Makefile.orig	2021-09-04 02:33:12.704720000 +0200
++++ usr.sbin/bhyvectl/Makefile	2021-09-04 02:34:28.740202000 +0200
+@@ -8,7 +8,8 @@
+ 
+ MAN=	bhyvectl.8
+ 
+-LIBADD=	vmmapi util
++LIBADD=	util
++LDADD=	../../lib/libvmmapi/libvmmapi.so.5.1
+ 
+ WARNS?=	3
+ 
+--- usr.sbin/bhyveload/Makefile.orig	2021-09-04 02:33:20.942548000 +0200
++++ usr.sbin/bhyveload/Makefile	2021-09-04 02:35:58.712996000 +0200
+@@ -5,7 +5,7 @@
+ MAN=	bhyveload.8
+ PACKAGE=	bhyve
+ 
+-LIBADD=	vmmapi
++LDADD=	../../lib/libvmmapi/libvmmapi.so.5.1
+ 
+ WARNS?=	3
+ 
diff --git a/sysutils/bhyve+/files/patch-msi-x-mappings b/sysutils/bhyve+/files/patch-msi-x-mappings
new file mode 100644
index 000000000000..4c8f453abbbb
--- /dev/null
+++ b/sysutils/bhyve+/files/patch-msi-x-mappings
@@ -0,0 +1,132 @@
+--- usr.sbin/bhyve/pci_emul.h.orig	2021-08-19 23:00:57 UTC
++++ usr.sbin/bhyve/pci_emul.h
+@@ -146,6 +146,7 @@ struct pci_devinst {
+ 		struct msix_table_entry *table;	/* allocated at runtime */
+ 		void	*pba_page;
+ 		int	pba_page_offset;
++		void	*table_page;
+ 	} pi_msix;
+ 
+ 	void      *pi_arg;		/* devemu-private data */
+--- usr.sbin/bhyve/pci_passthru.c.orig	2021-08-19 23:00:57 UTC
++++ usr.sbin/bhyve/pci_passthru.c
+@@ -324,13 +324,14 @@ msix_table_read(struct passthru_softc *sc, uint64_t of
+ 		return (data);
+ 	}
+ 
++	/* Should make this an assert. */
+ 	if (offset < pi->pi_msix.table_offset)
+ 		return (-1);
+ 
+ 	offset -= pi->pi_msix.table_offset;
+ 	index = offset / MSIX_TABLE_ENTRY_SIZE;
+ 	if (index >= pi->pi_msix.table_count)
+-		return (-1);
++		goto readbar;
+ 
+ 	entry = &pi->pi_msix.table[index];
+ 	entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
+@@ -357,6 +358,33 @@ msix_table_read(struct passthru_softc *sc, uint64_t of
+ 	}
+ 
+ 	return (data);
++
++readbar:
++	if (pi->pi_msix.table_page != NULL && offset < 4096) {
++		switch(size) {
++		case 1:
++			src8 = (uint8_t *)(pi->pi_msix.table_page + offset);
++			data = *src8;
++			break;
++		case 2:
++			src16 = (uint16_t *)(pi->pi_msix.table_page + offset);
++			data = *src16;
++			break;
++		case 4:
++			src32 = (uint32_t *)(pi->pi_msix.table_page + offset);
++			data = *src32;
++			break;
++		case 8:
++			src64 = (uint64_t *)(pi->pi_msix.table_page + offset);
++			data = *src64;
++			break;
++		default:
++			return (-1);
++		}
++		return (data);
++	}
++
++	return (-1);
+ }
+ 
+ static void
+@@ -403,13 +431,14 @@ msix_table_write(struct vmctx *ctx, int vcpu, struct p
+ 		return;
+ 	}
+ 
++	/* Should make this an assert. */
+ 	if (offset < pi->pi_msix.table_offset)
+ 		return;
+ 
+ 	offset -= pi->pi_msix.table_offset;
+ 	index = offset / MSIX_TABLE_ENTRY_SIZE;
+ 	if (index >= pi->pi_msix.table_count)
+-		return;
++		goto writebar;
+ 
+ 	entry = &pi->pi_msix.table[index];
+ 	entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
+@@ -432,6 +461,31 @@ msix_table_write(struct vmctx *ctx, int vcpu, struct p
+ 			    entry->msg_data, entry->vector_control);
+ 		}
+ 	}
++
++writebar:
++	if (pi->pi_msix.table_page != NULL && offset < 4096) {
++		switch(size) {
++		case 1:
++			dest8 = (uint8_t *)(pi->pi_msix.table_page + offset);
++			*dest8 = data;
++			break;
++		case 2:
++			dest16 = (uint16_t *)(pi->pi_msix.table_page + offset);
++			*dest16 = data;
++			break;
++		case 4:
++			dest32 = (uint32_t *)(pi->pi_msix.table_page + offset);
++			*dest32 = data;
++			break;
++		case 8:
++			dest64 = (uint64_t *)(pi->pi_msix.table_page + offset);
++			*dest64 = data;
++			break;
++		default:
++			break;
++		}
++		return;
++	}
+ }
+ 
+ static int
+@@ -466,6 +520,21 @@ init_msix_table(struct vmctx *ctx, struct passthru_sof
+ 	idx = pi->pi_msix.table_bar;
+ 	start = pi->pi_bar[idx].addr;
+ 	remaining = pi->pi_bar[idx].size;
++
++	/*
++	 * Some device (against better documentation of the spec)
++	 * are mapping other usable address space into the same page
++	 * as the end of the MSI-X tables.
++	 * At least Intel AX200 being one of them apparently.
++	 * Map the page and fall back to it for any reads/writes outside
++	 * the MSI-X table in msix_table_{read,write}.
++	 */
++	pi->pi_msix.table_page = mmap(NULL, 4096, PROT_READ | PROT_WRITE,
++	    MAP_SHARED, memfd, sc->psc_bar[idx].addr + table_offset);
++	if (pi->pi_msix.table_page == MAP_FAILED) {
++		warn("Failed to map table page for MSI-X on %d/%d/%d", b, s, f);
++		return (-1);
++	}
+ 
+ 	if (pi->pi_msix.pba_bar == pi->pi_msix.table_bar) {
+ 		pba_offset = pi->pi_msix.pba_offset;
diff --git a/sysutils/bhyve+/pkg-descr b/sysutils/bhyve+/pkg-descr
new file mode 100644
index 000000000000..b19a5c1e5766
--- /dev/null
+++ b/sysutils/bhyve+/pkg-descr
@@ -0,0 +1,4 @@
+A patched version of BHyVe, the BSD Hypervisor from the base system
+that includes features and fixes that are not yet officially there.
+
+WWW: https://github.com/pgj/freebsd-bhyve-plus-port
diff --git a/sysutils/bhyve+/pkg-plist b/sysutils/bhyve+/pkg-plist
new file mode 100644
index 000000000000..4744907bb592
--- /dev/null
+++ b/sysutils/bhyve+/pkg-plist
@@ -0,0 +1,9 @@
+sbin/bhyve
+sbin/bhyvectl
+sbin/bhyveload
+lib/lib9p.so.1.1
+lib/libvmmapi.so.5.1
+man/man8/bhyve.8.gz
+man/man8/bhyvectl.8.gz
+man/man8/bhyveload.8.gz
+/%%KMODDIR%%/vmm.ko