hyperv/vmbus: Reorganize vmbus device tree

For GEN1 Hyper-V, vmbus is attached to pcib0, which contains the
resources for PCI passthrough and SR-IOV.  There is no
acpi_syscontainer0 on GEN1 Hyper-V.

For GEN2 Hyper-V, vmbus is attached to acpi_syscontainer0, which
contains the resources for PCI passthrough and SR-IOV.  There is
no pcib0 on GEN2 Hyper-V.

The ACPI VMBUS device now only holds its _CRS, which is empty as
of this commit; its existence is mainly for upward compatibility.

Device tree structure is suggested by jhb@.

Tested-by:	dexuan@
Collabrated-wth:	dexuan@
MFC after:	1 week
Sponsored by:	Microsoft
Differential Revision:	https://reviews.freebsd.org/D10565

8 files changed, 329 insertions, 42 deletions

diff --git a/sys/conf/files b/sys/conf/files
index c63c4e17711a..250c7515b1e6 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -673,6 +673,7 @@ dev/acpica/acpi_perf.c		optional acpi
 dev/acpica/acpi_powerres.c	optional acpi
 dev/acpica/acpi_quirk.c		optional acpi
 dev/acpica/acpi_resource.c	optional acpi
+dev/acpica/acpi_container.c	optional acpi
 dev/acpica/acpi_smbat.c		optional acpi
 dev/acpica/acpi_thermal.c	optional acpi
 dev/acpica/acpi_throttle.c	optional acpi
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index f0c46128176f..b50f5fe0ffea 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -323,6 +323,7 @@ dev/hyperv/vmbus/vmbus_br.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_chan.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_et.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_if.m				optional	hyperv
+dev/hyperv/vmbus/vmbus_res.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_xact.c				optional	hyperv
 dev/hyperv/vmbus/amd64/hyperv_machdep.c			optional	hyperv
 dev/hyperv/vmbus/amd64/vmbus_vector.S			optional	hyperv
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index e51ee82ee23e..3784d87f44b5 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -244,6 +244,7 @@ dev/hyperv/vmbus/vmbus_br.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_chan.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_et.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_if.m				optional	hyperv
+dev/hyperv/vmbus/vmbus_res.c				optional	hyperv
 dev/hyperv/vmbus/vmbus_xact.c				optional	hyperv
 dev/hyperv/vmbus/i386/hyperv_machdep.c			optional	hyperv
 dev/hyperv/vmbus/i386/vmbus_vector.S			optional	hyperv
diff --git a/sys/dev/acpica/acpi_container.c b/sys/dev/acpica/acpi_container.c
new file mode 100644
index 000000000000..26855bd732d7
--- /dev/null
+++ b/sys/dev/acpica/acpi_container.c
@@ -0,0 +1,166 @@
+/*-
+ * Copyright (c) 2017 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/module.h>
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+
+#include "pcib_if.h"
+
+ACPI_MODULE_NAME("CONTAINER")
+
+static int			acpi_syscont_probe(device_t);
+static int			acpi_syscont_attach(device_t);
+static int			acpi_syscont_detach(device_t);
+static int			acpi_syscont_alloc_msi(device_t, device_t,
+				    int count, int maxcount, int *irqs);
+static int			acpi_syscont_release_msi(device_t bus, device_t dev,
+				    int count, int *irqs);
+static int			acpi_syscont_alloc_msix(device_t bus, device_t dev,
+				    int *irq);
+static int			acpi_syscont_release_msix(device_t bus, device_t dev,
+				    int irq);
+static int			acpi_syscont_map_msi(device_t bus, device_t dev,
+				    int irq, uint64_t *addr, uint32_t *data);
+
+static device_method_t acpi_syscont_methods[] = {
+    /* Device interface */
+    DEVMETHOD(device_probe,		acpi_syscont_probe),
+    DEVMETHOD(device_attach,		acpi_syscont_attach),
+    DEVMETHOD(device_detach,		acpi_syscont_detach),
+
+    /* Bus interface */
+    DEVMETHOD(bus_add_child,		bus_generic_add_child),
+    DEVMETHOD(bus_print_child,		bus_generic_print_child),
+    DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+    DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+    DEVMETHOD(bus_activate_resource,	bus_generic_activate_resource),
+    DEVMETHOD(bus_deactivate_resource,	bus_generic_deactivate_resource),
+    DEVMETHOD(bus_setup_intr,		bus_generic_setup_intr),
+    DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+#if __FreeBSD_version >= 1100000
+    DEVMETHOD(bus_get_cpus,		bus_generic_get_cpus),
+#endif
+
+    /* pcib interface */
+    DEVMETHOD(pcib_alloc_msi,		acpi_syscont_alloc_msi),
+    DEVMETHOD(pcib_release_msi,		acpi_syscont_release_msi),
+    DEVMETHOD(pcib_alloc_msix,		acpi_syscont_alloc_msix),
+    DEVMETHOD(pcib_release_msix,	acpi_syscont_release_msix),
+    DEVMETHOD(pcib_map_msi,		acpi_syscont_map_msi),
+
+    DEVMETHOD_END
+};
+
+static driver_t acpi_syscont_driver = {
+    "acpi_syscontainer",
+    acpi_syscont_methods,
+    0,
+};
+
+static devclass_t acpi_syscont_devclass;
+
+DRIVER_MODULE(acpi_syscontainer, acpi, acpi_syscont_driver,
+    acpi_syscont_devclass, NULL, NULL);
+MODULE_DEPEND(acpi_syscontainer, acpi, 1, 1, 1);
+
+static int
+acpi_syscont_probe(device_t dev)
+{
+    static char *syscont_ids[] = { "ACPI0004", "PNP0A05", "PNP0A06", NULL };
+
+    if (acpi_disabled("syscontainer") ||
+	ACPI_ID_PROBE(device_get_parent(dev), dev, syscont_ids) == NULL)
+	return (ENXIO);
+
+    device_set_desc(dev, "System Container");
+    return (BUS_PROBE_DEFAULT);
+}
+
+static int
+acpi_syscont_attach(device_t dev)
+{
+
+    bus_generic_probe(dev);
+    return (bus_generic_attach(dev));
+}
+
+static int
+acpi_syscont_detach(device_t dev)
+{
+
+    return (bus_generic_detach(dev));
+}
+
+static int
+acpi_syscont_alloc_msi(device_t bus, device_t dev, int count, int maxcount,
+    int *irqs)
+{
+    device_t parent = device_get_parent(bus);
+
+    return (PCIB_ALLOC_MSI(device_get_parent(parent), dev, count, maxcount,
+	irqs));
+}
+
+static int
+acpi_syscont_release_msi(device_t bus, device_t dev, int count, int *irqs)
+{
+    device_t parent = device_get_parent(bus);
+
+    return (PCIB_RELEASE_MSI(device_get_parent(parent), dev, count, irqs));
+}
+
+static int
+acpi_syscont_alloc_msix(device_t bus, device_t dev, int *irq)
+{
+    device_t parent = device_get_parent(bus);
+
+    return (PCIB_ALLOC_MSIX(device_get_parent(parent), dev, irq));
+}
+
+static int
+acpi_syscont_release_msix(device_t bus, device_t dev, int irq)
+{
+    device_t parent = device_get_parent(bus);
+
+    return (PCIB_RELEASE_MSIX(device_get_parent(parent), dev, irq));
+}
+
+static int
+acpi_syscont_map_msi(device_t bus, device_t dev, int irq, uint64_t *addr,
+    uint32_t *data)
+{
+    device_t parent = device_get_parent(bus);
+
+    return (PCIB_MAP_MSI(device_get_parent(parent), dev, irq, addr, data));
+}
diff --git a/sys/dev/acpica/acpi_pcib_acpi.c b/sys/dev/acpica/acpi_pcib_acpi.c
index d26ee0c37cd6..a1c94fb66c8a 100644
--- a/sys/dev/acpica/acpi_pcib_acpi.c
+++ b/sys/dev/acpica/acpi_pcib_acpi.c
@@ -537,6 +537,7 @@ acpi_pcib_acpi_attach(device_t dev)
 
     acpi_pcib_fetch_prt(dev, &sc->ap_prt);
 
+    bus_generic_probe(dev);
     if (device_add_child(dev, "pci", -1) == NULL) {
 	device_printf(device_get_parent(dev), "couldn't attach pci bus\n");
 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
diff --git a/sys/dev/hyperv/vmbus/vmbus.c b/sys/dev/hyperv/vmbus/vmbus.c
index 23ae6ed592c9..999990182239 100644
--- a/sys/dev/hyperv/vmbus/vmbus.c
+++ b/sys/dev/hyperv/vmbus/vmbus.c
@@ -71,6 +71,7 @@ struct vmbus_msghc {
 	struct hypercall_postmsg_in	mh_inprm_save;
 };
 
+static void			vmbus_identify(driver_t *, device_t);
 static int			vmbus_probe(device_t);
 static int			vmbus_attach(device_t);
 static int			vmbus_detach(device_t);
@@ -144,6 +145,7 @@ vmbus_chanmsg_handlers[VMBUS_CHANMSG_TYPE_MAX] = {
 
 static device_method_t vmbus_methods[] = {
 	/* Device interface */
+	DEVMETHOD(device_identify,		vmbus_identify),
 	DEVMETHOD(device_probe,			vmbus_probe),
 	DEVMETHOD(device_attach,		vmbus_attach),
 	DEVMETHOD(device_detach,		vmbus_detach),
@@ -190,7 +192,10 @@ static driver_t vmbus_driver = {
 
 static devclass_t vmbus_devclass;
 
-DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL);
+DRIVER_MODULE(vmbus, pcib, vmbus_driver, vmbus_devclass, NULL, NULL);
+DRIVER_MODULE(vmbus, acpi_syscontainer, vmbus_driver, vmbus_devclass,
+    NULL, NULL);
+
 MODULE_DEPEND(vmbus, acpi, 1, 1, 1);
 MODULE_DEPEND(vmbus, pci, 1, 1, 1);
 MODULE_VERSION(vmbus, 1);
@@ -1066,43 +1071,41 @@ vmbus_alloc_resource(device_t dev, device_t child, int type, int *rid,
 	return (res);
 }
 
-static device_t
-get_nexus(device_t vmbus)
-{
-	device_t acpi = device_get_parent(vmbus);
-	device_t nexus = device_get_parent(acpi);
-	return (nexus);
-}
-
 static int
 vmbus_alloc_msi(device_t bus, device_t dev, int count, int maxcount, int *irqs)
 {
-	return (PCIB_ALLOC_MSI(get_nexus(bus), dev, count, maxcount, irqs));
+
+	return (PCIB_ALLOC_MSI(device_get_parent(bus), dev, count, maxcount,
+	    irqs));
 }
 
 static int
 vmbus_release_msi(device_t bus, device_t dev, int count, int *irqs)
 {
-	return (PCIB_RELEASE_MSI(get_nexus(bus), dev, count, irqs));
+
+	return (PCIB_RELEASE_MSI(device_get_parent(bus), dev, count, irqs));
 }
 
 static int
 vmbus_alloc_msix(device_t bus, device_t dev, int *irq)
 {
-	return (PCIB_ALLOC_MSIX(get_nexus(bus), dev, irq));
+
+	return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
 }
 
 static int
 vmbus_release_msix(device_t bus, device_t dev, int irq)
 {
-	return (PCIB_RELEASE_MSIX(get_nexus(bus), dev, irq));
+
+	return (PCIB_RELEASE_MSIX(device_get_parent(bus), dev, irq));
 }
 
 static int
 vmbus_map_msi(device_t bus, device_t dev, int irq, uint64_t *addr,
 	uint32_t *data)
 {
-	return (PCIB_MAP_MSI(get_nexus(bus), dev, irq, addr, data));
+
+	return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
 }
 
 static uint32_t
@@ -1216,36 +1219,44 @@ vmbus_get_crs(device_t dev, device_t vmbus_dev, enum parse_pass pass)
 static void
 vmbus_get_mmio_res_pass(device_t dev, enum parse_pass pass)
 {
-	device_t acpi0, pcib0 = NULL;
-	device_t *children;
-	int i, count;
-
-	/* Try to find _CRS on VMBus device */
-	vmbus_get_crs(dev, dev, pass);
+	device_t acpi0, parent;
 
-	/* Try to find _CRS on VMBus device's parent */
-	acpi0 = device_get_parent(dev);
-	vmbus_get_crs(acpi0, dev, pass);
-
-	/* Try to locate pcib0 and find _CRS on it */
-	if (device_get_children(acpi0, &children, &count) != 0)
-		return;
+	parent = device_get_parent(dev);
 
-	for (i = 0; i < count; i++) {
-		if (!device_is_attached(children[i]))
-			continue;
+	acpi0 = device_get_parent(parent);
+	if (strcmp("acpi0", device_get_nameunit(acpi0)) == 0) {
+		device_t *children;
+		int count;
 
-		if (strcmp("pcib0", device_get_nameunit(children[i])))
-			continue;
+		/*
+		 * Try to locate VMBUS resources and find _CRS on them.
+		 */
+		if (device_get_children(acpi0, &children, &count) == 0) {
+			int i;
+
+			for (i = 0; i < count; ++i) {
+				if (!device_is_attached(children[i]))
+					continue;
+
+				if (strcmp("vmbus_res",
+				    device_get_name(children[i])) == 0)
+					vmbus_get_crs(children[i], dev, pass);
+			}
+			free(children, M_TEMP);
+		}
 
-		pcib0 = children[i];
-		break;
+		/*
+		 * Try to find _CRS on acpi.
+		 */
+		vmbus_get_crs(acpi0, dev, pass);
+	} else {
+		device_printf(dev, "not grandchild of acpi\n");
 	}
 
-	if (pcib0)
-		vmbus_get_crs(pcib0, dev, pass);
-
-	free(children, M_TEMP);
+	/*
+	 * Try to find _CRS on parent.
+	 */
+	vmbus_get_crs(parent, dev, pass);
 }
 
 static void
@@ -1275,18 +1286,25 @@ vmbus_free_mmio_res(device_t dev)
 }
 #endif	/* NEW_PCIB */
 
+static void
+vmbus_identify(driver_t *driver, device_t parent)
+{
+
+	if (device_get_unit(parent) != 0 || vm_guest != VM_GUEST_HV ||
+	    (hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
+		return;
+	device_add_child(parent, "vmbus", -1);
+}
+
 static int
 vmbus_probe(device_t dev)
 {
-	char *id[] = { "VMBUS", NULL };
 
-	if (ACPI_ID_PROBE(device_get_parent(dev), dev, id) == NULL ||
-	    device_get_unit(dev) != 0 || vm_guest != VM_GUEST_HV ||
+	if (device_get_unit(dev) != 0 || vm_guest != VM_GUEST_HV ||
 	    (hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
 		return (ENXIO);
 
 	device_set_desc(dev, "Hyper-V Vmbus");
-
 	return (BUS_PROBE_DEFAULT);
 }
 
diff --git a/sys/dev/hyperv/vmbus/vmbus_res.c b/sys/dev/hyperv/vmbus/vmbus_res.c
new file mode 100644
index 000000000000..164ebeb9175e
--- /dev/null
+++ b/sys/dev/hyperv/vmbus/vmbus_res.c
@@ -0,0 +1,98 @@
+/*-
+ * Copyright (c) 2017 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/module.h>
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+
+#include <dev/hyperv/include/hyperv.h>
+
+#include "acpi_if.h"
+#include "bus_if.h"
+
+static int		vmbus_res_probe(device_t);
+static int		vmbus_res_attach(device_t);
+static int		vmbus_res_detach(device_t);
+
+static device_method_t vmbus_res_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,			vmbus_res_probe),
+	DEVMETHOD(device_attach,		vmbus_res_attach),
+	DEVMETHOD(device_detach,		vmbus_res_detach),
+	DEVMETHOD(device_shutdown,		bus_generic_shutdown),
+	DEVMETHOD(device_suspend,		bus_generic_suspend),
+	DEVMETHOD(device_resume,		bus_generic_resume),
+
+	DEVMETHOD_END
+};
+
+static driver_t vmbus_res_driver = {
+	"vmbus_res",
+	vmbus_res_methods,
+	1
+};
+
+static devclass_t vmbus_res_devclass;
+
+DRIVER_MODULE(vmbus_res, acpi, vmbus_res_driver, vmbus_res_devclass,
+    NULL, NULL);
+MODULE_DEPEND(vmbus_res, acpi, 1, 1, 1);
+MODULE_VERSION(vmbus_res, 1);
+
+static int
+vmbus_res_probe(device_t dev)
+{
+	char *id[] = { "VMBUS", NULL };
+
+	if (ACPI_ID_PROBE(device_get_parent(dev), dev, id) == NULL ||
+	    device_get_unit(dev) != 0 || vm_guest != VM_GUEST_HV ||
+	    (hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "Hyper-V Vmbus Resource");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vmbus_res_attach(device_t dev __unused)
+{
+
+	return (0);
+}
+
+static int
+vmbus_res_detach(device_t dev __unused)
+{
+
+	return (0);
+}
diff --git a/sys/modules/hyperv/vmbus/Makefile b/sys/modules/hyperv/vmbus/Makefile
index e05ce851a4f8..944bf75b0e7b 100644
--- a/sys/modules/hyperv/vmbus/Makefile
+++ b/sys/modules/hyperv/vmbus/Makefile
@@ -12,6 +12,7 @@ SRCS=	hyperv.c \
 	vmbus_chan.c \
 	vmbus_et.c \
 	vmbus_if.c \
+	vmbus_res.c \
 	vmbus_xact.c
 SRCS+=	acpi_if.h bus_if.h device_if.h opt_acpi.h pci_if.h pcib_if.h vmbus_if.h