ofw_pcibus: Inherit PF locality for SR-IOV VFs

PCI VFs are allocated dynamically and have no corresponding OFW node.
The zero-filled OFW PCI devinfo currently leaves obd_node as 0, which
is not the invalid-node sentinel and can send NUMA lookup through an
unrelated firmware node.

Initialize dynamically allocated devinfo with an invalid OFW node. For
VF locality queries, use the owning PF's node when it exists. Fall back
to the PCI bus when neither the VF nor PF has a firmware node.

This preserves existing CPU-locality behavior for ordinary PCI devices
while making VF domain and interrupt placement follow their PF.

Reviewed by:	PowerPC (jhibbits)
MFC after:	2 weeks
Sponsored by:	BBOX.io
Differential Revision:	https://reviews.freebsd.org/D59064

1 files changed, 32 insertions, 7 deletions

diff --git a/sys/powerpc/ofw/ofw_pcibus.c b/sys/powerpc/ofw/ofw_pcibus.c
index 22a1634a0389..2c759f35cb69 100644
--- a/sys/powerpc/ofw/ofw_pcibus.c
+++ b/sys/powerpc/ofw/ofw_pcibus.c
@@ -156,6 +156,7 @@ ofw_pcibus_alloc_devinfo(device_t dev)
 	struct ofw_pcibus_devinfo *dinfo;
 
 	dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO);
+	dinfo->opd_obdinfo.obd_node = -1;
 	return (&dinfo->opd_dinfo);
 }
 
@@ -381,13 +382,31 @@ ofw_pcibus_get_devinfo(device_t bus, device_t dev)
 	return (&dinfo->opd_obdinfo);
 }
 
+/* Return a VF's owning PF node, or -1 if none is available. */
+static phandle_t
+ofw_pcibus_get_pf_node(device_t child)
+{
+	device_t pf;
+
+	pf = pci_iov_get_pf(child);
+	if (pf == NULL)
+		return (-1);
+	return (ofw_bus_get_node(pf));
+}
+
 int
 ofw_pcibus_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize,
     cpuset_t *cpuset)
 {
+	phandle_t node;
 	int d, error;
 
-	d = platform_node_numa_domain(ofw_bus_get_node(dev));
+	node = ofw_pcibus_get_pf_node(child);
+	if (node == -1)
+		node = ofw_bus_get_node(dev);
+	if (node == -1)
+		return (bus_generic_get_cpus(dev, child, op, setsize, cpuset));
+	d = platform_node_numa_domain(node);
 
 	switch (op) {
 	case LOCAL_CPUS:
@@ -410,17 +429,23 @@ ofw_pcibus_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsi
 }
 
 /*
- * Fetch the NUMA domain for the given device 'dev'.
+ * Fetch the NUMA domain for the given device 'child'.
  *
- * If a device has a _PXM method, map that to a NUMA domain.
- * Otherwise, pass the request up to the parent.
- * If there's no matching domain or the domain cannot be
- * determined, return ENOENT.
+ * VFs have no firmware node, so use the owning PF's node when available.
+ * Otherwise, pass the request up to the parent when the device is not
+ * represented in the firmware tree.
  */
 int
 ofw_pcibus_get_domain(device_t dev, device_t child, int *domain)
 {
-	*domain = platform_node_numa_domain(ofw_bus_get_node(child));
+	phandle_t node;
+
+	node = ofw_pcibus_get_pf_node(child);
+	if (node == -1)
+		node = ofw_bus_get_node(child);
+	if (node == -1)
+		return (bus_generic_get_domain(dev, child, domain));
+	*domain = platform_node_numa_domain(node);
 
 	return (0);
 }