Add malloc_domainset(9) and _domainset variants to other allocator KPIs.

Remove malloc_domain(9) and most other _domain KPIs added in r327900.
The new functions allow the caller to specify a general NUMA domain
selection policy, rather than specifically requesting an allocation from
a specific domain.  The latter policy tends to interact poorly with
M_WAITOK, resulting in situations where a caller is blocked indefinitely
because the specified domain is depleted.  Most existing consumers of
the _domain KPIs are converted to instead use a DOMAINSET_PREF() policy,
in which we fall back to other domains to satisfy the allocation
request.

This change also defines a set of DOMAINSET_FIXED() policies, which
only permit allocations from the specified domain.

Discussed with:	gallatin, jeff
Reported and tested by:	pho (previous version)
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D17418

20 files changed, 231 insertions, 144 deletions

diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index bef88ef99502..02d0b463834b 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,8 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20181030: malloc_domain(9) KPI change
+OLD_FILES+=share/man/man9/malloc_domain.9.gz
 # 20181026: joy(4) removal
 OLD_FILES+=usr/share/man/man4/joy.4.gz
 # 20181025: OpenSSL libraries version bump to avoid conflict with ports
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index f39220cd09b4..e8254780ea87 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -812,7 +812,8 @@ MLINKS+=condvar.9 cv_broadcast.9 \
 MLINKS+=config_intrhook.9 config_intrhook_disestablish.9 \
 	config_intrhook.9 config_intrhook_establish.9 \
 	config_intrhook.9 config_intrhook_oneshot.9
-MLINKS+=contigmalloc.9 contigfree.9
+MLINKS+=contigmalloc.9 contigmalloc_domainset.9 \
+	contigmalloc.9 contigfree.9
 MLINKS+=casuword.9 casueword.9 \
 	casuword.9 casueword32.9 \
 	casuword.9 casuword32.9
@@ -1289,7 +1290,7 @@ MLINKS+=make_dev.9 destroy_dev.9 \
 	make_dev.9 make_dev_p.9 \
 	make_dev.9 make_dev_s.9
 MLINKS+=malloc.9 free.9 \
-	malloc.9 malloc_domain.9 \
+	malloc.9 malloc_domainset.9 \
 	malloc.9 free_domain.9 \
 	malloc.9 mallocarray.9 \
 	malloc.9 MALLOC_DECLARE.9 \
diff --git a/share/man/man9/contigmalloc.9 b/share/man/man9/contigmalloc.9
index a1d447fe1bbf..5d6a531ecf98 100644
--- a/share/man/man9/contigmalloc.9
+++ b/share/man/man9/contigmalloc.9
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 29, 2015
+.Dd October 30, 2018
 .Dt CONTIGMALLOC 9
 .Os
 .Sh NAME
@@ -50,6 +50,19 @@
 .Fa "unsigned long size"
 .Fa "struct malloc_type *type"
 .Fc
+.In sys/param.h
+.In sys/domainset.h
+.Ft "void *"
+.Fo contigmalloc_domainset
+.Fa "unsigned long size"
+.Fa "struct malloc_type *type"
+.Fa "struct domainset *ds"
+.Fa "int flags"
+.Fa "vm_paddr_t low"
+.Fa "vm_paddr_t high"
+.Fa "unsigned long alignment"
+.Fa "vm_paddr_t boundary"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn contigmalloc
@@ -70,6 +83,15 @@ address range of
 bytes allocated from the kernel virtual address (KVA) map.
 .Pp
 The
+.Fn contigmalloc_domainset
+variant allows the caller to additionally specify a
+.Xr numa 4
+domain selection policy.
+See
+.Xr domainset 9
+for some example policies.
+.Pp
+The
 .Fa flags
 parameter modifies
 .Fn contigmalloc Ns 's
@@ -90,7 +112,9 @@ Other flags (if present) are ignored.
 The
 .Fn contigfree
 function deallocates memory allocated by a previous call to
-.Fn contigmalloc .
+.Fn contigmalloc
+or
+.Fn contigmalloc_domainset .
 .Sh IMPLEMENTATION NOTES
 The
 .Fn contigmalloc
diff --git a/share/man/man9/domainset.9 b/share/man/man9/domainset.9
index 470f79b250a7..5a3c763f4aa5 100644
--- a/share/man/man9/domainset.9
+++ b/share/man/man9/domainset.9
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 20, 2018
+.Dd October 30, 2018
 .Dt DOMAINSET 9
 .Os
 .Sh NAME
@@ -43,7 +43,11 @@ struct domainset {
 };
 .Ed
 .Pp
+.Ft struct domainset *
+.Fn DOMAINSET_FIXED domain
+.Ft struct domainset *
 .Fn DOMAINSET_RR
+.Ft struct domainset *
 .Fn DOMAINSET_PREF domain
 .Ft struct domainset *
 .Fn domainset_create "const struct domainset *key"
@@ -98,11 +102,26 @@ efficiency higher and is preferential to round-robin for general use.
 .El
 .Pp
 The
+.Fn DOMAINSET_FIXED ,
 .Fn DOMAINSET_RR
 and
 .Fn DOMAINSET_PREF
-provide pointers to global pre-defined policies for use when the
+macros provide pointers to global pre-defined policies for use when the
 desired policy is known at compile time.
+.Fn DOMAINSET_FIXED
+is a policy which only permits allocations from the specified domain.
+.Fn DOMAINSET_RR
+provides round-robin selection among all domains in the system.
+The
+.Fn DOMAINSET_PREF
+policies attempt allocation from the specified domain, but unlike
+.Fn DOMAINSET_FIXED
+will fall back to other domains to satisfy the request.
+These policies should be used in preference to
+.Fn DOMAINSET_FIXED
+to avoid blocking indefinitely on a
+.Dv M_WAITOK
+request.
 The
 .Fn domainset_create
 function takes a partially filled in domainset as a key and returns a
diff --git a/share/man/man9/malloc.9 b/share/man/man9/malloc.9
index 26ea111641c2..020b729028de 100644
--- a/share/man/man9/malloc.9
+++ b/share/man/man9/malloc.9
@@ -29,7 +29,7 @@
 .\" $NetBSD: malloc.9,v 1.3 1996/11/11 00:05:11 lukem Exp $
 .\" $FreeBSD$
 .\"
-.Dd June 13, 2018
+.Dd October 30, 2018
 .Dt MALLOC 9
 .Os
 .Sh NAME
@@ -46,13 +46,9 @@
 .Ft void *
 .Fn malloc "size_t size" "struct malloc_type *type" "int flags"
 .Ft void *
-.Fn malloc_domain "size_t size" "struct malloc_type *type" "int domain" "int flags"
-.Ft void *
 .Fn mallocarray "size_t nmemb" "size_t size" "struct malloc_type *type" "int flags"
 .Ft void
 .Fn free "void *addr" "struct malloc_type *type"
-.Ft void
-.Fn free_domain "void *addr" "struct malloc_type *type"
 .Ft void *
 .Fn realloc "void *addr" "size_t size" "struct malloc_type *type" "int flags"
 .Ft void *
@@ -62,6 +58,12 @@
 .In sys/malloc.h
 .In sys/kernel.h
 .Fn MALLOC_DEFINE type shortdesc longdesc
+.In sys/param.h
+.In sys/domainset.h
+.Ft void *
+.Fn malloc_domainset "size_t size" "struct malloc_type *type" "struct domainset *ds" "int flags"
+.Ft void
+.Fn free_domain "void *addr" "struct malloc_type *type"
 .Sh DESCRIPTION
 The
 .Fn malloc
@@ -70,12 +72,15 @@ object whose size is specified by
 .Fa size .
 .Pp
 The
-.Fn malloc_domain
-variant allocates the object from the specified memory domain.  Memory allocated
-with this function should be returned with
-.Fn free_domain .
+.Fn malloc_domainset
+variant allocates memory from a specific
+.Xr numa 4
+domain using the specified domain selection policy.
 See
-.Xr numa 9 for more details.
+.Xr domainset 9
+for some example policies.
+Memory allocated with this function should be returned with
+.Fn free_domain .
 .Pp
 The
 .Fn mallocarray
@@ -310,7 +315,9 @@ functions.
 Failing consistency checks will cause a panic or a system console
 message.
 .Sh SEE ALSO
+.Xr numa 4 ,
 .Xr vmstat 8 ,
 .Xr contigmalloc 9 ,
+.Xr domainset 9 ,
 .Xr memguard 9 ,
 .Xr vnode 9
diff --git a/sys/dev/hwpmc/hwpmc_logging.c b/sys/dev/hwpmc/hwpmc_logging.c
index 8764ac9e922c..e76aa9b710eb 100644
--- a/sys/dev/hwpmc/hwpmc_logging.c
+++ b/sys/dev/hwpmc/hwpmc_logging.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/capsicum.h>
+#include <sys/domainset.h>
 #include <sys/file.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
@@ -1231,8 +1232,8 @@ pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
 void
 pmclog_initialize()
 {
-	int domain;
 	struct pmclog_buffer *plb;
+	int domain, ncpus, total;
 
 	if (pmclog_buffer_size <= 0 || pmclog_buffer_size > 16*1024) {
 		(void) printf("hwpmc: tunable logbuffersize=%d must be "
@@ -1253,16 +1254,17 @@ pmclog_initialize()
 		pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
 	}
 	for (domain = 0; domain < vm_ndomains; domain++) {
-		int ncpus = pmc_dom_hdrs[domain]->pdbh_ncpus;
-		int total = ncpus*pmc_nlogbuffers_pcpu;
+		ncpus = pmc_dom_hdrs[domain]->pdbh_ncpus;
+		total = ncpus * pmc_nlogbuffers_pcpu;
 
-		plb = malloc_domain(sizeof(struct pmclog_buffer)*total, M_PMC, domain, M_WAITOK|M_ZERO);
+		plb = malloc_domainset(sizeof(struct pmclog_buffer) * total,
+		    M_PMC, DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 		pmc_dom_hdrs[domain]->pdbh_plbs = plb;
-		for (int i = 0; i < total; i++, plb++) {
+		for (; total > 0; total--, plb++) {
 			void *buf;
 
-			buf = malloc_domain(1024 * pmclog_buffer_size, M_PMC, domain,
-								M_WAITOK|M_ZERO);
+			buf = malloc_domainset(1024 * pmclog_buffer_size, M_PMC,
+			    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 			PMCLOG_INIT_BUFFER_DESCRIPTOR(plb, buf, domain);
 			pmc_plb_rele_unlocked(plb);
 		}
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index 1fad25c55702..f94f645b2e39 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -36,6 +36,7 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/domainset.h>
 #include <sys/eventhandler.h>
 #include <sys/gtaskqueue.h>
 #include <sys/jail.h>
@@ -78,14 +79,6 @@ __FBSDID("$FreeBSD$");
 
 #include "hwpmc_soft.h"
 
-#ifdef NUMA
-#define NDOMAINS vm_ndomains
-#else
-#define NDOMAINS 1
-#define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
-#define free_domain(addr, type) free(addr, type)
-#endif
-
 #define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
 #define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)
 
@@ -5643,15 +5636,16 @@ pmc_initialize(void)
 			continue;
 		pc = pcpu_find(cpu);
 		domain = pc->pc_domain;
-		sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
-			pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
-		    M_WAITOK|M_ZERO);
+		sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
+		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 
 		KASSERT(pmc_pcpu[cpu] != NULL,
 		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
 
-		sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
-			sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
+		sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
+		    pmc_nsamples * sizeof(uintptr_t), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 
 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
 			ps->ps_pc = sb->ps_callchains +
@@ -5659,35 +5653,27 @@ pmc_initialize(void)
 
 		pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
 
-		sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
-			pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
-		    M_WAITOK|M_ZERO);
-
-		KASSERT(pmc_pcpu[cpu] != NULL,
-		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
-
-		sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
-			sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
+		sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
+		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 
+		sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
+		    pmc_nsamples * sizeof(uintptr_t), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
 			ps->ps_pc = sb->ps_callchains +
 			    (n * pmc_callchaindepth);
 
 		pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
 
-		sb = malloc_domain(sizeof(struct pmc_samplebuffer) +
-			pmc_nsamples * sizeof(struct pmc_sample), M_PMC, domain,
-		    M_WAITOK|M_ZERO);
-
-		KASSERT(pmc_pcpu[cpu] != NULL,
-		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
-
-		sb->ps_callchains = malloc_domain(pmc_callchaindepth * pmc_nsamples *
-		    sizeof(uintptr_t), M_PMC, domain, M_WAITOK|M_ZERO);
-
+		sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
+		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
+		sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
+		    pmc_nsamples * sizeof(uintptr_t), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
-			ps->ps_pc = sb->ps_callchains +
-			    (n * pmc_callchaindepth);
+			ps->ps_pc = sb->ps_callchains + n * pmc_callchaindepth;
 
 		pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
 	}
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 80f10f92a2ce..ff9b8022193a 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -864,8 +864,8 @@ pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
 
 	/* Inform UMA that this allocator uses kernel_map/object. */
 	*flags = UMA_SLAB_KERNEL;
-	return ((void *)kmem_alloc_contig_domain(domain, bytes, wait, 0x0ULL,
-	    0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
+	return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
+	    bytes, wait, 0x0ULL, 0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
 }
 #endif
 
diff --git a/sys/kern/kern_cpuset.c b/sys/kern/kern_cpuset.c
index e6e9458c543e..87f9333bf43b 100644
--- a/sys/kern/kern_cpuset.c
+++ b/sys/kern/kern_cpuset.c
@@ -119,6 +119,7 @@ __FBSDID("$FreeBSD$");
  */
 
 LIST_HEAD(domainlist, domainset);
+struct domainset __read_mostly domainset_fixed[MAXMEMDOM];
 struct domainset __read_mostly domainset_prefer[MAXMEMDOM];
 struct domainset __read_mostly domainset_roundrobin;
 
@@ -1402,6 +1403,12 @@ domainset_init(void)
 	_domainset_create(dset, NULL);
 
 	for (i = 0; i < vm_ndomains; i++) {
+		dset = &domainset_fixed[i];
+		DOMAINSET_ZERO(&dset->ds_mask);
+		DOMAINSET_SET(i, &dset->ds_mask);
+		dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
+		_domainset_create(dset, NULL);
+
 		dset = &domainset_prefer[i];
 		DOMAINSET_COPY(&all_domains, &dset->ds_mask);
 		dset->ds_policy = DOMAINSET_POLICY_PREFER;
diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index a8d42a85d949..d4ac906714b9 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -68,6 +68,7 @@ __FBSDID("$FreeBSD$");
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#include <vm/vm_domainset.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_param.h>
 #include <vm/vm_kern.h>
@@ -453,13 +454,13 @@ contigmalloc(unsigned long size, struct malloc_type *type, int flags,
 }
 
 void *
-contigmalloc_domain(unsigned long size, struct malloc_type *type,
-    int domain, int flags, vm_paddr_t low, vm_paddr_t high,
+contigmalloc_domainset(unsigned long size, struct malloc_type *type,
+    struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high,
     unsigned long alignment, vm_paddr_t boundary)
 {
 	void *ret;
 
-	ret = (void *)kmem_alloc_contig_domain(domain, size, flags, low, high,
+	ret = (void *)kmem_alloc_contig_domainset(ds, size, flags, low, high,
 	    alignment, boundary, VM_MEMATTR_DEFAULT);
 	if (ret != NULL)
 		malloc_type_allocated(type, round_page(size));
@@ -595,9 +596,8 @@ void *
 	return ((void *) va);
 }
 
-void *
-malloc_domain(size_t size, struct malloc_type *mtp, int domain,
-    int flags)
+static void *
+malloc_domain(size_t size, struct malloc_type *mtp, int domain, int flags)
 {
 	int indx;
 	caddr_t va;
@@ -641,6 +641,24 @@ malloc_domain(size_t size, struct malloc_type *mtp, int domain,
 }
 
 void *
+malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds,
+    int flags)
+{
+	struct vm_domainset_iter di;
+	void *ret;
+	int domain;
+
+	vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
+	do {
+		ret = malloc_domain(size, mtp, domain, flags);
+		if (ret != NULL)
+			break;
+	} while (vm_domainset_iter_policy(&di, &domain) == 0);
+
+	return (ret);
+}
+
+void *
 mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags)
 {
 
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 289cfddd1852..100e01a4a069 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/conf.h>
+#include <sys/domainset.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
 #include <sys/mbuf.h>
@@ -590,8 +591,9 @@ mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
 
 	/* Inform UMA that this allocator uses kernel_map/object. */
 	*flags = UMA_SLAB_KERNEL;
-	return ((void *)kmem_alloc_contig_domain(domain, bytes, wait,
-	    (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
+	return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
+	    bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0,
+	    VM_MEMATTR_DEFAULT));
 }
 
 /*
diff --git a/sys/kern/kern_pmc.c b/sys/kern/kern_pmc.c
index 5ff3caebc8ce..4b73f134b2d4 100644
--- a/sys/kern/kern_pmc.c
+++ b/sys/kern/kern_pmc.c
@@ -35,8 +35,9 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_hwpmc_hooks.h"
 
-#include <sys/types.h>
+#include <sys/param.h>
 #include <sys/ctype.h>
+#include <sys/domainset.h>
 #include <sys/param.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
@@ -330,22 +331,6 @@ pmc_soft_ev_release(struct pmc_soft *ps)
 	mtx_unlock_spin(&pmc_softs_mtx);
 }
 
-#ifdef NUMA
-#define NDOMAINS vm_ndomains
-
-static int
-getdomain(int cpu)
-{
-	struct pcpu *pc;
-
-	pc = pcpu_find(cpu);
-	return (pc->pc_domain);
-}
-#else
-#define NDOMAINS 1
-#define malloc_domain(size, type, domain, flags) malloc((size), (type), (flags))
-#define getdomain(cpu) 0
-#endif
 /*
  *  Initialise hwpmc.
  */
@@ -362,14 +347,16 @@ init_hwpmc(void *dummy __unused)
 	}
 	pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS,
 	    M_WAITOK | M_ZERO);
-	for (domain = 0; domain < NDOMAINS; domain++) {
-		pmc_dom_hdrs[domain] = malloc_domain(sizeof(struct pmc_domain_buffer_header), M_PMC, domain,
-										M_WAITOK|M_ZERO);
+
+	for (domain = 0; domain < vm_ndomains; domain++) {
+		pmc_dom_hdrs[domain] = malloc_domainset(
+		    sizeof(struct pmc_domain_buffer_header), M_PMC,
+		    DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
 		mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN);
 		TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head);
 	}
 	CPU_FOREACH(cpu) {
-		domain = getdomain(cpu);
+		domain = pcpu_find(cpu)->pc_domain;
 		KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain));
 		pmc_dom_hdrs[domain]->pdbh_ncpus++;
 	}
diff --git a/sys/kern/subr_busdma_bufalloc.c b/sys/kern/subr_busdma_bufalloc.c
index 52d1388ba7fc..91445aa79950 100644
--- a/sys/kern/subr_busdma_bufalloc.c
+++ b/sys/kern/subr_busdma_bufalloc.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/busdma_bufalloc.h>
+#include <sys/domainset.h>
 #include <sys/malloc.h>
 
 #include <vm/vm.h>
@@ -152,18 +153,15 @@ void *
 busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, vm_size_t size, int domain,
     uint8_t *pflag, int wait)
 {
-#ifdef VM_MEMATTR_UNCACHEABLE
 
+#ifdef VM_MEMATTR_UNCACHEABLE
 	/* Inform UMA that this allocator uses kernel_arena/object. */
 	*pflag = UMA_SLAB_KERNEL;
 
-	return ((void *)kmem_alloc_attr_domain(domain, size, wait, 0,
-	    BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE));
-
+	return ((void *)kmem_alloc_attr_domainset(DOMAINSET_FIXED(domain), size,
+	    wait, 0, BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE));
 #else
-
 	panic("VM_MEMATTR_UNCACHEABLE unavailable");
-
 #endif	/* VM_MEMATTR_UNCACHEABLE */
 }
 
diff --git a/sys/sys/domainset.h b/sys/sys/domainset.h
index 32b35ac5c1e9..5a00347f29d3 100644
--- a/sys/sys/domainset.h
+++ b/sys/sys/domainset.h
@@ -96,7 +96,8 @@ struct domainset {
 	domainid_t	ds_order[MAXMEMDOM];  /* nth domain table. */
 };
 
-extern struct domainset domainset_prefer[MAXMEMDOM];
+extern struct domainset domainset_fixed[MAXMEMDOM], domainset_prefer[MAXMEMDOM];
+#define	DOMAINSET_FIXED(domain)	(&domainset_fixed[(domain)])
 #define	DOMAINSET_PREF(domain)	(&domainset_prefer[(domain)])
 extern struct domainset domainset_roundrobin;
 #define	DOMAINSET_RR()		(&domainset_roundrobin)
diff --git a/sys/sys/malloc.h b/sys/sys/malloc.h
index 520f18c5a969..be24f854b661 100644
--- a/sys/sys/malloc.h
+++ b/sys/sys/malloc.h
@@ -160,6 +160,7 @@ MALLOC_DECLARE(M_TEMP);
  */
 MALLOC_DECLARE(M_IOV);
 
+struct domainset;
 extern struct mtx malloc_mtx;
 
 /*
@@ -172,8 +173,8 @@ void	*contigmalloc(unsigned long size, struct malloc_type *type, int flags,
 	    vm_paddr_t low, vm_paddr_t high, unsigned long alignment,
 	    vm_paddr_t boundary) __malloc_like __result_use_check
 	    __alloc_size(1) __alloc_align(6);
-void	*contigmalloc_domain(unsigned long size, struct malloc_type *type,
-	    int domain, int flags, vm_paddr_t low, vm_paddr_t high,
+void	*contigmalloc_domainset(unsigned long size, struct malloc_type *type,
+	    struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high,
 	    unsigned long alignment, vm_paddr_t boundary)
 	    __malloc_like __result_use_check __alloc_size(1) __alloc_align(6);
 void	free(void *addr, struct malloc_type *type);
@@ -230,8 +231,9 @@ void	*malloc(size_t size, struct malloc_type *type, int flags) __malloc_like
 	_malloc_item;							\
 })
 
-void	*malloc_domain(size_t size, struct malloc_type *type, int domain,
-	    int flags) __malloc_like __result_use_check __alloc_size(1);
+void	*malloc_domainset(size_t size, struct malloc_type *type,
+	    struct domainset *ds, int flags) __malloc_like __result_use_check
+	    __alloc_size(1);
 void	*mallocarray(size_t nmemb, size_t size, struct malloc_type *type,
 	    int flags) __malloc_like __result_use_check
 	    __alloc_size2(1, 2);
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 8e0e0dc297ec..9db39ea2ac80 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -1172,7 +1172,7 @@ page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
 	void *p;	/* Returned page */
 
 	*pflag = UMA_SLAB_KERNEL;
-	p = (void *) kmem_malloc_domain(domain, bytes, wait);
+	p = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait);
 
 	return (p);
 }
@@ -3718,6 +3718,7 @@ uma_zone_exhausted_nolock(uma_zone_t zone)
 void *
 uma_large_malloc_domain(vm_size_t size, int domain, int wait)
 {
+	struct domainset *policy;
 	vm_offset_t addr;
 	uma_slab_t slab;
 
@@ -3729,10 +3730,9 @@ uma_large_malloc_domain(vm_size_t size, int domain, int wait)
 	slab = zone_alloc_item(slabzone, NULL, domain, wait);
 	if (slab == NULL)
 		return (NULL);
-	if (domain == UMA_ANYDOMAIN)
-		addr = kmem_malloc(size, wait);
-	else
-		addr = kmem_malloc_domain(domain, size, wait);
+	policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() :
+	    DOMAINSET_FIXED(domain);
+	addr = kmem_malloc_domainset(policy, size, wait);
 	if (addr != 0) {
 		vsetslab(addr, slab);
 		slab->us_data = (void *)addr;
diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h
index 3174e645c8be..fb06076e6c43 100644
--- a/sys/vm/vm_extern.h
+++ b/sys/vm/vm_extern.h
@@ -44,6 +44,7 @@ struct vmem;
 #ifdef _KERNEL
 struct cdev;
 struct cdevsw;
+struct domainset;
 
 /* These operate on kernel virtual addresses only. */
 vm_offset_t kva_alloc(vm_size_t);
@@ -56,16 +57,17 @@ void kmap_free_wakeup(vm_map_t, vm_offset_t, vm_size_t);
 /* These operate on virtual addresses backed by memory. */
 vm_offset_t kmem_alloc_attr(vm_size_t size, int flags,
     vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
-vm_offset_t kmem_alloc_attr_domain(int domain, vm_size_t size, int flags,
-    vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
+vm_offset_t kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size,
+    int flags, vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr);
 vm_offset_t kmem_alloc_contig(vm_size_t size, int flags,
     vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
     vm_memattr_t memattr);
-vm_offset_t kmem_alloc_contig_domain(int domain, vm_size_t size, int flags,
-    vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
-    vm_memattr_t memattr);
+vm_offset_t kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size,
+    int flags, vm_paddr_t low, vm_paddr_t high, u_long alignment,
+    vm_paddr_t boundary, vm_memattr_t memattr);
 vm_offset_t kmem_malloc(vm_size_t size, int flags);
-vm_offset_t kmem_malloc_domain(int domain, vm_size_t size, int flags);
+vm_offset_t kmem_malloc_domainset(struct domainset *ds, vm_size_t size,
+    int flags);
 void kmem_free(vm_offset_t addr, vm_size_t size);
 
 /* This provides memory for previously allocated address space. */
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index f57a58383d6f..79108ce1ef64 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -175,7 +175,7 @@ kva_free(vm_offset_t addr, vm_size_t size)
  *	necessarily physically contiguous.  If M_ZERO is specified through the
  *	given flags, then the pages are zeroed before they are mapped.
  */
-vm_offset_t
+static vm_offset_t
 kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
     vm_paddr_t high, vm_memattr_t memattr)
 {
@@ -231,11 +231,20 @@ vm_offset_t
 kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
     vm_memattr_t memattr)
 {
+
+	return (kmem_alloc_attr_domainset(DOMAINSET_RR(), size, flags, low,
+	    high, memattr));
+}
+
+vm_offset_t
+kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size, int flags,
+    vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr)
+{
 	struct vm_domainset_iter di;
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
+	vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
 	do {
 		addr = kmem_alloc_attr_domain(domain, size, flags, low, high,
 		    memattr);
@@ -254,7 +263,7 @@ kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
  *	through the given flags, then the pages are zeroed before they are
  *	mapped.
  */
-vm_offset_t
+static vm_offset_t
 kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
     vm_memattr_t memattr)
@@ -315,11 +324,21 @@ vm_offset_t
 kmem_alloc_contig(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high,
     u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr)
 {
+
+	return (kmem_alloc_contig_domainset(DOMAINSET_RR(), size, flags, low,
+	    high, alignment, boundary, memattr));
+}
+
+vm_offset_t
+kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size, int flags,
+    vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
+    vm_memattr_t memattr)
+{
 	struct vm_domainset_iter di;
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
+	vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
 	do {
 		addr = kmem_alloc_contig_domain(domain, size, flags, low, high,
 		    alignment, boundary, memattr);
@@ -368,11 +387,11 @@ kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
 }
 
 /*
- *	kmem_malloc:
+ *	kmem_malloc_domain:
  *
  *	Allocate wired-down pages in the kernel's address space.
  */
-vm_offset_t
+static vm_offset_t
 kmem_malloc_domain(int domain, vm_size_t size, int flags)
 {
 	vmem_t *arena;
@@ -402,11 +421,18 @@ kmem_malloc_domain(int domain, vm_size_t size, int flags)
 vm_offset_t
 kmem_malloc(vm_size_t size, int flags)
 {
+
+	return (kmem_malloc_domainset(DOMAINSET_RR(), size, flags));
+}
+
+vm_offset_t
+kmem_malloc_domainset(struct domainset *ds, vm_size_t size, int flags)
+{
 	struct vm_domainset_iter di;
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
+	vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
 	do {
 		addr = kmem_malloc_domain(domain, size, flags);
 		if (addr != 0)
diff --git a/sys/x86/iommu/busdma_dmar.c b/sys/x86/iommu/busdma_dmar.c
index 40786d46621c..c4bcef01b461 100644
--- a/sys/x86/iommu/busdma_dmar.c
+++ b/sys/x86/iommu/busdma_dmar.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/domainset.h>
 #include <sys/malloc.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
@@ -373,16 +374,16 @@ dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__);
 
 	tag = (struct bus_dma_tag_dmar *)dmat;
-	map = malloc_domain(sizeof(*map), M_DMAR_DMAMAP,
-	    tag->common.domain, M_NOWAIT | M_ZERO);
+	map = malloc_domainset(sizeof(*map), M_DMAR_DMAMAP,
+	    DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
 	if (map == NULL) {
 		*mapp = NULL;
 		return (ENOMEM);
 	}
 	if (tag->segments == NULL) {
-		tag->segments = malloc_domain(sizeof(bus_dma_segment_t) *
+		tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
 		    tag->common.nsegments, M_DMAR_DMAMAP,
-		    tag->common.domain, M_NOWAIT);
+		    DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
 		if (tag->segments == NULL) {
 			free_domain(map, M_DMAR_DMAMAP);
 			*mapp = NULL;
@@ -447,13 +448,13 @@ dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	if (tag->common.maxsize < PAGE_SIZE &&
 	    tag->common.alignment <= tag->common.maxsize &&
 	    attr == VM_MEMATTR_DEFAULT) {
-		*vaddr = malloc_domain(tag->common.maxsize, M_DEVBUF,
-		    tag->common.domain, mflags);
+		*vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
+		    DOMAINSET_PREF(tag->common.domain), mflags);
 		map->flags |= BUS_DMAMAP_DMAR_MALLOC;
 	} else {
-		*vaddr = (void *)kmem_alloc_attr_domain(tag->common.domain,
-		    tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR,
-		    attr);
+		*vaddr = (void *)kmem_alloc_attr_domainset(
+		    DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
+		    mflags, 0ul, BUS_SPACE_MAXADDR, attr);
 		map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
 	}
 	if (*vaddr == NULL) {
diff --git a/sys/x86/x86/busdma_bounce.c b/sys/x86/x86/busdma_bounce.c
index 1c76804a61d5..a40af04f436d 100644
--- a/sys/x86/x86/busdma_bounce.c
+++ b/sys/x86/x86/busdma_bounce.c
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/domainset.h>
 #include <sys/malloc.h>
 #include <sys/bus.h>
 #include <sys/interrupt.h>
@@ -294,9 +295,9 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 	error = 0;
 
 	if (dmat->segments == NULL) {
-		dmat->segments = (bus_dma_segment_t *)malloc_domain(
+		dmat->segments = (bus_dma_segment_t *)malloc_domainset(
 		    sizeof(bus_dma_segment_t) * dmat->common.nsegments,
-		    M_DEVBUF, dmat->common.domain, M_NOWAIT);
+		    M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), M_NOWAIT);
 		if (dmat->segments == NULL) {
 			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 			    __func__, dmat, ENOMEM);
@@ -317,8 +318,8 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 		}
 		bz = dmat->bounce_zone;
 
-		*mapp = (bus_dmamap_t)malloc_domain(sizeof(**mapp), M_DEVBUF,
-		    dmat->common.domain, M_NOWAIT | M_ZERO);
+		*mapp = (bus_dmamap_t)malloc_domainset(sizeof(**mapp), M_DEVBUF,
+		    DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO);
 		if (*mapp == NULL) {
 			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 			    __func__, dmat, ENOMEM);
@@ -411,9 +412,9 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	*mapp = NULL;
 
 	if (dmat->segments == NULL) {
-		dmat->segments = (bus_dma_segment_t *)malloc_domain(
+		dmat->segments = (bus_dma_segment_t *)malloc_domainset(
 		    sizeof(bus_dma_segment_t) * dmat->common.nsegments,
-		    M_DEVBUF, dmat->common.domain, mflags);
+		    M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), mflags);
 		if (dmat->segments == NULL) {
 			CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 			    __func__, dmat, dmat->common.flags, ENOMEM);
@@ -452,20 +453,21 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	   (dmat->common.alignment <= dmat->common.maxsize) &&
 	    dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
 	    attr == VM_MEMATTR_DEFAULT) {
-		*vaddr = malloc_domain(dmat->common.maxsize, M_DEVBUF,
-		    dmat->common.domain, mflags);
+		*vaddr = malloc_domainset(dmat->common.maxsize, M_DEVBUF,
+		    DOMAINSET_PREF(dmat->common.domain), mflags);
 	} else if (dmat->common.nsegments >=
 	    howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
 	    dmat->common.alignment <= PAGE_SIZE &&
 	    (dmat->common.boundary % PAGE_SIZE) == 0) {
 		/* Page-based multi-segment allocations allowed */
-		*vaddr = (void *)kmem_alloc_attr_domain(dmat->common.domain,
-		    dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
-		    attr);
+		*vaddr = (void *)kmem_alloc_attr_domainset(
+		    DOMAINSET_PREF(dmat->common.domain), dmat->common.maxsize,
+		    mflags, 0ul, dmat->common.lowaddr, attr);
 		dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
 	} else {
-		*vaddr = (void *)kmem_alloc_contig_domain(dmat->common.domain,
-		    dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
+		*vaddr = (void *)kmem_alloc_contig_domainset(
+		    DOMAINSET_PREF(dmat->common.domain), dmat->common.maxsize,
+		    mflags, 0ul, dmat->common.lowaddr,
 		    dmat->common.alignment != 0 ? dmat->common.alignment : 1ul,
 		    dmat->common.boundary, attr);
 		dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
@@ -1149,14 +1151,14 @@ alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 	while (numpages > 0) {
 		struct bounce_page *bpage;
 
-		bpage = (struct bounce_page *)malloc_domain(sizeof(*bpage),
-		    M_DEVBUF, dmat->common.domain, M_NOWAIT | M_ZERO);
+		bpage = malloc_domainset(sizeof(*bpage), M_DEVBUF,
+		    DOMAINSET_PREF(dmat->common.domain), M_NOWAIT | M_ZERO);
 
 		if (bpage == NULL)
 			break;
-		bpage->vaddr = (vm_offset_t)contigmalloc_domain(PAGE_SIZE,
-		    M_DEVBUF, dmat->common.domain, M_NOWAIT, 0ul,
-		    bz->lowaddr, PAGE_SIZE, 0);
+		bpage->vaddr = (vm_offset_t)contigmalloc_domainset(PAGE_SIZE,
+		    M_DEVBUF, DOMAINSET_PREF(dmat->common.domain), M_NOWAIT,
+		    0ul, bz->lowaddr, PAGE_SIZE, 0);
 		if (bpage->vaddr == 0) {
 			free_domain(bpage, M_DEVBUF);
 			break;