1 files changed, 143 insertions, 122 deletions

diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c
index b44bdb96b0d4..9fa17da954f7 100644
--- a/sys/vm/vm_domainset.c
+++ b/sys/vm/vm_domainset.c
@@ -58,6 +58,9 @@
 
 static int vm_domainset_default_stride = 64;
 
+static bool vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain);
+
+
 /*
  * Determine which policy is to be used for this allocation.
  */
@@ -93,28 +96,15 @@ vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
 			pindex += (((uintptr_t)obj) / sizeof(*obj));
 		di->di_offset = pindex;
 	}
-	/* Skip domains below min on the first pass. */
-	di->di_minskip = true;
 }
 
 static void
 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
 {
 
+	/* Grab the next domain in 'ds_order'. */
 	*domain = di->di_domain->ds_order[
-	    ++(*di->di_iter) % di->di_domain->ds_cnt];
-}
-
-static void
-vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
-{
-	int d;
-
-	do {
-		d = di->di_domain->ds_order[
-		    ++(*di->di_iter) % di->di_domain->ds_cnt];
-	} while (d == di->di_domain->ds_prefer);
-	*domain = d;
+	    (*di->di_iter)++ % di->di_domain->ds_cnt];
 }
 
 static void
@@ -127,79 +117,144 @@ vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)
 	*domain = di->di_domain->ds_order[d];
 }
 
-static void
-vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
+/*
+ * Internal function determining the current phase's first candidate domain.
+ *
+ * Returns whether these is an eligible domain, which is returned through
+ * '*domain'.  '*domain' can be modified even if there is no eligible domain.
+ *
+ * See herald comment of vm_domainset_iter_first() below about phases.
+ */
+static bool
+vm_domainset_iter_phase_first(struct vm_domainset_iter *di, int *domain)
 {
-
-	KASSERT(di->di_n > 0, ("%s: Invalid n %d", __func__, di->di_n));
 	switch (di->di_policy) {
 	case DOMAINSET_POLICY_FIRSTTOUCH:
-		/*
-		 * To prevent impossible allocations we convert an invalid
-		 * first-touch to round-robin.
-		 */
-		/* FALLTHROUGH */
-	case DOMAINSET_POLICY_INTERLEAVE:
-		/* FALLTHROUGH */
+		*domain = PCPU_GET(domain);
+		break;
 	case DOMAINSET_POLICY_ROUNDROBIN:
 		vm_domainset_iter_rr(di, domain);
 		break;
 	case DOMAINSET_POLICY_PREFER:
-		vm_domainset_iter_prefer(di, domain);
+		*domain = di->di_domain->ds_prefer;
+		break;
+	case DOMAINSET_POLICY_INTERLEAVE:
+		vm_domainset_iter_interleave(di, domain);
 		break;
 	default:
 		panic("%s: Unknown policy %d", __func__, di->di_policy);
 	}
 	KASSERT(*domain < vm_ndomains,
 	    ("%s: Invalid domain %d", __func__, *domain));
+
+	/*
+	 * Has the policy's start domain already been visited?
+	 */
+	if (!DOMAINSET_ISSET(*domain, &di->di_remain_mask))
+		return (vm_domainset_iter_next(di, domain));
+
+	DOMAINSET_CLR(*domain, &di->di_remain_mask);
+
+	/* Does it have enough free pages (phase 1)? */
+	if (di->di_minskip && vm_page_count_min_domain(*domain)) {
+		/* Mark the domain as eligible for phase 2. */
+		DOMAINSET_SET(*domain, &di->di_min_mask);
+		return (vm_domainset_iter_next(di, domain));
+	}
+
+	return (true);
 }
 
-static void
+/*
+ * Resets an iterator to point to the first candidate domain.
+ *
+ * Returns whether there is an eligible domain to start with.  '*domain' may be
+ * modified even if there is none.
+ *
+ * There must have been one call to vm_domainset_iter_init() before.
+ *
+ * This function must be called at least once before calling
+ * vm_domainset_iter_next().  Note that functions wrapping
+ * vm_domainset_iter_init() usually do that themselves.
+ *
+ * This function may be called again to reset the iterator to the policy's first
+ * candidate domain.  After each reset, the iterator will visit the same domains
+ * as in the previous iteration minus those on which vm_domainset_iter_ignore()
+ * has been called.  Note that the first candidate domain may change at each
+ * reset (at time of this writing, only on the DOMAINSET_POLICY_ROUNDROBIN
+ * policy).
+ *
+ * Domains which have a number of free pages over 'v_free_min' are always
+ * visited first (this is called the "phase 1" in comments, "phase 2" being the
+ * examination of the remaining domains; no domains are ever visited twice).
+ */
+static bool
 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
 {
+	/* Initialize the mask of domains to visit. */
+	DOMAINSET_COPY(&di->di_valid_mask, &di->di_remain_mask);
+	/*
+	 * No candidate domains for phase 2 at start.  This will be filled by
+	 * phase 1.
+	 */
+	DOMAINSET_ZERO(&di->di_min_mask);
+	/* Skip domains below 'v_free_min' on phase 1. */
+	di->di_minskip = true;
 
-	switch (di->di_policy) {
-	case DOMAINSET_POLICY_FIRSTTOUCH:
-		*domain = PCPU_GET(domain);
-		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask)) {
-			/*
-			 * Add an extra iteration because we will visit the
-			 * current domain a second time in the rr iterator.
-			 */
-			di->di_n = di->di_domain->ds_cnt + 1;
-			break;
-		}
-		/*
-		 * To prevent impossible allocations we convert an invalid
-		 * first-touch to round-robin.
-		 */
-		/* FALLTHROUGH */
-	case DOMAINSET_POLICY_ROUNDROBIN:
-		di->di_n = di->di_domain->ds_cnt;
+	return (vm_domainset_iter_phase_first(di, domain));
+}
+
+/*
+ * Advances the iterator to the next candidate domain.
+ *
+ * Returns whether there was another domain to visit.  '*domain' may be modified
+ * even if there is none.
+ *
+ * vm_domainset_iter_first() must have been called at least once before using
+ * this function (see its herald comment for more details on iterators).
+ */
+static bool
+vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
+{
+	/* Loop while there remains domains to visit in the current phase. */
+	while (!DOMAINSET_EMPTY(&di->di_remain_mask)) {
+		/* Grab the next domain in 'ds_order'. */
 		vm_domainset_iter_rr(di, domain);
-		break;
-	case DOMAINSET_POLICY_PREFER:
-		*domain = di->di_domain->ds_prefer;
-		di->di_n = di->di_domain->ds_cnt;
-		break;
-	case DOMAINSET_POLICY_INTERLEAVE:
-		vm_domainset_iter_interleave(di, domain);
-		di->di_n = di->di_domain->ds_cnt;
-		break;
-	default:
-		panic("%s: Unknown policy %d", __func__, di->di_policy);
+		KASSERT(*domain < vm_ndomains,
+		    ("%s: Invalid domain %d", __func__, *domain));
+
+		if (DOMAINSET_ISSET(*domain, &di->di_remain_mask)) {
+			DOMAINSET_CLR(*domain, &di->di_remain_mask);
+			if (!di->di_minskip || !vm_page_count_min_domain(*domain))
+				return (true);
+			DOMAINSET_SET(*domain, &di->di_min_mask);
+		}
 	}
-	KASSERT(di->di_n > 0, ("%s: Invalid n %d", __func__, di->di_n));
-	KASSERT(*domain < vm_ndomains,
-	    ("%s: Invalid domain %d", __func__, *domain));
+
+	/*
+	 * If phase 1 (skip low memory domains) is over, start phase 2 (consider
+	 * low memory domains).
+	 */
+	if (di->di_minskip) {
+		di->di_minskip = false;
+		/* Browse domains that were under 'v_free_min'. */
+		DOMAINSET_COPY(&di->di_min_mask, &di->di_remain_mask);
+		return (vm_domainset_iter_phase_first(di, domain));
+	}
+
+	return (false);
 }
 
-void
+int
 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
-    vm_pindex_t pindex, int *domain, int *req, struct pctrie_iter *pages)
+    vm_pindex_t pindex, int *domain, int *req)
 {
 	struct domainset_ref *dr;
 
+	di->di_flags = *req;
+	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
+	    VM_ALLOC_NOWAIT;
+
 	/*
 	 * Object policy takes precedence over thread policy.  The policies
 	 * are immutable and unsynchronized.  Updates can race but pointer
@@ -209,36 +264,21 @@ vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
 		dr = &obj->domain;
 	else
 		dr = &curthread->td_domain;
+
 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);
-	di->di_flags = *req;
-	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
-	    VM_ALLOC_NOWAIT;
-	vm_domainset_iter_first(di, domain);
-	if (vm_page_count_min_domain(*domain))
-		vm_domainset_iter_page(di, obj, domain, pages);
+	/*
+	 * XXXOC: Shouldn't we just panic on 'false' if VM_ALLOC_WAITOK was
+	 * passed?
+	 */
+	return (vm_domainset_iter_first(di, domain) ? 0 : ENOMEM);
 }
 
 int
 vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
     int *domain, struct pctrie_iter *pages)
 {
-	if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))
-		return (ENOMEM);
-
-	/* If there are more domains to visit we run the iterator. */
-	while (--di->di_n != 0) {
-		vm_domainset_iter_next(di, domain);
-		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
-		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))
-			return (0);
-	}
-
-	/* If we skipped domains below min restart the search. */
-	if (di->di_minskip) {
-		di->di_minskip = false;
-		vm_domainset_iter_first(di, domain);
+	if (vm_domainset_iter_next(di, domain))
 		return (0);
-	}
 
 	/* If we visited all domains and this was a NOWAIT we return error. */
 	if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
@@ -257,61 +297,43 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
 		return (ENOMEM);
 
 	/* Restart the search. */
-	vm_domainset_iter_first(di, domain);
-
-	return (0);
+	/* XXXOC: Shouldn't we just panic on 'false'? */
+	return (vm_domainset_iter_first(di, domain) ? 0 : ENOMEM);
 }
 
-static void
+static int
 _vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,
     int *flags)
 {
-
 	di->di_flags = *flags;
 	*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
-	vm_domainset_iter_first(di, domain);
-	if (vm_page_count_min_domain(*domain))
-		vm_domainset_iter_policy(di, domain);
+	/* XXXOC: Shouldn't we just panic on 'false' if M_WAITOK was passed? */
+	return (vm_domainset_iter_first(di, domain) ? 0 : ENOMEM);
 }
 
-void
+int
 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
     struct domainset *ds, int *domain, int *flags)
 {
 
 	vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);
-	_vm_domainset_iter_policy_init(di, domain, flags);
+	return (_vm_domainset_iter_policy_init(di, domain, flags));
 }
 
-void
+int
 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
     struct domainset_ref *dr, int *domain, int *flags)
 {
 
 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);
-	_vm_domainset_iter_policy_init(di, domain, flags);
+	return (_vm_domainset_iter_policy_init(di, domain, flags));
 }
 
 int
 vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
 {
-	if (DOMAINSET_EMPTY(&di->di_valid_mask))
-		return (ENOMEM);
-
-	/* If there are more domains to visit we run the iterator. */
-	while (--di->di_n != 0) {
-		vm_domainset_iter_next(di, domain);
-		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
-		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))
-			return (0);
-	}
-
-	/* If we skipped domains below min restart the search. */
-	if (di->di_minskip) {
-		di->di_minskip = false;
-		vm_domainset_iter_first(di, domain);
+	if (vm_domainset_iter_next(di, domain))
 		return (0);
-	}
 
 	/* If we visited all domains and this was a NOWAIT we return error. */
 	if ((di->di_flags & M_WAITOK) == 0)
@@ -321,9 +343,8 @@ vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
 	vm_wait_doms(&di->di_valid_mask, 0);
 
 	/* Restart the search. */
-	vm_domainset_iter_first(di, domain);
-
-	return (0);
+	/* XXXOC: Shouldn't we just panic on 'false'? */
+	return (vm_domainset_iter_first(di, domain) ? 0 : ENOMEM);
 }
 
 void
@@ -345,12 +366,12 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
 	return (EJUSTRETURN);
 }
 
-void
+int
 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
-    vm_pindex_t pindex, int *domain, int *flags, struct pctrie_iter *pages)
+    vm_pindex_t pindex, int *domain, int *flags)
 {
-
 	*domain = 0;
+	return (0);
 }
 
 int
@@ -360,20 +381,20 @@ vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
 	return (EJUSTRETURN);
 }
 
-void
+int
 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
     struct domainset *ds, int *domain, int *flags)
 {
-
 	*domain = 0;
+	return (0);
 }
 
-void
+int
 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
     struct domainset_ref *dr, int *domain, int *flags)
 {
-
 	*domain = 0;
+	return (0);
 }
 
 void