sched_ule(4): Fix hang with steal_thresh < 2.

e745d729be60 caused infinite loop with interrupts disabled in load
stealing code if steal_thresh set below 2.  Such configuration should
not generally be used, but appeared some people are using it to
workaround some problems.

To fix the problem explicitly pass to sched_highest() minimum number
of transferrable threads, supported by the caller, instead of guessing.

MFC after:	25 days

1 files changed, 36 insertions, 34 deletions

diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 49199b6f319f..0f873a6a30b6 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -638,12 +638,13 @@ struct cpu_search {
 	int	cs_prefer;	/* Prefer this CPU and groups including it. */
 	int	cs_running;	/* The thread is now running at cs_prefer. */
 	int	cs_pri;		/* Min priority for low. */
-	int	cs_limit;	/* Max load for low, min load for high. */
+	int	cs_load;	/* Max load for low, min load for high. */
+	int	cs_trans;	/* Min transferable load for high. */
 };
 
 struct cpu_search_res {
-	int	cs_cpu;		/* The best CPU found. */
-	int	cs_load;	/* The load of cs_cpu. */
+	int	csr_cpu;	/* The best CPU found. */
+	int	csr_load;	/* The load of cs_cpu. */
 };
 
 /*
@@ -663,7 +664,7 @@ cpu_search_lowest(const struct cpu_group *cg, const struct cpu_search *s,
 
 	total = 0;
 	bload = INT_MAX;
-	r->cs_cpu = -1;
+	r->csr_cpu = -1;
 
 	/* Loop through children CPU groups if there are any. */
 	if (cg->cg_children > 0) {
@@ -681,11 +682,11 @@ cpu_search_lowest(const struct cpu_group *cg, const struct cpu_search *s,
 			    load >= 128 && (load & 128) != 0)
 				load += 128;
 
-			if (lr.cs_cpu >= 0 && (load < bload ||
-			    (load == bload && lr.cs_load < r->cs_load))) {
+			if (lr.csr_cpu >= 0 && (load < bload ||
+			    (load == bload && lr.csr_load < r->csr_load))) {
 				bload = load;
-				r->cs_cpu = lr.cs_cpu;
-				r->cs_load = lr.cs_load;
+				r->csr_cpu = lr.csr_cpu;
+				r->csr_load = lr.csr_load;
 			}
 		}
 		return (total);
@@ -711,7 +712,7 @@ cpu_search_lowest(const struct cpu_group *cg, const struct cpu_search *s,
 		 * If the threads is already on the CPU, don't look on the TDQ
 		 * priority, since it can be the priority of the thread itself.
 		 */
-		if (l > s->cs_limit || (tdq->tdq_lowpri <= s->cs_pri &&
+		if (l > s->cs_load || (tdq->tdq_lowpri <= s->cs_pri &&
 		     (!s->cs_running || c != s->cs_prefer)) ||
 		    !CPU_ISSET(c, s->cs_mask))
 			continue;
@@ -727,8 +728,8 @@ cpu_search_lowest(const struct cpu_group *cg, const struct cpu_search *s,
 		load -= sched_random() % 128;
 		if (bload > load - p) {
 			bload = load - p;
-			r->cs_cpu = c;
-			r->cs_load = load;
+			r->csr_cpu = c;
+			r->csr_load = load;
 		}
 	}
 	return (total);
@@ -744,18 +745,18 @@ cpu_search_highest(const struct cpu_group *cg, const struct cpu_search *s,
 
 	total = 0;
 	bload = INT_MIN;
-	r->cs_cpu = -1;
+	r->csr_cpu = -1;
 
 	/* Loop through children CPU groups if there are any. */
 	if (cg->cg_children > 0) {
 		for (c = cg->cg_children - 1; c >= 0; c--) {
 			load = cpu_search_highest(&cg->cg_child[c], s, &lr);
 			total += load;
-			if (lr.cs_cpu >= 0 && (load > bload ||
-			    (load == bload && lr.cs_load > r->cs_load))) {
+			if (lr.csr_cpu >= 0 && (load > bload ||
+			    (load == bload && lr.csr_load > r->csr_load))) {
 				bload = load;
-				r->cs_cpu = lr.cs_cpu;
-				r->cs_load = lr.cs_load;
+				r->csr_cpu = lr.csr_cpu;
+				r->csr_load = lr.csr_load;
 			}
 		}
 		return (total);
@@ -772,21 +773,18 @@ cpu_search_highest(const struct cpu_group *cg, const struct cpu_search *s,
 
 		/*
 		 * Check this CPU is acceptable.
-		 * If requested minimum load is 1, then caller must know how
-		 * to handle running threads, not counted in tdq_transferable.
 		 */
-		if (l < s->cs_limit || (tdq->tdq_transferable == 0 &&
-		    (s->cs_limit > 1 || l > 1)) ||
+		if (l < s->cs_load || (tdq->tdq_transferable < s->cs_trans) ||
 		    !CPU_ISSET(c, s->cs_mask))
 			continue;
 
 		load -= sched_random() % 256;
 		if (load > bload) {
 			bload = load;
-			r->cs_cpu = c;
+			r->csr_cpu = c;
 		}
 	}
-	r->cs_load = bload;
+	r->csr_load = bload;
 	return (total);
 }
 
@@ -806,24 +804,26 @@ sched_lowest(const struct cpu_group *cg, cpuset_t *mask, int pri, int maxload,
 	s.cs_running = running;
 	s.cs_mask = mask;
 	s.cs_pri = pri;
-	s.cs_limit = maxload;
+	s.cs_load = maxload;
 	cpu_search_lowest(cg, &s, &r);
-	return (r.cs_cpu);
+	return (r.csr_cpu);
 }
 
 /*
  * Find the cpu with the highest load via the highest loaded path.
  */
 static inline int
-sched_highest(const struct cpu_group *cg, cpuset_t *mask, int minload)
+sched_highest(const struct cpu_group *cg, cpuset_t *mask, int minload,
+    int mintrans)
 {
 	struct cpu_search s;
 	struct cpu_search_res r;
 
 	s.cs_mask = mask;
-	s.cs_limit = minload;
+	s.cs_load = minload;
+	s.cs_trans = mintrans;
 	cpu_search_highest(cg, &s, &r);
-	return (r.cs_cpu);
+	return (r.csr_cpu);
 }
 
 static void
@@ -836,7 +836,7 @@ sched_balance_group(struct cpu_group *cg)
 
 	CPU_FILL(&hmask);
 	for (;;) {
-		high = sched_highest(cg, &hmask, 1);
+		high = sched_highest(cg, &hmask, 1, 0);
 		/* Stop if there is no more CPU with transferrable threads. */
 		if (high == -1)
 			break;
@@ -1008,7 +1008,7 @@ tdq_idled(struct tdq *tdq)
     restart:
 	switchcnt = tdq->tdq_switchcnt + tdq->tdq_oldswitchcnt;
 	for (cg = tdq->tdq_cg, goup = 0; ; ) {
-		cpu = sched_highest(cg, &mask, steal_thresh);
+		cpu = sched_highest(cg, &mask, steal_thresh, 1);
 		/*
 		 * We were assigned a thread but not preempted.  Returning
 		 * 0 here will cause our caller to switch to it.
@@ -1968,7 +1968,8 @@ tdq_trysteal(struct tdq *tdq)
 	cpuset_t mask;
 	int cpu, i, goup;
 
-	if (smp_started == 0 || trysteal_limit == 0 || tdq->tdq_cg == NULL)
+	if (smp_started == 0 || steal_idle == 0 || trysteal_limit == 0 ||
+	    tdq->tdq_cg == NULL)
 		return;
 	CPU_FILL(&mask);
 	CPU_CLR(PCPU_GET(cpuid), &mask);
@@ -1976,7 +1977,7 @@ tdq_trysteal(struct tdq *tdq)
 	spinlock_enter();
 	TDQ_UNLOCK(tdq);
 	for (i = 1, cg = tdq->tdq_cg, goup = 0; ; ) {
-		cpu = sched_highest(cg, &mask, steal_thresh);
+		cpu = sched_highest(cg, &mask, steal_thresh, 1);
 		/*
 		 * If a thread was added while interrupts were disabled don't
 		 * steal one here.
@@ -2019,7 +2020,9 @@ tdq_trysteal(struct tdq *tdq)
 		steal = TDQ_CPU(cpu);
 		/*
 		 * The data returned by sched_highest() is stale and
-                 * the chosen CPU no longer has an eligible thread.
+		 * the chosen CPU no longer has an eligible thread.
+		 * At this point unconditonally exit the loop to bound
+		 * the time spent in the critcal section.
 		 */
 		if (steal->tdq_load < steal_thresh ||
 		    steal->tdq_transferable == 0)
@@ -2028,8 +2031,7 @@ tdq_trysteal(struct tdq *tdq)
 		 * Try to lock both queues. If we are assigned a thread while
 		 * waited for the lock, switch to it now instead of stealing.
 		 * If we can't get the lock, then somebody likely got there
-		 * first.  At this point unconditonally exit the loop to
-		 * bound the time spent in the critcal section.
+		 * first.
 		 */
 		TDQ_LOCK(tdq);
 		if (tdq->tdq_load > 0)