Add a new kernel option, MALLOC_MAKE_FAILURES, which compiles

in a debugging feature causing M_NOWAIT allocations to fail at
a specified rate.  This can be useful for detecting poor
handling of M_NOWAIT: the most frequent problems I've bumped
into are unconditional deference of the pointer even though
it's NULL, and hangs as a result of a lost event where memory
for the event couldn't be allocated.  Two sysctls are added:

debug.malloc.failure_rate

  How often to generate a failure: if set to 0 (default), this
  feature is disabled.  Otherwise, the frequency of failures --
  I've been using 10 (one in ten mallocs fails), but other
  popular settings might be much lower or much higher.

debug.malloc.failure_count

  Number of times a coerced malloc failure has occurred as a
  result of this feature.  Useful for tracking what might have
  happened and whether failures are being generated.

Useful possible additions: tying failure rate to malloc type,
printfs indicating the thread that experienced the coerced
failure.

Reviewed by:	jeffr, jhb

1 files changed, 26 insertions, 0 deletions

diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index 34f791ab2f6e..048c4b34d095 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -138,6 +138,23 @@ static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
 /* time_uptime of last malloc(9) failure */
 static time_t t_malloc_fail;
 
+#ifdef MALLOC_MAKE_FAILURES
+/*
+ * Causes malloc failures every (n) mallocs with M_NOWAIT.  If set to 0,
+ * doesn't cause failures.
+ */
+SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
+    "Kernel malloc debugging options");
+
+static int malloc_failure_rate;
+static int malloc_nowait_count;
+static int malloc_failure_count;
+SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RW,
+    &malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail");
+SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
+    &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
+#endif
+
 int
 malloc_last_fail(void)
 {
@@ -188,6 +205,15 @@ malloc(size, type, flags)
 	if (size == 0)
 		Debugger("zero size malloc");
 #endif
+#ifdef MALLOC_MAKE_FAILURES
+	if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) {
+		atomic_add_int(&malloc_nowait_count, 1);
+		if ((malloc_nowait_count % malloc_failure_rate) == 0) {
+			atomic_add_int(&malloc_failure_count, 1);
+			return (NULL);
+		}
+	}
+#endif
 	if (flags & M_WAITOK)
 		KASSERT(curthread->td_intr_nesting_level == 0,
 		   ("malloc(M_WAITOK) in interrupt context"));