Add a kernel test framework. The callout_test is a demonstration and will only

work with the upcoming async-drain functionality. Tests can be added
to the tests directory and then the framework can be used to launch
those tests.

MFC after:	1 month
Sponsored by:	Netflix Inc.
Differential Revision:	https://reviews.freebsd.org/D1755

7 files changed, 742 insertions, 0 deletions

diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index fadde8d46c4f..006e0fa53ba9 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -343,6 +343,8 @@ SUBDIR=	\
 	${_syscons} \
 	sysvipc \
 	${_ti} \
+	tests/framework \
+	tests/callout_test \
 	tl \
 	tmpfs \
 	${_toecore} \
diff --git a/sys/modules/tests/callout_test/Makefile b/sys/modules/tests/callout_test/Makefile
new file mode 100644
index 000000000000..4781488af3b8
--- /dev/null
+++ b/sys/modules/tests/callout_test/Makefile
@@ -0,0 +1,15 @@
+#
+# $FreeBSD$
+#
+
+.PATH: ${.CURDIR}/../../../tests/callout_test
+
+KMOD=	callout_test
+SRCS=	callout_test.c
+
+#
+# Enable full debugging
+#
+#CFLAGS += -g
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/tests/framework/Makefile b/sys/modules/tests/framework/Makefile
new file mode 100644
index 000000000000..7b64cd1b0190
--- /dev/null
+++ b/sys/modules/tests/framework/Makefile
@@ -0,0 +1,15 @@
+#
+# $FreeBSD$
+#
+
+.PATH: ${.CURDIR}/../../../tests/framework
+
+KMOD=	kern_testfrwk
+SRCS=	kern_testfrwk.c
+
+#
+# Enable full debugging
+#
+#CFLAGS += -g
+
+.include <bsd.kmod.mk>
diff --git a/sys/tests/callout_test.h b/sys/tests/callout_test.h
new file mode 100644
index 000000000000..2f470dc8bbca
--- /dev/null
+++ b/sys/tests/callout_test.h
@@ -0,0 +1,34 @@
+#ifndef __callout_test_h__
+#define __callout_test_h__
+/*-
+ * Copyright (c) 2015
+ *	Netflix Incorporated, All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *__FBSDID("$FreeBSD$");
+ *
+ */
+struct callout_test {
+	int number_of_callouts;
+	int test_number;
+};
+#endif
diff --git a/sys/tests/callout_test/callout_test.c b/sys/tests/callout_test/callout_test.c
new file mode 100644
index 000000000000..f3d3a6714db8
--- /dev/null
+++ b/sys/tests/callout_test/callout_test.c
@@ -0,0 +1,284 @@
+/*-
+ * Copyright (c) 2015 Netflix Inc. All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/ioccom.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/priv.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <sys/sched.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/libkern.h>
+#include <sys/uio.h>
+#include <sys/pcpu.h>
+#include <sys/smp.h>
+#include <sys/pmckern.h>
+#include <sys/cpuctl.h>
+#include <tests/kern_testfrwk.h>
+#include <tests/callout_test.h>
+#ifdef SMP
+#include <machine/cpu.h>
+#else 
+#define cpu_spinwait()
+#endif
+
+MALLOC_DEFINE(M_CALLTMP, "Temp callout Memory", "CalloutTest");
+
+struct callout_run {
+	struct mtx lock;
+	struct callout *co_array;
+	int co_test;
+	int co_number_callouts;
+	int co_return_npa;
+	int co_completed;
+	int callout_waiting;
+	int drain_calls;
+	int cnt_zero;
+	int cnt_one;
+	int index;
+};
+
+static struct callout_run *comaster[MAXCPU];
+
+uint64_t callout_total=0;
+
+static void execute_the_co_test();
+
+static void
+co_saydone(void *arg)
+{
+	struct callout_run *rn;
+	rn = (struct callout_run *)arg;
+	printf("The callout test is now complete for thread %d\n",
+	       rn->index);
+	printf("number_callouts:%d\n",
+	       rn->co_number_callouts);
+	printf("Callouts that bailed (Not PENDING or ACTIVE cleared):%d\n",
+	       rn->co_return_npa);
+	printf("Callouts that completed:%d\n", rn->co_completed);
+	printf("Drain calls:%d\n", rn->drain_calls);
+	printf("Zero returns:%d non-zero:%d\n",
+	       rn->cnt_zero,
+	       rn->cnt_one);
+
+}
+
+static void
+drainit(void *arg)
+{
+	struct callout_run *rn;
+	rn = (struct callout_run *)arg;
+	mtx_lock(&rn->lock);
+	rn->drain_calls++;
+	mtx_unlock(&rn->lock);
+}
+
+static void
+test_callout(void *arg)
+{
+	struct callout_run *rn;
+	int cpu;
+	
+	critical_enter();
+	cpu = curcpu;
+	critical_exit();
+	rn = (struct callout_run *)arg;
+	atomic_add_int(&rn->callout_waiting, 1);
+	mtx_lock(&rn->lock);
+	if (callout_pending(&rn->co_array[cpu]) ||
+	    !callout_active(&rn->co_array[cpu])) {
+		rn->co_return_npa++;
+		atomic_subtract_int(&rn->callout_waiting, 1);
+		mtx_unlock(&rn->lock);
+		return;
+	}
+	callout_deactivate(&rn->co_array[cpu]);
+	rn->co_completed++;
+	mtx_unlock(&rn->lock);	
+	atomic_subtract_int(&rn->callout_waiting, 1);
+}
+
+void
+execute_the_co_test(struct callout_run *rn)
+{
+	int i, ret, cpu;
+	uint32_t tk_s, tk_e, tk_d;
+
+	mtx_lock(&rn->lock);
+	rn->callout_waiting = 0;
+	for(i=0; i<rn->co_number_callouts; i++) {
+		if (rn->co_test == 1) {
+			/* start all on spread out cpu's */
+			cpu = i % mp_ncpus;
+			callout_reset_sbt_on(&rn->co_array[i], 3, 0, test_callout, rn, 
+					     cpu, 0);
+		} else {
+			/* Start all on the same CPU */
+			callout_reset_sbt_on(&rn->co_array[i], 3, 0, test_callout, rn, 
+					     rn->index, 0);
+		}
+	}
+	tk_s = ticks;
+	while (rn->callout_waiting != rn->co_number_callouts) {
+		cpu_spinwait();
+		tk_e = ticks;
+		tk_d = tk_e - tk_s;
+		if (tk_d > 100) {
+			break;
+		}
+	}
+	/* OK everyone is waiting and we have the lock */
+	for(i=0; i<rn->co_number_callouts; i++) {
+		ret = callout_async_drain(&rn->co_array[i], drainit);
+		if (ret) {
+			rn->cnt_one++;
+		} else {
+			rn->cnt_zero++;
+		}
+	}
+	rn->callout_waiting -= rn->cnt_one;
+	mtx_unlock(&rn->lock);
+	/* Now wait until all are done */
+	tk_s = ticks;
+	while (rn->callout_waiting > 0) {
+		cpu_spinwait();
+		tk_e = ticks;
+		tk_d = tk_e - tk_s;
+		if (tk_d > 100) {
+			break;
+		}
+	}
+	co_saydone((void *)rn);
+}
+
+
+static void
+run_callout_test(struct kern_test *test)
+{
+	struct callout_test *u;
+	size_t sz;
+	int i;
+	struct callout_run *rn;
+	int index = test->tot_threads_running;
+
+	u = (struct callout_test *)test->test_options;
+	if (comaster[index] == NULL) {
+		rn = comaster[index] = malloc(sizeof(struct callout_run), M_CALLTMP, M_WAITOK);
+		memset(comaster[index], 0, sizeof(struct callout_run));
+		mtx_init(&rn->lock, "callouttest",  NULL, MTX_DUPOK);
+		rn->index = index;
+	} else {
+		rn = comaster[index];
+		rn->co_number_callouts = rn->co_return_npa = 0;
+		rn->co_completed = rn->callout_waiting = 0;
+		rn->drain_calls = rn->cnt_zero = rn->cnt_one = 0;
+		if (rn->co_array) {
+			free(rn->co_array, M_CALLTMP);
+			rn->co_array = NULL;
+		}
+	}
+	rn->co_number_callouts = u->number_of_callouts;
+	rn->co_test = u->test_number;
+	sz = sizeof(struct callout) * rn->co_number_callouts;
+	rn->co_array = malloc(sz, M_CALLTMP, M_WAITOK);
+	for(i=0; i<rn->co_number_callouts; i++) {
+		callout_init(&rn->co_array[i], CALLOUT_MPSAFE);
+	}
+	execute_the_co_test(rn);
+}
+
+int callout_test_is_loaded=0;
+
+static void
+cocleanup(void)
+{
+	int i;
+	for(i=0; i<MAXCPU; i++) {
+		if (comaster[i]) {
+			if (comaster[i]->co_array) {
+				free(comaster[i]->co_array, M_CALLTMP);
+				comaster[i]->co_array = NULL;
+			}
+			free(comaster[i], M_CALLTMP);
+			comaster[i] = NULL;
+		}
+	}
+}
+
+static int
+callout_test_modevent(module_t mod, int type, void *data)
+{
+	int err=0;
+
+	switch (type) {
+	case MOD_LOAD:
+		err = kern_testframework_register("callout_test",
+						  run_callout_test);
+		if (err) {
+			printf("Can't load callout_test err:%d returned\n",
+			       err);
+		} else {
+			memset(comaster, 0, sizeof(comaster));
+			callout_test_is_loaded = 1;
+		}
+		break;
+	case MOD_QUIESCE:
+		err = kern_testframework_deregister("callout_test");
+		if (err == 0) {
+			callout_test_is_loaded = 0;
+			cocleanup();
+		}
+		break;
+	case MOD_UNLOAD:
+		if (callout_test_is_loaded) {
+			err = kern_testframework_deregister("callout_test");
+			if (err == 0) {
+				cocleanup();
+				callout_test_is_loaded = 0;
+			}
+		}
+		break;
+	default:
+		return (EOPNOTSUPP);
+	}
+	return (err);
+}
+
+static moduledata_t callout_test_mod = {
+	.name = "callout_test",
+	.evhand = callout_test_modevent,
+	.priv = 0
+};
+
+MODULE_DEPEND(callout_test, kern_testframework, 1, 1, 1);
+DECLARE_MODULE(callout_test, callout_test_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
diff --git a/sys/tests/framework/kern_testfrwk.c b/sys/tests/framework/kern_testfrwk.c
new file mode 100644
index 000000000000..a6f2de550863
--- /dev/null
+++ b/sys/tests/framework/kern_testfrwk.c
@@ -0,0 +1,343 @@
+/*-
+ * Copyright (c) 2015
+ *	Netflix Incorporated, All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/callout.h>
+#include <sys/module.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/sdt.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+#include <sys/smp.h>
+#include <sys/queue.h>
+#include <tests/kern_testfrwk.h>
+#ifdef SMP
+#include <machine/cpu.h>
+#endif
+
+struct kern_test_list {
+	TAILQ_ENTRY(kern_test_list) next;
+	char name[TEST_NAME_LEN];
+	kerntfunc func;
+};
+
+TAILQ_HEAD(ktestlist, kern_test_list); 
+
+struct kern_test_entry {
+	TAILQ_ENTRY(kern_test_entry) next;
+	struct kern_test_list 	*kt_e;
+	struct kern_test 	kt_data;
+};
+TAILQ_HEAD(ktestqueue, kern_test_entry); 
+
+MALLOC_DEFINE(M_KTFRWK, "kern_tfrwk", "Kernel Test Framework");
+struct kern_totfrwk {
+	struct taskqueue	*kfrwk_tq;
+	struct task		kfrwk_que;
+	struct ktestlist 	kfrwk_testlist;
+	struct ktestqueue 	kfrwk_testq;
+	struct mtx 		kfrwk_mtx;
+	int 			kfrwk_waiting;
+};
+
+struct kern_totfrwk kfrwk;
+static int ktest_frwk_inited=0;
+
+#define KTFRWK_MUTEX_INIT() mtx_init(&kfrwk.kfrwk_mtx, "kern_test_frwk", "tfrwk", MTX_DEF)
+
+#define KTFRWK_DESTROY() mtx_destroy(&kfrwk.kfrwk_mtx)
+
+#define KTFRWK_LOCK() mtx_lock(&kfrwk.kfrwk_mtx)
+
+#define KTFRWK_UNLOCK()	mtx_unlock(&kfrwk.kfrwk_mtx)
+
+static void
+kfrwk_task(void *context, int pending)
+{
+	struct kern_totfrwk *tf;
+	struct kern_test_entry *wk;
+	int free_mem=0;
+	struct kern_test kt_data;
+	kerntfunc ktf;
+
+	memset(&kt_data, 0, sizeof(kt_data));
+	ktf = NULL;
+	tf = (struct kern_totfrwk *)context;
+	KTFRWK_LOCK();
+	wk = TAILQ_FIRST(&tf->kfrwk_testq);
+	if (wk) {
+		wk->kt_data.tot_threads_running--;
+		tf->kfrwk_waiting--;
+		memcpy(&kt_data, &wk->kt_data, sizeof(kt_data));
+		if (wk->kt_data.tot_threads_running == 0) {
+			TAILQ_REMOVE(&tf->kfrwk_testq, wk, next);
+			free_mem = 1;
+		} else {
+			/* Wake one of my colleages up to help too */
+			taskqueue_enqueue(tf->kfrwk_tq, &tf->kfrwk_que);
+		}
+		if (wk->kt_e) {
+			ktf = wk->kt_e->func;
+		}
+        }
+	KTFRWK_UNLOCK();
+	if (wk && free_mem) {
+		free(wk, M_KTFRWK);
+	}
+	/* Execute the test */
+	if (ktf){
+		(*ktf)(&kt_data);
+	}
+	/* We are done */
+	atomic_add_int(&tf->kfrwk_waiting, 1);
+}
+
+static int
+kerntest_frwk_init(void)
+{
+	u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
+
+	KTFRWK_MUTEX_INIT();
+	TAILQ_INIT(&kfrwk.kfrwk_testq);
+	TAILQ_INIT(&kfrwk.kfrwk_testlist);
+	/* Now lets start up a number of tasks to do the work */
+	TASK_INIT(&kfrwk.kfrwk_que, 0, kfrwk_task,  &kfrwk);
+	kfrwk.kfrwk_tq = taskqueue_create_fast("sbtls_task", M_NOWAIT,
+					       taskqueue_thread_enqueue, &kfrwk.kfrwk_tq);
+	if (kfrwk.kfrwk_tq == NULL) {
+		printf("Can't start taskqueue for Kernel Test Framework\n");
+		panic("Taskqueue init fails for kfrwk");
+	}
+	taskqueue_start_threads(&kfrwk.kfrwk_tq, ncpus, PI_NET, "[kt_frwk task]");
+	kfrwk.kfrwk_waiting = ncpus;
+	ktest_frwk_inited = 1;
+	return(0);
+}
+
+static int
+kerntest_frwk_fini(void)
+{
+	KTFRWK_LOCK();	
+	if (!TAILQ_EMPTY(&kfrwk.kfrwk_testlist)) {
+		/* Still modules registered */
+		KTFRWK_UNLOCK();	
+		return (EBUSY);
+	}
+	ktest_frwk_inited = 0;
+	KTFRWK_UNLOCK();
+	taskqueue_free(kfrwk.kfrwk_tq);
+	/* Ok lets destroy the mutex on the way outs */
+	KTFRWK_DESTROY();
+	return (0);
+}
+
+
+static int kerntest_execute(SYSCTL_HANDLER_ARGS);
+
+SYSCTL_DECL(_kern);
+SYSCTL_NODE(_kern, OID_AUTO, testfrwk,  CTLFLAG_RW, 0, "Kernel Test Framework");
+SYSCTL_PROC(_kern_testfrwk, OID_AUTO, runtest, (CTLTYPE_STRUCT | CTLFLAG_RW),
+	    0, 0, kerntest_execute, "IU", "Execute a kernel test");
+
+int 
+kerntest_execute(SYSCTL_HANDLER_ARGS)
+{
+	struct kern_test kt;
+	struct kern_test_list *li, *te=NULL;
+	struct kern_test_entry *kte=NULL;
+	int error = 0;
+
+	if (ktest_frwk_inited == 0) {
+		return(ENOENT);
+	}
+	/* Find the entry if possible */
+	error = SYSCTL_IN(req, &kt, sizeof(struct kern_test));
+	if (error) {
+		return(error);
+	}
+	if (kt.num_threads <= 0) {
+		return (EINVAL);
+	}
+	/* Grab some memory */
+	kte = malloc(sizeof(struct kern_test_entry), M_KTFRWK, M_WAITOK);
+	if (kte == NULL) {
+		error = ENOMEM;
+		goto out;
+	}
+	KTFRWK_LOCK();
+	TAILQ_FOREACH(li, &kfrwk.kfrwk_testlist, next) {
+		if (strcmp(li->name, kt.name) == 0) {
+			te = li;
+			break;
+		}
+	}
+	if (te == NULL) {
+		printf("Can't find the test %s\n", kt.name);
+		error = ENOENT;
+		free(kte, M_KTFRWK);
+		goto out;
+	}
+	/* Ok we have a test item to run, can we? */
+	if (!TAILQ_EMPTY(&kfrwk.kfrwk_testq)) {
+		/* We don't know if there is enough threads */
+		error = EAGAIN;
+		free(kte, M_KTFRWK);
+		goto out;
+	}
+	if (kfrwk.kfrwk_waiting < kt.num_threads) {
+		error = E2BIG;
+		free(kte, M_KTFRWK);
+		goto out;
+	}
+	kt.tot_threads_running = kt.num_threads;
+	/* Ok it looks like we can do it, lets get an entry */
+	kte->kt_e = li;
+	memcpy(&kte->kt_data, &kt, sizeof(kt));
+	TAILQ_INSERT_TAIL(&kfrwk.kfrwk_testq, kte, next);
+	taskqueue_enqueue(kfrwk.kfrwk_tq, &kfrwk.kfrwk_que);
+out:
+	KTFRWK_UNLOCK();
+	return(error);
+}
+
+int
+kern_testframework_register(const char *name, kerntfunc func)
+{
+	int error = 0;
+	struct kern_test_list *li, *te=NULL;
+	int len;
+
+	len = strlen(name);
+	if (len >= TEST_NAME_LEN) {
+		return (E2BIG);
+	}
+	te = malloc(sizeof(struct kern_test_list), M_KTFRWK, M_WAITOK);	
+	if (te == NULL) {
+		error = ENOMEM;
+		goto out;
+	}
+	KTFRWK_LOCK();
+	/* First does it already exist? */
+	TAILQ_FOREACH(li, &kfrwk.kfrwk_testlist, next) {
+		if (strcmp(li->name, name) == 0) {
+			error = EALREADY;
+			free(te, M_KTFRWK);
+			goto out;
+		}
+	}
+	/* Ok we can do it, lets add it to the list */
+	te->func = func;
+	strcpy(te->name, name);
+	TAILQ_INSERT_TAIL(&kfrwk.kfrwk_testlist, te, next);
+out:
+	KTFRWK_UNLOCK();	
+	return(error);
+}
+
+int
+kern_testframework_deregister(const char *name)
+{
+	struct kern_test_list *li, *te=NULL;
+	u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
+	int error = 0;
+
+	KTFRWK_LOCK();
+	/* First does it already exist? */
+	TAILQ_FOREACH (li, &kfrwk.kfrwk_testlist, next) {
+		if (strcmp(li->name, name) == 0) {
+			te = li;
+			break;
+		}
+	}
+	if (te == NULL) {
+		/* It is not registered so no problem */
+		goto out;
+	}
+	if (ncpus != kfrwk.kfrwk_waiting) {
+		/* We are busy executing something -- can't unload */
+		error = EBUSY;
+		goto out;
+	}
+	if (!TAILQ_EMPTY(&kfrwk.kfrwk_testq)) {
+		/* Something still to execute */
+		error = EBUSY;
+		goto out;
+	}
+	/* Ok we can remove the dude safely */
+	TAILQ_REMOVE(&kfrwk.kfrwk_testlist, te, next);
+	memset(te, 0, sizeof(struct kern_test_list));
+	free(te, M_KTFRWK);
+out:
+	KTFRWK_UNLOCK();	
+	return(error);
+}
+
+static int
+kerntest_mod_init(module_t mod, int type, void *data)
+{
+	int err;
+
+	switch (type) {
+	case MOD_LOAD:
+		err = kerntest_frwk_init();
+		break;
+	case MOD_QUIESCE:
+		KTFRWK_LOCK();
+		if (TAILQ_EMPTY(&kfrwk.kfrwk_testlist)) {
+			err = 0;
+		} else {
+			err = EBUSY;
+		}
+		KTFRWK_UNLOCK();		
+		break;
+	case MOD_UNLOAD:
+		err = kerntest_frwk_fini();
+		break;
+	default:
+		return (EOPNOTSUPP);
+	}
+	return (err);
+}
+
+static moduledata_t kern_test_framework = {
+	.name = "kernel_testfrwk",
+	.evhand = kerntest_mod_init,
+	.priv = 0
+};
+
+MODULE_VERSION(kern_testframework, 1);
+DECLARE_MODULE(kern_testframework, kern_test_framework, SI_SUB_PSEUDO, SI_ORDER_ANY);
diff --git a/sys/tests/kern_testfrwk.h b/sys/tests/kern_testfrwk.h
new file mode 100644
index 000000000000..844eea411313
--- /dev/null
+++ b/sys/tests/kern_testfrwk.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2015
+ *	Netflix Incorporated, All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *__FBSDID("$FreeBSD$");
+ *
+ */
+#ifndef _SYS_KERN_TESTFRWKT_H_
+#define _SYS_KERN_TESTFRWKT_H_
+
+#define TEST_NAME_LEN 32
+#define TEST_OPTION_SPACE 256
+
+struct kern_test {
+	char name[TEST_NAME_LEN];
+	int num_threads;	       	/* Fill in how many threads you want */
+	int tot_threads_running;	/* For framework */
+	uint8_t test_options[TEST_OPTION_SPACE];
+};
+
+
+typedef void (*kerntfunc)(struct kern_test *);
+
+#ifdef _KERNEL
+int kern_testframework_register(const char *name, kerntfunc);
+
+int kern_testframework_deregister(const char *name);
+#endif
+#endif