Make the LinuxKPI task struct persistent accross system calls.

A set of helper functions have been added to manage the life of the
LinuxKPI task struct. When an external system call or task is invoked,
a check is made to create the task struct by demand. A thread
destructor callback is registered to free the task struct when a
thread exits to avoid memory leaks.

This change lays the ground for emulating the Linux kernel more
closely which is a dependency by the code using the LinuxKPI APIs.

Add new dedicated td_lkpi_task field has been added to struct thread
instead of abusing td_retval[1].

Fix some header file inclusions to make LINT kernel build properly
after this change.

Bump the __FreeBSD_version to force a rebuild of all kernel modules.

MFC after:		1 week
Sponsored by:		Mellanox Technologies

14 files changed, 66 insertions, 101 deletions

diff --git a/sys/compat/linuxkpi/common/include/linux/bitops.h b/sys/compat/linuxkpi/common/include/linux/bitops.h
index 9e1fa2bc4569..2c521318804f 100644
--- a/sys/compat/linuxkpi/common/include/linux/bitops.h
+++ b/sys/compat/linuxkpi/common/include/linux/bitops.h
@@ -31,6 +31,7 @@
 #ifndef	_LINUX_BITOPS_H_
 #define	_LINUX_BITOPS_H_
 
+#include <sys/param.h>
 #include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/errno.h>
diff --git a/sys/compat/linuxkpi/common/include/linux/compat.h b/sys/compat/linuxkpi/common/include/linux/compat.h
index 01b8a85b9760..62ea3363394b 100644
--- a/sys/compat/linuxkpi/common/include/linux/compat.h
+++ b/sys/compat/linuxkpi/common/include/linux/compat.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,29 @@
 #ifndef	_LINUX_COMPAT_H_
 #define	_LINUX_COMPAT_H_
 
+#include <sys/param.h>
+#include <sys/proc.h>
+#include <sys/malloc.h>
+
 struct thread;
 struct task_struct;
 
-void linux_set_current(struct thread *td, struct task_struct *t);
-void linux_clear_current(struct thread *td);
+extern int linux_alloc_current(struct thread *, int flags);
+extern void linux_free_current(struct task_struct *);
+
+static inline void
+linux_set_current(struct thread *td)
+{
+	if (__predict_false(td->td_lkpi_task == NULL))
+		linux_alloc_current(td, M_WAITOK);
+}
+
+static inline int
+linux_set_current_flags(struct thread *td, int flags)
+{
+	if (__predict_false(td->td_lkpi_task == NULL))
+		return (linux_alloc_current(td, flags));
+	return (0);
+}
 
 #endif	/* _LINUX_COMPAT_H_ */
diff --git a/sys/compat/linuxkpi/common/include/linux/file.h b/sys/compat/linuxkpi/common/include/linux/file.h
index 559ac0437681..0661e70a5697 100644
--- a/sys/compat/linuxkpi/common/include/linux/file.h
+++ b/sys/compat/linuxkpi/common/include/linux/file.h
@@ -39,6 +39,7 @@
 #include <sys/proc.h>
 
 #include <linux/fs.h>
+#include <linux/slab.h>
 
 struct linux_file;
 
diff --git a/sys/compat/linuxkpi/common/include/linux/jiffies.h b/sys/compat/linuxkpi/common/include/linux/jiffies.h
index 9a85f616152a..a95e6064f41c 100644
--- a/sys/compat/linuxkpi/common/include/linux/jiffies.h
+++ b/sys/compat/linuxkpi/common/include/linux/jiffies.h
@@ -32,7 +32,6 @@
 #define	_LINUX_JIFFIES_H_
 
 #include <linux/types.h>
-#include <linux/kernel.h>
 #include <linux/time.h>
 
 #include <sys/time.h>
diff --git a/sys/compat/linuxkpi/common/include/linux/kdev_t.h b/sys/compat/linuxkpi/common/include/linux/kdev_t.h
index c0bb97e5ba42..447d7af2f842 100644
--- a/sys/compat/linuxkpi/common/include/linux/kdev_t.h
+++ b/sys/compat/linuxkpi/common/include/linux/kdev_t.h
@@ -31,6 +31,8 @@
 #ifndef	_LINUX_KDEV_T_H_
 #define	_LINUX_KDEV_T_H_
 
+#include <sys/types.h>
+
 #define MAJOR(dev)      major((dev))
 #define MINOR(dev)      minor((dev))
 #define MKDEV(ma, mi)   makedev((ma), (mi))
diff --git a/sys/compat/linuxkpi/common/include/linux/kernel.h b/sys/compat/linuxkpi/common/include/linux/kernel.h
index ead6a8a72bfe..fabdba5df517 100644
--- a/sys/compat/linuxkpi/common/include/linux/kernel.h
+++ b/sys/compat/linuxkpi/common/include/linux/kernel.h
@@ -45,7 +45,7 @@
 #include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/errno.h>
-#include <linux/kthread.h>
+#include <linux/sched.h>
 #include <linux/types.h>
 #include <linux/jiffies.h>
 #include <linux/wait.h>
diff --git a/sys/compat/linuxkpi/common/include/linux/kthread.h b/sys/compat/linuxkpi/common/include/linux/kthread.h
index 2e0da123d528..eaf2b31a71ab 100644
--- a/sys/compat/linuxkpi/common/include/linux/kthread.h
+++ b/sys/compat/linuxkpi/common/include/linux/kthread.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013-2016 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,74 +31,27 @@
 #ifndef	_LINUX_KTHREAD_H_
 #define	_LINUX_KTHREAD_H_
 
-#include <sys/param.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/sleepqueue.h>
-
-#include <linux/slab.h>
 #include <linux/sched.h>
 
-static inline void
-linux_kthread_fn(void *arg)
-{
-	struct task_struct *task;
-	struct thread *td = curthread;
-
-	task = arg;
-	task_struct_fill(td, task);
-	task_struct_set(td, task);
-	if (task->should_stop == 0)
-		task->task_ret = task->task_fn(task->task_data);
-	PROC_LOCK(td->td_proc);
-	task->should_stop = TASK_STOPPED;
-	wakeup(task);
-	PROC_UNLOCK(td->td_proc);
-	task_struct_set(td, NULL);
-	kthread_exit();
-}
-
-static inline struct task_struct *
-linux_kthread_create(int (*threadfn)(void *data), void *data)
-{
-	struct task_struct *task;
-
-	task = kzalloc(sizeof(*task), GFP_KERNEL);
-	task->task_fn = threadfn;
-	task->task_data = data;
-
-	return (task);
-}
+#include <sys/unistd.h>
+#include <sys/kthread.h>
 
-#define	kthread_run(fn, data, fmt, ...)					\
-({									\
-	struct task_struct *_task;					\
+#define	kthread_run(fn, data, fmt, ...)	({				\
+	struct task_struct *__task;					\
+	struct thread *__td;						\
 									\
-	_task = linux_kthread_create((fn), (data));			\
-	if (kthread_add(linux_kthread_fn, _task, NULL, &_task->task_thread,	\
-	    0, 0, fmt, ## __VA_ARGS__)) {				\
-		kfree(_task);						\
-		_task = NULL;						\
-	}								\
-	_task;								\
+	if (kthread_add(linux_kthread_fn, NULL, NULL, &__td,		\
+	    RFSTOPPED, 0, fmt, ## __VA_ARGS__))				\
+		__task = NULL;						\
+	else								\
+		__task = linux_kthread_setup_and_run(__td, fn, data);	\
+	__task;								\
 })
 
-#define	kthread_should_stop()	current->should_stop
-
-static inline int
-kthread_stop(struct task_struct *task)
-{
-
-	PROC_LOCK(task->task_thread->td_proc);
-	task->should_stop = TASK_SHOULD_STOP;
-	wake_up_process(task);
-	while (task->should_stop != TASK_STOPPED)
-		msleep(task, &task->task_thread->td_proc->p_mtx, PWAIT,
-		    "kstop", hz);
-	PROC_UNLOCK(task->task_thread->td_proc);
-	return task->task_ret;
-}
+extern int kthread_stop(struct task_struct *);
+extern bool kthread_should_stop_task(struct task_struct *);
+extern bool kthread_should_stop(void);
+extern void linux_kthread_fn(void *);
+extern struct task_struct *linux_kthread_setup_and_run(struct thread *, linux_task_fn_t *, void *arg);
 
 #endif	/* _LINUX_KTHREAD_H_ */
diff --git a/sys/compat/linuxkpi/common/include/linux/rwlock.h b/sys/compat/linuxkpi/common/include/linux/rwlock.h
index 54c53dc94988..4c9529e843ef 100644
--- a/sys/compat/linuxkpi/common/include/linux/rwlock.h
+++ b/sys/compat/linuxkpi/common/include/linux/rwlock.h
@@ -34,6 +34,7 @@
 #include <sys/types.h>
 #include <sys/lock.h>
 #include <sys/rwlock.h>
+#include <sys/libkern.h>
 
 typedef struct {
 	struct rwlock rw;
diff --git a/sys/compat/linuxkpi/common/include/linux/rwsem.h b/sys/compat/linuxkpi/common/include/linux/rwsem.h
index 22ad4dc62a94..7ca066125a48 100644
--- a/sys/compat/linuxkpi/common/include/linux/rwsem.h
+++ b/sys/compat/linuxkpi/common/include/linux/rwsem.h
@@ -34,6 +34,7 @@
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/sx.h>
+#include <sys/libkern.h>
 
 struct rw_semaphore {
 	struct sx sx;
diff --git a/sys/compat/linuxkpi/common/include/linux/sched.h b/sys/compat/linuxkpi/common/include/linux/sched.h
index c9f2a399904e..04abc8230775 100644
--- a/sys/compat/linuxkpi/common/include/linux/sched.h
+++ b/sys/compat/linuxkpi/common/include/linux/sched.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013-2016 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,6 +37,12 @@
 #include <sys/sched.h>
 #include <sys/sleepqueue.h>
 
+#include <linux/types.h>
+#include <linux/completion.h>
+#include <linux/slab.h>
+
+#include <asm/atomic.h>
+
 #define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
 
 #define	TASK_RUNNING		0
@@ -46,41 +52,22 @@
 #define	TASK_WAKEKILL		128
 #define	TASK_WAKING		256
 
-#define	TASK_SHOULD_STOP	1
-#define	TASK_STOPPED		2
-
-/*
- * A task_struct is only provided for threads created by kthread() and
- * file operation callbacks.
- *
- * Using these routines outside the above mentioned contexts will
- * cause panics because no task_struct is assigned and td_retval[1] is
- * overwritten by syscalls.
- */
 struct task_struct {
-	struct	thread *task_thread;
-	int	(*task_fn)(void *data);
-	void	*task_data;
+	struct thread *task_thread;
+	linux_task_fn_t *task_fn;
+	void   *task_data;
 	int	task_ret;
 	int	state;
-	int	should_stop;
+	atomic_t kthread_flags;
 	pid_t	pid;
 	const char    *comm;
-	void	*bsd_ioctl_data;
-	unsigned	bsd_ioctl_len;
+	void   *bsd_ioctl_data;
+	unsigned bsd_ioctl_len;
+	struct completion parked;
+	struct completion exited;
 };
 
-#define	current			task_struct_get(curthread)
-#define	task_struct_get(x)	((struct task_struct *)(uintptr_t)(x)->td_retval[1])
-#define	task_struct_fill(x, y) do {		\
-  	(y)->task_thread = (x);			\
-	(y)->comm = (x)->td_name;		\
-	(y)->pid = (x)->td_tid;			\
-} while (0)
-#define	task_struct_set(x, y)	(x)->td_retval[1] = (uintptr_t)(y)
-
-/* ensure the task_struct pointer fits into the td_retval[1] field */
-CTASSERT(sizeof(((struct thread *)0)->td_retval[1]) >= sizeof(uintptr_t));
+#define	current		((struct task_struct *)curthread->td_lkpi_task)
 
 #define	set_current_state(x)						\
 	atomic_store_rel_int((volatile int *)&current->state, (x))
diff --git a/sys/compat/linuxkpi/common/include/linux/semaphore.h b/sys/compat/linuxkpi/common/include/linux/semaphore.h
index 022a0164840f..59a35311a5cc 100644
--- a/sys/compat/linuxkpi/common/include/linux/semaphore.h
+++ b/sys/compat/linuxkpi/common/include/linux/semaphore.h
@@ -34,6 +34,7 @@
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/sema.h>
+#include <sys/libkern.h>
 
 /*
  * XXX BSD semaphores are disused and slow.  They also do not provide a
diff --git a/sys/compat/linuxkpi/common/include/linux/spinlock.h b/sys/compat/linuxkpi/common/include/linux/spinlock.h
index 97c83e0ed034..4beb6fe45f2d 100644
--- a/sys/compat/linuxkpi/common/include/linux/spinlock.h
+++ b/sys/compat/linuxkpi/common/include/linux/spinlock.h
@@ -35,9 +35,9 @@
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/libkern.h>
 
 #include <linux/compiler.h>
-#include <linux/kernel.h>
 #include <linux/rwlock.h>
 
 typedef struct {
diff --git a/sys/compat/linuxkpi/common/include/linux/types.h b/sys/compat/linuxkpi/common/include/linux/types.h
index c9c37284a706..28abc9ec269d 100644
--- a/sys/compat/linuxkpi/common/include/linux/types.h
+++ b/sys/compat/linuxkpi/common/include/linux/types.h
@@ -63,4 +63,6 @@ typedef u64 phys_addr_t;
 #define	DECLARE_BITMAP(n, bits)						\
 	unsigned long n[howmany(bits, sizeof(long) * 8)]
 
+typedef int linux_task_fn_t(void *data);
+
 #endif	/* _LINUX_TYPES_H_ */
diff --git a/sys/compat/linuxkpi/common/include/linux/wait.h b/sys/compat/linuxkpi/common/include/linux/wait.h
index 7ae6464c6d4e..14da6d264cec 100644
--- a/sys/compat/linuxkpi/common/include/linux/wait.h
+++ b/sys/compat/linuxkpi/common/include/linux/wait.h
@@ -32,8 +32,6 @@
 #define	_LINUX_WAIT_H_
 
 #include <linux/compiler.h>
-#include <linux/spinlock.h>
-#include <linux/sched.h>
 #include <linux/list.h>
 #include <linux/jiffies.h>