Add locking to the kqueue subsystem. This also makes the kqueue subsystem

a more complete subsystem, and removes the knowlege of how things are
implemented from the drivers.  Include locking around filter ops, so a
module like aio will know when not to be unloaded if there are outstanding
knotes using it's filter ops.

Currently, it uses the MTX_DUPOK even though it is not always safe to
aquire duplicate locks.  Witness currently doesn't support the ability
to discover if a dup lock is ok (in some cases).

Reviewed by:	green, rwatson (both earlier versions)

6 files changed, 73 insertions, 24 deletions

diff --git a/sys/sys/event.h b/sys/sys/event.h
index 4c7c0fe9c02d..1a7956c4e8e3 100644
--- a/sys/sys/event.h
+++ b/sys/sys/event.h
@@ -118,9 +118,16 @@ struct kevent {
  * This is currently visible to userland to work around broken
  * programs which pull in <sys/proc.h>.
  */
-#include <sys/queue.h>
+#include <sys/queue.h> 
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
 struct knote;
 SLIST_HEAD(klist, knote);
+struct knlist {
+	struct	mtx	*kl_lock;	/* lock to protect kll_list */
+	struct	klist	kl_list;
+};
+
 
 #ifdef _KERNEL
 
@@ -128,8 +135,14 @@ SLIST_HEAD(klist, knote);
 MALLOC_DECLARE(M_KQUEUE);
 #endif
 
-#define KNOTE(list, hint)	\
-	do { if ((list) != NULL) knote(list, hint); } while (0)
+struct kqueue;
+SLIST_HEAD(kqlist, kqueue);
+
+#define KNOTE(list, hist, lock)		knote(list, hist, lock)
+#define KNOTE_LOCKED(list, hint)	knote(list, hint, 1)
+#define KNOTE_UNLOCKED(list, hint)	knote(list, hint, 0)
+#define KNOTE_STATUS_BEGIN(kn)		knote_status(kn, 1)
+#define KNOTE_STATUS_END(kn)		knote_status(kn, 0)
 
 /*
  * Flag indicating hint is a signal.  Used by EVFILT_SIGNAL, and also
@@ -144,13 +157,28 @@ struct filterops {
 	int	(*f_event)(struct knote *kn, long hint);
 };
 
+/*
+ * Setting the KN_INFLUX flag enables you to unlock the kq that this knote
+ * is on, and modify kn_status as if you had the KQ lock.
+ *
+ * kn_sfflags, kn_sdata, and kn_kevent are protected by the knlist lock.
+ */
 struct knote {
-	SLIST_ENTRY(knote)	kn_link;	/* for fd */
+	SLIST_ENTRY(knote)	kn_link;	/* for kq */
 	SLIST_ENTRY(knote)	kn_selnext;	/* for struct selinfo */
+	struct			knlist *kn_knlist;	/* f_attach populated */
 	TAILQ_ENTRY(knote)	kn_tqe;
 	struct			kqueue *kn_kq;	/* which queue we are on */
 	struct 			kevent kn_kevent;
-	int			kn_status;
+	int			kn_status;	/* protected by kq lock */
+#define KN_ACTIVE	0x01			/* event has been triggered */
+#define KN_QUEUED	0x02			/* event is on queue */
+#define KN_DISABLED	0x04			/* event is disabled */
+#define KN_DETACHED	0x08			/* knote is detached */
+#define KN_INFLUX	0x10			/* knote is in flux */
+#define KN_MARKER	0x20			/* ignore this knote */
+#define KN_KQUEUE	0x40			/* this knote belongs to a kq */
+#define KN_HASKQLOCK	0x80			/* for _inevent */
 	int			kn_sfflags;	/* saved filter flags */
 	intptr_t		kn_sdata;	/* saved data field */
 	union {
@@ -159,10 +187,6 @@ struct knote {
 	} kn_ptr;
 	struct			filterops *kn_fop;
 	void			*kn_hook;
-#define KN_ACTIVE	0x01			/* event has been triggered */
-#define KN_QUEUED	0x02			/* event is on queue */
-#define KN_DISABLED	0x04			/* event is disabled */
-#define KN_DETACHED	0x08			/* knote is detached */
 
 #define kn_id		kn_kevent.ident
 #define kn_filter	kn_kevent.filter
@@ -174,12 +198,20 @@ struct knote {
 
 struct thread;
 struct proc;
-
-extern void	knote(struct klist *list, long hint);
-extern void	knote_remove(struct thread *p, struct klist *list);
+struct knlist;
+
+extern void	knote(struct knlist *list, long hint, int islocked);
+extern void	knote_status(struct knote *kn, int begin);
+extern void	knlist_add(struct knlist *knl, struct knote *kn, int islocked);
+extern void	knlist_remove(struct knlist *knl, struct knote *kn, int islocked);
+extern void	knlist_remove_inevent(struct knlist *knl, struct knote *kn);
+extern int	knlist_empty(struct knlist *knl);
+extern void	knlist_init(struct knlist *knl, struct mtx *mtx);
+extern void	knlist_destroy(struct knlist *knl);
+extern void	knlist_clear(struct knlist *knl, int islocked);
 extern void	knote_fdclose(struct thread *p, int fd);
 extern int 	kqueue_register(struct kqueue *kq,
-		    struct kevent *kev, struct thread *p);
+		    struct kevent *kev, struct thread *p, int waitok);
 extern int	kqueue_add_filteropts(int filt, struct filterops *filtops);
 extern int	kqueue_del_filteropts(int filt);
 
diff --git a/sys/sys/eventvar.h b/sys/sys/eventvar.h
index ef9087b61614..bdb3adf57271 100644
--- a/sys/sys/eventvar.h
+++ b/sys/sys/eventvar.h
@@ -29,11 +29,20 @@
 #ifndef _SYS_EVENTVAR_H_
 #define _SYS_EVENTVAR_H_
 
+#ifndef _KERNEL
+#error "no user-servicable parts inside"
+#endif
+
+#include <sys/_task.h>
+
 #define KQ_NEVENTS	8		/* minimize copy{in,out} calls */
 #define KQEXTENT	256		/* linear growth by this amount */
 
 struct kqueue {
-	TAILQ_HEAD(kqlist, knote) kq_head;	/* list of pending event */
+	struct		mtx kq_lock;
+	int		kq_refcnt;
+	SLIST_ENTRY(kqueue)	kq_list;
+	TAILQ_HEAD(, knote)	kq_head;	/* list of pending event */
 	int		kq_count;		/* number of pending events */
 	struct		selinfo kq_sel;
 	struct		sigio *kq_sigio;
@@ -41,8 +50,16 @@ struct kqueue {
 	int		kq_state;
 #define KQ_SEL		0x01
 #define KQ_SLEEP	0x02
-#define KQ_ASYNC	0x04
-	struct		kevent kq_kev[KQ_NEVENTS];
+#define KQ_FLUXWAIT	0x04			/* waiting for a in flux kn */
+#define KQ_ASYNC	0x08
+#define KQ_CLOSING	0x10
+#define	KQ_TASKSCHED	0x20			/* task scheduled */
+#define	KQ_TASKDRAIN	0x40			/* waiting for task to drain */
+	int		kq_knlistsize;		/* size of knlist */
+	struct		klist *kq_knlist;	/* list of knotes */
+	u_long		kq_knhashmask;		/* size of knhash */
+	struct		klist *kq_knhash;	/* hash table for knotes */
+	struct		task kq_task;
 };
 
 #endif /* !_SYS_EVENTVAR_H_ */
diff --git a/sys/sys/filedesc.h b/sys/sys/filedesc.h
index b813c32976bd..7b7c6b996137 100644
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@@ -34,6 +34,7 @@
 #define	_SYS_FILEDESC_H_
 
 #include <sys/queue.h>
+#include <sys/event.h>
 #include <sys/_lock.h>
 #include <sys/_mutex.h>
 
@@ -71,11 +72,8 @@ struct filedesc {
 	u_short	fd_cmask;		/* mask for file creation */
 	u_short	fd_refcnt;		/* reference count */
 
-	int	fd_knlistsize;		/* size of knlist */
-	struct	klist *fd_knlist;	/* list of attached knotes */
-	u_long	fd_knhashmask;		/* size of knhash */
-	struct	klist *fd_knhash;	/* hash table for attached knotes */
 	struct	mtx fd_mtx;		/* protects members of this struct */
+	struct	kqlist fd_kqlist;	/* list of kqueues on this filedesc */
 	int	fd_holdleaderscount;	/* block fdfree() for shared close() */
 	int	fd_holdleaderswakeup;	/* fdfree() needs wakeup */
 };
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 1b1f1f46b090..f5bd3c695c4a 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -608,7 +608,6 @@ struct proc {
 	struct vnode	*p_textvp;	/* (b) Vnode of executable. */
 	sigset_t	p_siglist;	/* (c) Sigs not delivered to a td. */
 	char		p_lock;		/* (c) Proclock (prevent swap) count. */
-	struct klist	p_klist;	/* (c) Knotes attached to this proc. */
 	struct sigiolst	p_sigiolst;	/* (c) List of sigio sources. */
 	int		p_sigparent;	/* (c) Signal to parent on exit. */
 	int		p_sig;		/* (n) For core dump/debugger XXX. */
@@ -638,6 +637,7 @@ struct proc {
 #define	p_endcopy	p_xstat
 
 	u_short		p_xstat;	/* (c) Exit status; also stop sig. */
+	struct knlist	p_klist;	/* (c) Knotes attached to this proc. */
 	int		p_numthreads;	/* (j) Number of threads. */
 	int		p_numksegrps;	/* (c) number of ksegrps */
 	struct mdproc	p_md;		/* Any machine-dependent fields. */
diff --git a/sys/sys/selinfo.h b/sys/sys/selinfo.h
index eb9d1ef6a3f6..946da8c9de44 100644
--- a/sys/sys/selinfo.h
+++ b/sys/sys/selinfo.h
@@ -42,7 +42,7 @@
 struct selinfo {
 	TAILQ_ENTRY(selinfo)	si_thrlist;	/* list hung off of thread */
 	struct	thread *si_thread;	/* thread waiting */
-	struct	klist si_note;	/* kernel note list */
+	struct	knlist si_note;	/* kernel note list */
 	short	si_flags;	/* see below */
 };
 #define	SI_COLL	0x0001		/* collision occurred */
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index df5ae0016fb9..166cd53f4c57 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -201,11 +201,13 @@ struct xvnode {
 			vn_pollevent((vp), (events));		\
 	} while (0)
 
-#define VN_KNOTE(vp, b)						\
+#define VN_KNOTE(vp, b, a)						\
 	do {							\
 		if ((vp)->v_pollinfo != NULL)			\
-			KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b)); \
+			KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b), (a)); \
 	} while (0)
+#define VN_KNOTE_LOCKED(vp, b)		VN_KNOTE(vp, b, 1)
+#define VN_KNOTE_UNLOCKED(vp, b)	VN_KNOTE(vp, b, 0)
 
 /*
  * Vnode flags.