netmap: don't schedule kqueue notify task when kqueue is not used

This change adds a counter (kqueue_users) to keep track of how many
kqueue users are referencing a given struct nm_selinfo.
In this way, nm_os_selwakeup() can schedule the kevent notification
task only when kqueue is actually being used.
This is important to avoid wasting CPU in the common case where
kqueue is not used.

Reviewed by:	Aleksandr Fedorov <aleksandr.fedorov@itglobal.com>
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D19177

2 files changed, 25 insertions, 7 deletions

diff --git a/sys/dev/netmap/netmap_freebsd.c b/sys/dev/netmap/netmap_freebsd.c
index d99e526f3d10..c6cd24461b6b 100644
--- a/sys/dev/netmap/netmap_freebsd.c
+++ b/sys/dev/netmap/netmap_freebsd.c
@@ -105,6 +105,7 @@ int nm_os_selinfo_init(NM_SELINFO_T *si, const char *name) {
 	snprintf(si->mtxname, sizeof(si->mtxname), "nmkl%s", name);
 	mtx_init(&si->m, si->mtxname, NULL, MTX_DEF);
 	knlist_init_mtx(&si->si.si_note, &si->m);
+	si->kqueue_users = 0;
 
 	return (0);
 }
@@ -1351,7 +1352,9 @@ void
 nm_os_selwakeup(struct nm_selinfo *si)
 {
 	selwakeuppri(&si->si, PI_NET);
-	taskqueue_enqueue(si->ntfytq, &si->ntfytask);
+	if (si->kqueue_users > 0) {
+		taskqueue_enqueue(si->ntfytq, &si->ntfytask);
+	}
 }
 
 void
@@ -1364,20 +1367,28 @@ static void
 netmap_knrdetach(struct knote *kn)
 {
 	struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook;
-	struct selinfo *si = &priv->np_si[NR_RX]->si;
+	struct nm_selinfo *si = priv->np_si[NR_RX];
 
-	nm_prinf("remove selinfo %p", si);
-	knlist_remove(&si->si_note, kn, /*islocked=*/0);
+	knlist_remove(&si->si.si_note, kn, /*islocked=*/0);
+	NMG_LOCK();
+	KASSERT(si->kqueue_users > 0, ("kqueue_user underflow on %s",
+	    si->mtxname));
+	si->kqueue_users--;
+	nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users);
+	NMG_UNLOCK();
 }
 
 static void
 netmap_knwdetach(struct knote *kn)
 {
 	struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook;
-	struct selinfo *si = &priv->np_si[NR_TX]->si;
+	struct nm_selinfo *si = priv->np_si[NR_TX];
 
-	nm_prinf("remove selinfo %p", si);
-	knlist_remove(&si->si_note, kn, /*islocked=*/0);
+	knlist_remove(&si->si.si_note, kn, /*islocked=*/0);
+	NMG_LOCK();
+	si->kqueue_users--;
+	nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users);
+	NMG_UNLOCK();
 }
 
 /*
@@ -1465,6 +1476,10 @@ netmap_kqfilter(struct cdev *dev, struct knote *kn)
 	kn->kn_fop = (ev == EVFILT_WRITE) ?
 		&netmap_wfiltops : &netmap_rfiltops;
 	kn->kn_hook = priv;
+	NMG_LOCK();
+	si->kqueue_users++;
+	nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users);
+	NMG_UNLOCK();
 	knlist_add(&si->si.si_note, kn, /*islocked=*/0);
 
 	return 0;
diff --git a/sys/dev/netmap/netmap_kern.h b/sys/dev/netmap/netmap_kern.h
index f8db7a38ac9c..ef2b1bcf0fb5 100644
--- a/sys/dev/netmap/netmap_kern.h
+++ b/sys/dev/netmap/netmap_kern.h
@@ -132,11 +132,14 @@ struct netmap_adapter *netmap_getna(if_t ifp);
 #define MBUF_QUEUED(m)		1
 
 struct nm_selinfo {
+	/* Support for select(2) and poll(2). */
 	struct selinfo si;
+	/* Support for kqueue(9). See comments in netmap_freebsd.c */
 	struct taskqueue *ntfytq;
 	struct task ntfytask;
 	struct mtx m;
 	char mtxname[32];
+	int kqueue_users;
 };