Add a little, but very useful script for use with programs that work using an

event loop and should sleep only when waiting for events (eg. via kevent(2)).
When a program is going to sleep in the kernel, the script will show its name,
PID, kernel stack trace and userland stack trace. Sleeping in kevent(2) is
ignored as it is expected to be valid.

Sample output:

# ./blocking lynxd

lynxd(15042) is blocking...

              kernel`_cv_wait_sig+0x124
              kernel`seltdwait+0xae
              kernel`sys_poll+0x3a3
              kernel`amd64_syscall+0x343
              kernel`0xffffffff806c79ab

              lynxd`poll+0xa
              lynxd`pqSocketCheck+0xa2
              lynxd`pqWaitTimed+0x29
              lynxd`connectDBComplete+0xd7
              lynxd`PQsetdbLogin+0x2ec
              lynxd`db_connect+0x3c
              lynxd`main+0x198
              lynxd`_start+0x16f
              0x2

lynxd(1925) is blocking...

              kernel`_cv_wait+0x125
              zfs.ko`zio_wait+0x5b
              zfs.ko`dmu_buf_hold_array_by_dnode+0x1dc
              zfs.ko`dmu_read+0xcb
              zfs.ko`zfs_freebsd_getpages+0x37b
              kernel`VOP_GETPAGES_APV+0xa7
              kernel`vnode_pager_getpages+0x9a
              kernel`vm_fault_hold+0x885
              kernel`vm_fault+0x77
              kernel`trap_pfault+0x211
              kernel`trap+0x506
              kernel`0xffffffff806c76c2

              lynxd`EVP_add_cipher+0x13
              lynxd`SSL_library_init+0x11
              lynxd`main+0x94
              lynxd`_start+0x16f
              0x2

lynxd(1925) is blocking...

              kernel`_cv_wait+0x125
              zfs.ko`zio_wait+0x5b
              zfs.ko`dbuf_read+0x791
              zfs.ko`dbuf_findbp+0x12f
              zfs.ko`dbuf_hold_impl+0xa2
              zfs.ko`dbuf_hold+0x1b
              zfs.ko`dmu_buf_hold_array_by_dnode+0x153
              zfs.ko`dmu_read_uio+0x66
              zfs.ko`zfs_freebsd_read+0x3a3
              kernel`VOP_READ_APV+0xa1
              kernel`vn_read+0x13a
              kernel`vn_io_fault+0x10b
              kernel`dofileread+0x95
              kernel`kern_readv+0x68
              kernel`sys_read+0x63
              kernel`amd64_syscall+0x343
              kernel`0xffffffff806c79ab

              lynxd`_read+0xa
              lynxd`__srefill+0x122
              lynxd`fgets+0x78
              lynxd`file_gets+0x1d
              lynxd`BIO_gets+0x64
              lynxd`PEM_read_bio+0xf5
              lynxd`PEM_X509_INFO_read_bio+0x90
              lynxd`X509_load_cert_crl_file+0x47
              lynxd`by_file_ctrl+0x2e
              lynxd`X509_STORE_load_locations+0x4a
              lynxd`sslctx_init+0x255
              lynxd`main+0x215
              lynxd`_start+0x16f
              0x2

Requested by:	gnn
Obtained from:	Wheel Systems http://wheelsystems.com

2 files changed, 59 insertions, 1 deletions

diff --git a/share/dtrace/Makefile b/share/dtrace/Makefile
index 11fd1afb3faf..c0c3f5ec0be8 100644
--- a/share/dtrace/Makefile
+++ b/share/dtrace/Makefile
@@ -12,7 +12,8 @@ SUBDIR= ${_toolkit}
 _toolkit=	toolkit
 .endif
 
-SCRIPTS=	disklatency \
+SCRIPTS=	blocking \
+		disklatency \
 		disklatencycmd \
 		hotopen \
 		nfsattrstats \
diff --git a/share/dtrace/blocking b/share/dtrace/blocking
new file mode 100755
index 000000000000..006500877087
--- /dev/null
+++ b/share/dtrace/blocking
@@ -0,0 +1,57 @@
+#!/usr/sbin/dtrace -s
+/*-
+ * Copyright (c) 2015 Pawel Jakub Dawidek <pawel@dawidek.net>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ * This little script is for use with programs that use event loop and should
+ * sleep only when waiting for events (eg. via kevent(2)). When a program is
+ * going to sleep in the kernel, the script will show its name, PID, kernel
+ * stack trace and userland stack trace. Sleeping in kevent(2) is ignored.
+ *
+ * usage: blocking <execname>
+ */
+
+#pragma D option quiet
+
+syscall::kevent:entry
+/execname == $$1/
+{
+	self->inkevent = 1;
+}
+
+fbt::sleepq_add:entry
+/!self->inkevent && execname == $$1/
+{
+	printf("\n%s(%d) is blocking...\n", execname, pid);
+	stack();
+	ustack();
+}
+
+syscall::kevent:return
+/execname == $$1/
+{
+	self->inkevent = 0;
+}