Introduce dwatch(1) as a tool for making DTrace more useful

Reviewed by:	markj, gnn, bdrewery (earlier version)
Relnotes:	yes
Sponsored by:	Smule, Inc.
Differential Revision:	https://reviews.freebsd.org/D10006

23 files changed, 4241 insertions, 0 deletions

diff --git a/cddl/usr.sbin/Makefile b/cddl/usr.sbin/Makefile
index a7ee242662a1..a736a75a73ec 100644
--- a/cddl/usr.sbin/Makefile
+++ b/cddl/usr.sbin/Makefile
@@ -3,6 +3,7 @@
 .include <src.opts.mk>
 
 SUBDIR=	${_dtrace} \
+	${_dwatch} \
 	${_lockstat} \
 	${_plockstat} \
 	${_zdb} \
@@ -23,6 +24,7 @@ _zfsd=	zfsd
 
 .if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386"
 _dtrace=	dtrace
+_dwatch=	dwatch
 _lockstat=	lockstat
 _plockstat=	plockstat
 .endif
@@ -30,15 +32,18 @@ _plockstat=	plockstat
 .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "arm" || \
     ${MACHINE_CPUARCH} == "riscv"
 _dtrace=	dtrace
+_dwatch=	dwatch
 _lockstat=	lockstat
 .endif
 
 .if ${MACHINE_CPUARCH} == "mips"
 _dtrace=	dtrace
+_dwatch=	dwatch
 .endif
 
 .if ${MACHINE_CPUARCH} == "powerpc"
 _dtrace=	dtrace
+_dwatch=	dwatch
 _lockstat=	lockstat
 .endif
 
diff --git a/cddl/usr.sbin/dwatch/Makefile b/cddl/usr.sbin/dwatch/Makefile
new file mode 100644
index 000000000000..647f07388890
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/Makefile
@@ -0,0 +1,15 @@
+# $FreeBSD$
+
+.include <src.opts.mk>
+
+SUBDIR= libexec
+
+.if ${MK_EXAMPLES} != "no"
+SUBDIR+= examples
+.endif
+
+SCRIPTS= dwatch
+
+MAN= dwatch.1
+
+.include <bsd.prog.mk>
diff --git a/cddl/usr.sbin/dwatch/dwatch b/cddl/usr.sbin/dwatch/dwatch
new file mode 100755
index 000000000000..a54d4d6bdaec
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/dwatch
@@ -0,0 +1,1291 @@
+#!/bin/sh
+#-
+# Copyright (c) 2014-2018 Devin Teske
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+############################################################ IDENT(1)
+#
+# $Title: Watch processes as they trigger a particular DTrace probe $
+# $FreeBSD$
+#
+############################################################ CONFIGURATION
+
+#
+# DTrace pragma settings
+#
+DTRACE_PRAGMA="
+	option quiet
+	option dynvarsize=16m
+	option switchrate=10hz
+" # END-QUOTE
+
+#
+# Profiles
+#
+: ${DWATCH_PROFILES_PATH="/usr/libexec/dwatch:/usr/local/libexec/dwatch"}
+
+############################################################ GLOBALS
+
+VERSION='$Version: 1.0-beta-91 $' # -V
+
+pgm="${0##*/}" # Program basename
+
+#
+# Command-line arguments
+#
+PROBE_ARG=
+
+#
+# Command-line options
+#
+CONSOLE=		# -y
+CONSOLE_FORCE=		# -y
+[ -t 1 ] && CONSOLE=1	# -y
+COUNT=0			# -N count
+CUSTOM_DETAILS=		# -E code
+CUSTOM_TEST=		# -t test
+DEBUG=			# -d
+DESTRUCTIVE_ACTIONS=	# -w
+EXECNAME=		# -k name
+EXECREGEX=		# -z regex
+EXIT_AFTER_COMPILE=	# -e
+FILTER=			# -r regex
+PROBE_COALESCE=		# -F
+GROUP=			# -g group
+JID=			# -j jail
+LIST=			# -l
+LIST_PROFILES=		# -Q
+MAX_ARGS=64		# -B num
+MAX_DEPTH=64		# -K num
+ONELINE=		# -1
+OUTPUT=			# -o file
+OUTPUT_CMD=		# -O cmd
+PID=			# -p pid
+PROBE_TYPE=		# -f -m -n -P
+PROFILE=		# -X profile
+PSTREE=			# -R
+QUIET=			# -q
+TIMEOUT=		# -T time
+TRACE=			# -x
+USER=			# -u user
+USE_PROFILE=		# -X profile
+VERBOSE=		# -v
+
+#
+# Global exit status
+#
+SUCCESS=0
+FAILURE=1
+
+#
+# Miscellaneous
+#
+ACTIONS=
+EVENT_DETAILS=
+EVENT_TAG='printf("%d.%d %s[%d]: ",
+		this->uid0, this->gid0, execname, this->pid0);'
+EVENT_TEST=
+FILE=
+ID=3
+MODULE_CHECKED=
+PROBE=
+PSARGS=1
+RGID=
+RUID=
+SUDO=
+export SUDO_PROMPT="[sudo] Password:"
+TITLE=\$Title:
+
+############################################################ FUNCTIONS
+
+ansi() { local fmt="$2 $4"; [ "$CONSOLE" ] && fmt="\\033[$1m$2\\033[$3m $4";
+	shift 4; printf "$fmt\n" "$@"; }
+die() { exec >&2; [ "$*" ] && echo "$pgm:" "$@"; exit $FAILURE; }
+info() { [ "$QUIET" ] || ansi 35 "INFO" 39 "$@" >&2; }
+
+usage()
+{
+	local optfmt="\t%-10s %s\n"
+	exec >&2
+	[ "$*" ] && printf "%s: %s\n" "$pgm" "$*"
+	printf "Usage: %s [-1defFmnPqRvVwxy] [%s] [%s] [%s] [%s]\n" "$pgm" \
+		"-B num" "-E code" "-g group" "-j jail"
+	printf "\t      [%s] [%s] [%s] [%s] [%s] [%s]\n" \
+		"-k name" "-K num" "-N count" "-o file" "-O cmd" "-p pid"
+	printf "\t      [%s] [%s] [%s] [%s] [%s] [%s]\n" \
+		"-r regex" "-t test" "-T time" "-u user" "-X profile" \
+		"-z regex"
+	printf "\t      probe[,...] [args ...]\n"
+	printf "       %s -l [-fmnPqy] [-r regex] [probe ...]\n" "$pgm"
+	printf "       %s -Q [-1qy] [-r regex]\n" "$pgm"
+	printf "\n"
+	printf "$optfmt" "-1" \
+		"Print one line per process/profile (Default; disables \`-R')."
+	printf "$optfmt" "-B num" \
+		"Maximum process arguments to display (Default $MAX_ARGS)."
+	printf "$optfmt" "-d" \
+		"Debug. Send dtrace(1) script to stdout instead of executing."
+	printf "$optfmt" "-e" \
+		"Exit after compiling request but prior to enabling probes."
+	printf "$optfmt" "-E code" \
+		"DTrace code for event details. If \`-', read from stdin."
+	printf "$optfmt" "-f" \
+		"Enable probe matching the specified function name."
+	printf "$optfmt" "-F" \
+		"Coalesce trace output by function."
+	printf "$optfmt" "-g group" \
+		"Group filter. Only show processes matching group name/gid."
+	printf "$optfmt" "-j jail" \
+		"Jail filter. Only show processes matching jail name/jid."
+	printf "$optfmt" "-k name" \
+		"Only show processes matching name."
+	printf "$optfmt" "-K num" \
+		"Maximum directory depth to display (Default $MAX_DEPTH)."
+	printf "$optfmt" "-l" \
+		"List available probes on standard output and exit."
+	printf "$optfmt" "-m" \
+		"Enable probe matching the specified module name."
+	printf "$optfmt" "-n" \
+		"Enable probe matching the specified probe name."
+	printf "$optfmt" "-N count" \
+		"Exit after count matching entries (Default 0 for disabled)."
+	printf "$optfmt" "-o file" \
+		"Set output file. If \`-', the path \`/dev/stdout' is used."
+	printf "$optfmt" "-O cmd" \
+		"Execute cmd for each event."
+	printf "$optfmt" "-p pid" \
+		"Process id filter. Only show processes with matching pid."
+	printf "$optfmt" "-P" \
+		"Enable probe matching the specified provider name."
+	printf "$optfmt" "-q" \
+		"Quiet. Hide informational messages and all dtrace(1) errors."
+	printf "$optfmt" "-Q" \
+		"List available profiles in DWATCH_PROFILES_PATH and exit."
+	printf "$optfmt" "-r regex" \
+		"Filter. Only show blocks matching awk(1) regular expression."
+	printf "$optfmt" "-R" \
+		"Show parent, grandparent, and ancestor of process."
+	printf "$optfmt" "-t test" \
+		"Test clause (predicate) to limit events (Default none)."
+	printf "$optfmt" "-T time" \
+		"Timeout. Format is \`\#[smhd]' or simply \`\#' for seconds."
+	printf "$optfmt" "-u user" \
+		"User filter. Only show processes matching user name/uid."
+	printf "$optfmt" "-v" \
+		"Verbose. Show all errors from dtrace(1)."
+	printf "$optfmt" "-V" \
+		"Report dwatch version on standard output and exit."
+	printf "$optfmt" "-w" \
+		"Permit destructive actions (copyout*, stop, panic, etc.)."
+	printf "$optfmt" "-x" \
+		"Trace. Print \`<probe-id>' when a probe is triggered."
+	printf "$optfmt" "-X profile" \
+		"Load profile name from DWATCH_PROFILES_PATH."
+	printf "$optfmt" "-y" \
+		"Always treat stdout as console (enable colors/columns/etc.)."
+	printf "$optfmt" "-z regex" \
+		"Only show processes matching awk(1) regular expression."
+	die
+}
+
+dtrace_cmd()
+{
+	local status stdout
+	local timeout=
+
+	if [ "$1" = "-t" ]; then
+		shift
+		[ "$TIMEOUT" ] && timeout=1
+	fi
+
+	exec 3>&1
+	stdout=3
+
+	#
+	# Filter dtrace(1) stderr while preserving exit status
+	#
+	status=$(
+		exec 4>&1
+		to_status=4
+		( trap 'echo $? >&$to_status' EXIT
+			eval $SUDO ${timeout:+timeout \"\$TIMEOUT\"} dtrace \
+				\"\$@\" 2>&1 ${QUIET:+2> /dev/null} >&$stdout
+		) | dtrace_stderr_filter >&2
+	)
+
+	return $status
+}
+
+dtrace_stderr_filter()
+{
+	if [ "$VERBOSE" ]; then
+		cat
+		return
+		# NOTREACHED
+	fi
+
+	awk ' # Start awk(1) stderr-filter
+	/[[:digit:]]+ drops? on CPU [[:digit:]]+/ { next }
+	/failed to write to <stdout>: No such file or directory/ { next }
+	/failed to write to <stdout>: Broken pipe/ { next }
+	/processing aborted: Broken pipe/ { next }
+	/invalid address \(0x[[:xdigit:]]+\) in action #[[:digit:]]+/ { next }
+	/out of scratch space in action #[[:digit:]]+/ { next }
+	/^Bus error$/ { next }
+	{ print; fflush() }
+	' # END-QUOTE
+}
+
+expand_probe()
+{
+	local OPTIND=1 OPTARG flag
+	local type=
+
+	while getopts t: flag; do
+		case "$flag" in
+		t) type="$OPTARG" ;;
+		esac
+	done
+	shift $(( $OPTIND - 1 ))
+
+	local probe="$1"
+	case "$probe" in
+	*:*)
+		echo "$probe"
+		return $SUCCESS
+		;;
+	esac
+
+	dtrace_cmd -l | awk -v probe="$probe" -v type="$type" '
+	# Start awk(1) processor
+	#################################################### BEGIN
+	BEGIN { getline dtrace_header }
+	#################################################### FUNCTIONS
+	function dump(unused1,unused2) {
+		if (n) {
+			if (NcF[n] == 1) f = N2F[n]
+			if (NcM[n] == 1) m = N2M[n]
+			if (NcP[n] == 1) p = N2P[n]
+		} else if (f) {
+			if (FcM[f] == 1) m = F2M[f]
+			if (FcP[f] == 1) p = F2P[f]
+			if (FcN[f] == 0 && found) n = "entry"
+		} else if (m) {
+			if (McP[m] == 1) p = M2P[m]
+		}
+		printf "%s:%s:%s:%s\n", p, m, f, n
+		exit !found
+	}
+	function inFMP() { return probe in F || probe in M || probe in P }
+	function inNMP() { return probe in N || probe in M || probe in P }
+	function inNFP() { return probe in N || probe in F || probe in P }
+	function inNFM() { return probe in N || probe in F || probe in M }
+	function diva(value, peerA, peerB, peerC) {
+		return value >= peerA && value >= peerB && value >= peerC
+	}
+	#################################################### MAIN
+	type == "name" && $NF != probe { next }
+	type == "function" && NF >=4 && $(NF-1) != probe { next }
+	type == "module" && NF == 5 && $(NF-2) != probe { next }
+	type == "provider" && $2 != probe { next }
+	type || $2 == probe || $3 == probe || $4 == probe || $5 == probe {
+		P[_p = $2]++
+		M[_m = (NF >= 5 ? $(NF-2) : "")]++
+		F[_f = (NF >= 4 ? $(NF-1) : "")]++
+		N[_n = $NF]++
+		if (N2F[_n] != _f) NcF[_n]++; N2F[_n] = _f
+		if (N2M[_n] != _m) NcM[_n]++; N2M[_n] = _m
+		if (N2P[_n] != _p) NcP[_n]++; N2P[_n] = _p
+		if (_n !~ /entry|return/) {
+			if (F2N[_f] != _n) FcN[_f]++
+			F2N[_f] = _n
+		}
+		if (F2M[_f] != _m) FcM[_f]++; F2M[_f] = _m
+		if (F2P[_f] != _p) FcP[_f]++; F2P[_f] = _p
+		if (M2P[_m] != _p) McP[_m]++; M2P[_m] = _p
+	}
+	#################################################### END
+	END {
+		if (type == "name")     dump(n = probe, found = probe in N)
+		if (type == "function") dump(f = probe, found = probe in F)
+		if (type == "module")   dump(m = probe, found = probe in M)
+		if (type == "provider") dump(p = probe, found = probe in P)
+		if (probe in N) {
+			found = 1
+			if (!inFMP()) dump(n = probe)
+			if (diva(F[probe], N[probe], M[probe], P[probe]))
+				dump(f = probe)
+			if (diva(M[probe], N[probe], F[probe], P[probe]))
+				dump(m = probe)
+			if (diva(P[probe], N[probe], F[probe], M[probe]))
+				dump(p = probe)
+			dump(n = probe) # N is the diva
+		} else if (probe in F) {
+			found = 1
+			if (!inNMP()) dump(f = probe)
+			if (diva(N[probe], F[probe], M[probe], P[probe]))
+				dump(n = probe)
+			if (diva(M[probe], F[probe], N[probe], P[probe]))
+				dump(m = probe)
+			if (diva(P[probe], F[probe], N[probe], M[probe]))
+				dump(p = probe)
+			dump(f = probe) # F is the diva
+		} else if (probe in M) {
+			found = 1
+			if (!inNFP()) dump(m = probe)
+			if (diva(N[probe], M[probe], F[probe], P[probe]))
+				dump(n = probe)
+			if (diva(F[probe], M[probe], N[probe], P[probe]))
+				dump(f = probe)
+			if (diva(P[probe], M[probe], N[probe], F[probe]))
+				dump(p = probe)
+			dump(m = probe) # M is the diva
+		} else if (probe in P) {
+			found = 1
+			if (!inNFM()) dump(p = probe)
+			if (diva(N[probe], P[probe], F[probe], M[probe]))
+				dump(n = probe)
+			if (diva(F[probe], P[probe], N[probe], M[probe]))
+				dump(f = probe)
+			if (diva(M[probe], P[probe], N[probe], F[probe]))
+				dump(m = probe)
+			dump(p = probe) # P is the diva
+		}
+		if (!found) print probe
+		exit !found
+	}
+	' # END-QUOTE
+}
+
+list_probes()
+{
+	local OPTIND=1 OPTARG flag
+	local column=0 header="PROVIDER:MODULE:FUNCTION:NAME"
+	local filter= quiet= type=
+
+	while getopts f:qt: flag; do
+		case "$flag" in
+		f) filter="$OPTARG" ;;
+		q) quiet=1 ;;
+		t) type="$OPTARG" ;;
+		esac
+	done
+	shift $(( $OPTIND - 1 ))
+
+	if [ $# -eq 0 ]; then
+		case "$type" in
+		provider) column=1 header="PROVIDER" ;;
+		module)   column=2 header="MODULE" ;;
+		function) column=3 header="FUNCTION" ;;
+		name)     column=4 header="NAME" ;;
+		esac
+	fi
+
+	[ "$quiet" ] || echo "$header"
+
+	local arg probe=
+	for arg in "$@"; do
+		arg=$( expand_probe -t "$type" -- "$arg" )
+		probe="$probe${probe:+, }$arg"
+	done
+
+	dtrace_cmd -l${probe:+n "$probe"} | awk -v pattern="$(
+		# Prevent backslashes from being lost
+		echo "$filter" | awk 'gsub(/\\/,"&&")||1'
+	)" -v want="$column" -v console="$CONSOLE" '
+		BEGIN { getline dtrace_header }
+		function ans(seq) { return console ? "\033[" seq "m" : "" }
+		NF > 3 && $(NF-1) ~ /^#/ { next }
+		!_[$0 = column[0] = sprintf("%s:%s:%s:%s",
+			column[1] = $2,
+			column[2] = (NF >= 5 ? $(NF-2) : ""),
+			column[3] = (NF >= 4 ? $(NF-1) : ""),
+			column[4] = $NF)]++ &&
+			!__[$0 = column[want]]++ &&
+			gsub(pattern, ans("31;1") "&" ans("39;22")) {
+				print | "sort"
+			}
+		END { close("sort") }
+	' # END-QUOTE
+
+	exit $SUCCESS
+}
+
+list_profiles()
+{
+	local OPTIND=1 OPTARG flag
+	local filter= oneline= quiet=
+
+	while getopts 1f:q flag; do
+		case "$flag" in
+		1) oneline=1 ;;
+		f) filter="$OPTARG" ;;
+		q) quiet=1 ;;
+		esac
+	done
+	shift $(( $OPTIND - 1 ))
+
+	# Prevent backslashes from being lost
+	filter=$( echo "$filter" | awk 'gsub(/\\/,"&&")||1' )
+
+	# Build a list of profiles available
+	local profiles
+	profiles=$( { IFS=:
+		for dir in $DWATCH_PROFILES_PATH; do
+			[ -d "$dir" ] || continue
+			for path in $dir/*; do
+				[ -f "$path" ] || continue
+				name="${path##*/}"
+				[ "$name" = "${name%%[!0-9A-Za-z_-]*}" ] ||
+					continue
+				echo $name
+			done
+		done
+	} | sort -u )
+
+	# Get the longest profile name
+	local longest_profile_name
+	longest_profile_name=$( echo "$profiles" |
+		awk -v N=0 '(L = length($0)) > N { N = L } END { print N }' )
+
+	# Get the width of the terminal
+	local max_size="$( stty size 2> /dev/null )"
+	: ${max_size:=24 80}
+	local max_width="${max_size#*[$IFS]}"
+
+	# Determine how many columns we can display
+	local x=$longest_profile_name ncols=1
+	[ "$QUIET" ] || x=$(( $x + 8 )) # Accommodate leading tab character
+	x=$(( $x + 3 + $longest_profile_name )) # Preload end of next column
+	while [ $x -lt $max_width ]; do
+		ncols=$(( $ncols + 1 ))
+		x=$(( $x + 3 + $longest_profile_name ))
+	done
+
+	# Output single lines if sent to a pipe
+	if [ "$oneline" ]; then
+		echo "$profiles" | awk -v filter="$filter" -v cons="$CONSOLE" '
+			function ans(s) { return cons ? "\033[" s "m" : "" }
+			gsub(filter, ans("31;1") "&" ans("39;22"))
+		' # END-QUOTE
+		return $SUCCESS
+		# NOTREACHED
+	fi
+
+	[ "$quiet" ] || echo PROFILES:
+	echo "$profiles" | awk \
+		-v colsize=$longest_profile_name \
+		-v console="$CONSOLE" \
+		-v ncols=$ncols \
+		-v quiet="$quiet" \
+		-v filter="$filter" \
+	' # Begin awk(1) processor
+		function ans(seq) { return console ? "\033[" seq "m" : "" }
+		BEGIN {
+			row_item[1] = ""
+			replace = ans("31;1") "&" ans("39;22")
+			ansi_offset = length(replace) - 1
+		}
+		function print_row()
+		{
+			cs = colsize + ansi_offset * \
+				gsub(filter, replace, row_item[1])
+			printf "%s%-*s", quiet ? "" : "\t", cs, row_item[1]
+			for (i = 2; i <= cur_col; i++) {
+				cs = colsize + ansi_offset * \
+					gsub(filter, replace, row_item[i])
+				printf "   %-*s", cs, row_item[i]
+			}
+			printf "\n"
+		}
+		$0 ~ filter {
+			n++
+			cur_col = ((n - 1) % ncols) + 1
+			row_item[cur_col] = $0
+			if (cur_col == ncols) print_row()
+		}
+		END { if (cur_col < ncols) print_row() }
+	' # END-QUOTE
+
+	exit $SUCCESS
+}
+
+load_profile()
+{
+	local profile="$1"
+
+	[ "$profile" ] ||
+		die "missing profile argument (\`$pgm -Q' to list profiles)"
+
+	local oldIFS="$IFS"
+	local dir found=
+
+	IFS=:
+	for dir in $DWATCH_PROFILES_PATH; do
+		[ -d "$dir" ] || continue
+		[ -f "$dir/$profile" ] || continue
+		PROFILE="$profile" found=1
+		info "Sourcing $profile profile [found in %s]" "$dir"
+		. "$dir/$profile"
+		break
+	done
+	IFS="$oldIFS"
+
+	[ "$found" ] ||
+		die "no module named \`$profile' (\`$pgm -Q' to list profiles)"
+}
+
+pproc()
+{
+	local OPTIND=1 OPTARG flag
+	local P= N=0
+
+	while getopts P: flag; do
+		case "$flag" in
+		P) P="$OPTARG" ;;
+		esac
+	done
+	shift $(( OPTIND - 1 ))
+
+	local proc=$1
+	if [ ! "$proc" ]; then
+		if [ "$P" = "0" ]; then
+			proc="curthread->td_proc"
+		else
+			proc="this->proc ? this->proc->p_pptr : NULL"
+		fi
+	fi
+
+	awk 'NR > 1 && $0 { $0 = "\t" $0 }
+		gsub(/\\\t/, "\t") || 1
+	' <<-EOFPREAMBLE
+	this->proc = $proc;
+	this->uid$P = this->proc ? this->proc->p_ucred->cr_uid : -1;
+	this->gid$P = this->proc ? this->proc->p_ucred->cr_rgid : -1;
+	this->pid$P = this->proc ? this->proc->p_pid : -1;
+	this->jid$P = this->proc ? this->proc->p_ucred->cr_prison->pr_id : -1;
+
+	this->p_args = this->proc ? this->proc->p_args : 0;
+	this->ar_length = this->p_args ? this->p_args->ar_length : 0;
+	this->ar_args = (char *)(this->p_args ? this->p_args->ar_args : 0);
+
+	this->args$P = this->arg${P}_$N = this->ar_length > 0 ?
+	\	this->ar_args : stringof(this->proc->p_comm);
+	this->len = this->ar_length > 0 ? strlen(this->ar_args) + 1 : 0;
+	this->ar_args += this->len;
+	this->ar_length -= this->len;
+
+	EOFPREAMBLE
+
+	awk -v P=$P -v MAX_ARGS=$MAX_ARGS '
+		$0 { $0 = "\t" $0 }
+		buf = buf $0 "\n" { }
+		END {
+			while (++N <= MAX_ARGS) {
+				$0 = buf
+				gsub(/P/, P)
+				gsub(/N/, N)
+				gsub(/\\\t/, "\t")
+				sub(/\n$/, "")
+				print
+			}
+		}
+	' <<-EOFARGS
+	this->argP_N = this->ar_length > 0 ? this->ar_args : "";
+	this->argsP = strjoin(this->argsP,
+	\	strjoin(this->argP_N != "" ? " " : "", this->argP_N));
+	this->len = this->ar_length > 0 ? strlen(this->ar_args) + 1 : 0;
+	this->ar_args += this->len;
+	this->ar_length -= this->len;
+
+	EOFARGS
+
+	N=$(( $MAX_ARGS + 1 ))
+	awk 'sub(/^\\\t/, "\t") || 1, $0 = "\t" $0' <<-EOFPROC
+	this->arg${P}_$N = this->ar_length > 0 ? "..." : "";
+	this->args$P = strjoin(this->args$P,
+	\	strjoin(this->arg${P}_$N != "" ? " " : "", this->arg${P}_$N));
+	EOFPROC
+}
+
+pproc_dump()
+{
+	local OPTIND=1 OPTARG flag
+	local verbose=
+
+	while getopts v flag; do
+		case "$flag" in
+		v) verbose=1 ;;
+		esac
+	done
+	shift $(( $OPTIND - 1 ))
+
+	local P=$1
+	if [ "$verbose" ]; then
+		awk -v P=$P '
+			BEGIN { printf "\t" }
+			NR > 1 && $0 { $0 = "\t" $0 }
+			buf = buf $0 "\n" { }
+			END {
+				$0 = buf
+				if (P < 3) S = sprintf("%" 7-2*(P+1) "s", "")
+				gsub(/S/, S)
+				gsub(/B/, P < 3 ? "\\" : "")
+				gsub(/\\\t/, "\t")
+				sub(/\n$/, "")
+				print
+			}
+		' <<-EOFPREAMBLE
+		printf(" SB-+= %05d %d.%d %s\n",
+		\	this->pid$P, this->uid$P, this->gid$P, this->args$P);
+		EOFPREAMBLE
+	else
+		cat <<-EOFPREAMBLE
+		printf("%s", this->args$P);
+		EOFPREAMBLE
+	fi
+}
+
+############################################################ MAIN
+
+# If we're running as root, no need for sudo(8)
+[ "$( id -u )" != 0 ] && type sudo > /dev/null 2>&1 && SUDO=sudo
+
+#
+# Process command-line options
+#
+while getopts 1B:deE:fFg:j:k:K:lmnN:o:O:p:PqQr:Rt:T:u:vVwxX:yz: flag; do
+	case "$flag" in
+	1) ONELINE=1 PSTREE= ;;
+	B) MAX_ARGS="$OPTARG" ;;
+	d) DEBUG=1 ;;
+	e) EXIT_AFTER_COMPILE=1 ;;
+	E) CUSTOM_DETAILS=1
+	   EVENT_DETAILS="${EVENT_DETAILS%;}"
+	   [ "$EVENT_DETAILS" ] && EVENT_DETAILS="$EVENT_DETAILS;
+		printf(\" \");
+		" # END-QUOTE
+	   # Read event code from stdin if `-' is argument
+	   [ "$OPTARG" = "-" ] && OPTARG=$( cat )
+	   EVENT_DETAILS="$EVENT_DETAILS$OPTARG" ;;
+	f) PROBE_TYPE=function ;;
+	F) PROBE_COALESCE=1 ;;
+	g) GROUP="$OPTARG" ;;
+	j) JID="$OPTARG" ;;
+	k) EXECNAME="$EXECNAME${EXECNAME:+ }$OPTARG"
+	   case "$OPTARG" in
+	   \**\*) name="${OPTARG%\*}"
+		predicate="strstr(execname, \"${name#\*}\") != NULL" ;;
+	   \**) name="${OPTARG#\*}"
+		predicate="strstr(execname, \"$name\") == (execname +"
+		predicate="$predicate strlen(execname) - ${#name})" ;;
+	   *\*) predicate="strstr(execname, \"${OPTARG%\*}\") == execname" ;;
+	   *) predicate="execname == \"$OPTARG\""
+	   esac
+	   EVENT_TEST="$predicate${EVENT_TEST:+ ||
+		($EVENT_TEST)}" ;;
+	K) MAX_DEPTH="$OPTARG" ;;
+	l) LIST=1 ;;
+	m) PROBE_TYPE=module ;;
+	n) PROBE_TYPE=name ;;
+	N) COUNT="$OPTARG" ;;
+	o) OUTPUT="$OPTARG" ;;
+	O) OUTPUT_CMD="$OPTARG" ;;
+	p) PID="$OPTARG" ;;
+	P) PROBE_TYPE=provider ;;
+	q) QUIET=1 ;;
+	Q) LIST_PROFILES=1 ;;
+	r) FILTER="$OPTARG" ;;
+	R) PSTREE=1 ;;
+	t) CUSTOM_TEST="${CUSTOM_TEST:+($CUSTOM_TEST) && }$OPTARG" ;;
+	T) TIMEOUT="$OPTARG" ;;
+	u) USER="$OPTARG" ;;
+	v) VERBOSE=1 ;;
+	V) vers="${VERSION#\$*[:\$]}"
+	   vers="${vers% \$}"
+	   printf "%s: %s\n" "$pgm" "${vers# }"
+	   exit ;;
+	w) DESTRUCTIVE_ACTIONS=1 ;;
+	x) TRACE=1 ;;
+	X) USE_PROFILE=1 PROFILE="$OPTARG" ;;
+	y) CONSOLE=1 CONSOLE_FORCE=1 ;;
+	z) EXECREGEX="$OPTARG" ;;
+	*) usage
+	   # NOTREACHED
+	esac
+done
+shift $(( $OPTIND - 1 ))
+
+#
+# List probes if `-l' was given
+#
+[ "$LIST" ] &&
+	list_probes -f "$FILTER" ${QUIET:+-q} -t "$PROBE_TYPE" -- "$@"
+	# NOTREACHED
+
+#
+# List profiles if `-Q' was given
+#
+[ "$LIST_PROFILES" ] &&
+	list_profiles ${ONELINE:+-1} -f "$FILTER" ${QUIET:+-q}
+	# NOTREACHED
+
+#
+# Validate number of arguments
+#
+if [ ! "$PROFILE" ]; then
+	# If not given `-X profile' then a probe argument is required
+	[ $# -gt 0 ] || usage # NOTREACHED
+fi
+
+#
+# Validate `-N count' option argument
+#
+case "$COUNT" in
+"") usage "-N option requires a number argument" ;; # NOTREACHED
+*[!0-9]*) usage "-N argument must be a number" ;; # NOTREACHED
+esac
+
+#
+# Validate `-B num' option argument
+#
+case "$MAX_ARGS" in
+"") usage "-B option requires a number argument" ;; # NOTREACHED
+*[!0-9]*) usage "-B argument must be a number" ;; # NOTREACHED
+esac
+
+#
+# Validate `-K num' option argument
+#
+case "$MAX_DEPTH" in
+"") usage "-K option requires a number argument" ;; # NOTREACHED
+*[!0-9]*) usage "-K argument must be a number" ;; # NOTREACHED
+esac
+
+#
+# Validate `-j jail' option argument
+#
+case "$JID" in
+"") : fall through ;;
+*[!0-9]*) JID=$( jls -j "$JID" jid ) || exit ;;
+esac
+
+#
+# Validate `-u user' option argument
+#
+case "$USER" in
+"") : fall through ;;
+*[![:alnum:]_-]*) RUID="$USER" ;;
+*[!0-9]*) RUID=$( id -u "$USER" 2> /dev/null ) || die "No such user: $USER" ;;
+*) RUID=$USER
+esac
+
+#
+# Validate `-g group' option argument
+#
+case "$GROUP" in
+"") : fall-through ;;
+*[![:alnum:]_-]*) RGID="$GROUP" ;;
+*[!0-9]*)
+	RGID=$( getent group | awk -F: -v group="$GROUP" '
+		$1 == group { print $3; exit found=1 }
+		END { exit !found }
+	' ) || die "No such group: $GROUP" ;;
+*) RGID=$GROUP
+esac
+
+#
+# Expand probe argument into probe(s)
+#
+case "$1" in
+-*) : Assume dtrace options such as "-c cmd" or "-p pid" ;; # No probe(s) given
+*)
+	PROBE_ARG="$1"
+	shift
+esac
+if [ "$PROBE_ARG" ]; then
+	oldIFS="$IFS"
+	IFS="$IFS,"
+	for arg in $PROBE_ARG; do
+		arg=$( expand_probe -t "$PROBE_TYPE" -- "$arg" )
+		PROBE="$PROBE${PROBE:+, }$arg"
+	done
+	IFS="$oldIFS"
+fi
+
+#
+# Set default event details if `-E code' was not given
+#
+[ "$CUSTOM_DETAILS" ] || EVENT_DETAILS=$( pproc_dump 0 )
+
+#
+# Load profile if given `-X profile'
+#
+[ "$USE_PROFILE" ] && load_profile "$PROFILE"
+[ "$PROBE" ] || die "PROBE not defined by profile and none given as argument"
+
+#
+# Show the user what's being watched
+#
+[ "$DEBUG$QUIET$EXIT_AFTER_COMPILE" ] || info "Watching '$PROBE' ..."
+
+#
+# Header for watched probe entry
+#
+case "$PROBE" in
+*,*) : fall-through ;;
+*:execve:entry|execve:entry)
+	ACTIONS=$( awk 'gsub(/\\\t/, "\t") || 1' <<-EOF
+		$PROBE /* probe ID $ID */
+		{${TRACE:+
+		\	printf("<$ID>");}
+		\	this->caller_execname = execname;
+		}
+		EOF
+	)
+	PROBE="${PROBE%entry}return"
+	ID=$(( $ID + 1 ))
+	EVENT_TEST="execname != this->caller_execname${EVENT_TEST:+ &&
+		($EVENT_TEST)}"
+	EVENT_TAG='printf("%d.%d %s[%d]: ",
+		this->uid1, this->gid1, this->caller_execname, this->pid1);'
+	;;
+esac
+
+#
+# Jail clause/predicate
+#
+if [ "$JID" ]; then
+	prison_id="curthread->td_proc->p_ucred->cr_prison->pr_id"
+	EVENT_TEST="$prison_id == $JID${EVENT_TEST:+ &&
+		($EVENT_TEST)}"
+fi
+
+#
+# Custom test clause/predicate
+#
+if [ "$CUSTOM_TEST" ]; then
+	case "$EVENT_TEST" in
+	"") EVENT_TEST="$CUSTOM_TEST" ;;
+	 *) EVENT_TEST="$EVENT_TEST &&
+		($CUSTOM_TEST)"
+	esac
+fi
+
+#
+# Make sure dynamic code has trailing semi-colons if non-NULL
+#
+EVENT_TAG="${EVENT_TAG%;}${EVENT_TAG:+;}"
+EVENT_DETAILS="${EVENT_DETAILS%;}${EVENT_DETAILS:+;}"
+
+#
+# DTrace script
+#
+# If `-d' is given, script is sent to stdout for debugging
+# If `-c count", `-g group', `-r regex', or `-u user' is given, run script with
+# dtrace and send output to awk(1) post-processor (making sure to preserve the
+# exit code returned by dtrace invocation). Otherwise, simply run script with
+# dtrace and then exit.
+#
+exec 9<<EOF
+$PROBE /* probe ID 2 */
+{${TRACE:+
+	printf("<2>");
+}
+	/*
+	 * Examine process, parent process, and grandparent process details
+	 */
+
+	/******************* CURPROC *******************/
+
+	$( pproc -P0 )
+
+	/******************* PPARENT *******************/
+
+	$( if [ "$PSTREE" ]; then pproc -P1; else echo -n \
+	"this->proc = this->proc ? this->proc->p_pptr : NULL;
+	this->pid1 = this->proc ? this->proc->p_pid : -1;
+	this->uid1 = this->proc ? this->proc->p_ucred->cr_uid : -1;
+	this->gid1 = this->proc ? this->proc->p_ucred->cr_rgid : -1;
+	this->jid1 = this->proc ? this->proc->p_ucred->cr_prison->pr_id : -1;"
+	fi )
+
+	/******************* GPARENT *******************/
+
+	$( [ "$PSTREE" ] && pproc -P2 )
+
+	/******************* APARENT *******************/
+
+	$( [ "$PSTREE" ] && pproc -P3 )
+}
+EOF
+PSARGS_ACTION=$( cat <&9 )
+[ "$OUTPUT" -a ! "$CONSOLE_FORCE" ] && CONSOLE=
+{
+	if [ "$DEBUG" ]; then
+		# Send script to stdout
+		cat
+		exit
+	fi
+
+	if [ "$CUSTOM_TEST$EXECNAME$JID$OUTPUT$TIMEOUT$TRACE$VERBOSE" -a \
+	    ! "$QUIET" ]
+	then
+		msg=Setting
+		[ "$CUSTOM_TEST" ] && msg="$msg test: $CUSTOM_TEST"
+		[ "$EXECNAME" ] && msg="$msg execname: $EXECNAME"
+		[ "$JID" ] && msg="$msg jid: $JID"
+		[ "$OUTPUT" ] && msg="$msg output: $OUTPUT"
+		[ "$TIMEOUT" ] && msg="$msg timeout: $TIMEOUT"
+		[ "$TRACE" ] && msg="$msg trace: $TRACE"
+		[ "$VERBOSE" ] && msg="$msg verbose: $VERBOSE"
+		info "$msg"
+	fi
+
+	exec 3>&1
+	console_stdout=3
+
+	if [ $COUNT -eq 0 -a ! "$EXECREGEX$FILTER$GROUP$OUTPUT_CMD$PID$USER" ]
+	then
+		case "$OUTPUT" in
+		-) output_path=/dev/stdout ;;
+		*) output_path="$OUTPUT"
+		esac
+
+		# Run script without pipe to awk post-processor
+		dtrace_cmd -t \
+			${DESTRUCTIVE_ACTIONS:+-w} \
+			${EXIT_AFTER_COMPILE:+-e} \
+			${OUTPUT:+-o "$output_path"} \
+			-s /dev/stdin \
+			"$@"
+		exit
+	fi
+
+	# Prevent backslashes from being lost
+	FILTER=$( echo "$FILTER" | awk 'gsub(/\\/,"&&")||1' )
+	EXECREGEX=$( echo "$EXECREGEX" | awk 'gsub(/\\/,"&&")||1' )
+
+	if [ ! "$QUIET" ]; then
+		msg=Filtering
+		[ "$EXECREGEX" ] && msg="$msg execregex: $EXECREGEX"
+		[ "$FILTER" ] && msg="$msg filter: $FILTER"
+		[ "$GROUP" ] && msg="$msg group: $GROUP"
+		[ "$OUTPUT_CMD" ] && msg="$msg cmd: $OUTPUT_CMD"
+		[ "$PID" ] && msg="$msg pid: $PID"
+		[ "$USER" ] && msg="$msg user: $USER"
+		[ $COUNT -gt 0 ] && msg="$msg count: $COUNT"
+		info "$msg"
+	fi
+
+	#
+	# Send script output to post-processor for filtering
+	#
+	status=$(
+		exec 4>&1
+		to_status=4
+		( exec 5>&1; to_dtrace_stderr_filter=5; (
+			trap 'echo $? >&$to_status' EXIT
+			eval $SUDO ${TIMEOUT:+timeout \"\$TIMEOUT\"} dtrace \
+				${EXIT_AFTER_COMPILE:+-e} \
+				${DESTRUCTIVE_ACTIONS:+-w} \
+				-s /dev/stdin \
+				\"\$@\" \
+				2>&$to_dtrace_stderr_filter \
+				${QUIET:+2> /dev/null}
+		) | $SUDO awk \
+			-v cmd="$OUTPUT_CMD" \
+			-v console="$CONSOLE" \
+			-v count=$COUNT \
+			-v execregex="$EXECREGEX" \
+			-v filter="$FILTER" \
+			-v gid="$RGID" \
+			-v output="$OUTPUT" \
+			-v pid="$PID" \
+			-v pstree=$PSTREE \
+			-v quiet=$QUIET \
+			-v tty=$( ps -o tty= -p $$ ) \
+			-v uid="$RUID" \
+		' # Start awk(1) post-processor
+		############################################ BEGIN
+		BEGIN {
+			true = 1
+			ansi = "(\\033\\[[[:digit:];]+m)?"
+			num = year = day = "[[:digit:]]+"
+			month = "[[:alpha:]]+"
+			date = year " " month " +" day
+			time = "[012][0-9]:[0-5][0-9]:[0-5][0-9]"
+			date_time = ansi date " +" time ansi
+			name1 = "[^\\[]*"
+			name2 = "[^\\n]*"
+			if (output == "-")
+				output = "/dev/stdout"
+
+			#
+			# Field definitions
+			#
+			nexecmatches = 2
+			execstart[1] = sprintf( \
+				"^(%s) (%s)\\.(%s) (%s)\\[(%s)\\]: ",
+				date_time, num, num, name1, num)
+			execstart[2] = sprintf( \
+				"\\n +\\\\?-\\+= (%s) (%s)\\.(%s) ",
+				num, num, num)
+			npidmatches = 2
+			pidstart[1] = sprintf("^(%s) (%s)\\.(%s) (%s)\\[",
+				date_time, num, num, name1)
+			pidstart[2] = "\\n +\\\\?-\\+= "
+			pidpreen[2] = "^0*"
+			piddeflt[2] = "0"
+			ngidmatches = 2
+			gidstart[1] = sprintf("^(%s) (%s)\\.", date_time, num)
+			gidstart[2] = sprintf("\\n +\\\\?-\\+= (%s) (%s)\\.",
+				ansi num ansi, num)
+			nuidmatches = 2
+			uidstart[1] = sprintf("^(%s) ", date_time)
+			uidstart[2] = sprintf("\\n +\\\\?-\\+= (%s) ",
+				ansi num ansi)
+		}
+		############################################ FUNCTIONS
+		function strip(s) { gsub(/\033\[[0-9;]*m/, "", s); return s }
+		function esc(str) { gsub(/'\''/, "&\\\\&&", str); return str }
+		function arg(str) { return "'\''" esc(str) "'\''" }
+		function env(var, str) { return var "=" arg(str) " " }
+		function ans(seq) { return console ? "\033[" seq "m" : "" }
+		function runcmd() {
+			return system(sprintf("%s/bin/sh -c %s",
+				env("TAG", strip(tag)) \
+					env("DETAILS", strip(details)),
+				arg(cmd)))
+		}
+		function filter_block() {
+			if (length(lines) < 1) return 0
+			block_match = 0
+			newstr = ""
+			start = 1
+			if (match(lines, "^(" date_time ") ")) {
+				newstr = newstr substr(lines, 1,
+					RSTART + RLENGTH - 1)
+				start = RSTART + RLENGTH
+			}
+			replace = ans("31;1") "&" ans("39;22")
+			workstr = substr(lines, start)
+			if (gsub(filter, replace, workstr)) block_match = 1
+			lines = newstr workstr
+			return block_match
+		}
+		function filter_field(startre, fieldre, matchre, isword,
+			preenre, defaultstr)
+		{
+			if (length(lines) < 1) return 0
+			field_match = 0
+			newstr = ""
+			start = 1
+			while ((workstr = substr(lines, start)) &&
+				(workstr ~ (startre fieldre)))
+			{
+				match(workstr, startre)
+				start += end = RSTART + RLENGTH - 1
+				newstr = newstr substr(workstr, 1, end)
+				workstr = substr(workstr, end + 1)
+				match(workstr, fieldre)
+				start += end = RSTART + RLENGTH - 1
+				field = matchstr = substr(workstr, 1, end)
+				sub(preenre, "", matchstr)
+				if (!matchstr) matchstr = defaultstr
+				if (isword) {
+					if (match(matchstr, matchre) &&
+						RSTART == 1 &&
+						RLENGTH == length(matchstr)) {
+						field_match = 1
+						field = ans(7) field ans(27)
+					}
+				} else {
+					replace = ans(7) "&" ans(27)
+					if (gsub(matchre, replace, matchstr)) {
+						field_match = 1
+						field = matchstr
+					}
+				}
+				newstr = newstr field
+			}
+			lines = newstr workstr
+			return field_match
+		}
+		function dump() {
+			lines = block
+			block = ""
+			found = 0
+			if (execregex != "") {
+				for (n = 1; n <= nexecmatches; n++)
+					if (filter_field(execstart[n], name2,
+						execregex)) found = 1
+				if (!found) return
+			}
+			if (pid != "") {
+				for (n = 1; n <= npidmatches; n++)
+					if (filter_field(pidstart[n], num, pid,
+						true, pidpreen[n],
+						piddeflt[n])) found = 1
+				if (!found) return
+			}
+			if (gid != "") {
+				for (n = 1; n <= ngidmatches; n++)
+					if (filter_field(gidstart[n], num,
+						gid, true)) found = 1
+				if (!found) return
+			}
+			if (uid != "") {
+				for (n = 1; n <= nuidmatches; n++)
+					if (filter_field(uidstart[n], num,
+						uid, true)) found = 1
+				if (!found) return
+			}
+			if (filter != "" && !filter_block()) return
+			if (lines) {
+				stdout = 1
+				if (output) {
+					stdout = 0
+					if (!console) lines = strip(lines)
+					print lines > output
+				} else if (cmd) {
+					if (!quiet) print lines
+					tag = details = lines
+					sub(/: .*/, "", tag)
+					sub(/.*: /, "", details)
+					if (!console) tag = strip(tag)
+					runcmd()
+				} else print lines
+			}
+			fflush()
+			++matches
+		}
+		############################################ MAIN
+		{ block = (block ? block "\n" : block) $0 }
+		!pstree { dump() }
+		$0 ~ sprintf("^%6s\\\\-\\+= %s ", "", num) { dump() }
+		count && matches >= count { exit }
+		############################################ END
+		END {
+			dump()
+			system(sprintf("pkill -t %s dtrace %s", tty,
+				quiet ? "2> /dev/null" : ""))
+		}
+		' >&$console_stdout ) | dtrace_stderr_filter >&2
+	) # status
+	exit $status
+
+} <<EOF
+#!/usr/sbin/dtrace -s
+/* -
+ * Copyright (c) 2014-2018 Devin Teske <dteske@FreeBSD.org>
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS \`\`AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $TITLE dtrace(1) script to log process(es) triggering $PROBE $
+ * \$FreeBSD$
+ */
+
+$( echo "$DTRACE_PRAGMA" | awk '
+	!/^[[:space:]]*(#|$)/, sub(/^[[:space:]]*/, "#pragma D ")||1
+' )
+
+int console;
+
+dtrace:::BEGIN { console = ${CONSOLE:-0} } /* probe ID 1 */
+
+/*********************************************************/
+
+${PSARGS:+$PSARGS_ACTION}
+${ACTIONS:+
+/*********************************************************/
+
+$ACTIONS
+}
+/*********************************************************/
+
+$PROBE${EVENT_TEST:+ /$EVENT_TEST/} /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/***********************************************/
+
+	printf("%s%Y%s ",
+		console ? "\033[32m" : "",
+		walltimestamp,
+		console ? "\033[39m" : "");
+
+	/****************** EVENT_TAG ******************/
+
+	${EVENT_TAG#[[:space:]]}
+${PROBE_COALESCE:+
+	/**************** PROBE_COALESCE ***************/
+
+	printf("%s%s:%s:%s:%s ", probename == "entry" ? "-> " :
+			probename == "return" ? "<- " :
+			probename == "start" ? "-> " :
+			probename == "done" ? "<- " : " | ",
+		probeprov, probemod, probefunc, probename);
+}
+	/**************** EVENT_DETAILS ****************/
+
+	${EVENT_DETAILS#[[:space:]]}
+
+	/***********************************************/
+
+	printf("\\n");
+${PSTREE:+
+	/*
+	 * Print process, parent, grandparent, and ancestor details
+	 */
+$(	pproc_dump -v 3
+	pproc_dump -v 2
+	pproc_dump -v 1
+	pproc_dump -v 0
+)}
+}
+EOF
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/dwatch.1 b/cddl/usr.sbin/dwatch/dwatch.1
new file mode 100644
index 000000000000..f70e4fcb43ac
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/dwatch.1
@@ -0,0 +1,766 @@
+.\" Copyright (c) 2014-2018 Devin Teske
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.\" DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+.\" INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+.\" ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd February 9, 2018
+.Dt DWATCH 1
+.Os
+.Sh NAME
+.Nm dwatch
+.Nd watch processes as they trigger a particular DTrace probe
+.Sh SYNOPSIS
+.Nm
+.Op Fl 1defFmnPqRvVwxy
+.Op Fl B Ar num
+.Op Fl E Ar code
+.Op Fl g Ar group
+.Op Fl j Ar jail
+.Op Fl k Ar name
+.Op Fl K Ar num
+.Op Fl N Ar count
+.Op Fl o Ar file
+.Op Fl O Ar cmd
+.Op Fl p Ar pid
+.Op Fl r Ar regex
+.Op Fl t Ar test
+.Op Fl T Ar time
+.Op Fl u Ar user
+.Op Fl X Ar profile
+.Op Fl z Ar regex
+.Op Fl -
+.Op probe[,...]
+.Op args ...
+.Nm
+.Fl l
+.Op Fl fmnPqy
+.Op Fl r Ar regex
+.Op probe ...
+.Nm
+.Fl Q
+.Op Fl 1qy
+.Op Fl r Ar regex
+.Sh DESCRIPTION
+The
+.Nm
+utility uses
+.Xr dtrace 1
+to display process info when a given DTrace probe point is triggered.
+Only the root user or users with
+.Xr sudo 8
+access can run this command.
+.Pp
+.Nm
+automates the process of generating DTrace scripts to coalesce trace output by
+date/time,
+process info,
+and
+.Op optionally
+probe-specific data.
+.Pp
+Output format without options is:
+.Pp
+.Dl date/time uid.gid execname[pid]: psargs
+.Pp
+For example,
+the command
+.Ql dwatch BEGIN
+produces:
+.Pp
+.Dl INFO Watching 'dtrace:::BEGIN' ...
+.Dl 2017 May 29 08:23:20 0.0 dtrace[60671]: dtrace -s /dev/stdin
+.Pp
+The
+.Fl F
+option causes
+.Nm
+to instead coalesce trace output by date/time,
+process info,
+and probe traversal.
+.Pp
+Output format with the
+.Ql Fl F
+option is:
+.Pp
+.Dl date/time uid.gid execname[pid]: {->,<-, |} prov:mod:func:name ...
+.Pp
+For example,
+the command
+.Ql dwatch -F BEGIN
+produces:
+.Pp
+.Dl INFO Watching 'dtrace:::BEGIN' ...
+.Dl 2017 May 29 21:34:41 0.0 dtrace[86593]:  | dtrace:::BEGIN ...
+.Pp
+The
+.Fl R
+option causes
+.Nm
+to display a process tree containing the parent,
+grandparent,
+and ancestor process info.
+.Pp
+Output format with the
+.Ql Fl R
+option is:
+.Pp
+.Dl date/time uid0.gid0 execname[pid0]: psargs0
+.Dl " -+= pid3 uid3.gid3 psargs3"
+.Dl "  \e\\-+= pid2 uid2.gid2 psargs2"
+.Dl "    \e\\-+= pid1 uid1.gid1 psargs1"
+.Dl "      \e\\-+= pid0 uid0.guid0 psargs0"
+.Pp
+For example,
+the command
+.Ql dwatch -R BEGIN
+produces:
+.Pp
+.Dl INFO Watching 'dtrace:::BEGIN' ...
+.Dl 2017 May 29 21:38:54 0.0 dtrace[86899]: dtrace -s /dev/stdin
+.Dl " -+= 86855 604.604 -bash"
+.Dl "  \e\\-+= 86857 604.604 /bin/sh /usr/sbin/dwatch -R BEGIN"
+.Dl "    \e\\-+= 86897 0.0 sudo dtrace -s /dev/stdin"
+.Dl "      \e\\-+= 86899 0.0 dtrace -s /dev/stdin"
+.Pp
+Of particular interest is the ability to filter using regular expressions.
+The
+.Ql Fl g Ar group ,
+.Ql Fl p Ar pid ,
+.Ql Fl r Ar regex ,
+.Ql Fl u Ar user ,
+and
+.Ql Fl z Ar regex
+options can be combined with
+.Ql Fl R
+to match on parent process criteria as well as current process info.
+.Pp
+In contrast,
+the
+.Ql Fl j Ar jail ,
+and
+.Ql Fl k Ar name
+options apply only to the current process even if
+.Ql Fl R
+is given.
+.Pp
+The
+.Ql Fl E Ar code
+option gives the ability to customize probe-specific data.
+For example,
+the command:
+.Pp
+.Dl dwatch -E 'printf("%s", copyinstr(arg0))' chdir
+.Pp
+displays the path argument sent to
+.Xr chdir 2
+calls.
+.Pp
+Profiles can be written for more complex routines and/or convenience.
+To list available profiles use the
+.Ql Fl Q
+option.
+Use the
+.Ql Fl X Ar profile
+option to use a particular profile.
+.Pp
+For example,
+the command
+.Ql dwatch -X kill
+displays arguments sent to
+.Xr kill 2 .
+.Sh OPTIONS
+If a
+.Ar probe
+argument does not contain colon
+.Pq Qo Li ":" Qc
+and none of
+.Ql Fl P ,
+.Ql Fl m ,
+.Ql Fl f ,
+or
+.Ql Fl n
+are given,
+the probe argument is intelligently mapped to its most-likely value.
+Use
+.Ql Nm Fl l Ar name
+to see what probes will match a given name.
+.Pp
+Multiple probes must be given as a single
+.Pq quoted
+argument,
+separated by comma and/or whitespace.
+Any/all arguments following said probes will be passed to
+.Xr dtrace 1
+unmodified.
+.Bl -tag -width "-c count"
+.It Fl 1
+Print one line per process/profile
+.Pq Default; disables Ql Fl R .
+.It Fl B Ar num
+Maximum number of arguments to display
+.Pq Default 64 .
+.It Fl d
+Debug.
+Send
+.Xr dtrace 1
+script to stdout instead of executing.
+.It Fl e
+Exit after compiling request but prior to enabling probes.
+.It Fl E Ar code
+DTrace
+.Ar code
+for event details.
+If `-',
+read from stdin.
+This allows customization of what is printed after date/time and process info.
+By default,
+the name and arguments of the program triggering the probe are shown.
+Can be specified multiple times.
+.It Fl f
+Enable probes matching the specified function names.
+.It Fl F
+Coalesce trace output by probe.
+.It Fl g Ar group
+Group filter.
+Only show processes matching
+.Ar group
+name/gid.
+This can be an
+.Xr awk 1
+regular expression to match a numerical gid.
+.It Fl j Ar jail
+Jail filter.
+Only show processes matching
+.Ar jail
+name/jid.
+.It Fl k Ar name
+Only show processes matching
+.Ar name .
+Can also be of the format
+.Ql Li name*
+to indicate
+.Dq Li begins with ,
+.Ql Li *name
+to indicate
+.Dq Li ends with ,
+or
+.Ql Li *name*
+to indicate
+.Dq Li contains .
+Can be specified multiple times.
+.It Fl K Ar num
+Maximum directory depth to display
+.Pq Default 64 .
+.It Fl l
+List available probes on standard output and exit.
+.It Fl m
+Enable probes matching the specified module names.
+.It Fl X Ar profile
+Load profile from DWATCH_PROFILES_PATH.
+.It Fl n
+Enable probes matching the specified probe names.
+.It Fl N Ar count
+Exit after
+.Ar count
+matching entries
+.Pq Default 0 for disabled .
+.It Fl o Ar file
+Set output file.
+If
+.Ql Li - ,
+the path
+.Ql Li /dev/stdout
+is used.
+.It Fl O Ar cmd
+Execute
+.Ar cmd
+for each event.
+This can be any valid
+.Xr sh 1
+command.
+The environment variables
+.Ql Li $TAG
+and
+.Ql Li $DETAILS
+are set for the given
+.Ar cmd .
+.It Fl p Ar pid
+Process id filter.
+Only show processes with matching
+.Ar pid .
+This can be an
+.Xr awk 1
+regular expression.
+.It Fl P
+Enable probe matching the specified provider name.
+.It Fl q
+Quiet.
+Hide informational messages and all dtrace(1) errors.
+.It Fl Q
+List available profiles in DWATCH_PROFILES_PATH and exit.
+.It Fl r Ar regex
+Filter.
+Only show blocks matching
+.Xr awk 1
+regular expression.
+.It Fl R
+Show parent,
+grandparent,
+and ancestor of process.
+.It Fl t Ar test
+Test clause
+.Pq predicate
+to limit events
+.Pq Default none .
+Can be specified multiple times.
+.It Fl T Ar time
+Timeout.
+The format is
+.Ql Li #[smhd]
+or just
+.Ql Li #
+for seconds.
+.It Fl u Ar user
+User filter.
+Only show processes matching
+.Ar user
+name/uid.
+This can be an
+.Xr awk 1
+regular expression to match a numerical UID.
+.It Fl v
+Verbose.
+Show all errors from
+.Xr dtrace 1 .
+.It Fl V
+Report
+.Nm
+version on standard output and exit.
+.It Fl w
+Permit destructive actions
+.Pq copyout*, stop, panic, etc. .
+.It Fl x
+Trace.
+Print
+.Ql Li <probe-id>
+when a probe is triggered.
+.It Fl y
+Always treat stdout as console
+.Pq enable colors/columns/etc. .
+.It Fl z Ar regex
+Only show processes matching
+.Xr awk 1
+regular expression.
+.El
+.Sh PROFILES
+Profiles customize the data printed during events.
+Profiles are loaded from a colon-separated list of directories in
+.Ev DWATCH_PROFILES_PATH .
+This is an incomplete list of profiles with basic descriptions:
+.Bl -tag -width "vop_readdir"
+.It chmod
+Print mode and path from
+.Xr chmod 2 ,
+.Xr lchmod 2 ,
+.Xr fchmodat 2
+.It errno
+Print non-zero errno results from system calls
+.It io
+Print disk I/O details provided by
+.Xr dtrace_io 4
+.It ip
+Print IPv4 and IPv6 details provided by
+.Xr dtrace_ip 4
+.It kill
+Print signal and pid from
+.Xr kill 2
+.It nanosleep
+Print requested time from
+.Xr nanosleep 2
+.It open
+Print path from
+.Xr open 2 ,
+.Xr openat 2
+.It proc
+Print process execution details provided by
+.Xr dtrace_proc 4
+.It proc-signal
+Print process signal details provided by
+.Xr dtrace_proc 4
+.It rw
+Print buffer contents from
+.Xr read 2 ,
+.Xr write 2
+.It sched
+Print CPU scheduling details provided by
+.Xr dtrace_sched 4
+.It tcp
+Print TCP address/port details provided by
+.Xr dtrace_tcp 4
+.It tcp-io
+Print TCP I/O details provided by
+.Xr dtrace_tcp 4
+.It udp
+Print UDP I/O details provided by
+.Xr dtrace_udp 4
+.It vop_create
+Print filesystem paths being created by
+.Xr VOP_CREATE 9
+.It vop_lookup
+Print filesystem paths being looked-up by
+.Xr VOP_LOOKUP 9
+.It vop_mkdir
+Print directory paths being created by
+.Xr VOP_MKDIR 9
+.It vop_mknod
+Print device node paths being created by
+.Xr VOP_MKNOD 9
+.It vop_readdir
+Print directory paths being read by
+.Xr VOP_READDIR 9
+.It vop_remove
+Print filesystem paths being removed by
+.Xr VOP_REMOVE 9
+.It vop_rename
+Print filesystem paths being renamed by
+.Xr VOP_RENAME 9
+.It vop_rmdir
+Print directory paths being removed by
+.Xr VOP_RMDIR 9
+.It vop_symlink
+Print symlink paths being created by
+.Xr VOP_SYMLINK 9
+.El
+.Sh ENVIRONMENT
+These environment variables affect the execution of
+.Nm :
+.Bl -tag -width "DWATCH_PROFILES_PATH"
+.It Ev DWATCH_PROFILES_PATH
+If
+.Ev DWATCH_PROFILES_PATH
+is set,
+.Nm
+searches for profiles in the colon-separated list of directories in that
+variable instead of the default
+.Ql Li /usr/libexec/dwatch:/usr/local/libexec/dwatch .
+If set to NULL,
+profiles are not loaded.
+.El
+.Sh EXIT STATUS
+.Ex -std
+.Sh EXAMPLES
+Watch processes entering system CPU scheduler.
+.Bd -literal -offset indent
+dwatch on-cpu
+.Ed
+.Pp
+List available profiles,
+one line per profile.
+.Bd -literal -offset indent
+dwatch -1 -Q
+.Ed
+.Pp
+Do not execute
+.Xr dtrace 1
+but display script on stdout and exit.
+.Bd -literal -offset indent
+dwatch -d fsync
+.Ed
+.Pp
+Compile and test but do not execute code generated with given probe.
+.Bd -literal -offset indent
+dwatch -e test_probe
+.Ed
+.Pp
+Print argument one being passed to each call of zfs_sync().
+.Bd -literal -offset indent
+dwatch -E 'printf("%i", arg1)' zfs_sync
+.Ed
+.Pp
+Watch all functions named
+.Ql Li read .
+.Bd -literal -offset indent
+dwatch -f read
+.Ed
+.Pp
+Watch all probe traversal.
+.Bd -literal -offset indent
+dwatch -F :
+.Ed
+.Pp
+Watch syscall probe traversal.
+.Bd -literal -offset indent
+dwatch -F syscall
+.Ed
+.Pp
+Display only processes belonging to wheel super-group.
+.Bd -literal -offset indent
+dwatch -g wheel execve
+.Ed
+.Pp
+Display only processes belonging to groups
+.Ql Li daemon
+or
+.Ql Li nobody .
+.Bd -literal -offset indent
+dwatch -g '1|65534' execve
+.Ed
+.Pp
+Ignore jails,
+displaying only base system processes.
+.Bd -literal -offset indent
+dwatch -j 0 execve
+.Ed
+.Pp
+Display only processes running inside the jail named
+.Ql Li myjail .
+.Bd -literal -offset indent
+dwatch -j myjail execve
+.Ed
+.Pp
+Watch syscall traversal by ruby processes.
+.Bd -literal -offset indent
+dwatch -k 'ruby*' -F syscall
+.Ed
+.Pp
+Watch syscall traversal by processes containing
+.Ql Li daemon
+in their name.
+.Bd -literal -offset indent
+dwatch -k '*daemon*' -F syscall
+.Ed
+.Pp
+Watch signals being passed to
+.Xr kill 2 .
+.Bd -literal -offset indent
+dwatch -X kill
+.Ed
+.Pp
+Watch signals being passed between
+.Xr bash 1
+and
+.Xr vi 1 .
+.Bd -literal -offset indent
+dwatch -k bash -k vi -X kill
+.Ed
+.Pp
+Display a list of unique functions available.
+.Bd -literal -offset indent
+dwatch -l -f
+.Ed
+.Pp
+List available probes for functions ending in
+.Ql Li read .
+.Bd -literal -offset indent
+dwatch -l -f '*read'
+.Ed
+.Pp
+List available probes ending in
+.Dq Li read .
+.Bd -literal -offset indent
+dwatch -l -r 'read$'
+.Ed
+.Pp
+Display a list of unique providers.
+.Bd -literal -offset indent
+dwatch -l -P
+.Ed
+.Pp
+Watch paths being removed by
+.Xr VOP_REMOVE 9 .
+.Bd -literal -offset indent
+dwatch -X vop_remove
+.Ed
+.Pp
+Watch the name
+.Ql Li read
+instead of the function
+.Ql Li read .
+The
+.Nm
+selection algorithm will commonly favor the function named
+.Ql Li read
+when not given a type
+.Pq using So Fl P Sc , So Fl m Sc , So Fl f Sc , or So Fl n Sc
+because there are more probes matching the function named
+.Ql Li read
+than probes matching
+.Ql Li read
+for any other type.
+.Bd -literal -offset indent
+dwatch -n read
+.Ed
+.Pp
+Display the first process to call
+.Xr kill 2
+and then exit.
+.Bd -literal -offset indent
+dwatch -N 1 kill
+.Ed
+.Pp
+Watch processes forked by pid 1234.
+.Bd -literal -offset indent
+dwatch -p 1234 execve
+.Ed
+.Pp
+Watch processes forked by either pid 1234 or pid 5678.
+.Bd -literal -offset indent
+dwatch -p '1234|5678' execve
+.Ed
+.Pp
+Watch the provider
+.Ql Li random
+instead of the function
+.Ql Li random .
+The
+.Nm
+selection algorithm will commonly favor the function named
+.Ql Li random
+when not given a type
+.Pq using So Fl P Sc , So Fl m Sc , So Fl f Sc , or So Fl n Sc
+because there are more probes matching the function named
+.Ql Li random
+than probes matching the provider named
+.Ql Li random .
+.Bd -literal -offset indent
+dwatch -P random
+.Ed
+.Pp
+Display available profiles matching
+.Ql Li vop .
+.Bd -literal -offset indent
+dwatch -Q -r vop
+.Ed
+.Pp
+Watch
+.Xr VOP_LOOKUP 9
+paths containing
+.Ql Li /lib/ .
+.Bd -literal -offset indent
+dwatch -r /lib/ -X vop_lookup
+.Ed
+.Pp
+Show process tree for each command as it is executed.
+.Bd -literal -offset indent
+dwatch -R execve
+.Ed
+.Pp
+Watch processes forked by pid 1234 or children thereof.
+.Bd -literal -offset indent
+dwatch -R -p 1234 execve
+.Ed
+.Pp
+Display processes calling
+.Xr write 2
+with
+.Dq nbytes
+less than 10.
+.Bd -literal -offset indent
+dwatch -t 'arg2<10' -E 'printf("%d",arg2)' write
+.Ed
+.Pp
+Display
+.Xr write 2
+buffer when
+.Dq execname
+is not
+.Ql Li dtrace
+and
+.Dq nbytes
+is less than 10.
+.Bd -literal -offset indent
+dwatch -X write -t 'execname != "dtrace" && this->nbytes < 10'
+.Ed
+.Pp
+Watch
+.Ql Li statfs
+for 5 minutes and exit.
+.Bd -literal -offset indent
+dwatch -T 5m statfs
+.Ed
+.Pp
+Display only processes belonging to the root super-user.
+.Bd -literal -offset indent
+dwatch -u root execve
+.Ed
+.Pp
+Display only processes belonging to users
+.Ql Li daemon
+or
+.Ql Li nobody .
+.Bd -literal -offset indent
+dwatch -u '1|65534' execve
+.Ed
+.Pp
+Print version and exit.
+.Bd -literal -offset indent
+dwatch -V
+.Ed
+.Pp
+View the first 100 scheduler preemptions.
+.Bd -literal -offset indent
+dwatch -y -N 100 preempt | less -R
+.Ed
+.Pp
+Display processes matching either
+.Dq Li mkdir
+or
+.Dq Li rmdir .
+.Bd -literal -offset indent
+dwatch -z '(mk|rm)dir' execve
+.Ed
+.Pp
+Run a command and watch network activity only while that command runs.
+.Bd -literal -offset indent
+dwatch -X tcp -- -c "nc -zvw10 google.com 22"
+.Ed
+.Pp
+Watch
+.Xr open 2
+and
+.Xr openat 2
+calls only while pid 1234 is active.
+.Bd -literal -offset indent
+dwatch -X open -- -p 1234
+.Ed
+.Pp
+Watch probe traversal for a given command.
+Note that
+.Dq Li -c true
+is passed to
+.Xr dtrace 1
+since it appears after the
+.Nm
+probe argument.
+.Bd -literal -offset indent
+dwatch -F 'pid$target:::entry' -c true
+.Ed
+.Sh SEE ALSO
+.Xr dtrace 1
+.Sh HISTORY
+.Nm
+first appeared in
+.Fx 12.0-CURRENT .
+.Sh AUTHORS
+.An Devin Teske Aq Mt dteske@FreeBSD.org
diff --git a/cddl/usr.sbin/dwatch/examples/Makefile b/cddl/usr.sbin/dwatch/examples/Makefile
new file mode 100644
index 000000000000..33364044942f
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/examples/Makefile
@@ -0,0 +1,6 @@
+# $FreeBSD$
+
+FILESDIR=	${SHAREDIR}/examples/dwatch
+FILES=		profile_template
+
+.include <bsd.prog.mk>
diff --git a/cddl/usr.sbin/dwatch/examples/profile_template b/cddl/usr.sbin/dwatch/examples/profile_template
new file mode 100644
index 000000000000..8bb3acfed2fb
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/examples/profile_template
@@ -0,0 +1,74 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) profile for XXX entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# XXX
+#
+############################################################ PRAGMAS
+
+# Optional: You can override the default pragmas (shown below)
+
+#DTRACE_PRAGMA="
+#	option quiet
+#	option dynvarsize=16m
+#	switchrate=10hz
+#" # END-QUOTE
+
+############################################################ PROBE
+
+# Optional: dwatch(8) initializes this to the expanded probe arguments
+
+#: ${PROBE:="XXX"}
+
+############################################################ ACTIONS
+
+# Optional actions to be performed before hitting the final print action
+
+#exec 9<<EOF
+#EOF
+#ACTIONS=$( cat <&9 )
+#ID=
+
+############################################################ EVENT ACTION
+
+# The default EVENT value is simply `entry'. This is paired with $PROBE.
+
+#EVENT=
+
+# Optional predicate which must be true before the event action will run
+
+#EVENT_TEST=
+
+############################################################ EVENT TAG
+
+# The EVENT_TAG is run inside the print action after the timestamp has been
+# printed. By default, `UID.GID CMD[PID]: ' of the process is printed.
+
+#exec 9<<EOF
+#EOF
+#EVENT_TAG=$( cat <&9 )
+
+############################################################ EVENT DETAILS
+
+# The DETAILS are run after the EVENT_TAG and by default, the program name and
+# arguments of the process hitting the EVENT action are shown. This can be
+# customized to call-specific information because the `-v' flag of dwatch(8)
+# can provide detailed process information for the EVENT action on lines below
+# the DETAILS.
+#
+# NB: Should produce a single-line and not print a trailing newline.
+
+#exec 9<<EOF
+#	printf("XXX");
+#EOF
+#DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/Makefile b/cddl/usr.sbin/dwatch/libexec/Makefile
new file mode 100644
index 000000000000..ceffca630b8b
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/Makefile
@@ -0,0 +1,80 @@
+# $FreeBSD$
+
+FILESDIR=	${LIBEXECDIR}/dwatch
+FILES=		chmod \
+		errno \
+		io \
+		ip \
+		kill \
+		nanosleep \
+		open \
+		proc \
+		rw \
+		sched \
+		tcp \
+		udp \
+		vop_create \
+		vop_readdir \
+		vop_rename \
+		vop_symlink
+
+LINKS=	${LIBEXECDIR}/dwatch/chmod ${LIBEXECDIR}/dwatch/fchmodat
+LINKS+=	${LIBEXECDIR}/dwatch/chmod ${LIBEXECDIR}/dwatch/lchmod
+LINKS+=	${LIBEXECDIR}/dwatch/io ${LIBEXECDIR}/dwatch/io-done
+LINKS+=	${LIBEXECDIR}/dwatch/io ${LIBEXECDIR}/dwatch/io-start
+LINKS+=	${LIBEXECDIR}/dwatch/ip ${LIBEXECDIR}/dwatch/ip-receive
+LINKS+=	${LIBEXECDIR}/dwatch/ip ${LIBEXECDIR}/dwatch/ip-send
+LINKS+=	${LIBEXECDIR}/dwatch/open ${LIBEXECDIR}/dwatch/openat
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-create
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exec
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exec-failure
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exec-success
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exit
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-clear
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-discard
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-send
+LINKS+=	${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-status
+LINKS+=	${LIBEXECDIR}/dwatch/rw ${LIBEXECDIR}/dwatch/read
+LINKS+=	${LIBEXECDIR}/dwatch/rw ${LIBEXECDIR}/dwatch/write
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-change-pri
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-cpu
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-dequeue
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-enqueue
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-exec
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-lend-pri
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-load-change
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-off-cpu
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-on-cpu
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-preempt
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-pri
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-queue
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-remain-cpu
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-sleep
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-surrender
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-tick
+LINKS+=	${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-wakeup
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept-established
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept-refused
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-connect
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-connect-established
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-connect-refused
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-connect-request
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-established
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-init
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-io
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-receive
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-refused
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-send
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-state-change
+LINKS+=	${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-status
+LINKS+=	${LIBEXECDIR}/dwatch/udp ${LIBEXECDIR}/dwatch/udp-receive
+LINKS+=	${LIBEXECDIR}/dwatch/udp ${LIBEXECDIR}/dwatch/udp-send
+LINKS+=	${LIBEXECDIR}/dwatch/vop_create ${LIBEXECDIR}/dwatch/vop_lookup
+LINKS+=	${LIBEXECDIR}/dwatch/vop_create ${LIBEXECDIR}/dwatch/vop_mkdir
+LINKS+=	${LIBEXECDIR}/dwatch/vop_create ${LIBEXECDIR}/dwatch/vop_mknod
+LINKS+=	${LIBEXECDIR}/dwatch/vop_create ${LIBEXECDIR}/dwatch/vop_remove
+LINKS+=	${LIBEXECDIR}/dwatch/vop_create ${LIBEXECDIR}/dwatch/vop_rmdir
+
+.include <bsd.prog.mk>
diff --git a/cddl/usr.sbin/dwatch/libexec/chmod b/cddl/usr.sbin/dwatch/libexec/chmod
new file mode 100644
index 000000000000..c2a3a93f193c
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/chmod
@@ -0,0 +1,63 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for [l]chmod(2), fchmodat(2), or similar entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print mode/path being passed to chmod(2), lchmod(2), fchmodat(2), or similar
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+chmod)
+	: ${PROBE:=$( echo \
+		syscall::chmod:entry, \
+		syscall::lchmod:entry, \
+		syscall::fchmodat:entry )}
+	;;
+*)
+	: ${PROBE:=syscall::$PROFILE:entry}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this mode_t mode;
+this string path;
+this u_char at;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * Should we expect the first argument to be a file descriptor?
+	 * NB: Based on probefunc ending in "at" (e.g., fchmodat(2))
+	 */
+	this->at = strstr(probefunc, "at") ==
+		(probefunc + strlen(probefunc) - 2) ? 1 : 0;
+
+	this->mode = (mode_t)(this->at ? arg2 : arg1);
+	this->path = copyinstr(this->at ? arg1 : arg0);
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print mode/path details
+	 */
+	printf("%04o %s", this->mode, this->path);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/errno b/cddl/usr.sbin/dwatch/libexec/errno
new file mode 100644
index 000000000000..e1d464144096
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/errno
@@ -0,0 +1,35 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for syscall errno logging $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print when syscall returns with non-zero errno (default) or other condition.
+# To override the default test condition, use (for example) `-t errno==2' to
+# test for specific value or simply `-t 1' to unconditionally show all values.
+#
+############################################################ PROBE
+
+: ${PROBE:=syscall:::return}
+
+############################################################ EVENT ACTION
+
+[ "$CUSTOM_TEST" ] || EVENT_TEST="errno > 0"
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print errno details
+	 */
+	printf("%s: %s (%i)", probefunc, strerror[errno], errno);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/io b/cddl/usr.sbin/dwatch/libexec/io
new file mode 100644
index 000000000000..0849d0d3a20b
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/io
@@ -0,0 +1,106 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_io(4) $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display activity related to disk I/O
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+io) : ${PROBE:=io:::start, io:::done} ;;
+ *) : ${PROBE:=io:::${PROFILE#io-}}
+esac
+
+############################################################ EVENT ACTION
+
+[ "$CUSTOM_TEST" ] || EVENT_TEST='this->devinfo.dev_name != ""'
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this bufinfo_t	bufinfo;
+this devinfo_t	devinfo;
+this int	b_flags;
+this long	bio_length;
+this string	bio_cmd;
+this string	bio_flags;
+this string	device_entry;
+this string	device_if;
+this string	device_type;
+this string	flow;
+
+inline string append_bio_flag[int flags, int flag] = this->bio_flags =
+	strjoin(this->bio_flags,
+	strjoin(this->bio_flags == "" ? "" : (flags & flag) == flag ? "|" : "",
+		bio_flag_string[flags & flag]));
+
+$PROBE /(struct bio *)args[0] != NULL/ /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * dtrace_io(4)
+	 */
+	this->flow = probefunc == "done" ? "<-" : "->";
+
+	/*
+	 * struct bio *
+	 */
+	this->bufinfo = xlate <bufinfo_t> ((struct bio *)args[0]);
+	this->bio_cmd = bio_cmd_string[(int)this->bufinfo.b_cmd];
+	this->b_flags = (int)this->bufinfo.b_flags;
+	this->bio_flags = bio_flag_string[this->b_flags & BIO_ERROR];
+	this->bio_flags = strjoin(this->bio_flags, this->bufinfo.b_error ?
+		strjoin(this->bio_flags == "" ?
+			bio_flag_string[BIO_ERROR] : "",
+			strjoin("#", lltostr(this->bufinfo.b_error))) :
+		"");
+	append_bio_flag[this->b_flags, BIO_DONE];
+	append_bio_flag[this->b_flags, BIO_ONQUEUE];
+	append_bio_flag[this->b_flags, BIO_ORDERED];
+	append_bio_flag[this->b_flags, BIO_UNMAPPED];
+	append_bio_flag[this->b_flags, BIO_TRANSIENT_MAPPING];
+	append_bio_flag[this->b_flags, BIO_VLIST];
+	this->bio_flags = this->bio_flags == "" ? "-" : this->bio_flags;
+	this->bio_length = (long)this->bufinfo.b_bcount;
+
+	/*
+	 * struct devstat *
+	 */
+	this->devinfo = xlate <devinfo_t> ((struct devstat *)args[1]);
+	this->device_type = device_type[(int)this->devinfo.dev_type];
+	this->device_if = device_if[(int)this->devinfo.dev_type];
+	this->device_entry = strjoin(this->devinfo.dev_name,
+		lltostr(this->devinfo.dev_minor));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print disk I/O details
+	 */
+	printf("%s %s %s %s %s %s %d byte%s",
+		this->flow,
+		this->device_type,
+		this->device_if,
+		this->device_entry,
+		this->bio_cmd,
+		this->bio_flags,
+		this->bio_length,
+		this->bio_length == 1 ? "" : "s");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/ip b/cddl/usr.sbin/dwatch/libexec/ip
new file mode 100644
index 000000000000..40ab601403fe
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/ip
@@ -0,0 +1,95 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_ip(4) $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display interface name and bytes sent/received when IP I/O occurs
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+ip) : ${PROBE:=ip:::send, ip:::receive} ;;
+ *) : ${PROBE:=ip:::${PROFILE#ip-}}
+esac
+
+############################################################ GLOBALS
+
+#
+# This profile does not support these dwatch features
+# NB: They are disabled here so they have no effect when profile is loaded
+#
+unset EXECNAME		# -k name
+unset EXECREGEX		# -z regex
+unset GROUP		# -g group
+unset PID		# -p pid
+unset PSARGS		# affects -d
+unset PSTREE		# -R
+unset USER		# -u user
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this string	flow;
+this string	if_name;
+this string	local;
+this string	remote;
+this u_char	recv;
+this uint32_t	length;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * dtrace_ip(4)
+	 */
+	this->recv = probename == "receive" ? 1 : 0;
+	this->flow = this->recv ? "<-" : "->";
+
+	/*
+	 * ipinfo_t *
+	 */
+	this->length = (uint32_t)args[2]->ip_plength;
+	this->local  = this->recv ? args[2]->ip_daddr : args[2]->ip_saddr;
+	this->remote = this->recv ? args[2]->ip_saddr : args[2]->ip_daddr;
+
+	/*
+	 * ifinfo_t *
+	 */
+	this->if_name = args[3]->if_name;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT TAG
+
+exec 9<<EOF
+	printf("%s: ", "$PROFILE");
+EOF
+EVENT_TAG=$( cat <&9 )
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print network I/O details
+	 */
+	printf("%s %s %s %s %u byte%s",
+		this->if_name,
+		this->local,
+		this->flow,
+		this->remote,
+		this->length,
+		this->length == 1 ? "" : "s");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/kill b/cddl/usr.sbin/dwatch/libexec/kill
new file mode 100644
index 000000000000..1f315a990dd7
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/kill
@@ -0,0 +1,45 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for kill(2) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print arguments being passed to kill(2) [or similar]
+#
+############################################################ PROBE
+
+: ${PROBE:=syscall::$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this int	sig;
+this pid_t	pid;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->pid = (pid_t)arg0;
+	this->sig = (int)arg1;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print signal/pid details
+	 */
+	printf("signal %i to pid %d", this->sig, this->pid);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/nanosleep b/cddl/usr.sbin/dwatch/libexec/nanosleep
new file mode 100644
index 000000000000..cd1f2aa01343
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/nanosleep
@@ -0,0 +1,52 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for nanosleep(2) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print arguments being passed to nanosleep(2) [or similar]
+#
+############################################################ PROBE
+
+: ${PROBE:=syscall::$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this struct timespec *	rqtp;
+this time_t		requested_sec;
+this long		requested_nsec;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	print("<$ID>");
+}
+	/*
+	 * const struct timespec *
+	 */
+	this->rqtp = (struct timespec *)copyin(arg0, sizeof(struct timespec));
+	this->requested_sec  = (time_t)this->rqtp->tv_sec;
+	this->requested_nsec = (long)this->rqtp->tv_nsec;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Dump nanosleep(2) arguments
+	 */
+	printf("%d.%d seconds",
+		this->requested_sec, this->requested_nsec / 100000);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/open b/cddl/usr.sbin/dwatch/libexec/open
new file mode 100644
index 000000000000..87eaa9b05b86
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/open
@@ -0,0 +1,55 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for open[at](2) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print path being passed to open(2), openat(2), or similar
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+open) : ${PROBE:=syscall::open:entry, syscall::openat:entry} ;;
+   *) : ${PROBE:=syscall::$PROFILE:entry}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this string path;
+this u_char at;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * Should we expect the first argument to be a file descriptor?
+	 * NB: Based on probefunc ending in "at" (e.g., openat(2))
+	 */
+	this->at = strstr(probefunc, "at") ==
+		(probefunc + strlen(probefunc) - 2) ? 1 : 0;
+
+	this->path = copyinstr(this->at ? arg1 : arg0);
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print path details
+	 */
+	printf("%s", this->path);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/proc b/cddl/usr.sbin/dwatch/libexec/proc
new file mode 100644
index 000000000000..3077527ff736
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/proc
@@ -0,0 +1,162 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_proc(4) activity $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display process activity
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+proc)
+	: ${PROBE:=$( echo \
+		proc:::create, \
+		proc:::exec, \
+		proc:::exec-failure, \
+		proc:::exec-success, \
+		proc:::exit, \
+		proc:::signal-clear, \
+		proc:::signal-discard, \
+		proc:::signal-send )}
+	;;
+proc-signal)
+	: ${PROBE:=$( echo \
+		proc:::signal-clear, \
+		proc:::signal-discard, \
+		proc:::signal-send )}
+	;;
+proc-status)
+	: ${PROBE:=$( echo \
+		proc:::create, \
+		proc:::exec, \
+		proc:::exec-failure, \
+		proc:::exec-success, \
+		proc:::exit )}
+	;;
+*)
+	: ${PROBE:=proc:::${PROFILE#proc-}}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this int	sig;
+this pid_t	pid;
+this string	details;
+this string	exec_arg0;
+
+inline string probealias[string name] =
+	name == "create" ?		"FORK" :
+	name == "exec" ?		"EXEC" :
+	name == "exec-failure" ?	"FAIL" :
+	name == "exec-success" ?	"INIT" :
+	name == "exit" ?		"EXIT" :
+	name == "signal-clear" ?	"CLEAR" :
+	name == "signal-discard" ?	"DISCARD" :
+	name == "signal-send" ?		"SEND" :
+	name;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->details = "";
+}
+
+proc:::create /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");
+}
+	$( pproc -P _create "(struct proc *)args[0]" )
+
+	/* details = "pid <pid of args[0]> -- <proc args of args[0]>" */
+	this->details = strjoin(
+		strjoin("pid ", lltostr(this->pid_create)),
+		strjoin(" -- ", this->args_create));
+}
+
+proc:::exec /* probe ID $(( $ID + 2 )) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))");}
+	this->details = this->exec_arg0 = stringof(arg0);
+}
+
+proc:::exec-failure /* probe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");
+}
+	/* details = "<arg0 from proc:::exec>: <strerror of arg0> (<arg0>)" */
+	this->details = strjoin(
+		strjoin(this->exec_arg0, ": "),
+		strjoin(strerror[(int)arg0],
+			strjoin(" (", strjoin(lltostr((int)arg0), ")"))));
+}
+
+proc:::exec-success /* probe ID $(( $ID + 4 )) */
+{${TRACE:+
+	printf("<$(( $ID + 4 ))>");}
+	this->details = this->args0;
+}
+
+proc:::exit /* probe ID $(( $ID + 5 )) */
+{${TRACE:+
+	printf("<$(( $ID + 5 ))>");}
+	this->details = child_signal_string[(int)arg0];
+}
+
+proc:::signal-clear /* probe ID $(( $ID + 6 )) */
+{${TRACE:+
+	printf("<$(( $ID + 6 ))>");}
+	this->pid = (pid_t)((ksiginfo_t *)args[1])->ksi_info.si_pid;
+	this->sig = (int)arg0;
+}
+
+proc:::signal-discard, proc:::signal-send /* probe ID $(( $ID + 7 )) */
+{${TRACE:+
+	printf("<$(( $ID + 7 ))>");}
+	this->pid = (pid_t)((struct proc *)args[1])->p_pid;
+	this->sig = (int)arg2;
+}
+
+proc:::signal-clear,
+proc:::signal-discard,
+proc:::signal-send /* probe ID $(( $ID + 8 )) */
+{${TRACE:+
+	printf("<$(( $ID + 8 ))>");
+}
+	/* details = "<signal>[<num>] pid <pid>" */
+	this->details = strjoin(strjoin(signal_string[this->sig], "["),
+		strjoin(strjoin(lltostr(this->sig), "] pid "),
+		lltostr(this->pid)));
+}
+
+proc:::signal-send, proc:::signal-discard /* probe ID $(( $ID + 9 )) */
+{${TRACE:+
+	printf("<$(( $ID + 9 ))>");
+}
+	$( pproc -P _signal "(struct proc *)args[1]" )
+
+	this->details = strjoin(this->details,
+		strjoin(" -- ", this->args_signal));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 10 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print details
+	 */
+	printf("%s %s", probealias[probename], this->details);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/rw b/cddl/usr.sbin/dwatch/libexec/rw
new file mode 100644
index 000000000000..7ba36ae98818
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/rw
@@ -0,0 +1,72 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for read(2), write(2), or similar entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display data sent/received when read(2)/write(2) occurs
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+rw) : ${PROBE:=syscall::read:entry, syscall::write:entry} ;;
+ *) : ${PROBE:=syscall::$PROFILE:entry}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this size_t nbytes;
+this string bufstr;
+this string flow;
+this void * buf;
+this void * data;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * R/W
+	 */
+	this->flow = probefunc == "read" ? "<-" : "->";
+	this->buf = (void *)arg1;
+	this->nbytes = (size_t)arg2;
+
+	/*
+	 * Allocate temporary memory for, copy, and NUL-terminate the data
+	 */
+	this->data = alloca(this->nbytes + 1);
+	copyinto((uintptr_t)this->buf, this->nbytes, this->data);
+	bcopy("\0", (void *)((uintptr_t)this->data + this->nbytes), 1);
+
+	/*
+	 * Extract string from temporary memory
+	 */
+	this->bufstr = stringof(this->data);
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print read/write details
+	 */
+	printf("%s \"%s\" %d byte%s",
+		this->flow,
+		this->bufstr,
+		this->nbytes,
+		this->nbytes == 1 ? "" : "s");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/sched b/cddl/usr.sbin/dwatch/libexec/sched
new file mode 100644
index 000000000000..c43ad31ea2b7
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/sched
@@ -0,0 +1,107 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_sched(4) $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display CPU scheduling activity
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+sched)
+	: ${PROBE:=sched:::} ;;
+sched-cpu)
+	: ${PROBE:=sched:::off-cpu, sched:::on-cpu, sched:::remain-cpu} ;;
+sched-exec)
+	: ${PROBE:=sched:::sleep, sched:::wakeup} ;;
+sched-pri)
+	: ${PROBE:=sched:::change-pri, sched:::lend-pri} ;;
+sched-queue)
+	: ${PROBE:=sched:::dequeue, sched:::enqueue, sched:::load-change} ;;
+*)
+	: ${PROBE:=sched:::${PROFILE#sched-}}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this pid_t	pid;
+this string	args;
+this string	details;
+this u_char	curprio;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->args = this->args0;
+	this->details = "";
+	this->pid = this->pid0;
+}
+
+sched:::change-pri, sched:::dequeue, sched:::enqueue,
+sched:::lend-pri, sched:::off-cpu, sched:::surrender,
+sched:::tick, sched:::wakeup /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");}
+	this->curprio = (u_char)((struct thread *)args[0])->td_priority;
+
+	$( pproc -P _sched "(struct proc *)args[1]" )
+
+	this->args = this->args_sched;
+	this->pid = this->pid_sched;
+}
+
+sched:::enqueue /* probe ID $(( $ID + 2 )) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");}
+	/* details = "head" or "tail" */
+	this->details = (int)arg3 == 0 ? "tail" : "head";
+}
+
+sched:::change-pri, sched:::lend-pri /* probe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	/* details = "<curprio> -> arg2" */
+	this->details = strjoin(lltostr(this->curprio),
+		strjoin("->", lltostr((uint8_t)arg2)));
+}
+
+sched:::load-change /* probe ID $(( $ID + 4 )) */
+{${TRACE:+
+	printf("<$(( $ID + 4 ))>");}
+	/* details = "CPU<arg0> queue <arg1>" */
+	this->details = strjoin(strjoin("CPU", lltostr((int)arg0)),
+		strjoin(" queue ", lltostr((int)arg1)));
+}
+
+$PROBE /* probe ID $(( $ID + 5 )) */
+{${TRACE:+
+	printf("<$(( $ID + 5 ))>");}
+	/* details += " pid <pid> -- <proc args of pid>" */
+	this->details = strjoin(this->details, this->details == "" ? "" : " ");
+	this->details = strjoin(this->details, strjoin(
+		strjoin("pid ", lltostr(this->pid_sched)),
+		strjoin(" -- ", this->args)));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 6 ))
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print scheduling details
+	 */
+	printf("%s %s", probename, this->details);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/tcp b/cddl/usr.sbin/dwatch/libexec/tcp
new file mode 100644
index 000000000000..e86e17ea04c9
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/tcp
@@ -0,0 +1,229 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_tcp(4) connections $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display local/remote TCP addresses/ports and bytes sent/received for TCP I/O
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+tcp)
+	: ${PROBE:=$( echo \
+		tcp:::accept-established, \
+		tcp:::accept-refused, \
+		tcp:::connect-established, \
+		tcp:::connect-refused, \
+		tcp:::connect-request, \
+		tcp:::receive, \
+		tcp:::send, \
+		tcp:::state-change )} ;;
+tcp-accept)
+	: ${PROBE:=tcp:::accept-established, tcp:::accept-refused} ;;
+tcp-connect)
+	: ${PROBE:=$( echo \
+		tcp:::connect-established, \
+		tcp:::connect-refused, \
+		tcp:::connect-request )} ;;
+tcp-established)
+	: ${PROBE:=tcp:::accept-established, tcp:::connect-established} ;;
+tcp-init)
+	: ${PROBE:=$( echo \
+		tcp:::accept-established, \
+		tcp:::accept-refused, \
+		tcp:::connect-established, \
+		tcp:::connect-refused, \
+		tcp:::connect-request )} ;;
+tcp-io)
+	: ${PROBE:=tcp:::send, tcp:::receive} ;;
+tcp-refused)
+	: ${PROBE:=tcp:::accept-refused, tcp:::connect-refused} ;;
+tcp-status)
+	: ${PROBE:=$( echo \
+		tcp:::accept-established, \
+		tcp:::accept-refused, \
+		tcp:::connect-established, \
+		tcp:::connect-refused, \
+		tcp:::connect-request, \
+		tcp:::state-change )} ;;
+*)
+	: ${PROBE:=tcp:::${PROFILE#tcp-}}
+esac
+
+############################################################ GLOBALS
+
+#
+# This profile does not support these dwatch features
+# NB: They are disabled here so they have no effect when profile is loaded
+#
+unset EXECNAME		# -k name
+unset EXECREGEX		# -z regex
+unset GROUP		# -g group
+unset PID		# -p pid
+unset PSARGS		# affects -d
+unset PSTREE		# -R
+unset USER		# -u user
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this int32_t	from_state;
+this int32_t	to_state;
+this string	details;
+this string	flow;
+this string	local;
+this string	remote;
+this u_char	local6;
+this u_char	remote6;
+this u_char	slocal;
+this uint16_t	lport;
+this uint16_t	rport;
+this uint32_t	length;
+
+inline string probeflow[string name] =
+	name == "accept-established" ?	"<-" :
+	name == "accept-refused" ?	"X-" :
+	name == "connect-refused" ?	"-X" :
+	name == "connect-request" ?	"-?" :
+	name == "receive" ?		"<-" :
+	"->";
+
+inline u_char srclocal[string name] =
+	name == "accept-refused" ?	1 :
+	name == "connect-request" ?	1 :
+	name == "send" ?		1 :
+	0;
+
+/*
+ * TCPSTATES from <sys/netinet/tcp_fsm.h> used by netstat(1)
+ */
+inline string tcpstate[int32_t state] =
+        state == TCPS_CLOSED ?		"CLOSED" :
+        state == TCPS_LISTEN ?		"LISTEN" :
+        state == TCPS_SYN_SENT ?	"SYN_SENT" :
+        state == TCPS_SYN_RECEIVED ?	"SYN_RCVD" :
+        state == TCPS_ESTABLISHED ?	"ESTABLISHED" :
+        state == TCPS_CLOSE_WAIT ?	"CLOSE_WAIT" :
+        state == TCPS_FIN_WAIT_1 ?	"FIN_WAIT_1" :
+        state == TCPS_CLOSING ?		"CLOSING" :
+        state == TCPS_LAST_ACK ?	"LAST_ACK" :
+        state == TCPS_FIN_WAIT_2 ?	"FIN_WAIT_2" :
+        state == TCPS_TIME_WAIT ?	"TIME_WAIT" :
+        strjoin("UNKNOWN(", strjoin(lltostr(state), ")"));
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->details = "";
+
+	/*
+	 * dtrace_tcp(4)
+	 */
+	this->flow = probeflow[probename];
+}
+
+tcp:::accept-established,
+tcp:::accept-refused,
+tcp:::connect-established,
+tcp:::connect-refused,
+tcp:::connect-request,
+tcp:::receive,
+tcp:::send /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");
+}
+	/*
+	 * dtrace_tcp(4)
+	 */
+	this->slocal = srclocal[probename];
+
+	/*
+	 * ipinfo_t *
+	 */
+	this->local  = this->slocal ? args[2]->ip_saddr : args[2]->ip_daddr;
+	this->remote = this->slocal ? args[2]->ip_daddr : args[2]->ip_saddr;
+
+	/*
+	 * tcpinfo_t *
+	 */
+	this->lport = this->slocal ? args[4]->tcp_sport : args[4]->tcp_dport;
+	this->rport = this->slocal ? args[4]->tcp_dport : args[4]->tcp_sport;
+
+	/*
+	 * IPv6 support
+	 */
+	this->local6 = strstr(this->local, ":") != NULL ? 1 : 0;
+	this->remote6 = strstr(this->remote, ":") != NULL ? 1 : 0;
+	this->local = strjoin(strjoin(this->local6 ? "[" : "",
+		this->local), this->local6 ? "]" : "");
+	this->remote = strjoin(strjoin(this->remote6 ? "[" : "",
+		this->remote), this->remote6 ? "]" : "");
+}
+
+tcp:::state-change /* probe ID $(( $ID + 2 )) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");
+}
+	/*
+	 * tcpsinfo_t *
+	 */
+	this->local    = args[3]->tcps_laddr;
+	this->lport    = (uint16_t)args[3]->tcps_lport;
+	this->remote   = args[3]->tcps_raddr;
+	this->rport    = (uint16_t)args[3]->tcps_rport;
+	this->to_state = (int32_t)args[3]->tcps_state;
+
+	/*
+	 * tcplsinfo_t *
+	 */
+	this->from_state = (int32_t)args[5]->tcps_state;
+
+	/* flow = "[from state]->[to state]" */
+	this->flow = strjoin(tcpstate[this->from_state],
+		strjoin("->", tcpstate[this->to_state]));
+}
+
+tcp:::send, tcp:::receive /* pribe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	this->length = (uint32_t)args[2]->ip_plength -
+		(uint8_t)args[4]->tcp_offset;
+
+	/* details = " <length> byte<s>" */
+	this->details = strjoin(
+		strjoin(" ", lltostr(this->length)),
+		strjoin(" byte", this->length == 1 ? "" : "s"));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 4 ))
+
+############################################################ EVENT TAG
+
+exec 9<<EOF
+	printf("%s: ", "$PROFILE");
+EOF
+EVENT_TAG=$( cat <&9 )
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print details
+	 */
+	printf("%s:%u %s %s:%u%s",
+		this->local, this->lport,
+		this->flow,
+		this->remote, this->rport,
+		this->details);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/udp b/cddl/usr.sbin/dwatch/libexec/udp
new file mode 100644
index 000000000000..d6e031025dae
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/udp
@@ -0,0 +1,108 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_udp(4) $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Display local/remote UDP addresses/ports and bytes sent/received for UDP I/O
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+udp) : ${PROBE:=udp:::send, udp:::receive} ;;
+  *) : ${PROBE:=udp:::${PROFILE#udp-}}
+esac
+
+############################################################ GLOBALS
+
+#
+# This profile does not support these dwatch features
+# NB: They are disabled here so they have no effect when profile is loaded
+#
+unset EXECNAME		# -k name
+unset EXECREGEX		# -z regex
+unset GROUP		# -g group
+unset PID		# -p pid
+unset PSARGS		# affects -d
+unset PSTREE		# -R
+unset USER		# -u user
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this string	flow;
+this string	local;
+this string	remote;
+this u_char	local6;
+this u_char	recv;
+this u_char	remote6;
+this uint16_t	length;
+this uint16_t	lport;
+this uint16_t	rport;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");
+}
+	/*
+	 * dtrace_udp(4)
+	 */
+	this->recv = probename == "receive" ? 1 : 0;
+	this->flow = this->recv ? "<-" : "->";
+
+	/*
+	 * ipinfo_t *
+	 */
+	this->local  = this->recv ? args[2]->ip_daddr : args[2]->ip_saddr;
+	this->remote = this->recv ? args[2]->ip_saddr : args[2]->ip_daddr;
+
+	/*
+	 * udpinfo_t *
+	 */
+	this->length = (uint16_t)args[4]->udp_length;
+	this->lport  = this->recv ? args[4]->udp_dport : args[4]->udp_sport;
+	this->rport  = this->recv ? args[4]->udp_sport : args[4]->udp_dport;
+
+	/*
+	 * IPv6 support
+	 */
+	this->local6 = strstr(this->local, ":") != NULL ? 1 : 0;
+	this->remote6 = strstr(this->remote, ":") != NULL ? 1 : 0;
+	this->local = strjoin(strjoin(this->local6 ? "[" : "",
+		this->local), this->local6 ? "]" : "");
+	this->remote = strjoin(strjoin(this->remote6 ? "[" : "",
+		this->remote), this->remote6 ? "]" : "");
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT TAG
+
+exec 9<<EOF
+	printf("%s: ", "$PROFILE");
+EOF
+EVENT_TAG=$( cat <&9 )
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print network I/O details
+	 */
+	printf("%s:%u %s %s:%u %d byte%s",
+		this->local, this->lport,
+		this->flow,
+		this->remote, this->rport,
+		this->length,
+		this->length == 1 ? "" : "s");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/vop_create b/cddl/usr.sbin/dwatch/libexec/vop_create
new file mode 100644
index 000000000000..fe971e71d1f0
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/vop_create
@@ -0,0 +1,194 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for VOP_CREATE(9) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print filesystem paths being operated-on by VOP_CREATE(9) [or similar]
+# NB: All paths are shown even if error prevents operation.
+#
+############################################################ PROBE
+
+: ${PROBE:=vfs:vop:$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->vp = (struct vnode *)arg0;
+	this->ncp = this->vp != NULL ?
+		this->vp->v_cache_dst.tqh_first : 0;
+	this->fi_name = args[1] ? (
+		args[1]->a_cnp != NULL ?
+			stringof(args[1]->a_cnp->cn_nameptr) : ""
+	) : "";
+	this->mount = this->vp != NULL ?
+		this->vp->v_mount : NULL; /* ptr to vfs we are in */
+	this->fi_fs = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_fstypename) : "";
+	this->fi_mount = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_mntonname) : "";
+	this->d_name = args[0]->v_cache_dd != NULL ?
+		stringof(args[0]->v_cache_dd->nc_name) : "";
+
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (DEPTH = 1; DEPTH <= MAX_DEPTH + 1; DEPTH++) {
+				gsub(/DEPTH/, DEPTH)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->nameDEPTH = "";
+	EOFDEPTH
+	)
+}
+
+$PROBE /this->vp == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == "" ||
+	this->fi_name == ""/ /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");}
+	this->ncp = 0;
+}
+
+/*********************************************************/
+
+$PROBE /this->ncp/ /* probe ID $(( $ID + 2 )) (depth 1) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");}
+	this->dvp = this->ncp->nc_dvp != NULL ?
+		this->ncp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name1 = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+$PROBE /this->name1 == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == "" ||
+	this->name1 == "/" || this->name1 == ""/ /* probe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	this->dvp = 0;
+}
+
+/*********************************************************/
+
+/*
+ * BEGIN Pathname-depth iterators
+ */
+
+$( awk -v ID=$(( $ID + 4 )) -v MAX_DEPTH=$MAX_DEPTH '
+	{ buf = buf $0 "\n" }
+	END {
+		sub(/\n$/, "", buf)
+		for (DEPTH = 2; DEPTH <= MAX_DEPTH; DEPTH++) {
+			$0 = buf
+			gsub(/DEPTH/, DEPTH)
+			gsub(/IDNUM/, ID++)
+			print
+		}
+	}
+' <<EOFDEPTH
+$PROBE /this->dvp/ /* probe ID IDNUM (depth DEPTH) */
+{${TRACE:+
+	printf("<IDNUM>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->nameDEPTH = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+EOFDEPTH
+)
+
+$PROBE /this->dvp/ /* probe ID $(( $ID + $MAX_DEPTH + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 3 ))>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name$(( $MAX_DEPTH + 1 )) = this->dvp != 0 ? (
+		this->dvp->nc_dvp != NULL ? "..." : ""
+	) : "";
+}
+
+/*
+ * END Pathname-depth iterators
+ */
+
+/*********************************************************/
+
+$PROBE /this->fi_mount != 0/ /* probe ID $(( $ID + $MAX_DEPTH + 4 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 4 ))>");
+}
+	/*
+	 * Join full path
+	 * NB: Up-to but not including the parent directory (joined below)
+	 */
+	this->path = this->fi_mount;
+	this->path = strjoin(this->path, this->fi_mount != 0 ? (
+		this->fi_mount == "/" ? "" : "/"
+	) : "/");
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (N = MAX_DEPTH + 1; N > 0; N--) {
+				gsub(/N/, N)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->path = strjoin(this->path,
+	\	strjoin(this->nameN, this->nameN != "" ? "/" : ""));
+	EOFDEPTH
+	)
+
+	/* Join the parent directory name */
+	this->path = strjoin(this->path, strjoin(this->name =
+		(this->d_name != 0 ? this->d_name : ""),
+		this->name != "" ? "/" : ""));
+
+	/* Join the entry name */
+	this->path = strjoin(this->path,
+		this->name = (this->fi_name != 0 ? this->fi_name : ""));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + $MAX_DEPTH + 5 ))
+
+############################################################ EVENT ACTION
+
+EVENT_TEST="this->fi_mount != 0"
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print full path
+	 */
+	printf("%s", this->path);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/vop_readdir b/cddl/usr.sbin/dwatch/libexec/vop_readdir
new file mode 100644
index 000000000000..c760ea8e49b6
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/vop_readdir
@@ -0,0 +1,186 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for VOP_READDIR(9) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print directory paths being read by VOP_READDIR(9) [or similar]
+# NB: All paths are shown even if error prevents their reading.
+#
+############################################################ PROBE
+
+: ${PROBE:=vfs:vop:$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->vp = (struct vnode *)arg0;
+	this->ncp = this->vp != NULL ?
+		this->vp->v_cache_dst.tqh_first : 0;
+	this->mount = this->vp != NULL ?
+		this->vp->v_mount : NULL; /* ptr to vfs we are in */
+	this->fi_fs = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_fstypename) : "";
+	this->fi_mount = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_mntonname) : "";
+	this->d_name = args[0]->v_cache_dd != NULL ?
+		stringof(args[0]->v_cache_dd->nc_name) : "";
+
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (DEPTH = 1; DEPTH <= MAX_DEPTH + 1; DEPTH++) {
+				gsub(/DEPTH/, DEPTH)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->nameDEPTH = "";
+	EOFDEPTH
+	)
+}
+
+$PROBE /this->vp == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == ""/ /* probe ID $((
+		$ID + 1
+	)) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");}
+	this->ncp = 0;
+}
+
+/*********************************************************/
+
+$PROBE /this->ncp/ /* probe ID $(( $ID + 2 )) (depth 1) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");}
+	this->dvp = this->ncp->nc_dvp != NULL ?
+		this->ncp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name1 = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+$PROBE /this->name1 == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == "" ||
+	this->name1 == "/" || this->name1 == ""/ /* probe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	this->dvp = 0;
+}
+
+/*********************************************************/
+
+/*
+ * BEGIN Pathname-depth iterators
+ */
+
+$( awk -v ID=$(( $ID + 4 )) -v MAX_DEPTH=$MAX_DEPTH '
+	{ buf = buf $0 "\n" }
+	END {
+		sub(/\n$/, "", buf)
+		for (DEPTH = 2; DEPTH <= MAX_DEPTH; DEPTH++) {
+			$0 = buf
+			gsub(/DEPTH/, DEPTH)
+			gsub(/IDNUM/, ID++)
+			print
+		}
+	}
+' <<EOFDEPTH
+$PROBE /this->dvp/ /* probe ID IDNUM (depth DEPTH) */
+{${TRACE:+
+	printf("<IDNUM>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->nameDEPTH = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+EOFDEPTH
+)
+
+$PROBE /this->dvp/ /* probe ID $(( $ID + $MAX_DEPTH + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 3 ))>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name$(( $MAX_DEPTH + 1 )) = this->dvp != 0 ? (
+		this->dvp->nc_dvp != NULL ? "..." : ""
+	) : "";
+}
+
+/*
+ * END Pathname-depth iterators
+ */
+
+/*********************************************************/
+
+$PROBE /this->fi_mount != 0/ /* probe ID $(( $ID + $MAX_DEPTH + 4 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 4 ))>");
+}
+	/*
+	 * Join full path
+	 * NB: Up-to but not including the parent directory (joined below)
+	 */
+	this->path = this->fi_mount;
+	this->path = strjoin(this->path, this->fi_mount != 0 ? (
+		this->fi_mount == "/" ? "" : "/"
+	) : "/");
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (N = MAX_DEPTH + 1; N > 0; N--) {
+				gsub(/N/, N)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->path = strjoin(this->path,
+	\	strjoin(this->nameN, this->nameN != "" ? "/" : ""));
+	EOFDEPTH
+	)
+
+	/* Join the parent directory name */
+	this->path = strjoin(this->path, strjoin(this->name =
+		(this->d_name != 0 ? this->d_name : ""),
+		this->name != "" ? "/" : ""));
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + $MAX_DEPTH + 5 ))
+
+############################################################ EVENT ACTION
+
+EVENT_TEST="this->fi_mount != 0"
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print full path
+	 */
+	printf("%s", this->path);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/vop_rename b/cddl/usr.sbin/dwatch/libexec/vop_rename
new file mode 100644
index 000000000000..a1263ba4578e
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/vop_rename
@@ -0,0 +1,300 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for VOP_RENAME(9) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print filesystem paths being renamed by VOP_RENAME(9) [or similar]
+# NB: All paths are shown even if error prevents their rename.
+#
+############################################################ PROBE
+
+: ${PROBE:=vfs:vop:$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->fvp = args[1] ? args[1]->a_fdvp : NULL;
+	this->fncp = this->fvp != NULL ?
+		this->fvp->v_cache_dst.tqh_first : 0;
+	this->ffi_name = args[1] ? (
+		args[1]->a_fcnp != NULL ?
+			stringof(args[1]->a_fcnp->cn_nameptr) : ""
+	) : "";
+	this->fmount = this->fvp != NULL ?
+		this->fvp->v_mount : NULL; /* ptr to vfs we are in */
+	this->ffi_fs = this->fmount != NULL ?
+		stringof(this->fmount->mnt_stat.f_fstypename) : "";
+	this->ffi_mount = this->fmount != NULL ?
+		stringof(this->fmount->mnt_stat.f_mntonname) : "";
+	this->fd_name = args[0]->v_cache_dd != NULL ?
+		stringof(args[0]->v_cache_dd->nc_name) : "";
+
+	this->tvp = args[1] ? args[1]->a_tdvp : NULL;
+	this->tncp = this->tvp != NULL ?
+		this->tvp->v_cache_dst.tqh_first : 0;
+	this->tfi_name = args[1] ? (
+		args[1]->a_tcnp != NULL ?
+			stringof(args[1]->a_tcnp->cn_nameptr) : ""
+	) : "";
+	this->tmount = this->tvp != NULL ?
+		this->tvp->v_mount : NULL; /* ptr to vfs we are in */
+	this->tfi_fs = this->tmount != NULL ?
+		stringof(this->tmount->mnt_stat.f_fstypename) : "";
+	this->tfi_mount = this->tmount != NULL ?
+		stringof(this->tmount->mnt_stat.f_mntonname) : "";
+	this->td_name = this->tvp != NULL ? (
+		this->tvp->v_cache_dd != NULL ?
+			stringof(this->tvp->v_cache_dd->nc_name) : ""
+	) : "";
+
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (DEPTH = 1; DEPTH <= MAX_DEPTH + 1; DEPTH++) {
+				gsub(/DEPTH/, DEPTH)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->fnameDEPTH = this->tnameDEPTH = "";
+	EOFDEPTH
+	)
+}
+
+$PROBE /this->fvp == 0 || this->ffi_fs == 0 ||
+	this->ffi_fs == "devfs" || this->ffi_fs == "" ||
+	this->ffi_name == ""/ /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");}
+	this->fncp = 0;
+}
+
+$PROBE /this->tvp == 0 || this->tfi_fs == 0 ||
+	this->tfi_fs == "devfs" || this->tfi_fs == "" ||
+	this->tfi_name == ""/ /* probe ID $(( $ID + 2 )) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");}
+	this->tncp = 0;
+}
+
+/*********************************************************/
+
+$PROBE /this->fncp/ /* probe ID $(( $ID + 3 )) (depth 1) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	this->fdvp = this->fncp->nc_dvp != NULL ?
+		this->fncp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->fname1 = this->fdvp != 0 ? (
+		this->fdvp->nc_name != 0 ? stringof(this->fdvp->nc_name) : ""
+	) : "";
+}
+
+$PROBE /this->tncp/ /* probe ID $(( $ID + 4 )) (depth 1) */
+{${TRACE:+
+	printf("<$(( $ID + 4 ))>");}
+	this->tdvp = this->tncp->nc_dvp != NULL ?
+		this->tncp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->tname1 = this->tdvp != 0 ? (
+		this->tdvp->nc_name != 0 ? stringof(this->tdvp->nc_name) : ""
+	) : "";
+}
+
+$PROBE /this->fname1 == 0 || this->ffi_fs == 0 ||
+	this->ffi_fs == "devfs" || this->ffi_fs == "" ||
+	this->fname1 == "/" || this->fname1 == ""/ /* probe ID $((
+		$ID + 5
+	)) */
+{${TRACE:+
+	printf("<$(( $ID + 5 ))>");}
+	this->fdvp = 0;
+}
+
+$PROBE /this->tname1 == 0 || this->tfi_fs == 0 ||
+	this->tfi_fs == "devfs" || this->tfi_fs == "" ||
+	this->tname1 == "/" || this->tname1 == ""/ /* probe ID $((
+		$ID + 6
+	)) */
+{${TRACE:+
+	printf("<$(( $ID + 6 ))>");}
+	this->tdvp = 0;
+}
+
+/*********************************************************/
+
+/*
+ * BEGIN Pathname-depth iterators
+ */
+
+$( awk -v ID=$(( $ID + 7 )) -v MAX_DEPTH=$MAX_DEPTH '
+	{ buf = buf $0 "\n" }
+	END {
+		sub(/\n$/, "", buf)
+		for (DEPTH = 2; DEPTH <= MAX_DEPTH; DEPTH++) {
+			$0 = buf
+			gsub(/DEPTH/, DEPTH)
+			gsub(/IDNUM1/, ID)
+			gsub(/IDNUM2/, ID + 1)
+			print
+			ID = ID + 2
+		}
+	}
+' <<EOFDEPTH
+$PROBE /this->fdvp/ /* probe ID IDNUM1 (depth DEPTH) */
+{${TRACE:+
+	printf("<IDNUM1>");}
+	this->fdvp = this->fdvp->nc_dvp != NULL ?
+		this->fdvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->fnameDEPTH = this->fdvp != 0 ? (
+		this->fdvp->nc_name != 0 ? stringof(this->fdvp->nc_name) : ""
+	) : "";
+}
+$PROBE /this->tdvp/ /* probe ID IDNUM2 (depth DEPTH) */
+{${TRACE:+
+	printf("<IDNUM2>");}
+	this->tdvp = this->tdvp->nc_dvp != NULL ?
+		this->tdvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->tnameDEPTH = this->tdvp != 0 ? (
+		this->tdvp->nc_name != 0 ? stringof(this->tdvp->nc_name) : ""
+	) : "";
+}
+
+EOFDEPTH
+)
+
+$PROBE /this->fdvp/ /* probe ID $(( $ID + $MAX_DEPTH * 2 + 5 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH * 2 + 5 ))>");}
+	this->fdvp = this->fdvp->nc_dvp != NULL ?
+		this->fdvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->fname$(( $MAX_DEPTH + 1 )) = this->fdvp != 0 ? (
+		this->fdvp->nc_dvp != NULL ? "..." : ""
+	) : "";
+}
+$PROBE /this->tdvp/ /* probe ID $(( $ID + $MAX_DEPTH * 2 + 6 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH * 2 + 6 ))>");}
+	this->tdvp = this->tdvp->nc_dvp != NULL ?
+		this->tdvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->tname$(( $MAX_DEPTH + 1 )) = this->tdvp != 0 ? (
+		this->tdvp->nc_dvp != NULL ? "..." : ""
+	) : "";
+}
+
+/*
+ * END Pathname-depth iterators
+ */
+
+/*********************************************************/
+
+$PROBE /this->ffi_mount != 0 && this->tfi_mount != 0/ /* probe ID $((
+	$ID + $MAX_DEPTH * 2 + 7
+)) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH * 2 + 7 ))>");
+}
+	/*
+	 * Join 'from' full path
+	 * NB: Up-to but not including the parent directory (joined below)
+	 */
+	this->fpath = this->ffi_mount;
+	this->fpath = strjoin(this->fpath, this->ffi_mount != 0 ? (
+		this->ffi_mount == "/" ? "" : "/"
+	) : "/");
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (N = MAX_DEPTH + 1; N > 0; N--) {
+				gsub(/N/, N)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->fpath = strjoin(this->fpath,
+	\	strjoin(this->fnameN, this->fnameN != "" ? "/" : ""));
+	EOFDEPTH
+	)
+
+	/* Join the 'from' parent directory name */
+	this->fpath = strjoin(this->fpath, strjoin(this->fname =
+		(this->fd_name != 0 ? this->fd_name : ""),
+		this->fname != "" ? "/" : ""));
+
+	/* Join the 'from' entry name */
+	this->fpath = strjoin(this->fpath,
+		this->fname = (this->ffi_name != 0 ? this->ffi_name : ""));
+
+	/*
+	 * Join 'to' full path
+	 * NB: Up-to but not including the parent directory (joined below)
+	 */
+	this->tpath = this->tfi_mount;
+	this->tpath = strjoin(this->tpath, this->tfi_mount != 0 ? (
+		this->tfi_mount == "/" ? "" : "/"
+	) : "/");
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (N = MAX_DEPTH + 1; N > 0; N--) {
+				gsub(/N/, N)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->tpath = strjoin(this->tpath,
+	\	strjoin(this->tnameN, this->tnameN != "" ? "/" : ""));
+	EOFDEPTH
+	)
+
+	/* Join the 'to' parent directory name */
+	this->tpath = strjoin(this->tpath, strjoin(this->tname =
+		(this->td_name != 0 ? this->td_name : ""),
+		this->tname != "" ? "/" : ""));
+
+	/* Join the 'to' entry name */
+	this->tpath = strjoin(this->tpath,
+		this->tname = (this->tfi_name != 0 ? this->tfi_name : ""));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + $MAX_DEPTH * 2 + 8 ))
+
+############################################################ EVENT ACTION
+
+EVENT_TEST="this->ffi_mount != 0 && this->tfi_mount != 0"
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print 'from' and 'to' full paths
+	 */
+	printf("%s -> %s", this->fpath, this->tpath);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/vop_symlink b/cddl/usr.sbin/dwatch/libexec/vop_symlink
new file mode 100644
index 000000000000..a2a155a41436
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/vop_symlink
@@ -0,0 +1,195 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for VOP_SYMLINK(9) [or similar] entry $
+# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $FreeBSD$
+#
+############################################################ DESCRIPTION
+#
+# Print symlink paths being created by VOP_SYMLINK(9) [or similar]
+# NB: All paths are shown even if error prevents their creation.
+#
+############################################################ PROBE
+
+: ${PROBE:=vfs:vop:$PROFILE:entry}
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+$PROBE /* probe ID $ID */
+{${TRACE:+
+	printf("<$ID>");}
+	this->vp = (struct vnode *)arg0;
+	this->ncp = this->vp != NULL ?
+		this->vp->v_cache_dst.tqh_first : 0;
+	this->target = args[1] ? args[1]->a_target : "";
+	this->fi_name = args[1] ? (
+		args[1]->a_cnp != NULL ?
+			stringof(args[1]->a_cnp->cn_nameptr) : ""
+	) : "";
+	this->mount = this->vp != NULL ?
+		this->vp->v_mount : NULL; /* ptr to vfs we are in */
+	this->fi_fs = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_fstypename) : "";
+	this->fi_mount = this->mount != NULL ?
+		stringof(this->mount->mnt_stat.f_mntonname) : "";
+	this->d_name = args[0]->v_cache_dd != NULL ?
+		stringof(args[0]->v_cache_dd->nc_name) : "";
+
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (DEPTH = 1; DEPTH <= MAX_DEPTH + 1; DEPTH++) {
+				gsub(/DEPTH/, DEPTH)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->nameDEPTH = "";
+	EOFDEPTH
+	)
+}
+
+$PROBE /this->vp == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == "" ||
+	this->fi_name == ""/ /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+	printf("<$(( $ID + 1 ))>");}
+	this->ncp = 0;
+}
+
+/*********************************************************/
+
+$PROBE /this->ncp/ /* probe ID $(( $ID + 2 )) (depth 1) */
+{${TRACE:+
+	printf("<$(( $ID + 2 ))>");}
+	this->dvp = this->ncp->nc_dvp != NULL ?
+		this->ncp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name1 = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+$PROBE /this->name1 == 0 || this->fi_fs == 0 ||
+	this->fi_fs == "devfs" || this->fi_fs == "" ||
+	this->name1 == "/" || this->name1 == ""/ /* probe ID $(( $ID + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + 3 ))>");}
+	this->dvp = 0;
+}
+
+/*********************************************************/
+
+/*
+ * BEGIN Pathname-depth iterators
+ */
+
+$( awk -v ID=$(( $ID + 4 )) -v MAX_DEPTH=$MAX_DEPTH '
+	{ buf = buf $0 "\n" }
+	END {
+		sub(/\n$/, "", buf)
+		for (DEPTH = 2; DEPTH <= MAX_DEPTH; DEPTH++) {
+			$0 = buf
+			gsub(/DEPTH/, DEPTH)
+			gsub(/IDNUM/, ID++)
+			print
+		}
+	}
+' <<EOFDEPTH
+$PROBE /this->dvp/ /* probe ID IDNUM (depth DEPTH) */
+{${TRACE:+
+	printf("<IDNUM>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->nameDEPTH = this->dvp != 0 ? (
+		this->dvp->nc_name != 0 ? stringof(this->dvp->nc_name) : ""
+	) : "";
+}
+
+EOFDEPTH
+)
+
+$PROBE /this->dvp/ /* probe ID $(( $ID + $MAX_DEPTH + 3 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 3 ))>");}
+	this->dvp = this->dvp->nc_dvp != NULL ?
+		this->dvp->nc_dvp->v_cache_dst.tqh_first : 0;
+	this->name$(( $MAX_DEPTH + 1 )) = this->dvp != 0 ? (
+		this->dvp->nc_dvp != NULL ? "..." : ""
+	) : "";
+}
+
+/*
+ * END Pathname-depth iterators
+ */
+
+/*********************************************************/
+
+$PROBE /this->fi_mount != 0/ /* probe ID $(( $ID + $MAX_DEPTH + 4 )) */
+{${TRACE:+
+	printf("<$(( $ID + $MAX_DEPTH + 4 ))>");
+}
+	/*
+	 * Join full path
+	 * NB: Up-to but not including the parent directory (joined below)
+	 */
+	this->path = this->fi_mount;
+	this->path = strjoin(this->path, this->fi_mount != 0 ? (
+		this->fi_mount == "/" ? "" : "/"
+	) : "/");
+	$( awk -v MAX_DEPTH=$MAX_DEPTH '
+		{ sub(/^\\\t/, "\t") }
+		{ buf = buf "\t" $0 "\n" }
+		END {
+			sub(/\n$/, "", buf)
+			$0 = buf
+			sub(/^[[:space:]]*/, "")
+			for (N = MAX_DEPTH + 1; N > 0; N--) {
+				gsub(/N/, N)
+				print
+				$0 = buf
+			}
+		}
+	' <<-EOFDEPTH
+	this->path = strjoin(this->path,
+	\	strjoin(this->nameN, this->nameN != "" ? "/" : ""));
+	EOFDEPTH
+	)
+
+	/* Join the parent directory name */
+	this->path = strjoin(this->path, strjoin(this->name =
+		(this->d_name != 0 ? this->d_name : ""),
+		this->name != "" ? "/" : ""));
+
+	/* Join the entry name */
+	this->path = strjoin(this->path,
+		this->name = (this->fi_name != 0 ? this->fi_name : ""));
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + $MAX_DEPTH + 5 ))
+
+############################################################ EVENT ACTION
+
+EVENT_TEST="this->fi_mount != 0"
+
+############################################################ EVENT DETAILS
+
+exec 9<<EOF
+	/*
+	 * Print full path and target
+	 */
+	printf("%s -> %s", this->path, this->target);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+
+################################################################################
+# END
+################################################################################