ctlstat: add prometheus output

When invoked by inetd, ctlstat -P will now produce output suitable for
ingestion into Prometheus.

It's a drop-in replacement for https://github.com/Gandi/ctld_exporter,
except that it doesn't report the number of initiators per target, and
it does report time and dma_time.

MFC after:	2 weeks
Sponsored by:	Axcient
Relnotes:	yes
Reviewed by: 	bapt, bcr
Differential Revision: https://reviews.freebsd.org/D29901

3 files changed, 257 insertions, 10 deletions

diff --git a/usr.bin/ctlstat/Makefile b/usr.bin/ctlstat/Makefile
index 31182374f381..53b7a59fae23 100644
--- a/usr.bin/ctlstat/Makefile
+++ b/usr.bin/ctlstat/Makefile
@@ -5,4 +5,6 @@ MAN=		ctlstat.8
 SDIR=		${SRCTOP}/sys
 CFLAGS+=	-I${SDIR}
 
+LIBADD=		sbuf bsdxml
+
 .include <bsd.prog.mk>
diff --git a/usr.bin/ctlstat/ctlstat.8 b/usr.bin/ctlstat/ctlstat.8
index 28e6d6fd1462..2512803f798a 100644
--- a/usr.bin/ctlstat/ctlstat.8
+++ b/usr.bin/ctlstat/ctlstat.8
@@ -34,7 +34,7 @@
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
 .\" $FreeBSD$
 .\"
-.Dd January 9, 2017
+.Dd April 22, 2021
 .Dt CTLSTAT 8
 .Os
 .Sh NAME
@@ -48,6 +48,7 @@
 .Op Fl d
 .Op Fl D
 .Op Fl j
+.Op Fl P
 .Op Fl l Ar lun
 .Op Fl n Ar numdevs
 .Op Fl p Ar port
@@ -83,6 +84,19 @@ Suppress display of the header.
 JSON dump mode.
 Dump statistics every 30 seconds in JavaScript Object Notation (JSON) format.
 No statistics are computed in this mode, only raw numbers are displayed.
+.It Fl P
+Prometheus dump mode.
+Dump statistics in a format suitable for ingestion into Prometheus.
+When invoked with this option,
+.Nm
+dumps once, regardless of the
+.Fl t
+option.
+This option is especially useful when invoked by
+.Xr inetd 8 .
+See the comments in
+.Pa /etc/inetd.conf
+for an example configuration.
 .It Fl l Ar lun
 Request statistics for the specified LUN.
 .It Fl n Ar numdevs
@@ -116,7 +130,13 @@ every 10 seconds.
 .Xr camcontrol 8 ,
 .Xr ctladm 8 ,
 .Xr ctld 8 ,
-.Xr iostat 8
+.Xr iostat 8 ,
+.Lk
+Prometheus project:
+.Pa https://prometheus.io/ .
+.Pp
+Prometheus exposition formats:
+.Lk https://prometheus.io/docs/instrumenting/exposition_formats/ .
 .Sh AUTHORS
 .An Ken Merry Aq Mt ken@FreeBSD.org
 .An Will Andrews Aq Mt will@FreeBSD.org
diff --git a/usr.bin/ctlstat/ctlstat.c b/usr.bin/ctlstat/ctlstat.c
index 6586430172ce..50b9e3b1445b 100644
--- a/usr.bin/ctlstat/ctlstat.c
+++ b/usr.bin/ctlstat/ctlstat.c
@@ -41,19 +41,23 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/ioctl.h>
-#include <sys/types.h>
 #include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysctl.h>
-#include <sys/resource.h>
-#include <sys/queue.h>
 #include <sys/callout.h>
+#include <sys/ioctl.h>
+#include <sys/queue.h>
+#include <sys/resource.h>
+#include <sys/sbuf.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include <bsdxml.h>
+#include <malloc_np.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
+#include <inttypes.h>
 #include <getopt.h>
 #include <string.h>
 #include <errno.h>
@@ -76,8 +80,8 @@ __FBSDID("$FreeBSD$");
 
 static int ctl_stat_bits;
 
-static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
-static const char *ctlstat_usage = "Usage:  ctlstat [-CDdjht] [-l lunnum]"
+static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:";
+static const char *ctlstat_usage = "Usage:  ctlstat [-CDPdjht] [-l lunnum]"
 				   "[-c count] [-n numdevs] [-w wait]\n";
 
 struct ctl_cpu_stats {
@@ -92,6 +96,7 @@ typedef enum {
 	CTLSTAT_MODE_STANDARD,
 	CTLSTAT_MODE_DUMP,
 	CTLSTAT_MODE_JSON,
+	CTLSTAT_MODE_PROMETHEUS,
 } ctlstat_mode_types;
 
 #define	CTLSTAT_FLAG_CPU		(1 << 0)
@@ -129,6 +134,15 @@ struct ctlstat_context {
 	int header_interval;
 };
 
+struct cctl_portlist_data {
+	int level;
+	struct sbuf *cur_sb[32];
+	int lun;
+	int ntargets;
+	char *target;
+	char **targets;
+};
+
 #ifndef min
 #define	min(x,y)	(((x) < (y)) ? (x) : (y))
 #endif
@@ -381,6 +395,200 @@ ctlstat_json(struct ctlstat_context *ctx) {
 	printf("]}");
 }
 
+#define CTLSTAT_PROMETHEUS_LOOP(field) \
+	for (i = n = 0; i < ctx->cur_items; i++) { \
+		if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+		    (int)stats[i].item) == 0) \
+			continue; \
+		for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+			int lun = stats[i].item; \
+			if (lun >= targdata.ntargets) \
+			    errx(1, "LUN %u out of range", lun); \
+			printf("iscsi_target_" #field "{" \
+			    "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+			    "\n", \
+			    lun, targdata.targets[lun], iotypes[iotype], \
+			    stats[i].field[iotype]); \
+		} \
+	} \
+
+#define CTLSTAT_PROMETHEUS_TIMELOOP(field) \
+	for (i = n = 0; i < ctx->cur_items; i++) { \
+		if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+		    (int)stats[i].item) == 0) \
+			continue; \
+		for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+			uint64_t us; \
+			struct timespec ts; \
+			int lun = stats[i].item; \
+			if (lun >= targdata.ntargets) \
+			    errx(1, "LUN %u out of range", lun); \
+			bintime2timespec(&stats[i].field[iotype], &ts); \
+			us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \
+			printf("iscsi_target_" #field "{" \
+			    "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+			    "\n", \
+			    lun, targdata.targets[lun], iotypes[iotype], us); \
+		} \
+	} \
+
+static void
+cctl_start_pelement(void *user_data, const char *name, const char **attr __unused)
+{
+	struct cctl_portlist_data* targdata = user_data;
+
+	targdata->level++;
+	if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) /
+	    sizeof(targdata->cur_sb[0])))
+		errx(1, "%s: too many nesting levels, %zd max", __func__,
+		     sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0]));
+
+	targdata->cur_sb[targdata->level] = sbuf_new_auto();
+	if (targdata->cur_sb[targdata->level] == NULL)
+		err(1, "%s: Unable to allocate sbuf", __func__);
+
+	if (strcmp(name, "targ_port") == 0) {
+		targdata->lun = -1;
+		free(targdata->target);
+		targdata->target = NULL;
+	}
+}
+
+static void
+cctl_char_phandler(void *user_data, const XML_Char *str, int len)
+{
+	struct cctl_portlist_data *targdata = user_data;
+
+	sbuf_bcat(targdata->cur_sb[targdata->level], str, len);
+}
+
+static void
+cctl_end_pelement(void *user_data, const char *name)
+{
+	struct cctl_portlist_data* targdata = user_data;
+	char *str;
+
+	if (targdata->cur_sb[targdata->level] == NULL)
+		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
+		     targdata->level, name);
+
+	if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0)
+		err(1, "%s: sbuf_finish", __func__);
+	str = strdup(sbuf_data(targdata->cur_sb[targdata->level]));
+	if (str == NULL)
+		err(1, "%s can't allocate %zd bytes for string", __func__,
+		    sbuf_len(targdata->cur_sb[targdata->level]));
+
+	sbuf_delete(targdata->cur_sb[targdata->level]);
+	targdata->cur_sb[targdata->level] = NULL;
+	targdata->level--;
+
+	if (strcmp(name, "target") == 0) {
+		free(targdata->target);
+		targdata->target = str;
+	} else if (strcmp(name, "lun") == 0) {
+		targdata->lun = atoi(str);
+		free(str);
+	} else if (strcmp(name, "targ_port") == 0) {
+		if (targdata->lun >= 0 && targdata->target != NULL) {
+			if (targdata->lun >= targdata->ntargets) {
+				/*
+				 * This can happen for example if there are
+				 * holes in CTL's lunlist.
+				 */
+				targdata->ntargets = MAX(targdata->ntargets * 2,
+					targdata->lun + 1);
+				size_t newsize = targdata->ntargets *
+					sizeof(char*);
+				targdata->targets = rallocx(targdata->targets,
+					newsize, MALLOCX_ZERO);
+			}
+			free(targdata->targets[targdata->lun]);
+			targdata->targets[targdata->lun] = targdata->target;
+			targdata->target = NULL;
+		}
+		free(str);
+	} else {
+		free(str);
+	}
+}
+
+static void
+ctlstat_prometheus(int fd, struct ctlstat_context *ctx) {
+	struct ctl_io_stats *stats = ctx->cur_stats;
+	struct ctl_lun_list list;
+	struct cctl_portlist_data targdata;
+	XML_Parser parser;
+	char *port_str = NULL;
+	int iotype, i, n, retval;
+	int port_len = 4096;
+
+	bzero(&targdata, sizeof(targdata));
+	targdata.ntargets = ctx->cur_items;
+	targdata.targets = calloc(targdata.ntargets, sizeof(char*));
+retry:
+	port_str = (char *)realloc(port_str, port_len);
+	bzero(&list, sizeof(list));
+	list.alloc_len = port_len;
+	list.status = CTL_LUN_LIST_NONE;
+	list.lun_xml = port_str;
+	if (ioctl(fd, CTL_PORT_LIST, &list) == -1)
+		err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__);
+	if (list.status == CTL_LUN_LIST_ERROR) {
+		warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s",
+		      __func__, list.error_str);
+	} else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
+		port_len <<= 1;
+		goto retry;
+	}
+
+	parser = XML_ParserCreate(NULL);
+	if (parser == NULL)
+		err(1, "%s: Unable to create XML parser", __func__);
+	XML_SetUserData(parser, &targdata);
+	XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
+	XML_SetCharacterDataHandler(parser, cctl_char_phandler);
+
+	retval = XML_Parse(parser, port_str, strlen(port_str), 1);
+	if (retval != 1) {
+		errx(1, "%s: Unable to parse XML: Error %d", __func__,
+		    XML_GetErrorCode(parser));
+	}
+	XML_ParserFree(parser);
+
+	/*
+	 * NB: Some clients will print a warning if we don't set Content-Length,
+	 * but they still work.  And the data still gets into Prometheus.
+	 */
+	printf("HTTP/1.1 200 OK\r\n"
+	       "Connection: close\r\n"
+	       "Content-Type: text/plain; version=0.0.4\r\n"
+	       "\r\n");
+
+	printf("# HELP iscsi_target_bytes Number of bytes\n"
+	       "# TYPE iscsi_target_bytes counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(bytes);
+	printf("# HELP iscsi_target_dmas Number of DMA\n"
+	       "# TYPE iscsi_target_dmas counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(dmas);
+	printf("# HELP iscsi_target_operations Number of operations\n"
+	       "# TYPE iscsi_target_operations counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(operations);
+	printf("# HELP iscsi_target_time Cumulative operation time in us\n"
+	       "# TYPE iscsi_target_time counter\n");
+	CTLSTAT_PROMETHEUS_TIMELOOP(time);
+	printf("# HELP iscsi_target_dma_time Cumulative DMA time in us\n"
+	       "# TYPE iscsi_target_dma_time counter\n");
+	CTLSTAT_PROMETHEUS_TIMELOOP(dma_time);
+
+	for (i = 0; i < targdata.ntargets; i++)
+		free(targdata.targets[i]);
+	free(targdata.target);
+	free(targdata.targets);
+
+	fflush(stdout);
+}
+
 static void
 ctlstat_standard(struct ctlstat_context *ctx) {
 	long double etime;
@@ -659,6 +867,9 @@ main(int argc, char **argv)
 			ctx.flags |= CTLSTAT_FLAG_PORTS;
 			break;
 		}
+		case 'P':
+			ctx.mode = CTLSTAT_MODE_PROMETHEUS;
+			break;
 		case 't':
 			ctx.flags |= CTLSTAT_FLAG_TOTALS;
 			break;
@@ -676,6 +887,17 @@ main(int argc, char **argv)
 	if (F_LUNS(&ctx) && F_PORTS(&ctx))
 		errx(1, "Options -p and -l are exclusive.");
 
+	if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) {
+		if ((count != -1) ||
+			(waittime != 1) ||
+			/* NB: -P could be compatible with -t in the future */
+			(ctx.flags & CTLSTAT_FLAG_TOTALS))
+		{
+			errx(1, "Option -P is exclusive with -c, -w, and -t");
+		}
+		count = 1;
+	}
+
 	if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) {
 		if (F_TOTALS(&ctx))
 			ctx.flags |= CTLSTAT_FLAG_PORTS;
@@ -712,6 +934,9 @@ main(int argc, char **argv)
 		case CTLSTAT_MODE_JSON:
 			ctlstat_json(&ctx);
 			break;
+		case CTLSTAT_MODE_PROMETHEUS:
+			ctlstat_prometheus(fd, &ctx);
+			break;
 		default:
 			break;
 		}