Add function to assert that the only descriptors we have open are the ones

we expect to be open. Also assert that they point at expected type.

Because openlog(3) API is unable to tell us descriptor number it is using, we
have to close syslog socket, remember assert message in local buffer and if we
fail on assertion, reopen syslog socket and log the message.

MFC after:	1 week

2 files changed, 142 insertions, 0 deletions

diff --git a/sbin/hastd/hastd.c b/sbin/hastd/hastd.c
index ad91131e8e8d..e38cde184ae2 100644
--- a/sbin/hastd/hastd.c
+++ b/sbin/hastd/hastd.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/linker.h>
 #include <sys/module.h>
+#include <sys/stat.h>
 #include <sys/wait.h>
 
 #include <assert.h>
@@ -119,6 +120,146 @@ descriptors_cleanup(struct hast_resource *res)
 	pjdlog_fini();
 }
 
+static const char *
+dtype2str(mode_t mode)
+{
+
+	if (S_ISBLK(mode))
+		return ("block device");
+	else if (S_ISCHR(mode)) 
+		return ("character device");
+	else if (S_ISDIR(mode)) 
+		return ("directory");
+	else if (S_ISFIFO(mode))
+		return ("pipe or FIFO");
+	else if (S_ISLNK(mode)) 
+		return ("symbolic link");
+	else if (S_ISREG(mode)) 
+		return ("regular file");
+	else if (S_ISSOCK(mode))
+		return ("socket");
+	else if (S_ISWHT(mode)) 
+		return ("whiteout");
+	else
+		return ("unknown");
+}
+
+void
+descriptors_assert(const struct hast_resource *res, int pjdlogmode)
+{
+	char msg[256];
+	struct stat sb;
+	long maxfd;
+	bool isopen;
+	mode_t mode;
+	int fd;
+
+	/*
+	 * At this point descriptor to syslog socket is closed, so if we want
+	 * to log assertion message, we have to first store it in 'msg' local
+	 * buffer and then open syslog socket and log it.
+	 */
+	msg[0] = '\0';
+
+	maxfd = sysconf(_SC_OPEN_MAX);
+	if (maxfd < 0) {
+		pjdlog_errno(LOG_WARNING, "sysconf(_SC_OPEN_MAX) failed");
+		maxfd = 16384;
+	}
+	for (fd = 0; fd <= maxfd; fd++) {
+		if (fstat(fd, &sb) == 0) {
+			isopen = true;
+			mode = sb.st_mode;
+		} else if (errno == EBADF) {
+			isopen = false;
+			mode = 0;
+		} else {
+			isopen = true;	/* silence gcc */
+			mode = 0;	/* silence gcc */
+			snprintf(msg, sizeof(msg),
+			    "Unable to fstat descriptor %d: %s", fd,
+			    strerror(errno));
+		}
+		if (fd == STDIN_FILENO || fd == STDOUT_FILENO ||
+		    fd == STDERR_FILENO) {
+			if (!isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (%s) is closed, but should be open.",
+				    fd, (fd == STDIN_FILENO ? "stdin" :
+				    (fd == STDOUT_FILENO ? "stdout" : "stderr")));
+				break;
+			}
+		} else if (fd == proto_descriptor(res->hr_event)) {
+			if (!isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (event) is closed, but should be open.",
+				    fd);
+				break;
+			}
+			if (!S_ISSOCK(mode)) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (event) is %s, but should be %s.",
+				    fd, dtype2str(mode), dtype2str(S_IFSOCK));
+				break;
+			}
+		} else if (fd == proto_descriptor(res->hr_ctrl)) {
+			if (!isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (ctrl) is closed, but should be open.",
+				    fd);
+				break;
+			}
+			if (!S_ISSOCK(mode)) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (ctrl) is %s, but should be %s.",
+				    fd, dtype2str(mode), dtype2str(S_IFSOCK));
+				break;
+			}
+		} else if (res->hr_role == HAST_ROLE_SECONDARY &&
+		    fd == proto_descriptor(res->hr_remotein)) {
+			if (!isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (remote in) is closed, but should be open.",
+				    fd);
+				break;
+			}
+			if (!S_ISSOCK(mode)) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (remote in) is %s, but should be %s.",
+				    fd, dtype2str(mode), dtype2str(S_IFSOCK));
+				break;
+			}
+		} else if (res->hr_role == HAST_ROLE_SECONDARY &&
+		    fd == proto_descriptor(res->hr_remoteout)) {
+			if (!isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (remote out) is closed, but should be open.",
+				    fd);
+				break;
+			}
+			if (!S_ISSOCK(mode)) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d (remote out) is %s, but should be %s.",
+				    fd, dtype2str(mode), dtype2str(S_IFSOCK));
+				break;
+			}
+		} else {
+			if (isopen) {
+				snprintf(msg, sizeof(msg),
+				    "Descriptor %d is open (%s), but should be closed.",
+				    fd, dtype2str(mode));
+				break;
+			}
+		}
+	}
+	if (msg[0] != '\0') {
+		pjdlog_init(pjdlogmode);
+		pjdlog_prefix_set("[%s] (%s) ", res->hr_name,
+		    role2str(res->hr_role));
+		PJDLOG_ABORT("%s", msg);
+	}
+}
+
 static void
 child_exit_log(unsigned int pid, int status)
 {
diff --git a/sbin/hastd/hastd.h b/sbin/hastd/hastd.h
index b0a4037a4c04..d23e85503907 100644
--- a/sbin/hastd/hastd.h
+++ b/sbin/hastd/hastd.h
@@ -44,6 +44,7 @@ extern bool sigexit_received;
 extern struct pidfh *pfh;
 
 void descriptors_cleanup(struct hast_resource *res);
+void descriptors_assert(const struct hast_resource *res, int pjdlogmode);
 
 void hastd_primary(struct hast_resource *res);
 void hastd_secondary(struct hast_resource *res, struct nv *nvin);