First cut at GEOM based multipath. This is an active/passive{/passive...}

arrangement that has no intrinsic internal knowledge of whether devices
it is given are truly multipath devices. As such, this is a simplistic
approach, but still a useful one.

The basic approach is to (at present- this will change soon) use camcontrol
to find likely identical devices and and label the trailing sector of the
first one. This label contains both a full UUID and a name. The name is
what is presented in /dev/multipath, but the UUID is used as a true
distinguishor at g_taste time, thus making sure we don't have chaos
on a shared SAN where everyone names their data multipath as "Fred".

The first of N identical devices (and N *may* be 1!) becomes the active
path until a BIO request is failed with EIO or ENXIO. When this occurs,
the active disk is ripped away and the next in a list is picked to
(retry and) continue with.

During g_taste events new disks that meet the match criteria for existing
multipath geoms get added to the tail end of the list.

Thus, this active/passive setup actually does work for devices which
go away and come back, as do (now) mpt(4) and isp(4) SAN based disks.

There is still a lot to do to improve this- like about 5 of the 12
recommendations I've received about it,  but it's been functional enough
for a while that it deserves a broader test base.

Reviewed by: pjd
Sponsored by: IronPort Systems
MFC: 2 months

3 files changed, 241 insertions, 0 deletions

diff --git a/sbin/geom/class/Makefile b/sbin/geom/class/Makefile
index 7adfe5e83684..f52695b8b672 100644
--- a/sbin/geom/class/Makefile
+++ b/sbin/geom/class/Makefile
@@ -10,6 +10,7 @@ SUBDIR+=eli
 SUBDIR+=journal
 SUBDIR+=label
 SUBDIR+=mirror
+SUBDIR+=multipath
 SUBDIR+=nop
 SUBDIR+=raid3
 SUBDIR+=shsec
diff --git a/sbin/geom/class/multipath/Makefile b/sbin/geom/class/multipath/Makefile
new file mode 100644
index 000000000000..7b418e1f20e8
--- /dev/null
+++ b/sbin/geom/class/multipath/Makefile
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../misc
+CLASS=	multipath
+NO_MAN=true
+
+
+.include <bsd.lib.mk>
+
+CFLAGS+=  -I${.CURDIR}/../../../../sys
diff --git a/sbin/geom/class/multipath/geom_multipath.c b/sbin/geom/class/multipath/geom_multipath.c
new file mode 100644
index 000000000000..d729af778e3c
--- /dev/null
+++ b/sbin/geom/class/multipath/geom_multipath.c
@@ -0,0 +1,230 @@
+/*-
+ * Copyright (c) 2006 Mathew Jacob <mjacob@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#include <sys/param.h>
+#include <errno.h>
+#include <paths.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <assert.h>
+#include <libgeom.h>
+#include <uuid.h>
+#include <geom/multipath/g_multipath.h>
+
+#include "core/geom.h"
+#include "misc/subr.h"
+
+uint32_t lib_version = G_LIB_VERSION;
+uint32_t version = G_MULTIPATH_VERSION;
+
+static void mp_main(struct gctl_req *, unsigned int);
+static void mp_label(struct gctl_req *);
+static void mp_clear(struct gctl_req *);
+
+struct g_command class_commands[] = {
+	{
+		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
+		"[-v] name prov ..."
+	},
+	{
+		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
+		"[-v] prov ..."
+	},
+	G_CMD_SENTINEL
+};
+
+static void
+mp_main(struct gctl_req *req, unsigned int flags __unused)
+{
+	const char *name;
+
+	name = gctl_get_ascii(req, "verb");
+	if (name == NULL) {
+		gctl_error(req, "No '%s' argument.", "verb");
+		return;
+	}
+	if (strcmp(name, "label") == 0) {
+		mp_label(req);
+	} else if (strcmp(name, "clear") == 0) {
+		mp_clear(req);
+	} else {
+		gctl_error(req, "Unknown command: %s.", name);
+	}
+}
+
+static void
+mp_label(struct gctl_req *req)
+{
+	struct g_multipath_metadata md;
+	off_t disksiz = 0, msize;
+	uint8_t *sector;
+	char *ptr;
+	uuid_t uuid;
+	uint32_t secsize = 0, ssize, status;
+	const char *name;
+	int error, i, nargs;
+
+	nargs = gctl_get_int(req, "nargs");
+	if (nargs < 2) {
+		gctl_error(req, "wrong number of arguments.");
+		return;
+	}
+
+	/*
+	 * First, check each provider to make sure it's the same size.
+	 * This also gets us our size and sectorsize for the metadata.
+	 */
+	for (i = 1; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+		msize = g_get_mediasize(name);
+		ssize = g_get_sectorsize(name);
+		if (msize == 0 || ssize == 0) {
+			gctl_error(req, "cannot get information about %s: %s.",
+			    name, strerror(errno));
+			return;
+		}
+		if (i == 1) {
+			secsize = ssize;
+			disksiz	= msize;
+		} else {
+			if (secsize != ssize) {
+				gctl_error(req, "%s sector size %u different.",
+				    name, ssize);
+				return;
+			}
+			if (disksiz != msize) {
+				gctl_error(req, "%s media size %ju different.",
+				    name, (intmax_t)msize);
+				return;
+			}
+		}
+		
+	}
+
+	/*
+	 * Allocate a sector to write as metadata.
+	 */
+	sector = malloc(secsize);
+	if (sector == NULL) {
+		gctl_error(req, "unable to allocate metadata buffer");
+		return;
+	}
+	memset(sector, 0, secsize);
+
+	/*
+	 * Generate metadata.
+	 */
+	strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
+	md.md_version = G_MULTIPATH_VERSION;
+	name = gctl_get_ascii(req, "arg0");
+	strlcpy(md.md_name, name, sizeof(md.md_name));
+	md.md_size = disksiz;
+	md.md_sectorsize = secsize;
+	uuid_create(&uuid, &status);
+	if (status != uuid_s_ok) {
+		gctl_error(req, "cannot create a UUID.");
+		return;
+	}
+	uuid_to_string(&uuid, &ptr, &status);
+	if (status != uuid_s_ok) {
+		gctl_error(req, "cannot stringify a UUID.");
+		return;
+	}
+	strlcpy(md.md_uuid, ptr, sizeof (md.md_uuid));
+	free(ptr);
+
+	/*
+	 * Clear last sector first for each provider to spoil anything extant
+	 */
+	for (i = 1; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+		error = g_metadata_clear(name, NULL);
+		if (error != 0) {
+			gctl_error(req, "cannot clear metadata on %s: %s.",
+			    name, strerror(error));
+			return;
+		}
+	}
+
+	multipath_metadata_encode(&md, sector);
+
+	/*
+	 * Ok, store metadata.
+	 */
+	for (i = 1; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+		error = g_metadata_store(name, sector, secsize);
+		if (error != 0) {
+			fprintf(stderr, "Can't store metadata on %s: %s.\n",
+			    name, strerror(error));
+			goto fail;
+		}
+	}
+	return;
+
+fail:
+	/*
+	 * Clear last sector first for each provider to spoil anything extant
+	 */
+	for (i = 1; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+		error = g_metadata_clear(name, NULL);
+		if (error != 0) {
+			gctl_error(req, "cannot clear metadata on %s: %s.",
+			    name, strerror(error));
+			continue;
+		}
+	}
+}
+
+static void
+mp_clear(struct gctl_req *req)
+{
+	const char *name;
+	int error, i, nargs;
+
+	nargs = gctl_get_int(req, "nargs");
+	if (nargs < 1) {
+		gctl_error(req, "Too few arguments.");
+		return;
+	}
+
+        for (i = 0; i < nargs; i++) {
+		name = gctl_get_ascii(req, "arg%d", i);
+                error = g_metadata_clear(name, G_MULTIPATH_MAGIC);
+		if (error != 0) {
+			fprintf(stderr, "Can't clear metadata on %s: %s.\n",
+			    name, strerror(error));
+			gctl_error(req, "Not fully done.");
+			continue;
+                }
+        }
+}