MFV r245512:

 * Illumos zfs issue #3035 [1] LZ4 compression support in ZFS.

LZ4 is a new high-speed BSD-licensed compression algorithm created
by Yann Collet that delivers very high compression and decompression
performance compared to lzjb (>50% faster on compression, >80% faster
on decompression and around 3x faster on compression of incompressible
data), while giving better compression ratio [1].

This version of LZ4 corresponds to upstream's [2] revision 85.

Please note that for obvious reasons this is not backward read
compatible.  This means once a pool have LZ4 compressed data, these
data can no longer be read by older ZFS implementations.

Local changes:

 - On-stack hash table disabled and using kernel slab allocator
   instead, at this time.  This requires larger kernel thread stack
   for zio workers.  This may change in the future should we adjusted
   the zio workers' thread stack size.
 - likely and unlikely will be undefined if they are already defined,
   this is required for i386 XEN build.
 - Removed De Bruijn sequence based __builtin_ctz family of builtins
   in favor of the latter.  Both GCC and clang supports these builtins.
 - Changed the way the LZ4 code detects endianness.
 - Manual pages modifications to mention the feature based on Illumos
   counterpart.
 - Boot loader changes to make it support LZ4 decompression.

[1] https://www.illumos.org/issues/3035
[2] http://code.google.com/p/lz4/source/list

Obtained from:	Illumos (13921:9d721847e469)
Tested on:	FreeBSD/amd64
MFC after:	1 month

4 files changed, 315 insertions, 0 deletions

diff --git a/sys/cddl/boot/zfs/README b/sys/cddl/boot/zfs/README
index d36e02e1c1b1..f7c045dab5d0 100644
--- a/sys/cddl/boot/zfs/README
+++ b/sys/cddl/boot/zfs/README
@@ -5,6 +5,7 @@ are used by the ZFS bootstrap:
 
     fletcher.c			checksum support
     sha256.c			checksum support
+    lz4.c			compression support
     lzjb.c			compression support
     zfssubr.c			checksum, compression and raidz support
     zfsimpl.h			mostly describing the physical layout
diff --git a/sys/cddl/boot/zfs/lz4.c b/sys/cddl/boot/zfs/lz4.c
new file mode 100644
index 000000000000..78af95942eee
--- /dev/null
+++ b/sys/cddl/boot/zfs/lz4.c
@@ -0,0 +1,308 @@
+/*
+ * LZ4 - Fast LZ compression algorithm
+ * Header File
+ * Copyright (C) 2011-2013, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ * - LZ4 source repository : http://code.google.com/p/lz4/
+ *
+ * $FreeBSD$
+ */
+
+static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
+					    int isize, int maxOutputSize);
+
+/* ARGSUSED */
+static int
+lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int dummy __unused)
+{
+	const uint8_t *src = s_start;
+	uint32_t bufsiz = htonl(*(uint32_t *)src);
+
+	/* invalid compressed buffer size encoded at start */
+	if (bufsiz + 4 > s_len)
+		return (1);
+
+	/*
+	 * Returns 0 on success (decompression function returned non-negative)
+	 * and non-zero on failure (decompression function returned negative).
+	 */
+	return (LZ4_uncompress_unknownOutputSize(s_start + 4, d_start, bufsiz,
+	    d_len) < 0);
+}
+
+/*
+ * CPU Feature Detection
+ */
+
+/* 32 or 64 bits ? */
+#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || \
+	defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || \
+	defined(__LP64__) || defined(_LP64))
+#define	LZ4_ARCH64	1
+#else
+#define	LZ4_ARCH64	0
+#endif
+
+/*
+ * Little Endian or Big Endian?
+ * Note: overwrite the below #define if you know your architecture endianess.
+ */
+#if BYTE_ORDER == BIG_ENDIAN
+#define	LZ4_BIG_ENDIAN	1
+#else
+	/*
+	 * Little Endian assumed. PDP Endian and other very rare endian format
+	 * are unsupported.
+	 */
+#endif
+
+/*
+ * Compiler Options
+ */
+#if __STDC_VERSION__ >= 199901L	/* C99 */
+/* "restrict" is a known keyword */
+#else
+/* Disable restrict */
+#define	restrict
+#endif
+
+#define	GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#define	lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) \
+	| (((x) & 0xffu) << 8)))
+
+#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+#define	expect(expr, value)    (__builtin_expect((expr), (value)))
+#else
+#define	expect(expr, value)    (expr)
+#endif
+
+#define	likely(expr)	expect((expr) != 0, 1)
+#define	unlikely(expr)	expect((expr) != 0, 0)
+
+/* Basic types */
+#define	BYTE	uint8_t
+#define	U16	uint16_t
+#define	U32	uint32_t
+#define	S32	int32_t
+#define	U64	uint64_t
+
+typedef struct _U16_S {
+	U16 v;
+} U16_S;
+typedef struct _U32_S {
+	U32 v;
+} U32_S;
+typedef struct _U64_S {
+	U64 v;
+} U64_S;
+
+#define	A64(x)	(((U64_S *)(x))->v)
+#define	A32(x)	(((U32_S *)(x))->v)
+#define	A16(x)	(((U16_S *)(x))->v)
+
+/*
+ * Constants
+ */
+#define	MINMATCH 4
+
+#define	COPYLENGTH 8
+#define	LASTLITERALS 5
+
+#define	ML_BITS 4
+#define	ML_MASK ((1U<<ML_BITS)-1)
+#define	RUN_BITS (8-ML_BITS)
+#define	RUN_MASK ((1U<<RUN_BITS)-1)
+
+/*
+ * Architecture-specific macros
+ */
+#if LZ4_ARCH64
+#define	STEPSIZE 8
+#define	UARCH U64
+#define	AARCH A64
+#define	LZ4_COPYSTEP(s, d)	A64(d) = A64(s); d += 8; s += 8;
+#define	LZ4_COPYPACKET(s, d)	LZ4_COPYSTEP(s, d)
+#define	LZ4_SECURECOPY(s, d, e)	if (d < e) LZ4_WILDCOPY(s, d, e)
+#define	HTYPE U32
+#define	INITBASE(base)		const BYTE* const base = ip
+#else
+#define	STEPSIZE 4
+#define	UARCH U32
+#define	AARCH A32
+#define	LZ4_COPYSTEP(s, d)	A32(d) = A32(s); d += 4; s += 4;
+#define	LZ4_COPYPACKET(s, d)	LZ4_COPYSTEP(s, d); LZ4_COPYSTEP(s, d);
+#define	LZ4_SECURECOPY		LZ4_WILDCOPY
+#define	HTYPE const BYTE*
+#define	INITBASE(base)		const int base = 0
+#endif
+
+#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
+#define	LZ4_READ_LITTLEENDIAN_16(d, s, p) \
+	{ U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
+#define	LZ4_WRITE_LITTLEENDIAN_16(p, i) \
+	{ U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p += 2; }
+#else
+#define	LZ4_READ_LITTLEENDIAN_16(d, s, p) { d = (s) - A16(p); }
+#define	LZ4_WRITE_LITTLEENDIAN_16(p, v)  { A16(p) = v; p += 2; }
+#endif
+
+/* Macros */
+#define	LZ4_WILDCOPY(s, d, e) do { LZ4_COPYPACKET(s, d) } while (d < e);
+
+/* Decompression functions */
+
+static int
+LZ4_uncompress_unknownOutputSize(const char *source,
+    char *dest, int isize, int maxOutputSize)
+{
+	/* Local Variables */
+	const BYTE *restrict ip = (const BYTE *) source;
+	const BYTE *const iend = ip + isize;
+	const BYTE *restrict ref;
+
+	BYTE *restrict op = (BYTE *) dest;
+	BYTE *const oend = op + maxOutputSize;
+	BYTE *cpy;
+
+	size_t dec[] = { 0, 3, 2, 3, 0, 0, 0, 0 };
+
+	/* Main Loop */
+	while (ip < iend) {
+		BYTE token;
+		int length;
+
+		/* get runlength */
+		token = *ip++;
+		if ((length = (token >> ML_BITS)) == RUN_MASK) {
+			int s = 255;
+			while ((ip < iend) && (s == 255)) {
+				s = *ip++;
+				length += s;
+			}
+		}
+		/* copy literals */
+		cpy = op + length;
+		if ((cpy > oend - COPYLENGTH) ||
+		    (ip + length > iend - COPYLENGTH)) {
+			if (cpy > oend)
+				/*
+				 * Error: request to write beyond destination
+				 * buffer.
+				 */
+				goto _output_error;
+			if (ip + length > iend)
+				/*
+				 * Error : request to read beyond source
+				 * buffer.
+				 */
+				goto _output_error;
+			memcpy(op, ip, length);
+			op += length;
+			ip += length;
+			if (ip < iend)
+				/* Error : LZ4 format violation */
+				goto _output_error;
+			/* Necessarily EOF, due to parsing restrictions. */
+			break;
+		}
+		LZ4_WILDCOPY(ip, op, cpy);
+		ip -= (op - cpy);
+		op = cpy;
+
+		/* get offset */
+		LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
+		ip += 2;
+		if (ref < (BYTE * const) dest)
+			/*
+			 * Error: offset creates reference outside of
+			 * destination buffer.
+			 */
+			goto _output_error;
+
+		/* get matchlength */
+		if ((length = (token & ML_MASK)) == ML_MASK) {
+			while (ip < iend) {
+				int s = *ip++;
+				length += s;
+				if (s == 255)
+					continue;
+				break;
+			}
+		}
+		/* copy repeated sequence */
+		if unlikely(op - ref < STEPSIZE) {
+#if LZ4_ARCH64
+			size_t dec2table[] = { 0, 0, 0, -1, 0, 1, 2, 3 };
+			size_t dec2 = dec2table[op - ref];
+#else
+			const int dec2 = 0;
+#endif
+			*op++ = *ref++;
+			*op++ = *ref++;
+			*op++ = *ref++;
+			*op++ = *ref++;
+			ref -= dec[op - ref];
+			A32(op) = A32(ref);
+			op += STEPSIZE - 4;
+			ref -= dec2;
+		} else {
+			LZ4_COPYSTEP(ref, op);
+		}
+		cpy = op + length - (STEPSIZE - 4);
+		if (cpy > oend - COPYLENGTH) {
+			if (cpy > oend)
+				/*
+				 * Error: request to write outside of
+				 * destination buffer.
+				 */
+				goto _output_error;
+			LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
+			while (op < cpy)
+				*op++ = *ref++;
+			op = cpy;
+			if (op == oend)
+				/*
+				 * Check EOF (should never happen, since last
+				 * 5 bytes are supposed to be literals).
+				 */
+				break;
+			continue;
+		}
+		LZ4_SECURECOPY(ref, op, cpy);
+		op = cpy;	/* correction */
+	}
+
+	/* end of decoding */
+	return (int)(((char *)op) - dest);
+
+	/* write overflow error detected */
+	_output_error:
+	return (int)(-(((char *)ip) - source));
+}
diff --git a/sys/cddl/boot/zfs/zfsimpl.h b/sys/cddl/boot/zfs/zfsimpl.h
index a684c3951624..394a5e39ea92 100644
--- a/sys/cddl/boot/zfs/zfsimpl.h
+++ b/sys/cddl/boot/zfs/zfsimpl.h
@@ -52,6 +52,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2013 by Saso Kiselkov. All rights reserved.
+ */
 
 #define	MAXNAMELEN	256
 
@@ -439,6 +442,7 @@ enum zio_compress {
 	ZIO_COMPRESS_GZIP_8,
 	ZIO_COMPRESS_GZIP_9,
 	ZIO_COMPRESS_ZLE,
+	ZIO_COMPRESS_LZ4,
 	ZIO_COMPRESS_FUNCTIONS
 };
 
diff --git a/sys/cddl/boot/zfs/zfssubr.c b/sys/cddl/boot/zfs/zfssubr.c
index b2d2e13da3ba..0f018487bcab 100644
--- a/sys/cddl/boot/zfs/zfssubr.c
+++ b/sys/cddl/boot/zfs/zfssubr.c
@@ -119,6 +119,7 @@ typedef struct zio_compress_info {
 
 #include "lzjb.c"
 #include "zle.c"
+#include "lz4.c"
 
 /*
  * Compression vectors.
@@ -139,6 +140,7 @@ static zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
 	{NULL,			NULL,			8,	"gzip-8"},
 	{NULL,			NULL,			9,	"gzip-9"},
 	{NULL,			zle_decompress,		64,	"zle"},
+	{NULL,			lz4_decompress,		0,	"lz4"},
 };
 
 static void