1 files changed, 249 insertions, 91 deletions

diff --git a/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c b/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c
index c74a35cded20..94249accd08b 100644
--- a/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c
+++ b/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2017 Sean Purcell
+ * Copyright (c) 2023-2024 Klara, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,12 +26,12 @@
 
 #include "archive_platform.h"
 
-__FBSDID("$FreeBSD$");
-
-
 #ifdef HAVE_ERRNO_H
 #include <errno.h>
 #endif
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
 #ifdef HAVE_STDLIB_H
 #include <stdlib.h>
 #endif
@@ -50,9 +51,24 @@ __FBSDID("$FreeBSD$");
 
 struct private_data {
 	int		 compression_level;
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	int		 threads;
+	int		 long_distance;
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
+	enum {
+		running,
+		finishing,
+		resetting,
+	} state;
+	int		 frame_per_file;
+	size_t		 min_frame_in;
+	size_t		 max_frame_in;
+	size_t		 min_frame_out;
+	size_t		 max_frame_out;
+	size_t		 cur_frame;
+	size_t		 cur_frame_in;
+	size_t		 cur_frame_out;
+	size_t		 total_in;
 	ZSTD_CStream	*cstream;
-	int64_t		 total_in;
 	ZSTD_outBuffer	 out;
 #else
 	struct archive_write_program_data *pdata;
@@ -66,17 +82,21 @@ struct private_data {
 #define CLEVEL_STD_MAX 19 /* without using --ultra */
 #define CLEVEL_MAX 22
 
+#define LONG_STD 27
+
 #define MINVER_NEGCLEVEL 10304
 #define MINVER_MINCLEVEL 10306
+#define MINVER_LONG 10302
 
 static int archive_compressor_zstd_options(struct archive_write_filter *,
 		    const char *, const char *);
 static int archive_compressor_zstd_open(struct archive_write_filter *);
 static int archive_compressor_zstd_write(struct archive_write_filter *,
 		    const void *, size_t);
+static int archive_compressor_zstd_flush(struct archive_write_filter *);
 static int archive_compressor_zstd_close(struct archive_write_filter *);
 static int archive_compressor_zstd_free(struct archive_write_filter *);
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 static int drive_compressor(struct archive_write_filter *,
 		    struct private_data *, int, const void *, size_t);
 #endif
@@ -102,12 +122,22 @@ archive_write_add_filter_zstd(struct archive *_a)
 	f->data = data;
 	f->open = &archive_compressor_zstd_open;
 	f->options = &archive_compressor_zstd_options;
+	f->flush = &archive_compressor_zstd_flush;
 	f->close = &archive_compressor_zstd_close;
 	f->free = &archive_compressor_zstd_free;
 	f->code = ARCHIVE_FILTER_ZSTD;
 	f->name = "zstd";
 	data->compression_level = CLEVEL_DEFAULT;
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	data->threads = 0;
+	data->long_distance = 0;
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
+	data->frame_per_file = 0;
+	data->min_frame_in = 0;
+	data->max_frame_in = SIZE_MAX;
+	data->min_frame_out = 0;
+	data->max_frame_out = SIZE_MAX;
+	data->cur_frame_in = 0;
+	data->cur_frame_out = 0;
 	data->cstream = ZSTD_createCStream();
 	if (data->cstream == NULL) {
 		free(data);
@@ -134,7 +164,7 @@ static int
 archive_compressor_zstd_free(struct archive_write_filter *f)
 {
 	struct private_data *data = (struct private_data *)f->data;
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 	ZSTD_freeCStream(data->cstream);
 	free(data->out.dst);
 #else
@@ -145,29 +175,54 @@ archive_compressor_zstd_free(struct archive_write_filter *f)
 	return (ARCHIVE_OK);
 }
 
-static int string_is_numeric (const char* value)
+static int
+string_to_number(const char *string, intmax_t *numberp)
+{
+	char *end;
+
+	if (string == NULL || *string == '\0')
+		return (ARCHIVE_WARN);
+	*numberp = strtoimax(string, &end, 10);
+	if (end == string || *end != '\0' || errno == EOVERFLOW) {
+		*numberp = 0;
+		return (ARCHIVE_WARN);
+	}
+	return (ARCHIVE_OK);
+}
+
+static int
+string_to_size(const char *string, size_t *numberp)
 {
-       size_t len = strlen(value);
-       size_t i;
-
-       if (len == 0) {
-               return (ARCHIVE_WARN);
-       }
-       else if (len == 1 && !(value[0] >= '0' && value[0] <= '9')) {
-               return (ARCHIVE_WARN);
-       }
-       else if (!(value[0] >= '0' && value[0] <= '9') &&
-                value[0] != '-' && value[0] != '+') {
-               return (ARCHIVE_WARN);
-       }
-
-       for (i = 1; i < len; i++) {
-               if (!(value[i] >= '0' && value[i] <= '9')) {
-                       return (ARCHIVE_WARN);
-               }
-       }
-
-       return (ARCHIVE_OK);
+	uintmax_t number;
+	char *end;
+	unsigned int shift = 0;
+
+	if (string == NULL || *string == '\0' || *string == '-')
+		return (ARCHIVE_WARN);
+	number = strtoumax(string, &end, 10);
+	if (end > string) {
+		if (*end == 'K' || *end == 'k') {
+			shift = 10;
+			end++;
+		} else if (*end == 'M' || *end == 'm') {
+			shift = 20;
+			end++;
+		} else if (*end == 'G' || *end == 'g') {
+			shift = 30;
+			end++;
+		}
+		if (*end == 'B' || *end == 'b') {
+			end++;
+		}
+	}
+	if (end == string || *end != '\0' || errno == EOVERFLOW) {
+		return (ARCHIVE_WARN);
+	}
+	if (number > (uintmax_t)SIZE_MAX >> shift) {
+		return (ARCHIVE_WARN);
+	}
+	*numberp = (size_t)(number << shift);
+	return (ARCHIVE_OK);
 }
 
 /*
@@ -180,14 +235,14 @@ archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
 	struct private_data *data = (struct private_data *)f->data;
 
 	if (strcmp(key, "compression-level") == 0) {
-		int level = atoi(value);
+		intmax_t level;
+		if (string_to_number(value, &level) != ARCHIVE_OK) {
+			return (ARCHIVE_WARN);
+		}
 		/* If we don't have the library, hard-code the max level */
 		int minimum = CLEVEL_MIN;
 		int maximum = CLEVEL_MAX;
-		if (string_is_numeric(value) != ARCHIVE_OK) {
-			return (ARCHIVE_WARN);
-		}
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 		maximum = ZSTD_maxCLevel();
 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
 		if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
@@ -202,7 +257,69 @@ archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
 		if (level < minimum || level > maximum) {
 			return (ARCHIVE_WARN);
 		}
-		data->compression_level = level;
+		data->compression_level = (int)level;
+		return (ARCHIVE_OK);
+	} else if (strcmp(key, "threads") == 0) {
+		intmax_t threads;
+		if (string_to_number(value, &threads) != ARCHIVE_OK) {
+			return (ARCHIVE_WARN);
+		}
+		if (threads < 0) {
+			return (ARCHIVE_WARN);
+		}
+		data->threads = (int)threads;
+		return (ARCHIVE_OK);
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
+	} else if (strcmp(key, "frame-per-file") == 0) {
+		data->frame_per_file = 1;
+		return (ARCHIVE_OK);
+	} else if (strcmp(key, "min-frame-in") == 0) {
+		if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) {
+			return (ARCHIVE_WARN);
+		}
+		return (ARCHIVE_OK);
+	} else if (strcmp(key, "min-frame-out") == 0 ||
+	    strcmp(key, "min-frame-size") == 0) {
+		if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) {
+			return (ARCHIVE_WARN);
+		}
+		return (ARCHIVE_OK);
+	} else if (strcmp(key, "max-frame-in") == 0 ||
+	    strcmp(key, "max-frame-size") == 0) {
+		if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK ||
+		    data->max_frame_in < 1024) {
+			return (ARCHIVE_WARN);
+		}
+		return (ARCHIVE_OK);
+	} else if (strcmp(key, "max-frame-out") == 0) {
+		if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK ||
+		    data->max_frame_out < 1024) {
+			return (ARCHIVE_WARN);
+		}
+		return (ARCHIVE_OK);
+#endif
+	}
+	else if (strcmp(key, "long") == 0) {
+		intmax_t long_distance;
+		if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
+			return (ARCHIVE_WARN);
+		}
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
+		ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
+		if (ZSTD_isError(bounds.error)) {
+			int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
+			if (((int)long_distance) < 10 || (int)long_distance > max_distance)
+				return (ARCHIVE_WARN);
+		} else {
+			if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound)
+				return (ARCHIVE_WARN);
+		}
+#else
+		int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
+		if (((int)long_distance) < 10 || (int)long_distance > max_distance)
+		    return (ARCHIVE_WARN);
+#endif
+		data->long_distance = (int)long_distance;
 		return (ARCHIVE_OK);
 	}
 
@@ -212,7 +329,7 @@ archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
 	return (ARCHIVE_WARN);
 }
 
-#if HAVE_ZSTD_H && HAVE_LIBZSTD
+#if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 /*
  * Setup callback.
  */
@@ -252,6 +369,12 @@ archive_compressor_zstd_open(struct archive_write_filter *f)
 		return (ARCHIVE_FATAL);
 	}
 
+	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);
+
+#if ZSTD_VERSION_NUMBER >= MINVER_LONG
+	ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance);
+#endif
+
 	return (ARCHIVE_OK);
 }
 
@@ -263,15 +386,25 @@ archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
     size_t length)
 {
 	struct private_data *data = (struct private_data *)f->data;
-	int ret;
 
-	/* Update statistics */
-	data->total_in += length;
+	return (drive_compressor(f, data, 0, buff, length));
+}
 
-	if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK)
-		return (ret);
+/*
+ * Flush the compressed stream.
+ */
+static int
+archive_compressor_zstd_flush(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
 
-	return (ARCHIVE_OK);
+	if (data->frame_per_file && data->state == running) {
+		if (data->cur_frame_in > data->min_frame_in &&
+		    data->cur_frame_out > data->min_frame_out) {
+			data->state = finishing;
+		}
+	}
+	return (drive_compressor(f, data, 1, NULL, 0));
 }
 
 /*
@@ -282,60 +415,77 @@ archive_compressor_zstd_close(struct archive_write_filter *f)
 {
 	struct private_data *data = (struct private_data *)f->data;
 
-	/* Finish zstd frame */
-	return drive_compressor(f, data, 1, NULL, 0);
+	if (data->state == running)
+		data->state = finishing;
+	return (drive_compressor(f, data, 1, NULL, 0));
 }
 
 /*
  * Utility function to push input data through compressor,
  * writing full output blocks as necessary.
- *
- * Note that this handles both the regular write case (finishing ==
- * false) and the end-of-archive case (finishing == true).
  */
 static int
 drive_compressor(struct archive_write_filter *f,
-    struct private_data *data, int finishing, const void *src, size_t length)
+    struct private_data *data, int flush, const void *src, size_t length)
 {
-	ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 };
+	ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
+	size_t ipos, opos, zstdret = 0;
+	int ret;
 
 	for (;;) {
-		if (data->out.pos == data->out.size) {
-			const int ret = __archive_write_filter(f->next_filter,
-			    data->out.dst, data->out.size);
-			if (ret != ARCHIVE_OK)
-				return (ARCHIVE_FATAL);
-			data->out.pos = 0;
+		ipos = in.pos;
+		opos = data->out.pos;
+		switch (data->state) {
+		case running:
+			if (in.pos == in.size)
+				return (ARCHIVE_OK);
+			zstdret = ZSTD_compressStream(data->cstream,
+			    &data->out, &in);
+			if (ZSTD_isError(zstdret))
+				goto zstd_fatal;
+			break;
+		case finishing:
+			zstdret = ZSTD_endStream(data->cstream, &data->out);
+			if (ZSTD_isError(zstdret))
+				goto zstd_fatal;
+			if (zstdret == 0)
+				data->state = resetting;
+			break;
+		case resetting:
+			ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only);
+			data->cur_frame++;
+			data->cur_frame_in = 0;
+			data->cur_frame_out = 0;
+			data->state = running;
+			break;
 		}
-
-		/* If there's nothing to do, we're done. */
-		if (!finishing && in.pos == in.size)
-			return (ARCHIVE_OK);
-
-		{
-			const size_t zstdret = !finishing ?
-			    ZSTD_compressStream(data->cstream, &data->out, &in)
-			    : ZSTD_endStream(data->cstream, &data->out);
-
-			if (ZSTD_isError(zstdret)) {
-				archive_set_error(f->archive,
-				    ARCHIVE_ERRNO_MISC,
-				    "Zstd compression failed: %s",
-				    ZSTD_getErrorName(zstdret));
-				return (ARCHIVE_FATAL);
-			}
-
-			/* If we're finishing, 0 means nothing left to flush */
-			if (finishing && zstdret == 0) {
-				const int ret = __archive_write_filter(f->next_filter,
-				    data->out.dst, data->out.pos);
-				return (ret);
+		data->total_in += in.pos - ipos;
+		data->cur_frame_in += in.pos - ipos;
+		data->cur_frame_out += data->out.pos - opos;
+		if (data->state == running) {
+			if (data->cur_frame_in >= data->max_frame_in ||
+			    data->cur_frame_out >= data->max_frame_out) {
+				data->state = finishing;
 			}
 		}
+		if (data->out.pos == data->out.size ||
+		    (flush && data->out.pos > 0)) {
+			ret = __archive_write_filter(f->next_filter,
+			    data->out.dst, data->out.pos);
+			if (ret != ARCHIVE_OK)
+				goto fatal;
+			data->out.pos = 0;
+		}
 	}
+zstd_fatal:
+	archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
+	    "Zstd compression failed: %s",
+	    ZSTD_getErrorName(zstdret));
+fatal:
+	return (ARCHIVE_FATAL);
 }
 
-#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */
+#else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
 
 static int
 archive_compressor_zstd_open(struct archive_write_filter *f)
@@ -349,23 +499,23 @@ archive_compressor_zstd_open(struct archive_write_filter *f)
 	archive_strcpy(&as, "zstd --no-check");
 
 	if (data->compression_level < CLEVEL_STD_MIN) {
-		struct archive_string as2;
-		archive_string_init(&as2);
-		archive_string_sprintf(&as2, " --fast=%d", -data->compression_level);
-		archive_string_concat(&as, &as2);
-		archive_string_free(&as2);
+		archive_string_sprintf(&as, " --fast=%d", -data->compression_level);
 	} else {
-		struct archive_string as2;
-		archive_string_init(&as2);
-		archive_string_sprintf(&as2, " -%d", data->compression_level);
-		archive_string_concat(&as, &as2);
-		archive_string_free(&as2);
+		archive_string_sprintf(&as, " -%d", data->compression_level);
 	}
 
 	if (data->compression_level > CLEVEL_STD_MAX) {
 		archive_strcat(&as, " --ultra");
 	}
 
+	if (data->threads != 0) {
+		archive_string_sprintf(&as, " --threads=%d", data->threads);
+	}
+
+	if (data->long_distance != 0) {
+		archive_string_sprintf(&as, " --long=%d", data->long_distance);
+	}
+
 	f->write = archive_compressor_zstd_write;
 	r = __archive_write_program_open(f, data->pdata, as.s);
 	archive_string_free(&as);
@@ -382,6 +532,14 @@ archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
 }
 
 static int
+archive_compressor_zstd_flush(struct archive_write_filter *f)
+{
+	(void)f; /* UNUSED */
+
+	return (ARCHIVE_OK);
+}
+
+static int
 archive_compressor_zstd_close(struct archive_write_filter *f)
 {
 	struct private_data *data = (struct private_data *)f->data;
@@ -389,4 +547,4 @@ archive_compressor_zstd_close(struct archive_write_filter *f)
 	return __archive_write_program_close(f, data->pdata);
 }
 
-#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */
+#endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */