libarchive: merge security fix from vendor branch

This commit fixes a couple of security vulnerabilities in the PAX writer:
1. Heap overflow in url_encode() in archive_write_set_format_pax.c
2. NULL dereference in archive_write_pax_header_xattrs()
3. Another NULL dereference in archive_write_pax_header_xattrs()
4. NULL dereference in archive_write_pax_header_xattr()

Security:	No known reference yet
Obtained from:	https://github.com/libarchive/libarchive/commit/1b4e0d0f9

(cherry picked from commit f10f65999fe56e92f00b5bc5d27ac342cfea5364)

1 files changed, 25 insertions, 10 deletions

diff --git a/contrib/libarchive/libarchive/archive_write_set_format_pax.c b/contrib/libarchive/libarchive/archive_write_set_format_pax.c
index 661b7013e0d2..8ce1d18c61e5 100644
--- a/contrib/libarchive/libarchive/archive_write_set_format_pax.c
+++ b/contrib/libarchive/libarchive/archive_write_set_format_pax.c
@@ -368,10 +368,12 @@ archive_write_pax_header_xattr(struct pax *pax, const char *encoded_name,
 	struct archive_string s;
 	char *encoded_value;
 
+	if (encoded_name == NULL)
+		return;
+
 	if (pax->flags & WRITE_LIBARCHIVE_XATTR) {
 		encoded_value = base64_encode((const char *)value, value_len);
-
-		if (encoded_name != NULL && encoded_value != NULL) {
+		if (encoded_value != NULL) {
 			archive_string_init(&s);
 			archive_strcpy(&s, "LIBARCHIVE.xattr.");
 			archive_strcat(&s, encoded_name);
@@ -404,17 +406,22 @@ archive_write_pax_header_xattrs(struct archive_write *a,
 
 		archive_entry_xattr_next(entry, &name, &value, &size);
 		url_encoded_name = url_encode(name);
-		if (url_encoded_name != NULL) {
+		if (url_encoded_name == NULL)
+			goto malloc_error;
+		else {
 			/* Convert narrow-character to UTF-8. */
 			r = archive_strcpy_l(&(pax->l_url_encoded_name),
 			    url_encoded_name, pax->sconv_utf8);
 			free(url_encoded_name); /* Done with this. */
 			if (r == 0)
 				encoded_name = pax->l_url_encoded_name.s;
-			else if (errno == ENOMEM) {
-				archive_set_error(&a->archive, ENOMEM,
-				    "Can't allocate memory for Linkname");
-				return (ARCHIVE_FATAL);
+			else if (r == -1)
+				goto malloc_error;
+			else {
+				archive_set_error(&a->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Error encoding pax extended attribute");
+				return (ARCHIVE_FAILED);
 			}
 		}
 
@@ -423,6 +430,9 @@ archive_write_pax_header_xattrs(struct archive_write *a,
 
 	}
 	return (ARCHIVE_OK);
+malloc_error:
+	archive_set_error(&a->archive, ENOMEM, "Can't allocate memory");
+	return (ARCHIVE_FATAL);
 }
 
 static int
@@ -1904,14 +1914,19 @@ url_encode(const char *in)
 {
 	const char *s;
 	char *d;
-	int out_len = 0;
+	size_t out_len = 0;
 	char *out;
 
 	for (s = in; *s != '\0'; s++) {
-		if (*s < 33 || *s > 126 || *s == '%' || *s == '=')
+		if (*s < 33 || *s > 126 || *s == '%' || *s == '=') {
+			if (SIZE_MAX - out_len < 4)
+				return (NULL);
 			out_len += 3;
-		else
+		} else {
+			if (SIZE_MAX - out_len < 2)
+				return (NULL);
 			out_len++;
+		}
 	}
 
 	out = (char *)malloc(out_len + 1);