diff options
| author | Xin LI <delphij@FreeBSD.org> | 2026-09-14 01:08:22 +0000 |
|---|---|---|
| committer | Xin LI <delphij@FreeBSD.org> | 2026-09-17 04:12:43 +0000 |
| commit | 8d58fa9fed63355d8921db13b899511ebff406c3 (patch) | |
| tree | b505cb9b85fffb3f5eb0271b93e79c6f723f7d6a | |
| parent | 84141e3e88c0bb52be5cff4c0647519f9a370d98 (diff) | |
MFV: xz 5.8.4.
(cherry picked from commit 7ffc4ec4860d01414b493cdf43738878a9ede538)
30 files changed, 2016 insertions, 287 deletions
diff --git a/contrib/xz/ChangeLog b/contrib/xz/ChangeLog index d592a60db513..2fa5d0bcf896 100644 --- a/contrib/xz/ChangeLog +++ b/contrib/xz/ChangeLog @@ -1,3 +1,1491 @@ +commit d3e650e63c110e830fd5391e7f8b45df0b91d3da +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 17:30:48 +0300 + + Bump version and soname for 5.8.4 + + src/liblzma/Makefile.am | 2 +- + src/liblzma/api/lzma/version.h | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +commit 44a7fbdbe4b6b75854df2f073cd0815abf6a6beb +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 20:31:20 +0300 + + Add NEWS for 5.8.4 + + NEWS | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 139 insertions(+) + +commit 0d30919e735a59ce0a029c00566c2f1463c7ac80 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 15:19:46 +0300 + + Translations: Run po4a/update-po + + This is to keep the files in release tarballs similar to the files + in the Git repository. + + po4a/fr.po | 9 ++------- + po4a/pt_BR.po | 14 +++++++------- + 2 files changed, 9 insertions(+), 14 deletions(-) + +commit b33ffadd16553bdc423b24f2529d290a8103ece4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 15:08:00 +0300 + + Translations: Run "make -C po update-po" + + 16 translations weren't updated after 102ea0ed1290 ("xz: Make it an error + if the totals in xz --list exceed UINT64_MAX"). + + po/ca.po | 6 +++++- + po/cs.po | 6 +++++- + po/da.po | 6 +++++- + po/eo.po | 6 +++++- + po/es.po | 6 +++++- + po/fi.po | 6 +++++- + po/fr.po | 6 +++++- + po/hu.po | 6 +++++- + po/ka.po | 6 +++++- + po/pt_BR.po | 6 +++++- + po/sr.po | 6 +++++- + po/sv.po | 6 +++++- + po/tr.po | 6 +++++- + po/vi.po | 6 +++++- + po/zh_CN.po | 6 +++++- + po/zh_TW.po | 6 +++++- + 16 files changed, 80 insertions(+), 16 deletions(-) + +commit df5bfb4547ec6f3e3ad28ba2ece5f89ebf8d38c6 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: Add a fuzzer for .xz Index decoding + + Normal .xz decoding uses lzma_index_hash, so this new fuzzer doesn't + overlap with the existing fuzzers. + + tests/ossfuzz/fuzz_decode_index.c | 66 +++++++++++++++++++++++++++++++++++++++ + 1 file changed, 66 insertions(+) + +commit 65198524f0e2a4a5670ca86b316e50dbd185fbd4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: Make some memory allocations fail in fuzzers + + tests/ossfuzz/fuzz_common.h | 37 +++++++++++++++++++++++++++++++++++ + tests/ossfuzz/fuzz_decode_alone.c | 6 ++++-- + tests/ossfuzz/fuzz_decode_stream.c | 6 ++++-- + tests/ossfuzz/fuzz_decode_stream_mt.c | 6 ++++-- + tests/ossfuzz/fuzz_encode_stream.c | 4 ++++ + 5 files changed, 53 insertions(+), 6 deletions(-) + +commit d4217668a79e6ca440e673115155559c81e89b84 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: Reinitialize decoders twice in fuzzers + + tests/ossfuzz/fuzz_decode_alone.c | 27 +++++++++------- + tests/ossfuzz/fuzz_decode_stream.c | 58 +++++++++++++++++++++-------------- + tests/ossfuzz/fuzz_decode_stream_mt.c | 30 ++++++++++++------ + 3 files changed, 71 insertions(+), 44 deletions(-) + +commit 9c3661926e8b7e9ee38b687cb197a0e98ecd3fb4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: fuzz_common.h: Reset next_out and avail_out + + It matters if the same lzma_stream is reused. + + tests/ossfuzz/fuzz_common.h | 3 +++ + 1 file changed, 3 insertions(+) + +commit 16f309314488c601daad2689924117a43a8ccbde +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: Adjust coding style in fuzzers + + tests/ossfuzz/fuzz_common.h | 3 ++- + tests/ossfuzz/fuzz_decode_alone.c | 1 + + tests/ossfuzz/fuzz_decode_stream.c | 1 + + 3 files changed, 4 insertions(+), 1 deletion(-) + +commit c3b290c3dc536d16884a6c695618eb08fdbc700a +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + liblzma: Make lzma_simple_coder_init() reinit-safe after alloc failure + + lzma_raw_coder_init() always calls lzma_next_end() if an error occurs + in filter initialization, and no other code path can call + lzma_simple_coder_init(), so this change doesn't fix any bug. + This is just to reduce likelyhood of future bugs. + + src/liblzma/simple/simple_coder.c | 20 +++++++++++--------- + 1 file changed, 11 insertions(+), 9 deletions(-) + +commit ff834f25ae4f9b3e6270d41b4c843b8b1183346c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + Tests: Test lzma_alone_decoder() reuse after dictionary allocation failure + + This test crashes if both of the two previous commits are reverted. + + tests/Makefile.am | 2 + + tests/test_alone_decoder.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++ + tests/tests.cmake | 1 + + 3 files changed, 99 insertions(+) + +commit e5e63d50eac1b4357a5a7a0abd3c5f99a5c47881 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + liblzma: Clean up after a filter chain initialization error + + Filter coders are initialized using lzma_next_filter_init(). If filter + chain initialization fails, the entire filter chain should be cleaned up + with lzma_next_end(). lzma_raw_encoder_init() and lzma_raw_decoder_init() + have always done this via lzma_raw_coder_init() in filter_common.c. + (Separate cleanup is weird, but it must have made sense to the young me.) + + The following decoders initialize LZMA1 filter directly without using the + raw filter chain API. This avoids needlessly pulling in all other filters + when a program is linked against static liblzma. These functions didn't + call lzma_next_end() after an initialization error, which left the state + available for later reinitialization. + + - lzma_alone_decoder() + - lzma_lzip_decoder() + - lzma_auto_decoder() [*] + - lzma_microlzma_decoder() + + [*] lzma_auto_decoder() is only indirectly affected due to the first two + functions. lzma_auto_decoder() itself doesn't need a fix. + + There are also encoder functions that initialize the LZMA1 encoder + directly. They don't have this issue because the whole lzma_stream + is cleaned up when encoder initialization fails. + + Reported-by: GitHub user christos-cantina-security (christos-spearbit) + + src/liblzma/common/alone_decoder.c | 8 ++++++-- + src/liblzma/common/lzip_decoder.c | 8 ++++++-- + src/liblzma/common/microlzma_decoder.c | 8 ++++++-- + 3 files changed, 18 insertions(+), 6 deletions(-) + +commit fe4d763d566a38ad61d4c5022520c25578a3a464 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-09 14:14:40 +0300 + + liblzma: Make lzma_lz_decoder_init() safe to reinit after alloc failure + + If memory allocation fails and the resulting coder state is reused, + ensure that memory allocation is attempted again. However, coders that + are initialized via lzma_next_filter_init() shouldn't be reinitialized + after failure; they should be cleaned up with lzma_next_end(). + + Reported-by: GitHub user christos-cantina-security (christos-spearbit) + + src/liblzma/lz/lz_decoder.c | 1 + + 1 file changed, 1 insertion(+) + +commit 9fc6f5cd8774ebef8d4e030f7081fb6984c0dc3f +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-09-04 12:13:43 +0300 + + liblzma: mt dec: Clarify comments about mutex usage + + src/liblzma/common/stream_decoder_mt.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +commit c6e3aadbb510e44cecfe870408ecfea1d1ca792c +Author: Trithem90 <149891156+Trithem90@users.noreply.github.com> +Date: 2026-09-03 19:52:23 +0200 + + liblzma: mt dec: Protect progress_in update with mutex + + Worker threads update coder->progress_in while holding coder->mutex, but + the main thread updated the same field without the mutex while decoding + the next Block Header. ThreadSanitizer reported the concurrent writes and + lzma_get_progress() under-reported input progress by one Block. + + All other coder->progress_in/progress_out accesses in stream_decode_mt() + are done when the worker threads aren't active, so only the access in + SEQ_BLOCK_HEADER needs to lock coder->mutex. + + Co-authored-by: Lasse Collin <lasse.collin@tukaani.org> + Fixes: 4cce3e27f529 ("liblzma: Add threaded .xz decompressor.") + Closes: https://github.com/tukaani-project/xz/pull/243 + + src/liblzma/common/stream_decoder_mt.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +commit 5121d1aed8107a113d7665b6d4f26b6187949f3d +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-31 20:35:04 +0300 + + Tests: Add tests for single-call .xz encoding + + Thanks-to: GitHub user androvonx95 + Closes: https://github.com/tukaani-project/xz/pull/231 + + tests/Makefile.am | 2 + + tests/test_stream_buffer_encode.c | 264 ++++++++++++++++++++++++++++++++++++++ + tests/tests.cmake | 1 + + 3 files changed, 267 insertions(+) + +commit 28a66a3d1b6af89ed6ee841384c339629fda8264 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-20 16:40:01 +0300 + + Translations: Update the Italian translation + + po/it.po | 17 ++++++++++------- + 1 file changed, 10 insertions(+), 7 deletions(-) + +commit c6c425829c8635a243211cbc14dd34a042a7f796 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-20 16:38:42 +0300 + + liblzma: LZMA2 decoder: Be more strict about compressed chunk size + + It wasn't really a bug, but it's good to be more strict. + + Side effects ("downsides" would be a too strong word): + + - Now the LZMA decoder has to use the slow path at the end of every + LZMA2 chunk even when the read position isn't close to &in[in_size]. + This doesn't matter in practice. + + - If a chunk header was corrupt exactly so that the compressed size is + too small, now fewer bytes of uncompressed data can be recovered. + + Reported-by: GitHub user christos-spearbit + + src/liblzma/lzma/lzma2_decoder.c | 25 +++++++++++++++++++++++-- + 1 file changed, 23 insertions(+), 2 deletions(-) + +commit c077edfd1867b473a38fb2283c6cac545a9ea857 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-19 22:58:43 +0300 + + Translations: Update the Polish translation + + po/pl.po | 17 +++++++++++------ + 1 file changed, 11 insertions(+), 6 deletions(-) + +commit 936e72e20e18cf0a637f2aa9d87f278d7a82df9d +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-19 20:26:37 +0300 + + Translations: Update the German translation + + po/de.po | 15 ++++++++++----- + 1 file changed, 10 insertions(+), 5 deletions(-) + +commit 11334a5d4d5ea3e8b2a3cbce74c1062d25cef772 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-19 19:07:34 +0300 + + Translations: Update the Korean translation + + po/ko.po | 17 +++++++++++------ + 1 file changed, 11 insertions(+), 6 deletions(-) + +commit 7b9955d0aaa10edbe9a71e54d16c28fdc8b60725 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-19 19:05:42 +0300 + + Translations: Update the Portuguese translation + + po/pt.po | 17 +++++++++++------ + 1 file changed, 11 insertions(+), 6 deletions(-) + +commit 9ed91f3afe50fd21051af3b31139466ebfa6b20a +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-19 14:03:55 +0300 + + Translations: Update the Dutch translation + + po/nl.po | 13 +++++++++---- + 1 file changed, 9 insertions(+), 4 deletions(-) + +commit e38f738e5cc5b72afc160285e65f5973f44db2fb +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-18 20:20:57 +0300 + + Translations: Update the Romanian translation + + po/ro.po | 16 +++++++++++----- + 1 file changed, 11 insertions(+), 5 deletions(-) + +commit f9a0f80e35aff05871bedea040b8b82fba386829 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-18 18:20:17 +0300 + + Translations: Update the Croatian translation + + po/hr.po | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +commit b1b54af595926744e5bf999c2a8c7a6a23ad238a +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-18 13:05:22 +0300 + + Translations: Update the Ukrainian translation + + po/uk.po | 16 ++++++++++------ + 1 file changed, 10 insertions(+), 6 deletions(-) + +commit 0072f6c9bcb69e11638f036fb7d6b54ff3c494e8 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-17 20:31:41 +0300 + + Translations: Restore SPDX license identifier to pt_BR.po + + Fixes: 8d13dd61ecb7 ("Translations: Update the Brazilian Portuguese translation") + + po/pt_BR.po | 2 ++ + 1 file changed, 2 insertions(+) + +commit 7f539d14484ee0fde7c1290ae22eebe739bc92cf +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:50:05 +0300 + + Tests: Test that integer overflow in xz --list totals is caught + + tests/test_files.sh | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +commit 490ca91153c24812bcef1f657220af1a4525eeb1 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:50:05 +0300 + + Tests: Add test file bad-1-index-huge-uncomp.xz + + tests/files/README | 5 +++++ + tests/files/bad-1-index-huge-uncomp.xz | Bin 0 -> 72 bytes + 2 files changed, 5 insertions(+) + +commit 102ea0ed1290869f3af53009f98def030c6b6cd0 +Author: xordanblu <jordygarciamorales@gmail.com> +Date: 2026-08-15 19:50:05 +0300 + + xz: Make it an error if the totals in xz --list exceed UINT64_MAX + + If at least three files are passed to xz --list, the totals might exceed + the range of uint64_t. Previously wrong totals were displayed in this + case. Now an error is shown as soon as an overflow would occur, and the + remaining files aren't processed. In most other cases an error doesn't + prevent xz from processing the remaining files, but in this case it + feels OK because (1) the error should be very rare, (2) it keeps the + code slightly simpler, and (3) with xz --robot --list the caller will + likely ignore the output anyway when xz exits with a non-zero status. + + Co-authored-by: Lasse Collin <lasse.collin@tukaani.org> + Fixes: https://github.com/tukaani-project/xz/pull/240 + + src/xz/list.c | 34 +++++++++++++++++++++++++++++----- + 1 file changed, 29 insertions(+), 5 deletions(-) + +commit 29f9ca3dbcebffd88fb55030db4fc0fcb1f66234 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:42:55 +0300 + + liblzma: index.c 4/4: Reverse the tree building for index_streams + + lzma_index_cat() is used by lzma_file_info_decoder() and by a few apps + that were written before lzma_file_info_decoder() existed (it was added + in 5.3.1alpha). The most common use case for these functions is xz --list. + The other use case is random access decoding. + + When decoding Index fields from multi-Stream .xz files, lzma_index_cat() + is used to concatenate the decoded lzma_index structures. Such decoding + starts from the last Stream in the file and stops at the first Stream in + the file, so lzma_index_cat() is called with the most recently decoded + Stream as the destination argument while the source argument contains all + other Streams that have been decoded so far. That is, the typical use case + is lzma_index_cat(one_stream, many_streams, NULL). + + With the old code, lzma_index_cat() appended Streams from src into dest + one at a time and updated their absolute offsets at the same time. + When a .xz file contained many Streams, the number of Streams in + the source argument grew for each call, which created a performance + problem if the file contained an unusually large number of Streams + (over 10,000). (A large number of Blocks isn't a problem.) + + Make lzma_index_cat() prepend the index_streams from dest to src + instead of appending from src to dest. Use the _bias members to adjust + the offsets of the existing index_streams in src. + + Change the numbering in index_stream.number from 1, 2, 3, ... to + ...., 2, 1, 0. That is, internally the last index_stream is now 0. + + Now lzma_index_cat() needs to update all Streams from dest instead from + src, which keeps the normal use case fast. Obviously the performance + problem still exists, but now it's in the opposite situation (repeated + calls of lzma_index_cat(many_streams, one_stream, NULL)). This should + be acceptable because there should be no use cases where doing it would + be useful. A full fix would require a more complex patch. + + As an alternative, I tried a rewrite of index.c that replaces the trees + with plain arrays and realloc(). It's smaller and simpler, but it's + still somewhat complex because the API is too flexible compared to what + functionality is actually needed. I didn't merge that version because + a rewrite has a higher risk of regressions, and in some corner cases + the peak memory usage was considerably higher than with the old code + (if a custom lzma_allocator is used, realloc() has to be emulated with + alloc + memcpy + free, and native realloc() might need to do that too), + and large preallocations make it unclear what lzma_index_memusage() and + lzma_index_memused() should return. + + Below is an example where xz --list became too slow before this commit. + Pass 18 or 20 instead of 16 to seq, and xz effectively hangs. + + true | xz > big.xz + for I in $(seq 16) + do + cat big.xz big.xz > tmp.xz + mv tmp.xz big.xz + echo ==== + echo $I + time xz -l big.xz + done + + Reported-by: Omkhar Arasaratnam + + src/liblzma/common/index.c | 105 +++++++++++++++++++++++---------------------- + 1 file changed, 54 insertions(+), 51 deletions(-) + +commit 869f0e85a8aa5ec30cc5f30c14e2a27cfa0485cf +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:42:55 +0300 + + liblzma: index.c 3/4: Add index_tree_prev() + + src/liblzma/common/index.c | 19 +++++++++++++++++++ + 1 file changed, 19 insertions(+) + +commit 0543606d156c3bd3bb5b1c08b848b4cbb794c094 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:42:55 +0300 + + liblzma: index.c 2/4: Make index_tree_append support prepending too + + src/liblzma/common/index.c | 72 +++++++++++++++++++++++++++++----------------- + 1 file changed, 46 insertions(+), 26 deletions(-) + +commit 59cac26b8002377fb0d931e9951607d0b84bc67d +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 19:42:55 +0300 + + liblzma: index.c 1/4: Add bias members + + In this commit the biases are still constant. + + src/liblzma/common/index.c | 54 ++++++++++++++++++++++++++++++++++------------ + 1 file changed, 40 insertions(+), 14 deletions(-) + +commit f8f53ca1fbe41e7a17844c762590c01eb701824c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 18:55:25 +0300 + + CI: MSVC: Build shared liblzma + + .github/workflows/msvc.yml | 3 +++ + 1 file changed, 3 insertions(+) + +commit 8697fcc82389255eb2f6de69105f05a2a7262155 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 18:55:25 +0300 + + Update .gitignore + + .gitignore | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 18769dd6927f16562008aee01ab1466c3035110e +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-15 18:55:25 +0300 + + CMake: Don't put test binaries in a subdirectory + + This can matter on Windows: + + - MSVC puts the binaries into subdirectories based on build type + (like "Debug" or "Release"), and then the test executables won't + find liblzma.dll because it's not in the working directory from + which the tests are run. + + - With mingw-w64, liblzma.dll was in the working directory so the test + executables worked. However, the if the system happened to have + liblzma.dll in, for example, the Windows directory, then the DLL + search order might pick that instead. + + Putting the shared library and test executables in the same directory + avoids these problems. + + Reported-by: Frank <mail@franku.nl> + Fixes: https://github.com/tukaani-project/xz/pull/239 + + tests/tests.cmake | 12 +++++------- + 1 file changed, 5 insertions(+), 7 deletions(-) + +commit c8b8ab2ef1eb0a0217ad2027d7f5d242ceb944d3 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-09 16:11:13 +0300 + + liblzma: Fix a theoretical integer overflow in lzma_index_cat() + + To trigger it, one would need to successfully allocate hundreds of + gigabytes of memory. + + src/liblzma/common/index.c | 5 +++++ + 1 file changed, 5 insertions(+) + +commit 1a4a4df3669453e9c80235baffa13e4dd1358bc8 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-09 16:11:13 +0300 + + Tests: test_index: Test lzma_index_memusage(STREAMS_MAX, 1) + + tests/test_index.c | 1 + + 1 file changed, 1 insertion(+) + +commit d31b473917a4270f5bb00fbf52598d4f3c143f7c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-09 16:11:03 +0300 + + liblzma: index.c: Define STREAMS_MAX + + src/liblzma/common/index.c | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +commit 93372479ec052e6cd1ef7b3d9ed8a96781fec155 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-24 21:23:31 +0300 + + Tests: test_index: Extend _dup + _cat testing + + tests/test_index.c | 19 ++++++++++++++++++- + 1 file changed, 18 insertions(+), 1 deletion(-) + +commit a967036afbf56208cbe30fc27ec732ced7ebe4c8 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-24 21:23:31 +0300 + + liblzma: Index decoder: Detect a too large allocation earlier + + Previously, if an Index had more than PREALLOC_MAX Records, then + allocations were done in PREALLOC_DEFAULT chunks (512 Records). + That makes little sense in practice because it's likely that one + would run out of address space later. Return LZMA_MEM_ERROR error + pre-emptively if PREALLOC_MAX would be exceeded. + + src/liblzma/common/index.c | 6 +++--- + src/liblzma/common/index.h | 4 +++- + src/liblzma/common/index_decoder.c | 5 ++++- + 3 files changed, 10 insertions(+), 5 deletions(-) + +commit 9be8d5c91fc33c755823c9f7aac7560dbfea7f91 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-08-04 17:59:43 +0300 + + Update THANKS + + THANKS | 1 + + 1 file changed, 1 insertion(+) + +commit d16c31196e4c1fd4a76c960a5c7db261075e4db1 +Author: Darren Carreras <283775510+carrerasdarren-cell@users.noreply.github.com> +Date: 2026-07-27 22:13:09 -0400 + + Tests: Add a test for the lzma_index_dup() + lzma_index_checks bug() + + Link: https://github.com/tukaani-project/xz/pull/238 + + tests/test_index.c | 14 ++++++++++++++ + 1 file changed, 14 insertions(+) + +commit c57500bda55f61936c9d3afb701d42e92fc8508b +Author: Darren Carreras <283775510+carrerasdarren-cell@users.noreply.github.com> +Date: 2026-07-27 22:13:09 -0400 + + liblzma: Preserve check statistics in lzma_index_dup() + + lzma_index_dup() copied the per-Stream flags but not the cached bitmask + for all Streams except the last one. As a result, lzma_index_checks() on + a duplicated multi-Stream Index reported only the Check type of its + final Stream. + + Fixes: https://github.com/tukaani-project/xz/pull/238 + + src/liblzma/common/index.c | 1 + + 1 file changed, 1 insertion(+) + +commit 6097929ae8f8c2a1714aacd844a8e9747b071cdc +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-21 13:56:42 +0300 + + Update THANKS + + THANKS | 2 ++ + 1 file changed, 2 insertions(+) + +commit 4b9b827117cecd7571c0952b501fa84c4bd7e5fb +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-21 13:50:13 +0300 + + xz: Free the filename read from the file specified with --files/--files0 + + Like the previous commit, this cleans up leak detection slightly. + Otherwise it doesn't matter because xz is going to exit soon anyway. + + src/xz/main.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +commit 7bd6d63b0c0cf2a0c4b609014c49e66d5199e385 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-21 13:32:05 +0300 + + xz: Free the filename allocated for --files or --files0 + + This cleans up leak detection slightly. Otherwise it doesn't matter + because xz is going to exit soon anyway. + + Reported-by: Phuc An + Fixes: https://github.com/tukaani-project/xz/pull/232 + + src/xz/args.c | 2 +- + src/xz/args.h | 2 +- + src/xz/main.c | 4 +++- + 3 files changed, 5 insertions(+), 3 deletions(-) + +commit 0877609467a81fb831a3bcbcea732d20582a58b9 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-21 12:58:45 +0300 + + liblzma: Fix input_size_max in lzma_index_buffer_decode() + + If *in_pos > 0, the condition became more relaxed than it needed to be. + + Fixes: 5005914f1acf ("liblzma: Index decoder: Reject an obviously-bad Number of> + Fixes: 0352dc7929b1 ("liblzma: Silence a compiler warning") + + src/liblzma/common/index_decoder.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +commit d4edae32b6f1d34d68f3ac8917c20d0191e0cfcd +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 22:29:58 +0300 + + Tests: Fix test_index.c when encoding support is disabled + + Fixes: 10dfae83fd3b ("Tests: test_index: Add a test for the Index decoder memusage bug") + + tests/test_index.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +commit 0352dc7929b19f17afcdafae49ec86156cc31462 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 22:14:05 +0300 + + liblzma: Silence a compiler warning + + Fixes: 5005914f1acf ("liblzma: Index decoder: Reject an obviously-bad Number of Records better") + + src/liblzma/common/index_decoder.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +commit 47c44e90089de546619d927dba190f0100df4eda +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + Tests: test_index: Test the stricter Number of Records condition + + tests/test_index.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 45 insertions(+) + +commit a9c92328bdee9b9afc2d8a0a300ad66d8edcf801 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + Tests: Add test file bad-0-index-1.xz + + tests/files/README | 3 +++ + tests/files/bad-0-index-1.xz | Bin 0 -> 32 bytes + 2 files changed, 3 insertions(+) + +commit 006126aca97b5f5b6bacd549787b18c1e9ce1908 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + Revert "liblzma: Add internal constant INDEX_RECORDS_MAX" + + This reverts commit 9ff4dadca96a5ac3cb478759a4fdb8663be114eb. + The INDEX_RECORDS_MAX constant isn't needed after the previous commit. + + src/liblzma/common/index.h | 6 ------ + 1 file changed, 6 deletions(-) + +commit 5005914f1acfdb9884bedc8100bce6185d0861b6 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + liblzma: Index decoder: Reject an obviously-bad Number of Records better + + File info decoder knows Backward Size, so pass it to the Index decoder. + Now the Index decoder, when called from the file info decoder, won't + allocate a large amount of memory when Backward Size is tiny and the + Number of Records field contains a huge invalid value. + + If a reasonable memory usage limit was set, the old code returned + LZMA_MEMLIMIT_ERROR if the value in Number of Records was too huge. + Now it results in LZMA_DATA_ERROR which is better. + + src/liblzma/common/file_info.c | 7 ++++--- + src/liblzma/common/index_decoder.c | 35 +++++++++++++++++++++++++---------- + src/liblzma/common/index_decoder.h | 2 +- + 3 files changed, 30 insertions(+), 14 deletions(-) + +commit 10dfae83fd3b63c32c0e170540e98c4b25bf02c6 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + Tests: test_index: Add a test for the Index decoder memusage bug + + tests/test_index.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 48 insertions(+) + +commit bd3bfe834620b664d73b15952894864660a9bef1 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + liblzma: Index decoder: Fix lzma_memusage() + + If lzma_stream was initialized using lzma_index_decoder() or + lzma_file_info_decoder(), and lzma_memusage() was called after a few + Records had already been decoded, the return value was too low because + it was calculated based on the number of Records that haven't been + decoded yet. So near the end of a big Index, the return value would be + tiny compared to the actual memory usage. + + It worked correctly in the primary use case, that is, after + LZMA_MEMLIMIT_ERROR. Thus, in practice this bug didn't matter much. + + src/liblzma/common/index_decoder.c | 21 +++++++++++++++++---- + 1 file changed, 17 insertions(+), 4 deletions(-) + +commit 2187b27c64ef23d6ffa2917c8d835e100969e3e6 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-20 20:00:06 +0300 + + Tests: test_index: Fix testing of _locate() with concatenated lzma_index + + tests/test_index.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +commit 8d13dd61ecb703cfabdf826ca0a282c2b5f60cd4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-19 10:44:49 +0300 + + Translations: Update the Brazilian Portuguese translation + + po/pt_BR.po | 453 +++++++++++++++++------------------------------------------- + 1 file changed, 125 insertions(+), 328 deletions(-) + +commit 791ce1bb6dcae5f424a5a40b877ec4abd9476399 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-15 22:12:07 +0300 + + liblzma: Remove an outdated API doc in hardware.h + + src/liblzma/api/lzma/hardware.h | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +commit f3b5688159c60495f48db3942a36509671dfce89 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-03 16:55:03 +0300 + + Tests: Comment that a sanitizer may be needed for lzma_index_append() bug + + Fixes: a3ea8832bec1 ("Tests: Add a test for the lzma_index_prealloc() + lzma_index_append() bug") + Link: https://github.com/tukaani-project/xz/issues/233 + + tests/test_index.c | 2 ++ + 1 file changed, 2 insertions(+) + +commit 6a07932a8787eab0b2e1d1a6856d4393e50bba67 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-02 21:56:33 +0300 + + Translations: Update the Croatian translation + + po/hr.po | 387 +++++++++++++++++++++++++++++++-------------------------------- + 1 file changed, 191 insertions(+), 196 deletions(-) + +commit b8d0707ae788b401161babf0fe771c859e3f996c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-07-02 21:09:36 +0300 + + NEWS: Clarify which bug is CVE-2026-34743 + + Fixes: https://salsa.debian.org/debian/xz-utils/-/merge_requests/5#note_775071 + + NEWS | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +commit e95cd90da9c620cf9ceca1f9e94025bcb437fca9 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-21 19:29:59 +0300 + + Translations: Fix LTR/RTL issue in Arabic xz man page tables + + The headings are RTL but in some tables the rows were LTR because the + first strong char was LTR, like the "e" in "-0e". Add {RLM}{LRM}...{RLM} + to those columns ("-0" to "-9", and "-0e" to "-9e"): + + sed -ri $'/^msgstr /{s/"(-[0-9]e?)"/"\u200f\u200e\\1\u200f"/}' ar.po + + It's not perfect because groff doesn't treat RLM and LRM as zero-width + chars, and thus the columns are slightly misaligned. It's good enough + in practice still. + + Thanks-to: Zayed Al-Saidi <zayed.alsaidi@gmail.com> + + po4a/ar.po | 40 ++++++++++++++++++++-------------------- + 1 file changed, 20 insertions(+), 20 deletions(-) + +commit b46944d73495b99148eebb1d9661574c2041f4db +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-21 20:02:49 +0300 + + CI: Add 'timeout: 10' to coverity.yml + + .github/workflows/coverity.yml | 1 + + 1 file changed, 1 insertion(+) + +commit 0def41200695c830c6da95a21ef7158f60a6af01 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-19 22:07:15 +0300 + + CI: Don't pin vmactions or MSYS2 actions to commit ID + + These workflows don't use any secrets other than GITHUB_TOKEN, which + only has the 'contents: read' permission. That is, if an action was + compromised, it shouldn't be able to modify the repository or steal any + secrets. I recognize that a compromised action isn't completely harmless + still. For example, the runners on GitHub have Internet access which the + action could abuse until the workflow timeout (currently 10 or 20 minutes) + expires. + + Now we always use the most recent actions instead of ones that are + sometimes several months old. The hassle of updating the pinned + commit IDs goes away. The benefits feel bigger than the risks. + + Link: https://github.com/tukaani-project/xz/pull/227 + + .github/workflows/dragonflybsd.yml | 2 +- + .github/workflows/freebsd.yml | 2 +- + .github/workflows/haiku.yml | 2 +- + .github/workflows/msys2.yml | 4 ++-- + .github/workflows/netbsd.yml | 2 +- + .github/workflows/openbsd.yml | 2 +- + .github/workflows/solaris.yml | 2 +- + 7 files changed, 8 insertions(+), 8 deletions(-) + +commit 3c0dbed89069f7a57f7ce5e0a9259081fcc4ba4d +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-19 22:07:15 +0300 + + CI: Upgrade actions/checkout from v6 to v7 + + .github/workflows/ci.yml | 2 +- + .github/workflows/coverity.yml | 2 +- + .github/workflows/dragonflybsd.yml | 2 +- + .github/workflows/freebsd.yml | 2 +- + .github/workflows/haiku.yml | 2 +- + .github/workflows/msvc.yml | 2 +- + .github/workflows/msys2.yml | 2 +- + .github/workflows/netbsd.yml | 2 +- + .github/workflows/openbsd.yml | 2 +- + .github/workflows/solaris.yml | 2 +- + 10 files changed, 10 insertions(+), 10 deletions(-) + +commit 0fab61de63f458a6f72ee1745790c17ac368787e +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-19 22:07:15 +0300 + + CI: Run CI workflows on push to 'ci_run' in addition to 'master' + + Make it convenient to run the CI workflows by pushing to a branch with + a specific name. + + .github/workflows/ci.yml | 2 +- + .github/workflows/cifuzz.yml | 2 +- + .github/workflows/dragonflybsd.yml | 2 +- + .github/workflows/freebsd.yml | 2 +- + .github/workflows/haiku.yml | 2 +- + .github/workflows/msvc.yml | 2 +- + .github/workflows/msys2.yml | 2 +- + .github/workflows/netbsd.yml | 2 +- + .github/workflows/openbsd.yml | 2 +- + .github/workflows/solaris.yml | 2 +- + 10 files changed, 10 insertions(+), 10 deletions(-) + +commit 8e8c6598a56d5d07daa9554cfac2456631afc1a9 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-19 22:07:15 +0300 + + CI: Remove the on 'pull_request:' entries + + I don't run CI from PRs, so these just add noise to the "Actions" tab + in the GH UI. + + .github/workflows/ci.yml | 2 -- + .github/workflows/dragonflybsd.yml | 2 -- + .github/workflows/freebsd.yml | 2 -- + .github/workflows/haiku.yml | 2 -- + .github/workflows/msvc.yml | 2 -- + .github/workflows/msys2.yml | 2 -- + .github/workflows/netbsd.yml | 2 -- + .github/workflows/openbsd.yml | 2 -- + .github/workflows/solaris.yml | 2 -- + 9 files changed, 18 deletions(-) + +commit 88ae12f7873c7d8731cbd1d05313dc4a016171f7 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-19 22:07:15 +0300 + + CI: Use 'contents: read' after all + + If someone makes a private fork, then it will work there too. + + Thanks-to: Eli Schwartz + + .github/workflows/ci.yml | 3 ++- + .github/workflows/cifuzz.yml | 3 ++- + .github/workflows/coverity.yml | 3 ++- + .github/workflows/dragonflybsd.yml | 3 ++- + .github/workflows/freebsd.yml | 3 ++- + .github/workflows/haiku.yml | 3 ++- + .github/workflows/msvc.yml | 3 ++- + .github/workflows/msys2.yml | 3 ++- + .github/workflows/netbsd.yml | 3 ++- + .github/workflows/openbsd.yml | 3 ++- + .github/workflows/solaris.yml | 3 ++- + 11 files changed, 22 insertions(+), 11 deletions(-) + +commit b70af4d5320750acda7cffbd75a3390eede600bd +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-17 16:02:45 +0300 + + CI: Add 'permissions' to coverity.yml + + GitHub's actions/checkout@v6 is documented to need 'contents: read' but + it works without it too, probably because the repository is public. + + Reported-by: GitHub user Alb3e3 + Fixes: https://github.com/tukaani-project/xz/pull/230 + + .github/workflows/coverity.yml | 2 ++ + 1 file changed, 2 insertions(+) + +commit 56c18ccfb7a707d4eba4b619e60324f84032b4fa +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-16 21:48:57 +0300 + + liblzma: file info decoder: Add an integer overflow check + + After the previous commit, the overflow should be impossible, but it's + better to add the check anyway. + + Fixes: https://github.com/tukaani-project/xz/pull/226 + + src/liblzma/common/file_info.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +commit a73f4c67154fe6111076ee673fd8991f5853d8d4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-03 00:20:28 +0300 + + liblzma: Index decoder: Reject an obviously-bad Number of Records early + + There no point to try decoding further when the Index is clearly not + from a valid .xz file. + + This also limits what index_decoder_memconfig() can return as the + memory usage. Very huge values aren't possible anymore. This affects + file_info_decoder_memconfig() in file_info.c where very huge values + aren't expected and could result in an integer overflow when not much + memory has actually been allocated. Now the overflow can only happen if + one manages to successfully allocate and fill a few exabytes of memory. + + Reported-by: GitHub user jmestwa-coder + Fixes: https://github.com/tukaani-project/xz/pull/226 + + src/liblzma/common/index_decoder.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +commit 9ff4dadca96a5ac3cb478759a4fdb8663be114eb +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-03 00:05:48 +0300 + + liblzma: Add internal constant INDEX_RECORDS_MAX + + It's the theoretical maximum number of Records that fit in a .xz file. + This can be used to reject obviously wrong values. + + src/liblzma/common/index.h | 6 ++++++ + 1 file changed, 6 insertions(+) + +commit 25a53ed6d1c91372fde1fe8631dcf93ac6625e5c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-17 22:04:17 +0300 + + Translation: Update Arabic man page translations + + In two tables, the column headings are now narrower and + + MANWIDTH=80 man --warnings po4a/man/ar/xz.1 > /dev/null + + no longer warns. + + Thanks-to: Zayed Al-Saidi <zayed.alsaidi@gmail.com> + Fixes: https://github.com/tukaani-project/xz/pull/220 + + po4a/ar.po | 32 ++++++++++++++++---------------- + 1 file changed, 16 insertions(+), 16 deletions(-) + +commit 225c46e99c3a3e6e0996649ee8d858b6e07111b3 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-06-04 23:33:37 +0300 + + xzdiff: Use LC_ALL=C with sed and expr + + This is required to handle strings that may contain invalid multibyte + sequences. For example, on GNU/Linux and FreeBSD 14 with UTF-8 locale: + + $ printf "\201'A\n" | sed s/./x/ | hexdump -C + 00000000 81 78 41 0a |.xA.| + + That is, the first byte remains untouched and the ' is replaced with x. + This kind of string would result in inbalanced quotes in xzdiff. In the + C locale it works safely: + + $ printf "\201'A\n" | LC_ALL=C sed s/./x/ | hexdump -C + 00000000 78 27 41 0a |x'A.| + + Similar changes were made in xzgrep in 2022 (see the commits 69d1b3fc2967 + and 6a4a4a7d2667). Somehow xzdiff was forgotten even though this issue + was fixed in zdiff in gzip in 2022. + + Co-authored-by: rootvector2 <dxbnaveed.k@gmail.com> + Fixes: https://github.com/tukaani-project/xz/pull/228 + + src/scripts/xzdiff.in | 14 +++++++------- + 1 file changed, 7 insertions(+), 7 deletions(-) + +commit 1aab7e9cab1293c2b8a481ed1beac94da21ef04f +Author: rootvector2 <dxbnaveed.k@gmail.com> +Date: 2026-05-28 12:29:23 +0530 + + xz: Fix --files/--files0 usage via XZ_OPT and XZ_DEFAULTS env vars + + If --files=FILELIST or --files0=FILELIST was specified via an enrivonment + variable, it worked if there were no errors when reading from the file + FILELIST. However, if an error occurred when reading from FILELIST, + there was a use-after-free bug when printing the error message because + the memory containing the string "FILELIST" had been freed. + + printf foo.xz > filelist.txt + XZ_OPT=--files=filelist.txt xz -l + + Because filelist.txt doesn't end in a newline, the file list is seen as + invalid/truncated, resulting in an error message like this: + + xz: ????????.txt: Unexpected end of input when reading filenames + + The question marks are because the garbage string is masked using + tuklib_mask_nonprint(). After this commit, it works: + + xz: filelist.txt: Unexpected end of input when reading filenames + + Co-authored-by: Lasse Collin <lasse.collin@tukaani.org> + Fixes: https://github.com/tukaani-project/xz/pull/223 + + src/xz/args.c | 11 ++++++++--- + 1 file changed, 8 insertions(+), 3 deletions(-) + +commit d0d380797e9f1c25c0a88ce5d4cfd092590a209d +Author: rootvector2 <dxbnaveed.k@gmail.com> +Date: 2026-05-27 23:51:52 +0530 + + xz: Mask the multiplier suffix in str_to_uint64() error message + + A similar error message in the same function was masked in the + previous commit 7e7bb6c21cf9. + + Fixes: https://github.com/tukaani-project/xz/pull/222 + + src/xz/util.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 7e7bb6c21cf9c78f95a5049242fa35aebce38ea4 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-26 19:30:29 +0300 + + xz: Use tuklib_mbstr_nonprint in a few more places + + Reported-by: GitHub user curious-rabbit + + src/xz/args.c | 22 +++++++++++++--------- + src/xz/coder.c | 2 +- + src/xz/util.c | 2 +- + 3 files changed, 15 insertions(+), 11 deletions(-) + +commit 1df00237ae36da1debd5f30ca76a78e28a10d6e2 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-25 18:32:15 +0300 + + xzgrep: Fix handling of the ' char at the end of an option + + For example, the following would run the "id" command: + + xzgrep "--a='" "--b=;id;'" "pattern" foo.xz + + The bug was inherited from gzip's zgrep. It was reported to bug-gzip by + Leenear. It was fixed in gzip in the commit cd74dc30f8cc by Paul Eggert. + + Link: https://lists.gnu.org/archive/html/bug-gzip/2026-04/msg00032.html + + src/scripts/xzgrep.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 2928248922a200eb3fbcc5e7dea9216d7f81a788 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-21 19:04:34 +0300 + + Update THANKS + + THANKS | 3 +++ + 1 file changed, 3 insertions(+) + +commit 8654b72980c774d4cb142e11a6012fe9a6a83122 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-21 18:57:36 +0300 + + Landlock: Support ABI version 9 + + Features added in versions 7 and 8 shouldn't be relevant here. + + src/common/my_landlock.h | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +commit f5665a445f08b031cd9807068654620b5a15e5eb +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-21 18:38:16 +0300 + + Landlock: Provide fallbacks for ABI 2, 3, and 5 macros + + This way the newer features are available on new enough kernels + even if the package is built against old <linux/landlock.h>. + + Landlock ABI versions 4 and 6 would require an extended + struct landlock_ruleset_attr. No fallback is provided (or planned) + for those features. + + Suggested-by: Omkhar Arasaratnam + + src/common/my_landlock.h | 32 ++++++++++++++++++++------------ + 1 file changed, 20 insertions(+), 12 deletions(-) + +commit 6237e4319fabe740d6f56ebf5960ce348bafdbcf +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 21:21:51 +0300 + + xz: Reduce indentation of three tables on the man page + + The extra indentation makes the tables stand out from the text, and + in the English man page it works well. However, some translations + have strings that require more space, and the table might become + too wide. In some translations this has been solved by splitting + the translated string on two lines, for example, Romanian translation + of "4096-byte alignment is best". + + On Debian 13 with groff 1.23.0, the "man" command shows a few warnings: + + $ LANGUAGE=it MANWIDTH=80 man --warnings xz > /dev/null + <standard input>:477: warning: table wider than line length minus indentation + <standard input>:545: warning: table wider than line length minus indentation + <standard input>:1182: warning: table wider than line length minus indentation + + German man page shows those warnings too. On Arch with groff 1.24.1, + there are fewer warnings because the newer groff indents less, and + fewer tables reach or exceed the right margin. Thus, some safety margin + is good to keep the output neat with different man page formatters. + + It feels silly to reduce the indentation on the English man page which + has plenty of safety margin, but it's simpler than trying to do it + only for the translated man pages. This doesn't affect readability; + it's just cosmetic. + + I tried using .IP instead of .RS + .PP for the tables, but on Solaris 10 + it results in an extra empty line before the table. It makes some + difference on AIX 7.3 too. With groff, both are fine. Stick to + .RS + .PP + .TS ... .TE + .RE for slightly better portability. + + Co-authored-by: Otto Kekäläinen <otto@debian.org> + Partially-fixes: https://github.com/tukaani-project/xz/pull/220 + + src/xz/xz.1 | 6 ------ + 1 file changed, 6 deletions(-) + +commit f9b243e2034a2d71af92fb4785e02d68d9f2216c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 13:42:45 +0300 + + Translations: Fix a syntax typo in the Serbian xz man page translation + + Testing with this: + + cd po4a + ./update-po + groff -K utf8 -man -t -ww -rCHECKSTYLE=3 -z man/sr/*.1 + + The output included this: + + an.tmac:man/sr/xz.1:751: style: 1 leading space(s) on input line + + The actual reason seemed to be an unwanted newline (likely a typo). + + po4a/sr.po | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +commit 625387f089655a8e4a2f4317e66b20a42e276d06 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 13:37:18 +0300 + + Translations: Fix a groff warning in the German xz man page translation + + Testing with this: + + cd po4a + ./update-po + groff -K utf8 -man -t -ww -rCHECKSTYLE=3 -z man/de/*.1 + + The output included this: + + an.tmac:man/de/xz.1:524: style: 1 leading space(s) on input line + + po4a/de.po | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 6365d4dadd86ceb6c02734cf1aac4c483043b733 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 13:32:43 +0300 + + Translations: Fix syntax in the Korean man page translations + + Testing with this: + + cd po4a + ./update-po + groff -K utf8 -man -t -ww -z man/ko/*.1 + + It produced warnings: + + troff:man/ko/xz.1:1405: warning: color 'i' not defined + troff:man/ko/xzdec.1:17: warning: cannot select font '\' + troff:man/ko/xzdec.1:19: warning: cannot select font '\' + + po4a/ko.po | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +commit 8e99c3934b31eb84e9a08329be41623deaf2de54 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 13:24:03 +0300 + + Translations: Fix a syntax typo in the French xz man page translation + + Testing with this: + + cd po4a + ./update-po + groff -K utf8 -man -t -ww -z man/fr/xz.1 + + It produced a warning: + + troff:man/fr/xz.1:516: warning: color 'i' not defined + + The backslash for non-breaking space was misplaced: + + Old: 8 \\Mio, + New: 8\\ Mio, + + po4a/fr.po | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +commit 784adc9263a9e5dee684fd5dd45a93fcaed98981 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 13:09:59 +0300 + + Man pages: Improve rendering in mandoc + + Don't use "" as an argument to .IP because it makes mandoc insert + an unwanted extra empty line. With the xz man page, this change makes + no difference in plain text output with GNU groff or on AIX or + Solaris 10. The list in the xzgrep man page has a different indentation + now because previously it had two arguments (.IP "" 4), but that's OK. + + I don't remember why '.IP' was changed to '.IP ""' in cec0ddc8ec4c. + Maybe it made a difference on some proprietary UNIX over 15 years ago + or maybe it was just a misunderstanding. + + src/scripts/xzgrep.1 | 18 +++--- + src/xz/xz.1 | 152 +++++++++++++++++++++++++-------------------------- + 2 files changed, 85 insertions(+), 85 deletions(-) + +commit 814e8828b57af30b529aee15132f4066ddb8fcf0 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-20 01:25:41 +0300 + + po4a/update-po: Workaround an issue with non-ASCII chars in tables + + Non-ASCII chars in tables (.TS ... .TE) result in warnings: + + man --warnings po4a/man/uk/xz.1 > /dev/null + troff:<standard input>:464: warning: special character 'u0411\[tbl' not defined + troff:<standard input>:1157: warning: special character 'u044\[tbl' not defined + troff:<standard input>:1682: warning: special character 'u0434\[tbl' not defined + + po4a generates groff Unicode escapes like '\u0435' followed immediately + by '[' in table contexts, causing groff to interpret the full string + 'u0435[tbl' as a single special character name. Fix this by inserting + a dummy character (\&) after every Unicode escape so that the bracket + is treated as literal text instead of part of the character name. + + Co-authored-by: Otto Kekäläinen <otto@debian.org> + Partially-fixes: https://github.com/tukaani-project/xz/pull/220 + + po4a/update-po | 6 ++++++ + 1 file changed, 6 insertions(+) + +commit bc96573a65008641783f25a2cbd877b9cccff1e9 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 18:12:07 +0300 + + Translations: Update the German man page translations + + po4a/de.po | 20 +++++++------------- + 1 file changed, 7 insertions(+), 13 deletions(-) + +commit 81faa2e653e81c305cc386c0a5280fef5924c9b7 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 17:36:15 +0300 + + liblzma: ARM64 & LoongArch: Avoid aligned_readXXle on unaligned pointers + + While most ARM64 and LoongArch processors support unaligned access, + the code wouldn't have worked on strict-align processors. + + Use byte-by-byte method if the input buffer is at most 7 bytes. This + way the code works on strict-align processors too. It might not be + the very best method for tiny buffers, but it shouldn't matter in + practice (the simple implementation for big buffers isn't amazing + either because it cannot execute more than one CRC32 instruction + at a time on superscalar processors). + + Reported-by: Guanni Qu + Fixes: 0ed893668554 ("liblzma: ARM64 CRC32: Align the buffer faster") + Fixes: 7baf6835cfbf ("liblzma: Speed up CRC32 calculation on 64-bit LoongArch") + + src/liblzma/check/crc32_arm64.h | 36 ++++++++++++++++++++---------------- + src/liblzma/check/crc32_loongarch.h | 32 +++++++++++++++++++------------- + 2 files changed, 39 insertions(+), 29 deletions(-) + +commit c2fe308792d97ba41b898a009e4d9d98d1e75720 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 18:07:47 +0300 + + liblzma: Document the lzma_stream_buffer_decode() bug + + src/liblzma/api/lzma/container.h | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +commit 6566e6251ac46e2a6caeb1dee504da1c241e2b7f +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 14:10:03 +0300 + + Tests: tuktest.h: Fix a comment + + tests/tuktest.h | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +commit 29584cdaf360e0528859cc962b380a93a6398590 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 14:04:43 +0300 + + Tests: Add a test for the lzma_stream_buffer_decode() bug + + tests/Makefile.am | 2 + + tests/test_stream_buffer_decode.c | 109 ++++++++++++++++++++++++++++++++++++++ + tests/tests.cmake | 1 + + 3 files changed, 112 insertions(+) + +commit b53838ae0caf9f09b976a8577cf4392910d6f02c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 13:52:41 +0300 + + liblzma: lzma_stream_buffer_decode: Fix wrong error code + + lzma_stream_buffer_decode() would often fail with LZMA_BUF_ERROR + when it should have failed with LZMA_DATA_ERROR. + In debug builds this also resulted in an assertion failure. + + In the _buffer_decode() functions, LZMA_BUF_ERROR is only used when + the output buffer is too small. Truncated input is LZMA_DATA_ERROR. + + Reported-by: Guanni Qu + Reported-by: Omkhar Arasaratnam + Fixes: c81f13ff2927 ("Added lzma_stream_buffer_decode() and made minor cleanups.") + + src/liblzma/common/stream_buffer_decoder.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +commit 327c906dcd6a9e40a39bf922f69560bd62f0b1a2 +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-05-19 12:34:53 +0300 + + xz: Fix use-after-free in verbose error message to a non-tty + + If --verbose was used when stderr isn't a terminal, the filename + contained a non-printable character, and an error occurred, + a use-after-free occurred when printing the error message. + + The bug exists in versions 5.7.1alpha to 5.8.3. + + Reported-by: Omkhar Arasaratnam + Fixes: d22f96921fd2 ("xz: Use tuklib_mbstr_nonprint") + + src/xz/message.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +commit ebb0e6789cefe3be71756881aa8f2009fda9938c +Author: Lasse Collin <lasse.collin@tukaani.org> +Date: 2026-03-31 18:41:04 +0300 + + doc/SHA256SUMS: Add 5.8.3 + + doc/SHA256SUMS | 6 ++++++ + 1 file changed, 6 insertions(+) + commit 4b73f2ec19a99ef465282fbce633e8deb33691b3 Author: Lasse Collin <lasse.collin@tukaani.org> Date: 2026-03-31 17:26:16 +0300 diff --git a/contrib/xz/THANKS b/contrib/xz/THANKS index d519147617e5..b1f9b0eebf47 100644 --- a/contrib/xz/THANKS +++ b/contrib/xz/THANKS @@ -65,6 +65,7 @@ has been important. :-) - Daniel Mealha Cabrita - Daniel Packard - Daniel Richard G. + - Darren Carreras - David Burklund - Denis Excoffier - Derwin McGeary @@ -96,6 +97,7 @@ has been important. :-) - Gabriela Gutierrez - Gilles Espinasse - Gregory Margo + - Guanni Qu - Guillaume Outters - Guiorgy Potskhishvili - H. Peter Anvin @@ -178,8 +180,10 @@ has been important. :-) - Nicholas Jackson - Nobuhiro Iwamatsu - Ole André Vadla Ravnås + - Omkhar Arasaratnam - Orange Tsai - Orgad Shaneh + - Otto Kekäläinen - Patrick J. Volkerding - Paul Eggert - Paul Townsend @@ -190,6 +194,7 @@ has been important. :-) - Peter O'Gorman - Peter Pallinger - Peter Seiderer + - Phuc An - Pierre-Yves Martin - Pilorz Wojciech - Pino Toscano @@ -245,6 +250,7 @@ has been important. :-) - Xi Ruoyao - Xin Li - Yifeng Li + - Zayed Al-Saidi - 榆柳松 (ZhengSen Wang) Companies: diff --git a/contrib/xz/src/common/my_landlock.h b/contrib/xz/src/common/my_landlock.h index 5f761695b6a5..35aee08e7712 100644 --- a/contrib/xz/src/common/my_landlock.h +++ b/contrib/xz/src/common/my_landlock.h @@ -24,6 +24,29 @@ #include <sys/utsname.h> +// In case we are being compiled against an old <linux/landlock.h>, +// provide fallbacks for handled_access_fs flags that were added in +// Landlock ABI versions 2, 3, 5, and 9. This way we need fewer #ifdefs +// later on and, more importantly, these flags are then supported if +// the program is run under a kernel that supports a newer Landlock ABI +// than the current <linux/landlock.h>. +// +// NOTE: There are no fallbacks for ABI 4 and 6 features because they +// require an extended struct landlock_ruleset_attr. +#ifndef LANDLOCK_ACCESS_FS_REFER +# define LANDLOCK_ACCESS_FS_REFER (1ULL << 13) +#endif +#ifndef LANDLOCK_ACCESS_FS_TRUNCATE +# define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) +#endif +#ifndef LANDLOCK_ACCESS_FS_IOCTL_DEV +# define LANDLOCK_ACCESS_FS_IOCTL_DEV (1ULL << 15) +#endif +#ifndef LANDLOCK_ACCESS_FS_RESOLVE_UNIX +# define LANDLOCK_ACCESS_FS_RESOLVE_UNIX (1ULL << 16) +#endif + + /// \brief Initialize Landlock ruleset attributes to forbid everything /// /// The supported Landlock ABI is checked at runtime and only the supported @@ -87,15 +110,10 @@ my_landlock_ruleset_attr_forbid_all(struct landlock_ruleset_attr *attr) | LANDLOCK_ACCESS_FS_MAKE_FIFO | LANDLOCK_ACCESS_FS_MAKE_BLOCK | LANDLOCK_ACCESS_FS_MAKE_SYM -#ifdef LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_REFER // ABI 2 -#endif -#ifdef LANDLOCK_ACCESS_FS_TRUNCATE | LANDLOCK_ACCESS_FS_TRUNCATE // ABI 3 -#endif -#ifdef LANDLOCK_ACCESS_FS_IOCTL_DEV | LANDLOCK_ACCESS_FS_IOCTL_DEV // ABI 5 -#endif + | LANDLOCK_ACCESS_FS_RESOLVE_UNIX // ABI 9 ; #ifdef LANDLOCK_ACCESS_NET_BIND_TCP @@ -115,15 +133,11 @@ my_landlock_ruleset_attr_forbid_all(struct landlock_ruleset_attr *attr) // Disable flags that require a new ABI version. switch (abi_version) { case 1: -#ifdef LANDLOCK_ACCESS_FS_REFER attr->handled_access_fs &= ~LANDLOCK_ACCESS_FS_REFER; -#endif FALLTHROUGH; case 2: -#ifdef LANDLOCK_ACCESS_FS_TRUNCATE attr->handled_access_fs &= ~LANDLOCK_ACCESS_FS_TRUNCATE; -#endif FALLTHROUGH; case 3: @@ -133,9 +147,7 @@ my_landlock_ruleset_attr_forbid_all(struct landlock_ruleset_attr *attr) FALLTHROUGH; case 4: -#ifdef LANDLOCK_ACCESS_FS_IOCTL_DEV attr->handled_access_fs &= ~LANDLOCK_ACCESS_FS_IOCTL_DEV; -#endif FALLTHROUGH; case 5: @@ -152,6 +164,11 @@ my_landlock_ruleset_attr_forbid_all(struct landlock_ruleset_attr *attr) FALLTHROUGH; + case 7: + case 8: + attr->handled_access_fs &= ~LANDLOCK_ACCESS_FS_RESOLVE_UNIX; + FALLTHROUGH; + default: // We only know about the features of the ABIs 1-6. break; diff --git a/contrib/xz/src/liblzma/api/lzma/container.h b/contrib/xz/src/liblzma/api/lzma/container.h index 37d9ab92f4b5..ea773946f1bb 100644 --- a/contrib/xz/src/liblzma/api/lzma/container.h +++ b/contrib/xz/src/liblzma/api/lzma/container.h @@ -935,7 +935,10 @@ extern LZMA_API(lzma_ret) lzma_lzip_decoder( * - LZMA_MEM_ERROR * - LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. * The minimum required memlimit value was stored to *memlimit. - * - LZMA_BUF_ERROR: Output buffer was too small. + * - LZMA_BUF_ERROR: Output buffer was too small. (liblzma 5.8.3 + * and older have a bug: LZMA_BUF_ERROR may be returned when + * the input seems to be truncated. The correct error code + * would be LZMA_DATA_ERROR.) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( diff --git a/contrib/xz/src/liblzma/api/lzma/hardware.h b/contrib/xz/src/liblzma/api/lzma/hardware.h index 7a1a84fcccfc..6b14a2e2036c 100644 --- a/contrib/xz/src/liblzma/api/lzma/hardware.h +++ b/contrib/xz/src/liblzma/api/lzma/hardware.h @@ -9,10 +9,7 @@ * ways to limit the resource usage. Applications linking against liblzma * need to do the actual decisions how much resources to let liblzma to use. * To ease making these decisions, liblzma provides functions to find out - * the relevant capabilities of the underlying hardware. Currently there - * is only a function to find out the amount of RAM, but in the future there - * will be also a function to detect how many concurrent threads the system - * can run. + * the relevant capabilities of the underlying hardware. * * \note On some operating systems, these function may temporarily * load a shared library or open file descriptor(s) to find out diff --git a/contrib/xz/src/liblzma/api/lzma/version.h b/contrib/xz/src/liblzma/api/lzma/version.h index 5e524f0a9742..db83389f3339 100644 --- a/contrib/xz/src/liblzma/api/lzma/version.h +++ b/contrib/xz/src/liblzma/api/lzma/version.h @@ -22,7 +22,7 @@ #define LZMA_VERSION_MINOR 8 /** \brief Patch version number of the liblzma release. */ -#define LZMA_VERSION_PATCH 3 +#define LZMA_VERSION_PATCH 4 /** * \brief Version stability marker diff --git a/contrib/xz/src/liblzma/check/crc32_arm64.h b/contrib/xz/src/liblzma/check/crc32_arm64.h index cce1131b336f..4a0818f17300 100644 --- a/contrib/xz/src/liblzma/check/crc32_arm64.h +++ b/contrib/xz/src/liblzma/check/crc32_arm64.h @@ -50,9 +50,15 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; - if (size >= 8) { - // Align the input buffer because this was shown to be - // significantly faster than unaligned accesses. + if (size < 8) { + while (size > 0) { + crc = __crc32b(crc, *buf++); + --size; + } + } else { + // We have at least 8 bytes of input. Align the input buffer. + // Aligned is faster than unaligned access and works also on + // strict-align targets. const size_t align = (0 - (uintptr_t)buf) & 7; if (align & 1) @@ -79,23 +85,21 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) buf < limit; buf += 8) crc = __crc32d(crc, aligned_read64le(buf)); - size &= 7; - } + // Process the remaining 0-7 bytes. + if (size & 4) { + crc = __crc32w(crc, aligned_read32le(buf)); + buf += 4; + } - // Process the remaining bytes that are not 8 byte aligned. - if (size & 4) { - crc = __crc32w(crc, aligned_read32le(buf)); - buf += 4; - } + if (size & 2) { + crc = __crc32h(crc, aligned_read16le(buf)); + buf += 2; + } - if (size & 2) { - crc = __crc32h(crc, aligned_read16le(buf)); - buf += 2; + if (size & 1) + crc = __crc32b(crc, *buf); } - if (size & 1) - crc = __crc32b(crc, *buf); - return ~crc; } diff --git a/contrib/xz/src/liblzma/check/crc32_loongarch.h b/contrib/xz/src/liblzma/check/crc32_loongarch.h index ec738b83d70a..62cad8520eec 100644 --- a/contrib/xz/src/liblzma/check/crc32_loongarch.h +++ b/contrib/xz/src/liblzma/check/crc32_loongarch.h @@ -21,7 +21,13 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc_unsigned) { int32_t crc = (int32_t)~crc_unsigned; - if (size >= 8) { + if (size < 8) { + while (size > 0) { + crc = __crc_w_b_w((int8_t)*buf++, crc); + --size; + } + } else { + // We have at least 8 bytes of input. Align the input buffer. const size_t align = (0 - (uintptr_t)buf) & 7; if (align & 1) @@ -39,26 +45,26 @@ crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc_unsigned) size -= align; + // Process 8 bytes at a time. for (const uint8_t *limit = buf + (size & ~(size_t)7); buf < limit; buf += 8) crc = __crc_w_d_w((int64_t)aligned_read64le(buf), crc); - size &= 7; - } + // Process the remaining 0-7 bytes. + if (size & 4) { + crc = __crc_w_w_w((int32_t)aligned_read32le(buf), crc); + buf += 4; + } - if (size & 4) { - crc = __crc_w_w_w((int32_t)aligned_read32le(buf), crc); - buf += 4; - } + if (size & 2) { + crc = __crc_w_h_w((int16_t)aligned_read16le(buf), crc); + buf += 2; + } - if (size & 2) { - crc = __crc_w_h_w((int16_t)aligned_read16le(buf), crc); - buf += 2; + if (size & 1) + crc = __crc_w_b_w((int8_t)*buf, crc); } - if (size & 1) - crc = __crc_w_b_w((int8_t)*buf, crc); - return (uint32_t)~crc; } diff --git a/contrib/xz/src/liblzma/common/alone_decoder.c b/contrib/xz/src/liblzma/common/alone_decoder.c index 8ebbbe8aa70f..174b3b1dfebb 100644 --- a/contrib/xz/src/liblzma/common/alone_decoder.c +++ b/contrib/xz/src/liblzma/common/alone_decoder.c @@ -152,8 +152,12 @@ alone_decode(void *coder_ptr, const lzma_allocator *allocator, } }; - return_if_error(lzma_next_filter_init(&coder->next, - allocator, filters)); + const lzma_ret ret = lzma_next_filter_init(&coder->next, + allocator, filters); + if (ret != LZMA_OK) { + lzma_next_end(&coder->next, allocator); + return ret; + } coder->sequence = SEQ_CODE; break; diff --git a/contrib/xz/src/liblzma/common/file_info.c b/contrib/xz/src/liblzma/common/file_info.c index 4b2eb5d0400b..57d720d32ad3 100644 --- a/contrib/xz/src/liblzma/common/file_info.c +++ b/contrib/xz/src/liblzma/common/file_info.c @@ -438,7 +438,8 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator, return_if_error(lzma_index_decoder_init( &coder->index_decoder, allocator, &coder->this_index, - coder->memlimit - memused)); + coder->memlimit - memused, + coder->footer_flags.backward_size)); coder->index_remaining = coder->footer_flags.backward_size; coder->sequence = SEQ_INDEX_DECODE; @@ -717,6 +718,13 @@ file_info_decoder_memconfig(void *coder_ptr, uint64_t *memusage, } } + // combined_index_memusage + this_index_memusage shouldn't overflow + // because combined_index_memusage is limited by how much can be + // successfully allocated, and this_index_memusage is limited by the + // input_size_max check in index_decoder.c. Check for overflow anyway. + if (UINT64_MAX - combined_index_memusage < this_index_memusage) + return LZMA_PROG_ERROR; + // Now we know the total memory usage/requirement. If we had neither // old Indexes nor a new Index, this will be zero which isn't // acceptable as lzma_memusage() has to return non-zero on success diff --git a/contrib/xz/src/liblzma/common/index.c b/contrib/xz/src/liblzma/common/index.c index c4aadb9b008d..3fbb5b1c74e8 100644 --- a/contrib/xz/src/liblzma/common/index.c +++ b/contrib/xz/src/liblzma/common/index.c @@ -14,6 +14,14 @@ #include "stream_flags_common.h" +/// \brief Maximum number of Streams supported in lzma_index +/// +/// The maximum number of Streams is UINT32_MAX, because index_tree.count +/// is uint32_t. This is unlikely to be reached in practice because it +/// would require allocating hundreds of gigabytes of memory. +#define STREAMS_MAX UINT32_MAX + + /// \brief How many Records to allocate at once /// /// This should be big enough to avoid making lots of tiny allocations @@ -28,12 +36,14 @@ /// \brief Base structure for index_stream and index_group structures typedef struct index_tree_node_s index_tree_node; struct index_tree_node_s { - /// Uncompressed start offset of this Stream (relative to the - /// beginning of the file) or Block (relative to the beginning - /// of the Stream) + /// Uncompressed start offset of this Stream (relative to + /// lzma_index.uncompressed_bias) or Block (relative to + /// the beginning of the Stream) lzma_vli uncompressed_base; - /// Compressed start offset of this Stream or Block + /// Compressed start offset of this Stream (relative to + /// lzma_index.compressed_bias) or Block (relative to + /// the beginning of the Stream) lzma_vli compressed_base; index_tree_node *parent; @@ -47,13 +57,10 @@ typedef struct { /// Root node index_tree_node *root; - /// Leftmost node. Since the tree will be filled sequentially, - /// this won't change after the first node has been added to - /// the tree. + /// Leftmost (first) node index_tree_node *leftmost; - /// The rightmost node in the tree. Since the tree is filled - /// sequentially, this is always the node where to add the new data. + /// Rightmost (last) node index_tree_node *rightmost; /// Number of nodes in the tree @@ -108,10 +115,14 @@ typedef struct { /// Every index_stream is a node in the tree of Streams. index_tree_node node; - /// Number of this Stream (first one is 1) + /// Number of this Stream counted in reverse: the last Stream + /// in the lzma_index has .number == 0. In the API, the first + /// Stream is 1, so the real number of this Stream is + /// lzma_index.streams.count - .number. uint32_t number; - /// Total number of Blocks before this Stream + /// Total number of Blocks before this Stream relative to + /// lzma_index.block_number_base. lzma_vli block_number_base; /// Record groups of this Stream are stored in a tree. @@ -148,6 +159,15 @@ struct lzma_index_s { /// are many concatenated Streams. index_tree streams; + /// In index_stream, node.uncompressed_base, node.compressed_base, + /// and block_number_base are relative to these _bias members + /// instead of being relative to 0. If a new index_stream is + /// prepended before the existing index_streams, the offsets of all + /// existing index_streams can be changed by updating these _biases. + lzma_vli uncompressed_bias; + lzma_vli compressed_bias; + lzma_vli block_number_bias; + /// Uncompressed size of all the Blocks in the Stream(s) lzma_vli uncompressed_size; @@ -226,10 +246,13 @@ index_tree_end(index_tree *tree, const lzma_allocator *allocator, /// Add a new node to the tree. node->uncompressed_base and /// node->compressed_base must have been set by the caller already. +/// +/// The tree is always filled sequentially: index_streams are prepended +/// and index_groups are appended. static void -index_tree_append(index_tree *tree, index_tree_node *node) +index_tree_append(index_tree *tree, index_tree_node *node, bool prepend) { - node->parent = tree->rightmost; + node->parent = prepend ? tree->leftmost : tree->rightmost; node->left = NULL; node->right = NULL; @@ -243,14 +266,22 @@ index_tree_append(index_tree *tree, index_tree_node *node) return; } - // The tree is always filled sequentially. - assert(tree->rightmost->uncompressed_base <= node->uncompressed_base); - assert(tree->rightmost->compressed_base < node->compressed_base); - - // Add the new node after the rightmost node. It's the correct - // place due to the reason above. - tree->rightmost->right = node; - tree->rightmost = node; + // Add the new node before the leftmost or after the rightmost node. + if (prepend) { + assert(tree->leftmost->uncompressed_base + >= node->uncompressed_base); + assert(tree->leftmost->compressed_base + > node->compressed_base); + tree->leftmost->left = node; + tree->leftmost = node; + } else { + assert(tree->rightmost->uncompressed_base + <= node->uncompressed_base); + assert(tree->rightmost->compressed_base + < node->compressed_base); + tree->rightmost->right = node; + tree->rightmost = node; + } // Balance the AVL-tree if needed. We don't need to keep the balance // factors in nodes, because we always fill the tree sequentially, @@ -265,11 +296,14 @@ index_tree_append(index_tree *tree, index_tree_node *node) node = node->parent; } while (--up > 0); - // Rotate left using node as the rotation root. - index_tree_node *pivot = node->right; + // Rotate right/left using node as the rotation root. + index_tree_node *pivot = prepend ? node->left : node->right; if (node->parent == NULL) { tree->root = pivot; + } else if (prepend) { + assert(node->parent->left == node); + node->parent->left = pivot; } else { assert(node->parent->right == node); node->parent->right = pivot; @@ -277,11 +311,20 @@ index_tree_append(index_tree *tree, index_tree_node *node) pivot->parent = node->parent; - node->right = pivot->left; - if (node->right != NULL) - node->right->parent = node; + if (prepend) { + node->left = pivot->right; + if (node->left != NULL) + node->left->parent = node; + + pivot->right = node; + } else { + node->right = pivot->left; + if (node->right != NULL) + node->right->parent = node; + + pivot->left = node; + } - pivot->left = node; node->parent = pivot; } @@ -308,6 +351,25 @@ index_tree_next(const index_tree_node *node) } +/// Get the previous node in the tree. Return NULL if there are no more nodes. +static void * +index_tree_prev(const index_tree_node *node) +{ + if (node->left != NULL) { + node = node->left; + while (node->right != NULL) + node = node->right; + + return (void *)(node); + } + + while (node->parent != NULL && node->parent->left == node) + node = node->parent; + + return (void *)(node->parent); +} + + /// Locate a node that contains the given uncompressed offset. It is /// caller's job to check that target is not bigger than the uncompressed /// size of the tree (the last node would be returned in that case still). @@ -317,9 +379,6 @@ index_tree_locate(const index_tree *tree, lzma_vli target) const index_tree_node *result = NULL; const index_tree_node *node = tree->root; - assert(tree->leftmost == NULL - || tree->leftmost->uncompressed_base == 0); - // Consecutive nodes may have the same uncompressed_base. // We must pick the rightmost one. while (node != NULL) { @@ -382,6 +441,9 @@ index_init_plain(const lzma_allocator *allocator) lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); if (i != NULL) { index_tree_init(&i->streams); + i->uncompressed_bias = LZMA_VLI_MAX; + i->compressed_bias = LZMA_VLI_MAX; + i->block_number_bias = LZMA_VLI_MAX; i->uncompressed_size = 0; i->total_size = 0; i->record_count = 0; @@ -401,13 +463,15 @@ lzma_index_init(const lzma_allocator *allocator) if (i == NULL) return NULL; - index_stream *s = index_stream_init(0, 0, 1, 0, allocator); + index_stream *s = index_stream_init( + i->compressed_bias, i->uncompressed_bias, + 0, i->block_number_bias, allocator); if (s == NULL) { lzma_free(i, allocator); return NULL; } - index_tree_append(&i->streams, &s->node); + index_tree_append(&i->streams, &s->node, true); return i; } @@ -427,11 +491,11 @@ lzma_index_end(lzma_index *i, const lzma_allocator *allocator) } -extern void +extern bool lzma_index_prealloc(lzma_index *i, lzma_vli records) { if (records > PREALLOC_MAX) - records = PREALLOC_MAX; + return true; // If index_decoder.c calls us with records == 0, it's decoding // an Index that has no Records. In that case the decoder won't call @@ -454,7 +518,7 @@ lzma_index_prealloc(lzma_index *i, lzma_vli records) records = INDEX_GROUP_SIZE; i->prealloc = (size_t)(records); - return; + return false; } @@ -494,10 +558,8 @@ lzma_index_memusage(lzma_vli streams, lzma_vli blocks) const uint64_t index_base = sizeof(lzma_index) + alloc_overhead; // Validate the arguments and catch integer overflows. - // Maximum number of Streams is "only" UINT32_MAX, because - // that limit is used by the tree containing the Streams. const uint64_t limit = UINT64_MAX - index_base; - if (streams == 0 || streams > UINT32_MAX || blocks > LZMA_VLI_MAX + if (streams == 0 || streams > STREAMS_MAX || blocks > LZMA_VLI_MAX || streams > limit / stream_base || groups > limit / group_base || limit - streams_mem < groups_mem) @@ -581,7 +643,7 @@ lzma_index_file_size(const lzma_index *i) { const index_stream *s = (const index_stream *)(i->streams.rightmost); const index_group *g = (const index_group *)(s->groups.rightmost); - return index_file_size(s->node.compressed_base, + return index_file_size(s->node.compressed_base - i->compressed_bias, g == NULL ? 0 : g->records[g->last].unpadded_sum, s->record_count, s->index_list_size, s->stream_padding); @@ -687,7 +749,7 @@ lzma_index_append(lzma_index *i, const lzma_allocator *allocator, return LZMA_DATA_ERROR; // Check that the file size will stay within limits. - if (index_file_size(s->node.compressed_base, + if (index_file_size(s->node.compressed_base - i->compressed_bias, compressed_base + unpadded_size, s->record_count + 1, s->index_list_size + index_list_size_add, s->stream_padding) == LZMA_VLI_UNKNOWN) @@ -725,7 +787,7 @@ lzma_index_append(lzma_index *i, const lzma_allocator *allocator, g->number_base = s->record_count + 1; // Add the new group to the Stream. - index_tree_append(&s->groups, &g->node); + index_tree_append(&s->groups, &g->node, false); } // Add the new Record to the group. @@ -749,23 +811,11 @@ lzma_index_append(lzma_index *i, const lzma_allocator *allocator, /// Structure to pass info to index_cat_helper() typedef struct { - /// Uncompressed size of the destination - lzma_vli uncompressed_size; - - /// Compressed file size of the destination - lzma_vli file_size; - - /// Same as above but for Block numbers - lzma_vli block_number_add; - - /// Number of Streams that were in the destination index before we - /// started appending new Streams from the source index. This is - /// used to fix the Stream numbering. - uint32_t stream_number_add; - - /// Destination index' Stream tree + lzma_vli uncompressed_adjust; + lzma_vli compressed_adjust; + lzma_vli block_number_adjust; + uint32_t stream_number_adjust; index_tree *streams; - } index_cat_info; @@ -778,18 +828,19 @@ index_cat_helper(const index_cat_info *info, index_stream *this) index_stream *left = (index_stream *)(this->node.left); index_stream *right = (index_stream *)(this->node.right); - if (left != NULL) - index_cat_helper(info, left); - - this->node.uncompressed_base += info->uncompressed_size; - this->node.compressed_base += info->file_size; - this->number += info->stream_number_add; - this->block_number_base += info->block_number_add; - index_tree_append(info->streams, &this->node); - if (right != NULL) index_cat_helper(info, right); + // The Stream number counts in reverse, thus += instead of -=. + this->node.uncompressed_base -= info->uncompressed_adjust; + this->node.compressed_base -= info->compressed_adjust; + this->number += info->stream_number_adjust; + this->block_number_base -= info->block_number_adjust; + index_tree_append(info->streams, &this->node, true); + + if (left != NULL) + index_cat_helper(info, left); + return; } @@ -801,6 +852,11 @@ lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, if (dest == NULL || src == NULL) return LZMA_PROG_ERROR; + // Check that we don't exceed the maximum number of Streams + // per lzma_index. + if (STREAMS_MAX - dest->streams.count < src->streams.count) + return LZMA_DATA_ERROR; + const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -870,29 +926,39 @@ lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, } // dest->checks includes the check types of all except the last Stream - // in dest. Set the bit for the check type of the last Stream now so - // that it won't get lost when Stream(s) from src are appended to dest. - dest->checks = lzma_index_checks(dest); + // in dest. Use lzma_index_checks() to get the check type of the + // last Stream too. This needs to be done before the Streams are + // moved from dest to src. + src->checks |= lzma_index_checks(dest); - // Add all the Streams from src to dest. Update the base offsets - // of each Stream from src. + // Update the biases. + src->uncompressed_bias -= dest->uncompressed_size; + src->compressed_bias -= dest_file_size; + src->block_number_bias -= dest->record_count; + + // Prepend all the Streams from dest to src. const index_cat_info info = { - .uncompressed_size = dest->uncompressed_size, - .file_size = dest_file_size, - .stream_number_add = dest->streams.count, - .block_number_add = dest->record_count, - .streams = &dest->streams, + .uncompressed_adjust = dest->uncompressed_bias + - src->uncompressed_bias, + .compressed_adjust = dest->compressed_bias + - src->compressed_bias, + .block_number_adjust = dest->block_number_bias + - src->block_number_bias, + .stream_number_adjust = src->streams.count, + .streams = &src->streams, }; - index_cat_helper(&info, (index_stream *)(src->streams.root)); + index_cat_helper(&info, (index_stream *)(dest->streams.root)); // Update info about all the combined Streams. - dest->uncompressed_size += src->uncompressed_size; - dest->total_size += src->total_size; - dest->record_count += src->record_count; - dest->index_list_size += src->index_list_size; - dest->checks |= src->checks; + src->uncompressed_size += dest->uncompressed_size; + src->total_size += dest->total_size; + src->record_count += dest->record_count; + src->index_list_size += dest->index_list_size; - // There's nothing else left in src than the base structure. + // There's nothing else left in dest than the base structure. + // The API is defined so that dest is modified and src is freed, + // so copy src to dest and free the base struct of src. + *dest = *src; lzma_free(src, allocator); return LZMA_OK; @@ -955,7 +1021,7 @@ index_dup_stream(const index_stream *src, const lzma_allocator *allocator) assert(i == destg->allocated); // Add the group to the new Stream. - index_tree_append(&dest->groups, &destg->node); + index_tree_append(&dest->groups, &destg->node, false); return dest; } @@ -970,14 +1036,18 @@ lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) return NULL; // Copy the totals. + dest->uncompressed_bias = src->uncompressed_bias; + dest->compressed_bias = src->compressed_bias; + dest->block_number_bias = src->block_number_bias; dest->uncompressed_size = src->uncompressed_size; dest->total_size = src->total_size; dest->record_count = src->record_count; dest->index_list_size = src->index_list_size; + dest->checks = src->checks; // Copy the Streams and the groups in them. const index_stream *srcstream - = (const index_stream *)(src->streams.leftmost); + = (const index_stream *)(src->streams.rightmost); do { index_stream *deststream = index_dup_stream( srcstream, allocator); @@ -986,9 +1056,9 @@ lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) return NULL; } - index_tree_append(&dest->streams, &deststream->node); + index_tree_append(&dest->streams, &deststream->node, true); - srcstream = index_tree_next(&srcstream->node); + srcstream = index_tree_prev(&srcstream->node); } while (srcstream != NULL); return dest; @@ -1052,11 +1122,14 @@ iter_set_info(lzma_index_iter *iter) } // NOTE: lzma_index_iter.stream.number is lzma_vli but we use uint32_t - // internally. - iter->stream.number = stream->number; + // internally. The internal value counts in reverse (last one is 0) + // but in lzma_index_iter.stream.number the first Stream is 1. + iter->stream.number = i->streams.count - stream->number; iter->stream.block_count = stream->record_count; - iter->stream.compressed_offset = stream->node.compressed_base; - iter->stream.uncompressed_offset = stream->node.uncompressed_base; + iter->stream.compressed_offset = stream->node.compressed_base + - i->compressed_bias; + iter->stream.uncompressed_offset = stream->node.uncompressed_base + - i->uncompressed_bias; // iter->stream.flags will be NULL if the Stream Flags haven't been // set with lzma_index_stream_flags(). @@ -1085,7 +1158,7 @@ iter_set_info(lzma_index_iter *iter) if (group != NULL) { iter->block.number_in_stream = group->number_base + record; iter->block.number_in_file = iter->block.number_in_stream - + stream->block_number_base; + + (stream->block_number_base - i->block_number_bias); iter->block.compressed_stream_offset = record == 0 ? group->node.compressed_base @@ -1250,14 +1323,21 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) { const lzma_index *i = iter->internal[ITER_INDEX].p; + assert(i->uncompressed_size <= LZMA_VLI_MAX); + // If the target is past the end of the file, return immediately. if (i->uncompressed_size <= target) return true; + assert(target < LZMA_VLI_MAX); + // Locate the Stream containing the target offset. + // NOTE: Adding the bias can make target >= LZMA_VLI_MAX. + target += i->uncompressed_bias; const index_stream *stream = index_tree_locate(&i->streams, target); assert(stream != NULL); target -= stream->node.uncompressed_base; + assert(target < LZMA_VLI_MAX); // Locate the group containing the target offset. const index_group *group = index_tree_locate(&stream->groups, target); diff --git a/contrib/xz/src/liblzma/common/index.h b/contrib/xz/src/liblzma/common/index.h index 007e1188f259..c06528b22c83 100644 --- a/contrib/xz/src/liblzma/common/index.h +++ b/contrib/xz/src/liblzma/common/index.h @@ -38,7 +38,9 @@ extern uint32_t lzma_index_padding_size(const lzma_index *i); /// Set for how many Records to allocate memory the next time /// lzma_index_append() needs to allocate space for a new Record. /// This is used only by the Index decoder. -extern void lzma_index_prealloc(lzma_index *i, lzma_vli records); +/// +/// \return false on success, true if records is too large (SIZE_MAX issue) +extern bool lzma_index_prealloc(lzma_index *i, lzma_vli records); /// Round the variable-length integer to the next multiple of four. diff --git a/contrib/xz/src/liblzma/common/index_decoder.c b/contrib/xz/src/liblzma/common/index_decoder.c index 4eab56d942e1..a59e4fbf619a 100644 --- a/contrib/xz/src/liblzma/common/index_decoder.c +++ b/contrib/xz/src/liblzma/common/index_decoder.c @@ -28,6 +28,10 @@ typedef struct { /// Memory usage limit uint64_t memlimit; + // Memory used for lzma_index once it has been allocated. + // Before that, this holds a tiny value. + uint64_t memused; + /// Target Index lzma_index *index; @@ -35,6 +39,12 @@ typedef struct { /// successful decoding. lzma_index **index_ptr; + /// If Number of Records contains a value higher than this, + /// the Index is immediately considered to be corrupt. + /// This prevents us from allocating a large amount of memory + /// when we know that the Index isn't big. + lzma_vli number_of_records_max; + /// Number of Records left to decode. lzma_vli count; @@ -91,19 +101,32 @@ index_decode(void *coder_ptr, const lzma_allocator *allocator, if (ret != LZMA_STREAM_END) goto out; + // Reject Number of Records values that are clearly invalid. + if (coder->count > coder->number_of_records_max) + return LZMA_DATA_ERROR; + coder->pos = 0; coder->sequence = SEQ_MEMUSAGE; FALLTHROUGH; case SEQ_MEMUSAGE: - if (lzma_index_memusage(1, coder->count) > coder->memlimit) { + coder->memused = lzma_index_memusage(1, coder->count); + + // We sanity-checked coder->count in SEQ_COUNT, which + // guarantees that lzma_index_memusage() won't fail. + assert(coder->memused != UINT64_MAX); + + if (coder->memused > coder->memlimit) { ret = LZMA_MEMLIMIT_ERROR; goto out; } // Tell the Index handling code how many Records this // Index has to allow it to allocate memory more efficiently. - lzma_index_prealloc(coder->index, coder->count); + // If the Record count is so high that memory allocation would + // fail later, return LZMA_MEM_ERROR now. + if (lzma_index_prealloc(coder->index, coder->count)) + return LZMA_MEM_ERROR; ret = LZMA_OK; coder->sequence = coder->count == 0 @@ -230,7 +253,10 @@ index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, { lzma_index_coder *coder = coder_ptr; - *memusage = lzma_index_memusage(1, coder->count); + // Index decoder is special in sense that the returned memusage + // doesn't include the memory usage of the decoder itself; only + // the resulting lzma_index is counted. + *memusage = coder->memused; *old_memlimit = coder->memlimit; if (new_memlimit != 0) { @@ -246,7 +272,7 @@ index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, static lzma_ret index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, - lzma_index **i, uint64_t memlimit) + lzma_index **i, uint64_t memlimit, uint64_t input_size_max) { // Remember the pointer given by the application. We will set it // to point to the decoded Index only if decoding is successful. @@ -260,10 +286,19 @@ index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, if (coder->index == NULL) return LZMA_MEM_ERROR; + // Approximate how many Records the Index may contain at most when + // we know that the Index won't be larger than input_size_max bytes. + // Each Record must consume at least two bytes. The CRC32 etc. + // consume a few bytes too, but we don't need an exact value. + // + // file_info.c knows Backward Size and passes it to us. When we have + // no information, input_size_max equals LZMA_BACKWARD_SIZE_MAX. + coder->number_of_records_max = input_size_max / 2; + // Initialize the rest. coder->sequence = SEQ_INDICATOR; coder->memlimit = my_max(1, memlimit); - coder->count = 0; // Needs to be initialized due to _memconfig(). + coder->memused = 1; // If _memconfig() is called before SEQ_MEMUSAGE. coder->pos = 0; coder->crc32 = 0; @@ -273,7 +308,7 @@ index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, extern lzma_ret lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - lzma_index **i, uint64_t memlimit) + lzma_index **i, uint64_t memlimit, uint64_t input_size_max) { lzma_next_coder_init(&lzma_index_decoder_init, next, allocator); @@ -295,7 +330,8 @@ lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index_end(coder->index, allocator); } - return index_decoder_reset(coder, allocator, i, memlimit); + return index_decoder_reset( + coder, allocator, i, memlimit, input_size_max); } @@ -308,7 +344,8 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) if (i != NULL) *i = NULL; - lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit); + lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit, + LZMA_BACKWARD_SIZE_MAX); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -334,7 +371,9 @@ lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, // Initialize the decoder. lzma_index_coder coder; - return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + const uint64_t input_size_max = in_size - *in_pos; + return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit, + my_min(input_size_max, LZMA_BACKWARD_SIZE_MAX))); // Store the input start position so that we can restore it in case // of an error. @@ -361,7 +400,7 @@ lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, } else if (ret == LZMA_MEMLIMIT_ERROR) { // Tell the caller how much memory would have // been needed. - *memlimit = lzma_index_memusage(1, coder.count); + *memlimit = coder.memused; } } diff --git a/contrib/xz/src/liblzma/common/index_decoder.h b/contrib/xz/src/liblzma/common/index_decoder.h index 5351d2f0dfa4..fa8dc4df129b 100644 --- a/contrib/xz/src/liblzma/common/index_decoder.h +++ b/contrib/xz/src/liblzma/common/index_decoder.h @@ -18,7 +18,7 @@ extern lzma_ret lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - lzma_index **i, uint64_t memlimit); + lzma_index **i, uint64_t memlimit, uint64_t input_size_max); #endif diff --git a/contrib/xz/src/liblzma/common/lzip_decoder.c b/contrib/xz/src/liblzma/common/lzip_decoder.c index 5630039f97b9..d6f929d2f449 100644 --- a/contrib/xz/src/liblzma/common/lzip_decoder.c +++ b/contrib/xz/src/liblzma/common/lzip_decoder.c @@ -237,8 +237,12 @@ lzip_decode(void *coder_ptr, const lzma_allocator *allocator, } }; - return_if_error(lzma_next_filter_init(&coder->lzma_decoder, - allocator, filters)); + const lzma_ret ret = lzma_next_filter_init( + &coder->lzma_decoder, allocator, filters); + if (ret != LZMA_OK) { + lzma_next_end(&coder->lzma_decoder, allocator); + return ret; + } coder->crc32 = 0; coder->sequence = SEQ_LZMA_STREAM; diff --git a/contrib/xz/src/liblzma/common/microlzma_decoder.c b/contrib/xz/src/liblzma/common/microlzma_decoder.c index 882cb2c808d1..a7720cc2ae32 100644 --- a/contrib/xz/src/liblzma/common/microlzma_decoder.c +++ b/contrib/xz/src/liblzma/common/microlzma_decoder.c @@ -108,8 +108,12 @@ microlzma_decode(void *coder_ptr, const lzma_allocator *allocator, } }; - return_if_error(lzma_next_filter_init(&coder->lzma, - allocator, filters)); + const lzma_ret ret = lzma_next_filter_init(&coder->lzma, + allocator, filters); + if (ret != LZMA_OK) { + lzma_next_end(&coder->lzma, allocator); + return ret; + } // Pass one dummy 0x00 byte to the LZMA decoder since that // is what it expects the first byte to be. diff --git a/contrib/xz/src/liblzma/common/stream_buffer_decoder.c b/contrib/xz/src/liblzma/common/stream_buffer_decoder.c index c4f91fb49839..bbf20046e0b4 100644 --- a/contrib/xz/src/liblzma/common/stream_buffer_decoder.c +++ b/contrib/xz/src/liblzma/common/stream_buffer_decoder.c @@ -50,10 +50,6 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, if (ret == LZMA_STREAM_END) { ret = LZMA_OK; } else { - // Something went wrong, restore the positions. - *in_pos = in_start; - *out_pos = out_start; - if (ret == LZMA_OK) { // Either the input was truncated or the // output buffer was too small. @@ -78,6 +74,10 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, stream_decoder.coder, memlimit, &memusage, 0); } + + // Something went wrong, restore the positions. + *in_pos = in_start; + *out_pos = out_start; } } diff --git a/contrib/xz/src/liblzma/common/stream_decoder_mt.c b/contrib/xz/src/liblzma/common/stream_decoder_mt.c index d51366e1b02c..44696bd9fb40 100644 --- a/contrib/xz/src/liblzma/common/stream_decoder_mt.c +++ b/contrib/xz/src/liblzma/common/stream_decoder_mt.c @@ -250,13 +250,13 @@ struct lzma_stream_coder { /// Amount of compressed data in Stream Header + Blocks that have /// already been finished. /// - /// \note Use mutex. + /// \note Use mutex when worker threads might be active. uint64_t progress_in; /// Amount of uncompressed data in Blocks that have already /// been finished. /// - /// \note Use mutex. + /// \note Use mutex when worker threads might be active. uint64_t progress_out; @@ -1068,7 +1068,12 @@ stream_decode_mt(void *coder_ptr, const lzma_allocator *allocator, const size_t in_old = *in_pos; const lzma_ret ret = decode_block_header(coder, allocator, in, in_pos, in_size); - coder->progress_in += *in_pos - in_old; + + // Worker threads may finish earlier Blocks and need to update + // coder->progress_in at the same time with us. + mythread_sync(coder->mutex) { + coder->progress_in += *in_pos - in_old; + } if (ret == LZMA_OK) { // We didn't decode the whole Block Header yet. diff --git a/contrib/xz/src/liblzma/lz/lz_decoder.c b/contrib/xz/src/liblzma/lz/lz_decoder.c index 1cb120ab3b09..6dc4cefb7679 100644 --- a/contrib/xz/src/liblzma/lz/lz_decoder.c +++ b/contrib/xz/src/liblzma/lz/lz_decoder.c @@ -278,6 +278,7 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, // Allocate and initialize the dictionary. if (coder->dict.size != alloc_size) { + coder->dict.size = 0; lzma_free(coder->dict.buf, allocator); // The LZ_DICT_EXTRA bytes at the end of the buffer aren't diff --git a/contrib/xz/src/liblzma/lzma/lzma2_decoder.c b/contrib/xz/src/liblzma/lzma/lzma2_decoder.c index 37ab253f5b0a..164897ad1333 100644 --- a/contrib/xz/src/liblzma/lzma/lzma2_decoder.c +++ b/contrib/xz/src/liblzma/lzma/lzma2_decoder.c @@ -165,11 +165,32 @@ lzma2_decode(void *coder_ptr, lzma_dict *restrict dict, // coder->compressed_size later. const size_t in_start = *in_pos; + // LZMA2 stream ends with the end marker (0x00), so there + // must be at least one byte after this chunk. Let the + // decoder read at most one byte past the end of the chunk. + // If the decoder reads the extra byte, then the input is + // corrupt. This way we won't produce (much) junk output + // from the input bytes that are past the end of this chunk. + // The extra byte makes things simpler, because we can ignore + // uncompressed size and not think about some corner cases. + // + // NOTE: It's not a security issue (information leak) to + // pass more input to the decoder than the chunk size. + // If an attacker can modify the compressed input, then the + // attacker can modify a chunk header so that it specifies + // a too large compressed size. liblzma <= 5.8.3 didn't + // have in_limit; in_size was passed to the decoder as is. + const size_t in_limit = *in_pos + my_min(in_size - *in_pos, + coder->compressed_size + 1); + // Decode from in[] to *dict. const lzma_ret ret = coder->lzma.code(coder->lzma.coder, - dict, in, in_pos, in_size); + dict, in, in_pos, in_limit); - // Validate and update coder->compressed_size. + // Validate and update coder->compressed_size. If the input + // is corrupt, let the caller still see the newly-decoded + // output even if it is (partially) corrupt. It might allow + // users to recover a small amount of useful data. const size_t in_used = *in_pos - in_start; if (in_used > coder->compressed_size) return LZMA_DATA_ERROR; diff --git a/contrib/xz/src/liblzma/simple/simple_coder.c b/contrib/xz/src/liblzma/simple/simple_coder.c index 5cbfa8227047..48ae90dfc093 100644 --- a/contrib/xz/src/liblzma/simple/simple_coder.c +++ b/contrib/xz/src/liblzma/simple/simple_coder.c @@ -252,6 +252,17 @@ lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, if (coder == NULL) return LZMA_MEM_ERROR; + // Allocate memory for the filter-specific data structure. + if (simple_size > 0) { + coder->simple = lzma_alloc(simple_size, allocator); + if (coder->simple == NULL) { + lzma_free(coder, allocator); + return LZMA_MEM_ERROR; + } + } else { + coder->simple = NULL; + } + next->coder = coder; next->code = &simple_code; next->end = &simple_coder_end; @@ -260,15 +271,6 @@ lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, coder->next = LZMA_NEXT_CODER_INIT; coder->filter = filter; coder->allocated = 2 * unfiltered_max; - - // Allocate memory for filter-specific data structure. - if (simple_size > 0) { - coder->simple = lzma_alloc(simple_size, allocator); - if (coder->simple == NULL) - return LZMA_MEM_ERROR; - } else { - coder->simple = NULL; - } } if (filters[0].options != NULL) { diff --git a/contrib/xz/src/xz/args.c b/contrib/xz/src/xz/args.c index c32ee5c11159..cf64380bec11 100644 --- a/contrib/xz/src/xz/args.c +++ b/contrib/xz/src/xz/args.c @@ -75,7 +75,8 @@ parse_block_list(const char *str_const) // It must be non-empty and not begin with a comma. if (str[0] == '\0' || str[0] == ',') - message_fatal(_("%s: Invalid argument to --block-list"), str); + message_fatal(_("%s: Invalid argument to --block-list"), + tuklib_mask_nonprint(str)); // Count the number of comma-separated strings. size_t count = 1; @@ -86,7 +87,7 @@ parse_block_list(const char *str_const) // Prevent an unlikely integer overflow. if (count > SIZE_MAX / sizeof(block_list_entry) - 1) message_fatal(_("%s: Too many arguments to --block-list"), - str); + tuklib_mask_nonprint(str)); // Allocate memory to hold all the sizes specified. // If --block-list was specified already, its value is forgotten. @@ -562,7 +563,8 @@ parse_real(args_info *args, int argc, char **argv) if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unknown file " "format type"), - optarg); + tuklib_mask_nonprint( + optarg)); opt_format = types[i].format; break; @@ -584,15 +586,16 @@ parse_real(args_info *args, int argc, char **argv) while (strcmp(types[i].str, optarg) != 0) { if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unsupported " - "integrity " - "check type"), optarg); + "integrity check type"), + tuklib_mask_nonprint(optarg)); } // Use a separate check in case we are using different // liblzma than what was used to compile us. if (!lzma_check_is_supported(types[i].check)) - message_fatal(_("%s: Unsupported integrity " - "check type"), optarg); + message_fatal(_("%s: Unsupported " + "integrity check type"), + tuklib_mask_nonprint(optarg)); coder_set_check(types[i].check); break; @@ -635,18 +638,24 @@ parse_real(args_info *args, int argc, char **argv) "or '--files0'.")); if (optarg == NULL) { - args->files_name = stdin_filename; + args->files_name = (char *)stdin_filename; args->files_file = stdin; } else { - args->files_name = optarg; - args->files_file = fopen(optarg, + // If we are called from parse_environment(), + // the memory pointed by optarg will be freed + // after we return to parse_environment(). + // We need to duplicate the string. + args->files_name = xstrdup(optarg); + args->files_file = fopen(args->files_name, c == OPT_FILES ? "r" : "rb"); if (args->files_file == NULL) // TRANSLATORS: This is a translatable // string because French needs a space // before the colon ("%s : %s"). - message_fatal(_("%s: %s"), optarg, - strerror(errno)); + message_fatal(_("%s: %s"), + tuklib_mask_nonprint( + args->files_name), + strerror(errno)); } break; diff --git a/contrib/xz/src/xz/args.h b/contrib/xz/src/xz/args.h index 7fdf37f1420f..19c1261620e8 100644 --- a/contrib/xz/src/xz/args.h +++ b/contrib/xz/src/xz/args.h @@ -19,7 +19,7 @@ typedef struct { /// Name of the file from which to read filenames. This is NULL /// if --files or --files0 was not used. - const char *files_name; + char *files_name; /// File opened for reading from which filenames are read. This is /// non-NULL only if files_name is non-NULL. diff --git a/contrib/xz/src/xz/coder.c b/contrib/xz/src/xz/coder.c index c28f874a25f7..92e35622a2c8 100644 --- a/contrib/xz/src/xz/coder.c +++ b/contrib/xz/src/xz/coder.c @@ -172,7 +172,7 @@ str_to_filters(const char *str, uint32_t index, uint32_t flags) // the messages are included in xz's translations. message(V_ERROR, _("Error in --filters%s=FILTERS option:"), filter_num); - message(V_ERROR, "%s", str); + message(V_ERROR, "%s", tuklib_mask_nonprint(str)); message(V_ERROR, "%*s^", error_pos, ""); message_fatal("%s", _(err)); } diff --git a/contrib/xz/src/xz/list.c b/contrib/xz/src/xz/list.c index 210f23ad1baa..56744829f7e1 100644 --- a/contrib/xz/src/xz/list.c +++ b/contrib/xz/src/xz/list.c @@ -1142,12 +1142,36 @@ print_info_robot(xz_file_info *xfi, file_pair *pair) static void update_totals(const xz_file_info *xfi) { - // TODO: Integer overflow checks + const uint64_t streams = lzma_index_stream_count(xfi->idx); + const uint64_t blocks = lzma_index_block_count(xfi->idx); + const uint64_t compressed_size = lzma_index_file_size(xfi->idx); + const uint64_t uncompressed_size + = lzma_index_uncompressed_size(xfi->idx); + + // Of these, totals.uncompressed_size is the most likely to overflow + // because it doesn't require a huge file. Even that is likely to + // happen only with invalid/crafted files. + // + // For simplicity, the error message refers to file sizes even though + // we are checking other things too. + if (totals.files == UINT64_MAX + || UINT64_MAX - totals.streams < streams + || UINT64_MAX - totals.blocks < blocks + || UINT64_MAX - totals.compressed_size + < compressed_size + || UINT64_MAX - totals.uncompressed_size + < uncompressed_size + || UINT64_MAX - totals.stream_padding + < xfi->stream_padding) { + message_fatal(_("Error: The total compressed or uncompressed " + "size of the files is too large")); + } + ++totals.files; - totals.streams += lzma_index_stream_count(xfi->idx); - totals.blocks += lzma_index_block_count(xfi->idx); - totals.compressed_size += lzma_index_file_size(xfi->idx); - totals.uncompressed_size += lzma_index_uncompressed_size(xfi->idx); + totals.streams += streams; + totals.blocks += blocks; + totals.compressed_size += compressed_size; + totals.uncompressed_size += uncompressed_size; totals.stream_padding += xfi->stream_padding; totals.checks |= lzma_index_checks(xfi->idx); diff --git a/contrib/xz/src/xz/main.c b/contrib/xz/src/xz/main.c index 8cc3b740f658..fe5ff4ad1028 100644 --- a/contrib/xz/src/xz/main.c +++ b/contrib/xz/src/xz/main.c @@ -89,7 +89,7 @@ read_name(const args_info *args) message_error(_("%s: Error reading filenames: %s"), tuklib_mask_nonprint(args->files_name), strerror(errno)); - return NULL; + break; } if (feof(args->files_file)) { @@ -99,7 +99,7 @@ read_name(const args_info *args) tuklib_mask_nonprint( args->files_name)); - return NULL; + break; } if (c == args->files_delim) { @@ -125,7 +125,7 @@ read_name(const args_info *args) "of '--files'?"), tuklib_mask_nonprint( args->files_name)); - return NULL; + break; } name[pos++] = c; @@ -149,6 +149,8 @@ read_name(const args_info *args) } } + free(name); + name = NULL; return NULL; } @@ -335,8 +337,10 @@ main(int argc, char **argv) run(name); } - if (args.files_name != stdin_filename) + if (args.files_name != stdin_filename) { (void)fclose(args.files_file); + free(args.files_name); + } } #ifdef HAVE_DECODERS diff --git a/contrib/xz/src/xz/message.c b/contrib/xz/src/xz/message.c index 7657e85648da..dfd3da76262f 100644 --- a/contrib/xz/src/xz/message.c +++ b/contrib/xz/src/xz/message.c @@ -650,7 +650,14 @@ progress_flush(bool finished) cols[4]); } else { // The filename is always printed. - fprintf(stderr, _("%s: "), tuklib_mask_nonprint(filename)); + // + // NOTE: This function is called from vmessage() whose + // caller may have used tuklib_mask_nonprint(). Thus, + // we must use the _r variant here. + char *mem = NULL; + fprintf(stderr, _("%s: "), + tuklib_mask_nonprint_r(filename, &mem)); + free(mem); // Percentage is printed only if we didn't finish yet. if (!finished) { diff --git a/contrib/xz/src/xz/util.c b/contrib/xz/src/xz/util.c index bb9b86ad382c..b9386c0353f9 100644 --- a/contrib/xz/src/xz/util.c +++ b/contrib/xz/src/xz/util.c @@ -111,7 +111,7 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) return max; if (*value < '0' || *value > '9') - message_fatal(_("%s: %s"), value, + message_fatal(_("%s: %s"), tuklib_mask_nonprint(value), _("Value is not a non-negative decimal integer")); do { @@ -154,7 +154,7 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) if (multiplier == 0) { message(V_ERROR, _("%s: Invalid multiplier suffix"), - value - 1); + tuklib_mask_nonprint(value - 1)); message_fatal(_("Valid suffixes are 'KiB' (2^10), " "'MiB' (2^20), and 'GiB' (2^30).")); } diff --git a/contrib/xz/src/xz/xz.1 b/contrib/xz/src/xz/xz.1 index 6e37b426a3b5..f787dd5bd93d 100644 --- a/contrib/xz/src/xz/xz.1 +++ b/contrib/xz/src/xz/xz.1 @@ -353,7 +353,7 @@ the command name (for example, .B unxz implies .BR \-\-decompress ). -.IP "" +.IP .\" The DESCRIPTION section already says this but it's good to repeat it .\" here because the default behavior is a bit dangerous and new users .\" in a hurry may skip reading the DESCRIPTION section. @@ -389,7 +389,7 @@ and no files are created or removed. In list mode, the program cannot read the compressed data from standard input or from other unseekable sources. -.IP "" +.IP The default listing shows basic information about .IR files , one file per line. @@ -404,7 +404,7 @@ The width of verbose output exceeds 80 characters, so piping the output to, for example, .B "less\ \-S" may be convenient if the terminal isn't wide enough. -.IP "" +.IP The exact output may vary between .B xz versions and different locales. @@ -416,7 +416,7 @@ should be used. .TP .BR \-k ", " \-\-keep Don't delete the input files. -.IP "" +.IP Since .B xz 5.2.6, @@ -485,7 +485,7 @@ silently ignore possible remaining input data following the stream. Normally such trailing garbage makes .B xz display an error. -.IP "" +.IP .B xz never decompresses more than one stream from .B .lzma @@ -494,12 +494,12 @@ files or raw streams, but this option still makes ignore the possible trailing data after the .B .lzma file or raw stream. -.IP "" +.IP This option has no effect if the operation mode is not .B \-\-decompress or .BR \-\-test . -.IP "" +.IP Since .B xz 5.7.1alpha, @@ -530,7 +530,7 @@ If not writing to standard output and the source file already has the suffix .IR .suf , a warning is displayed and the file is skipped. -.IP "" +.IP When decompressing, recognize files with the suffix .I .suf in addition to files with the @@ -544,7 +544,7 @@ suffix. If the source file has the suffix .IR .suf , the suffix is removed to get the target filename. -.IP "" +.IP When compressing or decompressing raw streams .RB ( \-\-format=raw ), the suffix must always be specified unless @@ -612,7 +612,7 @@ Accept only .B .lz files when decompressing. Compression is not supported. -.IP "" +.IP The .B .lz format versions 0 and 1 are supported. @@ -651,7 +651,7 @@ format doesn't support integrity checks. The integrity check (if any) is verified when the .B .xz file is decompressed. -.IP "" +.IP Supported .I check types: @@ -678,7 +678,7 @@ at detecting damaged files and the speed difference is negligible. Calculate SHA-256. This is somewhat slower than CRC32 and CRC64. .RE -.IP "" +.IP Integrity of the .B .xz headers is always verified with CRC32. @@ -689,7 +689,7 @@ Don't verify the integrity check of the compressed data when decompressing. The CRC32 values in the .B .xz headers will still be verified normally. -.IP "" +.IP .B "Do not use this option unless you know what you are doing." Possible reasons to use this option: .RS @@ -711,7 +711,7 @@ If multiple preset levels are specified, the last one takes effect. If a custom filter chain was already specified, setting a compression preset level clears the custom filter chain. -.IP "" +.IP The differences between the presets are more significant than with .BR gzip (1) and @@ -761,17 +761,16 @@ but with higher compressor and decompressor memory requirements. These are useful only when compressing files bigger than 8\ MiB, 16\ MiB, and 32\ MiB, respectively. .RE -.IP "" +.IP On the same hardware, the decompression speed is approximately a constant number of bytes of compressed data per second. In other words, the better the compression, the faster the decompression will usually be. This also means that the amount of uncompressed output produced per second can vary a lot. -.IP "" +.IP The following table summarises the features of the presets: .RS -.RS .PP .TS tab(;); @@ -790,8 +789,7 @@ Preset;DictSize;CompCPU;CompMem;DecMem \-9;64 MiB;6;674 MiB;65 MiB .TE .RE -.RE -.IP "" +.IP Column descriptions: .RS .IP \(bu 3 @@ -828,7 +826,7 @@ The exact decompressor memory usage is slightly more than the LZMA2 dictionary size, but the values in the table have been rounded up to the next full MiB. .RE -.IP "" +.IP Memory requirements of the multi-threaded mode are significantly higher than that of the single-threaded mode. With the default value of @@ -846,7 +844,7 @@ but with bad luck this can also make it worse. Decompressor memory usage is not affected, but compressor memory usage increases a little at preset levels .BR \-0 " ... " \-3 . -.IP "" +.IP Since there are two presets with dictionary sizes 4\ MiB and 8\ MiB, the presets .B \-3e @@ -859,7 +857,6 @@ and respectively. That way no two presets are identical. .RS -.RS .PP .TS tab(;); @@ -878,8 +875,7 @@ Preset;DictSize;CompCPU;CompMem;DecMem \-9e;64 MiB;8;674 MiB;65 MiB .TE .RE -.RE -.IP "" +.IP For example, there are a total of four presets that use 8\ MiB dictionary, whose order from the fastest to the slowest is .BR \-5 , @@ -914,7 +910,7 @@ makes limited random-access decompression possible. This option is typically used to override the default block size in multi-threaded mode, but this option can be used in single-threaded mode too. -.IP "" +.IP In multi-threaded mode about three times .I size bytes will be allocated in each thread for buffering input and output. @@ -931,7 +927,7 @@ because then the LZMA2 dictionary buffer will never get fully used. In multi-threaded mode, the sizes of the blocks are stored in the block headers. This size information is required for multi-threaded decompression. -.IP "" +.IP In single-threaded mode no block splitting is done by default. Setting this option doesn't affect memory usage. No size information is stored in block headers, @@ -946,7 +942,7 @@ When compressing to the .B .xz format, start a new block with an optional custom filter chain after the given intervals of uncompressed data. -.IP "" +.IP The .I items are a comma-separated list. @@ -956,7 +952,7 @@ between 0 and 9 followed by a colon and a required size of uncompressed data. Omitting an item (two or more consecutive commas) is a shorthand to use the size and filters of the previous item. -.IP "" +.IP If the input file is bigger than the sum of the sizes in .IR items , @@ -965,7 +961,7 @@ A special value of .B 0 may be used as the last size to indicate that the rest of the file should be encoded as a single block. -.IP "" +.IP An alternative filter chain for each block can be specified in combination with the .BI \-\-filters1= filters @@ -1005,7 +1001,7 @@ The default filter chain and 2 MiB input The default filter chain and 4 MiB input for every block until end of input. .RE -.IP "" +.IP If one specifies a size that exceeds the encoder's block size (either the default value in threaded mode or the value specified with \fB\-\-block\-size=\fIsize\fR), @@ -1018,7 +1014,7 @@ For example, if one specifies and the input file is 80 MiB, one will get 11 blocks: 5, 10, 8, 10, 2, 10, 10, 4, 10, 10, and 1 MiB. -.IP "" +.IP In multi-threaded mode the sizes of the blocks are stored in the block headers. This isn't done in single-threaded mode, @@ -1041,7 +1037,7 @@ values make the data available at the receiving end with a small delay, but large .I timeout values give better compression ratio. -.IP "" +.IP This feature is disabled by default. If this option is specified more than once, the last one takes effect. The special @@ -1049,9 +1045,9 @@ The special value of .B 0 can be used to explicitly disable this feature. -.IP "" +.IP This feature is not available on non-POSIX systems. -.IP "" +.IP .\" FIXME .B "This feature is still experimental." Currently @@ -1070,12 +1066,12 @@ it is possible that the target file was not written to the storage device but the delete operation was. In that case neither the original source file nor the target file is available. -.IP "" +.IP This option has an effect only when .B xz is going to remove the source file. In other cases synchronization is never done. -.IP "" +.IP The synchronization and .B \-\-no\-sync were added in @@ -1086,7 +1082,7 @@ were added in Set a memory usage limit for compression. If this option is specified multiple times, the last one takes effect. -.IP "" +.IP If the compression settings exceed the .IR limit , .B xz @@ -1099,7 +1095,7 @@ switching to single-threaded mode if even one thread in multi-threaded mode exceeds the .IR limit , and finally reducing the LZMA2 dictionary size. -.IP "" +.IP When compressing with .B \-\-format=raw or if @@ -1107,14 +1103,14 @@ or if has been specified, only the number of threads may be reduced since it can be done without affecting the compressed output. -.IP "" +.IP If the .I limit cannot be met even with the adjustments described above, an error is displayed and .B xz will exit with exit status 1. -.IP "" +.IP The .I limit can be specified in multiple ways: @@ -1151,7 +1147,7 @@ to .B max (no memory usage limit). .RE -.IP "" +.IP For 32-bit .B xz there is a special case: if the @@ -1174,7 +1170,7 @@ A similar feature doesn't exist for decompression.) This can be helpful when a 32-bit executable has access to 4\ GiB address space (2 GiB on MIPS32) while hopefully doing no harm in other situations. -.IP "" +.IP See also the section .BR "Memory usage" . .TP @@ -1213,14 +1209,14 @@ and so the effective .I limit for multi-threading will never be higher than the limit set with .BR \-\-memlimit\-decompress . -.IP "" +.IP In contrast to the other memory usage limit options, .BI \-\-memlimit\-mt\-decompress= limit has a system-specific default .IR limit . .B "xz \-\-info\-memory" can be used to see the current value. -.IP "" +.IP This option and its default value exist because without any limit the threaded decompressor could end up allocating an insane amount of memory with some input files. @@ -1236,7 +1232,7 @@ will attempt to use that amount of memory even with a low number of threads. Running out of memory or swapping will not improve decompression performance. -.IP "" +.IP See .BI \-\-memlimit\-compress= limit for possible ways to specify the @@ -1264,7 +1260,7 @@ from switching the encoder from multi-threaded mode to single-threaded mode and from reducing the LZMA2 dictionary size. Even when this option is used the number of threads may be reduced to meet the memory usage limit as that won't affect the compressed output. -.IP "" +.IP Automatic adjusting is always disabled when creating raw streams .RB ( \-\-format=raw ). .TP @@ -1282,7 +1278,7 @@ The actual number of threads can be fewer than if the input file is not big enough for threading with the given settings or if using more threads would exceed the memory usage limit. -.IP "" +.IP The single-threaded and multi-threaded compressors produce different output. Single-threaded compressor will give the smallest file size but only the output from the multi-threaded compressor can be decompressed @@ -1301,7 +1297,7 @@ even if the system supports only one hardware thread. .RB ( xz 5.2.x used single-threaded mode in this situation.) -.IP "" +.IP To use multi-threaded mode with only one thread, set .I threads to @@ -1320,7 +1316,7 @@ Support for the prefix was added in .B xz 5.4.0. -.IP "" +.IP If an automatic number of threads has been requested and no memory usage limit has been specified, then a system-specific default soft limit will be used to possibly @@ -1335,14 +1331,14 @@ This default soft limit will not make switch from multi-threaded mode to single-threaded mode. The active limits can be seen with .BR "xz \-\-info\-memory" . -.IP "" +.IP Currently the only threading method is to split the input into blocks and compress them independently from each other. The default block size depends on the compression level and can be overridden with the .BI \-\-block\-size= size option. -.IP "" +.IP Threaded decompression only works on files that contain multiple blocks with size information in block headers. All large enough files compressed in multi-threaded mode @@ -1350,7 +1346,7 @@ meet this condition, but files compressed in single-threaded mode don't even if .BI \-\-block\-size= size has been used. -.IP "" +.IP The default value for .I threads is @@ -1457,7 +1453,7 @@ to apply the same options as \fB\-\-filters1\fR=\fIfilters\fR ... \fB\-\-filters9\fR=\fIfilters Specify up to nine additional filter chains that can be used with .BR \-\-block\-list . -.IP "" +.IP For example, when compressing an archive with executable files followed by text files, the executable part could use a filter chain with a BCJ filter and the text part only the LZMA2 filter. @@ -1479,7 +1475,7 @@ options, and exit successfully. .PD Add LZMA1 or LZMA2 filter to the filter chain. These filters can be used only as the last filter in the chain. -.IP "" +.IP LZMA1 is a legacy filter, which is supported almost solely due to the legacy .B .lzma @@ -1491,7 +1487,7 @@ The format uses LZMA2 and doesn't support LZMA1 at all. Compression speed and ratios of LZMA1 and LZMA2 are practically the same. -.IP "" +.IP LZMA1 and LZMA2 share the same set of .IR options : .RS @@ -1542,7 +1538,7 @@ Thus, increasing dictionary .I size usually improves compression ratio, but a dictionary bigger than the uncompressed file is waste of memory. -.IP "" +.IP Typical dictionary .I size is from 64\ KiB to 64\ MiB. @@ -1551,7 +1547,7 @@ The maximum for compression is currently 1.5\ GiB (1536\ MiB). The decompressor already supports dictionaries up to one byte less than 4\ GiB, which is the maximum for the LZMA1 and LZMA2 stream formats. -.IP "" +.IP Dictionary .I size and match finder @@ -1587,12 +1583,12 @@ In addition, the sum of and .I lp must not exceed 4. -.IP "" +.IP All bytes that cannot be encoded as matches are encoded as literals. That is, literals are simply 8-bit bytes that are encoded one at a time. -.IP "" +.IP The literal coding makes an assumption that the highest .I lc bits of the previous uncompressed byte correlate @@ -1606,7 +1602,7 @@ When .I lc is at least 3, the literal coding can take advantage of this property in the uncompressed data. -.IP "" +.IP The default value (3) is usually good. If you want maximum compression, test .BR lc=4 . @@ -1619,7 +1615,7 @@ too. .BI lp= lp Specify the number of literal position bits. The minimum is 0 and the maximum is 4; the default is 0. -.IP "" +.IP .I Lp affects what kind of alignment in the uncompressed data is assumed when encoding literals. @@ -1630,14 +1626,14 @@ below for more information about alignment. .BI pb= pb Specify the number of position bits. The minimum is 0 and the maximum is 4; the default is 2. -.IP "" +.IP .I Pb affects what kind of alignment in the uncompressed data is assumed in general. The default means four-byte alignment .RI (2^ pb =2^2=4), which is often a good choice when there's no better guess. -.IP "" +.IP When the alignment is known, setting .I pb accordingly may reduce the file size a little. @@ -1651,7 +1647,7 @@ is a good choice. If the alignment is an odd number like 3 bytes, .B pb=0 might be the best choice. -.IP "" +.IP Even though the assumed alignment can be adjusted with .I pb and @@ -1674,7 +1670,7 @@ use .BR hc4 , and the rest use .BR bt4 . -.IP "" +.IP The following match finders are supported. The memory usage formulas below are rough approximations, which are closest to the reality when @@ -1790,7 +1786,7 @@ for for .I presets 4\(en9. -.IP "" +.IP Usually .B fast is used with Hash Chain match finders and @@ -1806,7 +1802,7 @@ Once a match of at least .I nice bytes is found, the algorithm stops looking for possibly better matches. -.IP "" +.IP .I Nice can be 2\(en273 bytes. Higher values tend to give better compression ratio @@ -1823,7 +1819,7 @@ from .I mf and .IR nice . -.IP "" +.IP Reasonable .I depth for Hash Chains is 4\(en100 and 16\(en1000 for Binary Trees. @@ -1835,7 +1831,7 @@ Avoid setting the over 1000 unless you are prepared to interrupt the compression in case it is taking far too long. .RE -.IP "" +.IP When decoding raw streams .RB ( \-\-format=raw ), LZMA2 needs only the dictionary @@ -1866,7 +1862,7 @@ and Add a branch/call/jump (BCJ) filter to the filter chain. These filters can be used only as a non-last filter in the filter chain. -.IP "" +.IP A BCJ filter converts relative addresses in the machine code to their absolute counterparts. This doesn't change the size of the data @@ -1880,7 +1876,7 @@ doesn't cause any data loss, although it may make the compression ratio slightly worse. The BCJ filters are very fast and use an insignificant amount of memory. -.IP "" +.IP These BCJ filters have known problems related to the compression ratio: .RS @@ -1901,12 +1897,11 @@ The contents of non-executable files in the same archive can matter too. In practice one has to try with and without a BCJ filter to see which is better in each situation. .RE -.IP "" +.IP Different instruction sets have different alignment: the executable file must be aligned to a multiple of this value in the input data to make the filter work. .RS -.RS .PP .TS tab(;); @@ -1923,8 +1918,7 @@ SPARC;4; RISC-V;2; .TE .RE -.RE -.IP "" +.IP Since the BCJ-filtered data is usually compressed with LZMA2, the compression ratio may be improved slightly if the LZMA2 options are set to match the @@ -1961,7 +1955,7 @@ It's usually good to stick to LZMA2's defaults .RB ( pb=2,lp=0,lc=3 ) when compressing x86 executables. .RE -.IP "" +.IP All BCJ filters support the same .IR options : .RS @@ -1985,7 +1979,7 @@ is almost never useful. Add the Delta filter to the filter chain. The Delta filter can be only used as a non-last filter in the filter chain. -.IP "" +.IP Currently only simple byte-wise delta calculation is supported. It can be useful when compressing, for example, uncompressed bitmap images or uncompressed PCM audio. @@ -1994,7 +1988,7 @@ results than Delta + LZMA2. This is true especially with audio, which compresses faster and better, for example, with .BR flac (1). -.IP "" +.IP Supported .IR options : .RS @@ -2006,7 +2000,7 @@ of the delta calculation in bytes. .I distance must be 1\(en256. The default is 1. -.IP "" +.IP For example, with .B dist=2 and eight-byte input A1 B1 A2 B3 A3 B5 A4 B7, the output will be @@ -2030,7 +2024,7 @@ will display a progress indicator. Specifying .B \-\-verbose twice will give even more verbose output. -.IP "" +.IP The progress indicator shows the following information: .RS .IP \(bu 3 @@ -2065,7 +2059,7 @@ started processing the file. The time is shown in a less precise format which never has any colons, for example, 2 min 30 s. .RE -.IP "" +.IP When standard error is not a terminal, .B \-\-verbose will make @@ -2641,7 +2635,7 @@ XZ_OPT=\-2v tar caf foo.tar.xz foo .fi .RE .RE -.IP "" +.IP Scripts may use .BR XZ_OPT , for example, to set script-specific default compression options. diff --git a/lib/liblzma/config.h b/lib/liblzma/config.h index db2d4e9c6a0d..22e8794e0a3d 100644 --- a/lib/liblzma/config.h +++ b/lib/liblzma/config.h @@ -409,7 +409,7 @@ #define PACKAGE_NAME "XZ Utils" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "XZ Utils 5.8.3" +#define PACKAGE_STRING "XZ Utils 5.8.4" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "xz" @@ -418,7 +418,7 @@ #define PACKAGE_URL "https://tukaani.org/xz/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "5.8.3" +#define PACKAGE_VERSION "5.8.4" /* Define to necessary symbol if this constant uses a non-standard name on your system. */ @@ -587,7 +587,7 @@ /* Version number of package */ -#define VERSION "5.8.3" +#define VERSION "5.8.4" /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ |
