diff options
Diffstat (limited to 'src/xz/coder.c')
| -rw-r--r-- | src/xz/coder.c | 648 |
1 files changed, 532 insertions, 116 deletions
diff --git a/src/xz/coder.c b/src/xz/coder.c index 91d40ed2bb7b..4efaa802b9bb 100644 --- a/src/xz/coder.c +++ b/src/xz/coder.c @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file coder.c /// \brief Compresses or uncompresses a file // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -26,25 +26,51 @@ enum format_type opt_format = FORMAT_AUTO; bool opt_auto_adjust = true; bool opt_single_stream = false; uint64_t opt_block_size = 0; -uint64_t *opt_block_list = NULL; - +block_list_entry *opt_block_list = NULL; /// Stream used to communicate with liblzma static lzma_stream strm = LZMA_STREAM_INIT; -/// Filters needed for all encoding all formats, and also decoding in raw data -static lzma_filter filters[LZMA_FILTERS_MAX + 1]; +/// Maximum number of filter chains. The first filter chain is the default, +/// and 9 other filter chains can be specified with --filtersX. +#define NUM_FILTER_CHAIN_MAX 10 + +/// The default filter chain is in filters[0]. It is used for encoding +/// in all supported formats and also for decdoing raw streams. The other +/// filter chains are set by --filtersX to support changing filters with +/// the --block-list option. +static lzma_filter filters[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1]; + +/// Bit mask representing the filters that are actually used when encoding +/// in the xz format. This is needed since a filter chain could be +/// specified in --filtersX (or the default filter chain), but never used +/// in --block-list. The default filter chain is always assumed to be used, +/// unless --block-list is specified and does not have a block using the +/// default filter chain. +static uint32_t filters_used_mask = 1; + +#ifdef HAVE_ENCODERS +/// Track the memory usage for all filter chains (default or --filtersX). +/// The memory usage may need to be scaled down depending on the memory limit. +static uint64_t filter_memusages[ARRAY_SIZE(filters)]; +#endif /// Input and output buffers static io_buf in_buf; static io_buf out_buf; -/// Number of filters. Zero indicates that we are using a preset. +/// Number of filters in the default filter chain. Zero indicates that +/// we are using a preset. static uint32_t filters_count = 0; /// Number of the preset (0-9) static uint32_t preset_number = LZMA_PRESET_DEFAULT; +/// True if the current default filter chain was set using the --filters +/// option. The filter chain is reset if a preset option (like -9) or an +/// old-style filter option (like --lzma2) is used after a --filters option. +static bool string_to_filter_used = false; + /// Integrity check type static lzma_check check; @@ -60,7 +86,6 @@ static bool allow_trailing_input; static lzma_mt mt_options = { .flags = 0, .timeout = 300, - .filters = filters, }; #endif @@ -77,14 +102,14 @@ coder_set_check(lzma_check new_check) static void forget_filter_chain(void) { - // Setting a preset makes us forget a possibly defined custom - // filter chain. - while (filters_count > 0) { - --filters_count; - free(filters[filters_count].options); - filters[filters_count].options = NULL; + // Setting a preset or using --filters makes us forget + // the earlier custom filter chain (if any). + if (filters_count > 0) { + lzma_filters_free(filters[0], NULL); + filters_count = 0; } + string_to_filter_used = false; return; } @@ -114,9 +139,14 @@ coder_add_filter(lzma_vli id, void *options) if (filters_count == LZMA_FILTERS_MAX) message_fatal(_("Maximum number of filters is four")); - filters[filters_count].id = id; - filters[filters_count].options = options; - ++filters_count; + if (string_to_filter_used) + forget_filter_chain(); + + filters[0][filters_count].id = id; + filters[0][filters_count].options = options; + // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify + // implementation of forget_filter_chain(). + filters[0][++filters_count].id = LZMA_VLI_UNKNOWN; // Setting a custom filter chain makes us forget the preset options. // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e" @@ -128,6 +158,69 @@ coder_add_filter(lzma_vli id, void *options) } +static void +str_to_filters(const char *str, uint32_t index, uint32_t flags) +{ + int error_pos; + const char *err = lzma_str_to_filters(str, &error_pos, + filters[index], flags, NULL); + + if (err != NULL) { + char filter_num[2] = ""; + if (index > 0) + filter_num[0] = '0' + index; + + // FIXME? The message in err isn't translated. + // Including the translations in the xz translations is + // slightly ugly but possible. Creating a new domain for + // liblzma might not be worth it especially since on some + // OSes it adds extra dependencies to translation libraries. + message(V_ERROR, _("Error in --filters%s=FILTERS option:"), + filter_num); + message(V_ERROR, "%s", str); + message(V_ERROR, "%*s^", error_pos, ""); + message_fatal("%s", err); + } +} + + +extern void +coder_add_filters_from_str(const char *filter_str) +{ + // Forget presets and previously defined filter chain. See + // coder_add_filter() above for why preset_number must be reset too. + forget_filter_chain(); + preset_number = LZMA_PRESET_DEFAULT; + + string_to_filter_used = true; + + // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw. + str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS); + + // Set the filters_count to be the number of filters converted from + // the string. + for (filters_count = 0; filters[0][filters_count].id + != LZMA_VLI_UNKNOWN; + ++filters_count) ; + + assert(filters_count > 0); + return; +} + + +extern void +coder_add_block_filters(const char *str, size_t slot) +{ + // Free old filters first, if they were previously allocated. + if (filters_used_mask & (1U << slot)) + lzma_filters_free(filters[slot], NULL); + + str_to_filters(str, slot, 0); + + filters_used_mask |= 1U << slot; +} + + tuklib_attr_noreturn static void memlimit_too_small(uint64_t memory_usage) @@ -139,6 +232,71 @@ memlimit_too_small(uint64_t memory_usage) } +#ifdef HAVE_ENCODERS +// For a given opt_block_list index, validate that the filter has been +// set. If it has not been set, we must exit with error to avoid using +// an uninitialized filter chain. +static void +validate_block_list_filter(const uint32_t filter_num) +{ + if (!(filters_used_mask & (1U << filter_num))) + message_fatal(_("filter chain %u used by --block-list but " + "not specified with --filters%u="), + (unsigned)filter_num, (unsigned)filter_num); +} + + +// Sets the memory usage for each filter chain. It will return the maximum +// memory usage of all of the filter chains. +static uint64_t +filters_memusage_max(const lzma_mt *mt, bool encode) +{ + uint64_t max_memusage = 0; + +#ifdef MYTHREAD_ENABLED + // Copy multithreaded options to a temporary struct since the + // filters member needs to be changed + lzma_mt mt_local; + if (mt != NULL) + mt_local = *mt; +#else + (void)mt; +#endif + + for (uint32_t i = 0; i < ARRAY_SIZE(filters); i++) { + if (!(filters_used_mask & (1U << i))) + continue; + + uint64_t memusage = UINT64_MAX; +#ifdef MYTHREAD_ENABLED + if (mt != NULL) { + mt_local.filters = filters[i]; + memusage = lzma_stream_encoder_mt_memusage(&mt_local); + filter_memusages[i] = memusage; + } + else +#endif + + if (encode) { + memusage = lzma_raw_encoder_memusage(filters[i]); + filter_memusages[i] = memusage; + } + +#ifdef HAVE_DECODERS + else { + memusage = lzma_raw_decoder_memusage(filters[i]); + } +#endif + + if (memusage > max_memusage) + max_memusage = memusage; + } + + return max_memusage; +} + +#endif + extern void coder_set_compression_settings(void) { @@ -147,6 +305,48 @@ coder_set_compression_settings(void) assert(opt_format != FORMAT_LZIP); #endif +#ifdef HAVE_ENCODERS +# ifdef MYTHREAD_ENABLED + // Represents the largest Block size specified with --block-list. This + // is needed to help reduce the Block size in the multithreaded encoder + // so memory is not wasted. + uint64_t max_block_list_size = 0; +# endif + + if (opt_block_list != NULL) { + // This mask tracks the filters actually referenced in + // --block-list. It is used to help remove bits from + // filters_used_mask when a filter chain was specified + // but never actually used. + uint32_t filters_ref_mask = 0; + + for (uint32_t i = 0; opt_block_list[i].size != 0; i++) { + validate_block_list_filter( + opt_block_list[i].filters_index); + + // Mark the current filter as referenced. + filters_ref_mask |= 1U << + opt_block_list[i].filters_index; + +# ifdef MYTHREAD_ENABLED + if (opt_block_list[i].size > max_block_list_size) + max_block_list_size = opt_block_list[i].size; +# endif + } + + assert(filters_ref_mask != 0); + // Note: The filters that were initialized but not used do + // not free their options and do not have the filter + // IDs set to LZMA_VLI_UNKNOWN. Filter chains are not + // freed outside of debug mode and the default filter + // chain is never freed. + filters_used_mask = filters_ref_mask; + } else { + // Reset filters used mask in case --block-list is not + // used, but --filtersX is used. + filters_used_mask = 1; + } +#endif // The default check type is CRC64, but fallback to CRC32 // if CRC64 isn't supported by the copy of liblzma we are // using. CRC32 is always supported. @@ -159,7 +359,11 @@ coder_set_compression_settings(void) // Options for LZMA1 or LZMA2 in case we are using a preset. static lzma_options_lzma opt_lzma; - if (filters_count == 0) { + // The first filter in the filters[] array is for the default + // filter chain. + lzma_filter *default_filters = filters[0]; + + if (filters_count == 0 && filters_used_mask & 1) { // We are using a preset. This is not a good idea in raw mode // except when playing around with things. Different versions // of this software may use different options in presets, and @@ -179,46 +383,59 @@ coder_set_compression_settings(void) message_bug(); // Use LZMA2 except with --format=lzma we use LZMA1. - filters[0].id = opt_format == FORMAT_LZMA + default_filters[0].id = opt_format == FORMAT_LZMA ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2; - filters[0].options = &opt_lzma; + default_filters[0].options = &opt_lzma; + filters_count = 1; - } - // Terminate the filter options array. - filters[filters_count].id = LZMA_VLI_UNKNOWN; + // Terminate the filter options array. + default_filters[1].id = LZMA_VLI_UNKNOWN; + } // If we are using the .lzma format, allow exactly one filter - // which has to be LZMA1. + // which has to be LZMA1. There is no need to check if the default + // filter chain is being used since it can only be disabled if + // --block-list is used, which is incompatible with FORMAT_LZMA. if (opt_format == FORMAT_LZMA && (filters_count != 1 - || filters[0].id != LZMA_FILTER_LZMA1)) + || default_filters[0].id != LZMA_FILTER_LZMA1)) message_fatal(_("The .lzma format supports only " "the LZMA1 filter")); // If we are using the .xz format, make sure that there is no LZMA1 // filter to prevent LZMA_PROG_ERROR. - if (opt_format == FORMAT_XZ) + if (opt_format == FORMAT_XZ && filters_used_mask & 1) for (size_t i = 0; i < filters_count; ++i) - if (filters[i].id == LZMA_FILTER_LZMA1) + if (default_filters[i].id == LZMA_FILTER_LZMA1) message_fatal(_("LZMA1 cannot be used " "with the .xz format")); - // Print the selected filter chain. - message_filters_show(V_DEBUG, filters); + if (filters_used_mask & 1) { + // Print the selected default filter chain. + message_filters_show(V_DEBUG, default_filters); + } // The --flush-timeout option requires LZMA_SYNC_FLUSH support - // from the filter chain. Currently threaded encoder doesn't support - // LZMA_SYNC_FLUSH so single-threaded mode must be used. + // from the filter chain. Currently the threaded encoder doesn't + // support LZMA_SYNC_FLUSH so single-threaded mode must be used. if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) { - for (size_t i = 0; i < filters_count; ++i) { - switch (filters[i].id) { - case LZMA_FILTER_LZMA2: - case LZMA_FILTER_DELTA: - break; + for (uint32_t i = 0; i < ARRAY_SIZE(filters); ++i) { + if (!(filters_used_mask & (1U << i))) + continue; + + const lzma_filter *fc = filters[i]; + for (size_t j = 0; fc[j].id != LZMA_VLI_UNKNOWN; j++) { + switch (fc[j].id) { + case LZMA_FILTER_LZMA2: + case LZMA_FILTER_DELTA: + break; - default: - message_fatal(_("The filter chain is " - "incompatible with --flush-timeout")); + default: + message_fatal(_("Filter chain %u is " + "incompatible with " + "--flush-timeout"), + (unsigned)i); + } } } @@ -229,11 +446,15 @@ coder_set_compression_settings(void) } } - // Get the memory usage. Note that if --format=raw was used, - // we can be decompressing. + // Get the memory usage and memory limit. The memory usage is the + // maximum of the default filters[] and any filters specified by + // --filtersX. + // Note that if --format=raw was used, we can be decompressing and + // do not need to account for any filter chains created + // with --filtersX. // - // If multithreaded .xz compression is done, this value will be - // replaced. + // If multithreaded .xz compression is done, the memory limit + // will be replaced. uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage = UINT64_MAX; if (opt_mode == MODE_COMPRESS) { @@ -242,10 +463,54 @@ coder_set_compression_settings(void) if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) { memory_limit = hardware_memlimit_mtenc_get(); mt_options.threads = hardware_threads_get(); - mt_options.block_size = opt_block_size; + + uint64_t block_size = opt_block_size; + // If opt_block_size is not set, find the maximum + // recommended Block size based on the filter chains + if (block_size == 0) { + for (uint32_t i = 0; i < ARRAY_SIZE(filters); + i++) { + if (!(filters_used_mask & (1U << i))) + continue; + + uint64_t size = lzma_mt_block_size( + filters[i]); + + // If this returns an error, then one + // of the filter chains in use is + // invalid, so there is no point in + // progressing further. + if (size == UINT64_MAX) + message_fatal(_("Unsupported " + "options in filter " + "chain %u"), + (unsigned)i); + + if (size > block_size) + block_size = size; + } + + // If the largest block size specified + // with --block-list is less than the + // recommended Block size, then it is a waste + // of RAM to use a larger Block size. It may + // even allow more threads to be used in some + // situations. If the special 0 Block size is + // used (encode all remaining data in 1 Block) + // then max_block_list_size will be set to + // UINT64_MAX, so the recommended Block size + // will always be used in this case. + if (max_block_list_size > 0 + && max_block_list_size + < block_size) + block_size = max_block_list_size; + } + + mt_options.block_size = block_size; mt_options.check = check; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + + memory_usage = filters_memusage_max( + &mt_options, true); if (memory_usage != UINT64_MAX) message(V_DEBUG, _("Using up to %" PRIu32 " threads."), @@ -253,12 +518,12 @@ coder_set_compression_settings(void) } else # endif { - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = filters_memusage_max(NULL, true); } #endif } else { #ifdef HAVE_DECODERS - memory_usage = lzma_raw_decoder_memusage(filters); + memory_usage = lzma_raw_decoder_memusage(default_filters); #endif } @@ -273,7 +538,16 @@ coder_set_compression_settings(void) message_mem_needed(V_DEBUG, memory_usage); #ifdef HAVE_DECODERS if (opt_mode == MODE_COMPRESS) { - const uint64_t decmem = lzma_raw_decoder_memusage(filters); +#ifdef HAVE_ENCODERS + const uint64_t decmem = + filters_memusage_max(NULL, false); +#else + // If encoders are not enabled, then --block-list is never + // usable, so the other filter chains 1-9 can never be used. + // So there is no need to find the maximum decoder memory + // required in this case. + const uint64_t decmem = lzma_raw_decoder_memusage(filters[0]); +#endif if (decmem != UINT64_MAX) message(V_DEBUG, _("Decompression will need " "%s MiB of memory."), uint64_to_str( @@ -300,8 +574,8 @@ coder_set_compression_settings(void) // Reduce the number of threads by one and check // the memory usage. --mt_options.threads; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + memory_usage = filters_memusage_max( + &mt_options, true); if (memory_usage == UINT64_MAX) message_bug(); @@ -353,7 +627,7 @@ coder_set_compression_settings(void) // the multithreaded mode but the output // is also different. hardware_threads_set(1); - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = filters_memusage_max(NULL, true); message(V_WARNING, _("Switching to single-threaded mode " "to not exceed the memory usage limit of %s MiB"), uint64_to_str(round_up_to_mib(memory_limit), 0)); @@ -368,55 +642,138 @@ coder_set_compression_settings(void) if (!opt_auto_adjust) memlimit_too_small(memory_usage); - // Look for the last filter if it is LZMA2 or LZMA1, so we can make - // it use less RAM. With other filters we don't know what to do. - size_t i = 0; - while (filters[i].id != LZMA_FILTER_LZMA2 - && filters[i].id != LZMA_FILTER_LZMA1) { - if (filters[i].id == LZMA_VLI_UNKNOWN) - memlimit_too_small(memory_usage); - - ++i; + // Decrease the dictionary size until we meet the memory usage limit. + // The struct is used to track data needed to correctly reduce the + // memory usage and report which filters were adjusted. + typedef struct { + // Pointer to the filter chain that needs to be reduced. + // NULL indicates that this filter chain was either never + // set or was never above the memory limit. + lzma_filter *filters; + + // Original dictionary sizes are used to show how each + // filter's dictionary was reduced. + uint64_t orig_dict_size; + + // Index of the LZMA filter in the filters member. We only + // adjust this filter's memusage because we don't know how + // to reduce the memory usage of the other filters. + uint32_t lzma_idx; + + // Indicates if the filter's dictionary size needs to be + // reduced to fit under the memory limit (true) or if the + // filter chain is unused or is already under the memory + // limit (false). + bool reduce_dict_size; + } memusage_reduction_data; + + memusage_reduction_data memusage_reduction[ARRAY_SIZE(filters)]; + + // Counter represents how many filter chains are above the memory + // limit. + size_t count = 0; + + for (uint32_t i = 0; i < ARRAY_SIZE(filters); i++) { + // The short var name "r" will reduce the number of lines + // of code needed since less lines will stretch past 80 + // characters. + memusage_reduction_data *r = &memusage_reduction[i]; + r->filters = NULL; + r->reduce_dict_size = false; + + if (!(filters_used_mask & (1U << i))) + continue; + + for (uint32_t j = 0; filters[i][j].id != LZMA_VLI_UNKNOWN; + j++) + if ((filters[i][j].id == LZMA_FILTER_LZMA2 + || filters[i][j].id + == LZMA_FILTER_LZMA1) + && filter_memusages[i] + > memory_limit) { + count++; + r->filters = filters[i]; + r->lzma_idx = j; + r->reduce_dict_size = true; + + lzma_options_lzma *opt = r->filters + [r->lzma_idx].options; + r->orig_dict_size = opt->dict_size; + opt->dict_size &= ~((UINT32_C(1) << 20) - 1); + } } - // Decrease the dictionary size until we meet the memory - // usage limit. First round down to full mebibytes. - lzma_options_lzma *opt = filters[i].options; - const uint32_t orig_dict_size = opt->dict_size; - opt->dict_size &= ~((UINT32_C(1) << 20) - 1); - while (true) { - // If it is below 1 MiB, auto-adjusting failed. We could be - // more sophisticated and scale it down even more, but let's - // see if many complain about this version. - // - // FIXME: Displays the scaled memory usage instead - // of the original. - if (opt->dict_size < (UINT32_C(1) << 20)) - memlimit_too_small(memory_usage); + // Loop until all filters use <= memory_limit, or exit. + while (count > 0) { + for (uint32_t i = 0; i < ARRAY_SIZE(memusage_reduction); i++) { + memusage_reduction_data *r = &memusage_reduction[i]; - memory_usage = lzma_raw_encoder_memusage(filters); - if (memory_usage == UINT64_MAX) - message_bug(); + if (!r->reduce_dict_size) + continue; - // Accept it if it is low enough. - if (memory_usage <= memory_limit) - break; + lzma_options_lzma *opt = + r->filters[r->lzma_idx].options; + + // If it is below 1 MiB, auto-adjusting failed. + // We could be more sophisticated and scale it + // down even more, but nobody has complained so far. + if (opt->dict_size < (UINT32_C(1) << 20)) + memlimit_too_small(memory_usage); + + uint64_t filt_mem_usage = + lzma_raw_encoder_memusage(r->filters); + + if (filt_mem_usage == UINT64_MAX) + message_bug(); - // Otherwise 1 MiB down and try again. I hope this - // isn't too slow method for cases where the original - // dict_size is very big. - opt->dict_size -= UINT32_C(1) << 20; + if (filt_mem_usage < memory_limit) { + r->reduce_dict_size = false; + count--; + } + else { + opt->dict_size -= UINT32_C(1) << 20; + } + } } - // Tell the user that we decreased the dictionary size. - message(V_WARNING, _("Adjusted LZMA%c dictionary size " - "from %s MiB to %s MiB to not exceed " - "the memory usage limit of %s MiB"), - filters[i].id == LZMA_FILTER_LZMA2 - ? '2' : '1', - uint64_to_str(orig_dict_size >> 20, 0), - uint64_to_str(opt->dict_size >> 20, 1), - uint64_to_str(round_up_to_mib(memory_limit), 2)); + // Tell the user that we decreased the dictionary size for + // each filter that was adjusted. + for (uint32_t i = 0; i < ARRAY_SIZE(memusage_reduction); i++) { + memusage_reduction_data *r = &memusage_reduction[i]; + + // If the filters were never set, then the memory usage + // was never adjusted. + if (r->filters == NULL) + continue; + + lzma_filter *filter_lzma = &(r->filters[r->lzma_idx]); + lzma_options_lzma *opt = filter_lzma->options; + + // The first index is the default filter chain. The message + // should be slightly different if the default filter chain + // or if --filtersX was adjusted. + if (i == 0) + message(V_WARNING, _("Adjusted LZMA%c dictionary " + "size from %s MiB to %s MiB to not exceed the " + "memory usage limit of %s MiB"), + filter_lzma->id == LZMA_FILTER_LZMA2 + ? '2' : '1', + uint64_to_str(r->orig_dict_size >> 20, 0), + uint64_to_str(opt->dict_size >> 20, 1), + uint64_to_str(round_up_to_mib( + memory_limit), 2)); + else + message(V_WARNING, _("Adjusted LZMA%c dictionary size " + "for --filters%u from %s MiB to %s MiB to not " + "exceed the memory usage limit of %s MiB"), + filter_lzma->id == LZMA_FILTER_LZMA2 + ? '2' : '1', + (unsigned)i, + uint64_to_str(r->orig_dict_size >> 20, 0), + uint64_to_str(opt->dict_size >> 20, 1), + uint64_to_str(round_up_to_mib( + memory_limit), 2)); + } #endif return; @@ -512,6 +869,13 @@ coder_init(file_pair *pair) // These will be handled later in this function. allow_trailing_input = false; + // Set the first filter chain. If the --block-list option is not + // used then use the default filter chain (filters[0]). + // Otherwise, use first filter chain from the block list. + lzma_filter *active_filters = opt_block_list == NULL + ? filters[0] + : filters[opt_block_list[0].filters_index]; + if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS switch (opt_format) { @@ -522,17 +886,19 @@ coder_init(file_pair *pair) case FORMAT_XZ: # ifdef MYTHREAD_ENABLED + mt_options.filters = active_filters; if (hardware_threads_is_mt()) ret = lzma_stream_encoder_mt( &strm, &mt_options); else # endif ret = lzma_stream_encoder( - &strm, filters, check); + &strm, active_filters, check); break; case FORMAT_LZMA: - ret = lzma_alone_encoder(&strm, filters[0].options); + ret = lzma_alone_encoder(&strm, + active_filters[0].options); break; # ifdef HAVE_LZIP_DECODER @@ -544,7 +910,7 @@ coder_init(file_pair *pair) # endif case FORMAT_RAW: - ret = lzma_raw_encoder(&strm, filters); + ret = lzma_raw_encoder(&strm, active_filters); break; } #endif @@ -668,7 +1034,7 @@ coder_init(file_pair *pair) case FORMAT_RAW: // Memory usage has already been checked in // coder_set_compression_settings(). - ret = lzma_raw_decoder(&strm, filters); + ret = lzma_raw_decoder(&strm, active_filters); break; } @@ -716,6 +1082,7 @@ coder_init(file_pair *pair) } +#ifdef HAVE_ENCODERS /// Resolve conflicts between opt_block_size and opt_block_list in single /// threaded mode. We want to default to opt_block_list, except when it is /// larger than opt_block_size. If this is the case for the current Block @@ -746,12 +1113,39 @@ split_block(uint64_t *block_remaining, } else { // The Block at *list_pos has been finished. Go to the next - // entry in the list. If the end of the list has been reached, - // reuse the size of the last Block. - if (opt_block_list[*list_pos + 1] != 0) + // entry in the list. If the end of the list has been + // reached, reuse the size and filters of the last Block. + if (opt_block_list[*list_pos + 1].size != 0) { ++*list_pos; - *block_remaining = opt_block_list[*list_pos]; + // Update the filters if needed. + if (opt_block_list[*list_pos - 1].filters_index + != opt_block_list[*list_pos].filters_index) { + const uint32_t filter_idx = opt_block_list + [*list_pos].filters_index; + const lzma_filter *next = filters[filter_idx]; + const lzma_ret ret = lzma_filters_update( + &strm, next); + + if (ret != LZMA_OK) { + // This message is only possible if + // the filter chain has unsupported + // options since the filter chain is + // validated using + // lzma_raw_encoder_memusage() or + // lzma_stream_encoder_mt_memusage(). + // Some options are not validated until + // the encoders are initialized. + message_fatal( + _("Error changing to " + "filter chain %u: %s"), + (unsigned)filter_idx, + message_strm(ret)); + } + } + } + + *block_remaining = opt_block_list[*list_pos].size; // If in single-threaded mode, split up the Block if needed. // This is not needed in multi-threaded mode because liblzma @@ -764,6 +1158,7 @@ split_block(uint64_t *block_remaining, } } } +#endif static bool @@ -796,6 +1191,7 @@ coder_normal(file_pair *pair) // Assume that something goes wrong. bool success = false; +#ifdef HAVE_ENCODERS // block_remaining indicates how many input bytes to encode before // finishing the current .xz Block. The Block size is set with // --block-size=SIZE and --block-list. They have an effect only when @@ -829,15 +1225,18 @@ coder_normal(file_pair *pair) // output is still not identical because in single-threaded // mode the size info isn't written into Block Headers. if (opt_block_list != NULL) { - if (block_remaining < opt_block_list[list_pos]) { + if (block_remaining < opt_block_list[list_pos].size) { assert(!hardware_threads_is_mt()); - next_block_remaining = opt_block_list[list_pos] + next_block_remaining = + opt_block_list[list_pos].size - block_remaining; } else { - block_remaining = opt_block_list[list_pos]; + block_remaining = + opt_block_list[list_pos].size; } } } +#endif strm.next_out = out_buf.u8; strm.avail_out = IO_BUFFER_SIZE; @@ -847,17 +1246,22 @@ coder_normal(file_pair *pair) // flushing or finishing. if (strm.avail_in == 0 && action == LZMA_RUN) { strm.next_in = in_buf.u8; - strm.avail_in = io_read(pair, &in_buf, - my_min(block_remaining, - IO_BUFFER_SIZE)); +#ifdef HAVE_ENCODERS + const size_t read_size = my_min(block_remaining, + IO_BUFFER_SIZE); +#else + const size_t read_size = IO_BUFFER_SIZE; +#endif + strm.avail_in = io_read(pair, &in_buf, read_size); if (strm.avail_in == SIZE_MAX) break; if (pair->src_eof) { action = LZMA_FINISH; - - } else if (block_remaining != UINT64_MAX) { + } +#ifdef HAVE_ENCODERS + else if (block_remaining != UINT64_MAX) { // Start a new Block after every // opt_block_size bytes of input. block_remaining -= strm.avail_in; @@ -867,17 +1271,18 @@ coder_normal(file_pair *pair) if (action == LZMA_RUN && pair->flush_needed) action = LZMA_SYNC_FLUSH; +#endif } // Let liblzma do the actual work. ret = lzma_code(&strm, action); // Write out if the output buffer became full. - if (strm.avail_out == 0) { + if (strm.avail_out == 0) if (coder_write_output(pair)) break; - } +#ifdef HAVE_ENCODERS if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH || action == LZMA_FULL_BARRIER)) { if (action == LZMA_SYNC_FLUSH) { @@ -907,8 +1312,9 @@ coder_normal(file_pair *pair) // Start a new Block after LZMA_FULL_FLUSH or continue // the same block after LZMA_SYNC_FLUSH. action = LZMA_RUN; - - } else if (ret != LZMA_OK) { + } else +#endif + if (ret != LZMA_OK) { // Determine if the return value indicates that we // won't continue coding. LZMA_NO_CHECK would be // here too if LZMA_TELL_ANY_CHECK was used. @@ -1103,6 +1509,16 @@ coder_run(const char *filename) extern void coder_free(void) { + // Free starting from the second filter chain since the default + // filter chain may have its options set from a static variable + // in coder_set_compression_settings(). Since this is only run in + // debug mode and will be freed when the process ends anyway, we + // don't worry about freeing it. + for (uint32_t i = 1; i < ARRAY_SIZE(filters); i++) { + if (filters_used_mask & (1U << i)) + lzma_filters_free(filters[i], NULL); + } + lzma_end(&strm); return; } |