diff options
Diffstat (limited to 'src/liblzma')
155 files changed, 3635 insertions, 1409 deletions
diff --git a/src/liblzma/api/lzma.h b/src/liblzma/api/lzma.h index de12f225859f..d55349f47ec9 100644 --- a/src/liblzma/api/lzma.h +++ b/src/liblzma/api/lzma.h @@ -1,31 +1,30 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file api/lzma.h * \brief The public API of liblzma data compression library * \mainpage * - * liblzma is a public domain general-purpose data compression library with - * a zlib-like API. The native file format is .xz, but also the old .lzma - * format and raw (no headers) streams are supported. Multiple compression - * algorithms (filters) are supported. Currently LZMA2 is the primary filter. + * liblzma is a general-purpose data compression library with a zlib-like API. + * The native file format is .xz, but also the old .lzma format and raw (no + * headers) streams are supported. Multiple compression algorithms (filters) + * are supported. Currently LZMA2 is the primary filter. + * + * liblzma is part of XZ Utils <https://xz.tukaani.org/xz-utils/>. XZ Utils + * includes a gzip-like command line tool named xz and some other tools. + * XZ Utils is developed and maintained by Lasse Collin and Jia Tan. * - * liblzma is part of XZ Utils <https://tukaani.org/xz/>. XZ Utils includes - * a gzip-like command line tool named xz and some other tools. XZ Utils - * is developed and maintained by Lasse Collin and Jia Tan. + * Major parts of liblzma are based on code written by Igor Pavlov, + * specifically the LZMA SDK <https://7-zip.org/sdk.html>. * - * Major parts of liblzma are based on Igor Pavlov's public domain LZMA SDK - * <https://7-zip.org/sdk.html>. + * The SHA-256 implementation in liblzma is based on code written by + * Wei Dai in Crypto++ Library <https://www.cryptopp.com/>. * - * The SHA-256 implementation is based on the public domain code found from - * 7-Zip <https://7-zip.org/>, which has a modified version of the public - * domain SHA-256 code found from Crypto++ <https://www.cryptopp.com/>. - * The SHA-256 code in Crypto++ was written by Kevin Springle and Wei Dai. + * liblzma is distributed under the BSD Zero Clause License (0BSD). */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h index 75cdd72acf20..20d485b97bec 100644 --- a/src/liblzma/api/lzma/base.h +++ b/src/liblzma/api/lzma/base.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/base.h * \brief Data types and functions used in many places in liblzma API @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -21,8 +20,8 @@ * * This is here because C89 doesn't have stdbool.h. To set a value for * variables having type lzma_bool, you can use - * - C99's `true' and `false' from stdbool.h; - * - C++'s internal `true' and `false'; or + * - C99's 'true' and 'false' from stdbool.h; + * - C++'s internal 'true' and 'false'; or * - integers one (true) and zero (false). */ typedef unsigned char lzma_bool; @@ -273,13 +272,13 @@ typedef enum { /** - * \brief The `action' argument for lzma_code() + * \brief The 'action' argument for lzma_code() * * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, - * or LZMA_FINISH, the same `action' must be used until lzma_code() returns + * or LZMA_FINISH, the same 'action' must be used until lzma_code() returns * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must * not be modified by the application until lzma_code() returns - * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * LZMA_STREAM_END. Changing the 'action' or modifying the amount of input * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { @@ -393,8 +392,8 @@ typedef enum { * Single-threaded mode only: liblzma doesn't make an internal copy of * lzma_allocator. Thus, it is OK to change these function pointers in * the middle of the coding process, but obviously it must be done - * carefully to make sure that the replacement `free' can deallocate - * memory allocated by the earlier `alloc' function(s). + * carefully to make sure that the replacement 'free' can deallocate + * memory allocated by the earlier 'alloc' function(s). * * Multithreaded mode: liblzma might internally store pointers to the * lzma_allocator given via the lzma_stream structure. The application @@ -422,7 +421,7 @@ typedef struct { * liblzma never sets this to zero. * * \return Pointer to the beginning of a memory block of - * `size' bytes, or NULL if allocation fails + * 'size' bytes, or NULL if allocation fails * for some reason. When allocation fails, functions * of liblzma return LZMA_MEM_ERROR. * @@ -622,7 +621,7 @@ typedef struct { * to and get output from liblzma. * * See the description of the coder-specific initialization function to find - * out what `action' values are supported by the coder. + * out what 'action' values are supported by the coder. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. diff --git a/src/liblzma/api/lzma/bcj.h b/src/liblzma/api/lzma/bcj.h index 0c84e0cff901..7f6611feb325 100644 --- a/src/liblzma/api/lzma/bcj.h +++ b/src/liblzma/api/lzma/bcj.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/bcj.h * \brief Branch/Call/Jump conversion filters @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -53,6 +52,11 @@ */ #define LZMA_FILTER_ARM64 LZMA_VLI_C(0x0A) +/** + * \brief Filter for RISC-V binaries + */ +#define LZMA_FILTER_RISCV LZMA_VLI_C(0x0B) + /** * \brief Options for BCJ filters diff --git a/src/liblzma/api/lzma/block.h b/src/liblzma/api/lzma/block.h index ec5e77a69ae9..05b77e59aabb 100644 --- a/src/liblzma/api/lzma/block.h +++ b/src/liblzma/api/lzma/block.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/block.h * \brief .xz Block handling @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h index b37197d2c7f7..e7a50ed3a3c3 100644 --- a/src/liblzma/api/lzma/check.h +++ b/src/liblzma/api/lzma/check.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/check.h * \brief Integrity checks @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h index 2849fbfd3c51..ad3f99057a32 100644 --- a/src/liblzma/api/lzma/container.h +++ b/src/liblzma/api/lzma/container.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/container.h * \brief File formats @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -297,7 +296,7 @@ extern LZMA_API(uint64_t) lzma_easy_decoder_memusage(uint32_t preset) * to call lzma_end() after failed initialization. * * If initialization succeeds, use lzma_code() to do the actual encoding. - * Valid values for `action' (the second argument of lzma_code()) are + * Valid values for 'action' (the second argument of lzma_code()) are * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, * there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. * @@ -436,6 +435,34 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( /** + * \brief Calculate recommended Block size for multithreaded .xz encoder + * + * This calculates a recommended Block size for multithreaded encoding given + * a filter chain. This is used internally by lzma_stream_encoder_mt() to + * determine the Block size if the block_size member is not set to the + * special value of 0 in the lzma_mt options struct. + * + * If one wishes to change the filters between Blocks, this function is + * helpful to set the block_size member of the lzma_mt struct before calling + * lzma_stream_encoder_mt(). Since the block_size member represents the + * maximum possible Block size for the multithreaded .xz encoder, one can + * use this function to find the maximum recommended Block size based on + * all planned filter chains. Otherwise, the multithreaded encoder will + * base its maximum Block size on the first filter chain used (if the + * block_size member is not set), which may unnecessarily limit the Block + * size for a later filter chain. + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Recommended Block size in bytes, or UINT64_MAX if + * an error occurred. + */ +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) + lzma_nothrow; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -651,13 +678,13 @@ extern LZMA_API(lzma_ret) lzma_microlzma_encoder( * supported by liblzma, only the .xz and .lz formats allow concatenated * files. Concatenated files are not allowed with the legacy .lzma format. * - * This flag also affects the usage of the `action' argument for lzma_code(). + * This flag also affects the usage of the 'action' argument for lzma_code(). * When LZMA_CONCATENATED is used, lzma_code() won't return LZMA_STREAM_END - * unless LZMA_FINISH is used as `action'. Thus, the application has to set + * unless LZMA_FINISH is used as 'action'. Thus, the application has to set * LZMA_FINISH in the same way as it does when encoding. * * If LZMA_CONCATENATED is not used, the decoders still accept LZMA_FINISH - * as `action' for lzma_code(), but the usage of LZMA_FINISH isn't required. + * as 'action' for lzma_code(), but the usage of LZMA_FINISH isn't required. */ #define LZMA_CONCATENATED UINT32_C(0x08) @@ -791,7 +818,7 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * diff --git a/src/liblzma/api/lzma/delta.h b/src/liblzma/api/lzma/delta.h index 7a725bc40742..5ebacef81584 100644 --- a/src/liblzma/api/lzma/delta.h +++ b/src/liblzma/api/lzma/delta.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/delta.h * \brief Delta filter @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/filter.h b/src/liblzma/api/lzma/filter.h index 1d887b4f2f43..e86809c4e395 100644 --- a/src/liblzma/api/lzma/filter.h +++ b/src/liblzma/api/lzma/filter.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/filter.h * \brief Common filter related types and functions @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -43,7 +42,7 @@ typedef struct { /** * \brief Filter ID * - * Use constants whose name begin with `LZMA_FILTER_' to specify + * Use constants whose name begin with 'LZMA_FILTER_' to specify * different filters. In an array of lzma_filter structures, use * LZMA_VLI_UNKNOWN to indicate end of filters. * @@ -199,7 +198,7 @@ extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) * * This function may be useful when implementing custom file formats. * - * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * The 'action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the * filter chain supports it), or LZMA_FINISH. * * \param strm Pointer to lzma_stream that is at least @@ -223,7 +222,7 @@ extern LZMA_API(lzma_ret) lzma_raw_encoder( * * The initialization of raw decoder goes similarly to raw encoder. * - * The `action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using + * The 'action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using * LZMA_FINISH is not required, it is supported just for convenience. * * \param strm Pointer to lzma_stream that is at least diff --git a/src/liblzma/api/lzma/hardware.h b/src/liblzma/api/lzma/hardware.h index f34897d8740f..7a1a84fcccfc 100644 --- a/src/liblzma/api/lzma/hardware.h +++ b/src/liblzma/api/lzma/hardware.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/hardware.h * \brief Hardware information @@ -23,9 +25,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h index 6eee4d68137d..7d8a9950ceaf 100644 --- a/src/liblzma/api/lzma/index.h +++ b/src/liblzma/api/lzma/index.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index.h * \brief Handling of .xz Index and related information @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -302,6 +301,28 @@ typedef enum { /** + * \brief Mask for return value from lzma_index_checks() for check none + * + * \note This and the other CHECK_MASK macros were added in 5.5.1alpha. + */ +#define LZMA_INDEX_CHECK_MASK_NONE (UINT32_C(1) << LZMA_CHECK_NONE) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC32 + */ +#define LZMA_INDEX_CHECK_MASK_CRC32 (UINT32_C(1) << LZMA_CHECK_CRC32) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC64 + */ +#define LZMA_INDEX_CHECK_MASK_CRC64 (UINT32_C(1) << LZMA_CHECK_CRC64) + +/** + * \brief Mask for return value from lzma_index_checks() for check SHA256 + */ +#define LZMA_INDEX_CHECK_MASK_SHA256 (UINT32_C(1) << LZMA_CHECK_SHA256) + +/** * \brief Calculate memory usage of lzma_index * * On disk, the size of the Index field depends on both the number of Records @@ -431,6 +452,7 @@ extern LZMA_API(lzma_ret) lzma_index_stream_flags( * showing the Check types to the user. * * The bitmask is 1 << check_id, e.g. CRC32 is 1 << 1 and SHA-256 is 1 << 10. + * These masks are defined for convenience as LZMA_INDEX_CHECK_MASK_XXX * * \param i Pointer to lzma_index structure * @@ -686,7 +708,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param strm Pointer to properly prepared lzma_stream * \param i Pointer to lzma_index which should be encoded. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. + * The valid 'action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. * It is enough to use only one of them (you can choose freely). * * \return Possible lzma_ret values: @@ -715,7 +737,7 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * don't allow 0 here and return LZMA_PROG_ERROR; * later versions treat 0 as if 1 had been specified. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * @@ -819,10 +841,10 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, * expect to see the same exact value for the same file if you change the * input buffer size or switch to a different liblzma version. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * You only need to use LZMA_RUN; LZMA_FINISH is only supported because it * might be convenient for some applications. If you use LZMA_FINISH and if - * lzma_code() asks the application to seek, remember to reset `action' back + * lzma_code() asks the application to seek, remember to reset 'action' back * to LZMA_RUN unless you hit the end of the file again. * * Possible return values from lzma_code(): diff --git a/src/liblzma/api/lzma/index_hash.h b/src/liblzma/api/lzma/index_hash.h index a2d4c4845b7c..68f9024eb3bc 100644 --- a/src/liblzma/api/lzma/index_hash.h +++ b/src/liblzma/api/lzma/index_hash.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index_hash.h * \brief Validate Index by using a hash function @@ -9,9 +11,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/lzma12.h b/src/liblzma/api/lzma/lzma12.h index 8ef6ea5b5010..05f5b66eb56a 100644 --- a/src/liblzma/api/lzma/lzma12.h +++ b/src/liblzma/api/lzma/lzma12.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -289,7 +288,7 @@ typedef struct { * \brief Number of literal context bits * * How many of the highest bits of the previous uncompressed - * eight-bit byte (also known as `literal') are taken into + * eight-bit byte (also known as 'literal') are taken into * account when predicting the bits of the next literal. * * E.g. in typical English text, an upper-case letter is diff --git a/src/liblzma/api/lzma/stream_flags.h b/src/liblzma/api/lzma/stream_flags.h index 7622a62120e3..a33fe4683760 100644 --- a/src/liblzma/api/lzma/stream_flags.h +++ b/src/liblzma/api/lzma/stream_flags.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/stream_flags.h * \brief .xz Stream Header and Stream Footer encoder and decoder @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h index 8dac38297273..c13a82d5f3b5 100644 --- a/src/liblzma/api/lzma/version.h +++ b/src/liblzma/api/lzma/version.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/version.h * \brief Version number @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -20,10 +19,10 @@ #define LZMA_VERSION_MAJOR 5 /** \brief Minor version number of the liblzma release. */ -#define LZMA_VERSION_MINOR 4 +#define LZMA_VERSION_MINOR 6 /** \brief Patch version number of the liblzma release. */ -#define LZMA_VERSION_PATCH 5 +#define LZMA_VERSION_PATCH 0 /** * \brief Version stability marker diff --git a/src/liblzma/api/lzma/vli.h b/src/liblzma/api/lzma/vli.h index f9ad15500dfe..7f3e398aae5b 100644 --- a/src/liblzma/api/lzma/vli.h +++ b/src/liblzma/api/lzma/vli.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/vli.h * \brief Variable-length integer handling @@ -17,9 +19,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c index 428ddaeb7798..7734ace1856e 100644 --- a/src/liblzma/check/check.c +++ b/src/liblzma/check/check.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h index 8ae95d59019f..f0eb1172d907 100644 --- a/src/liblzma/check/check.h +++ b/src/liblzma/check/check.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_CHECK_H diff --git a/src/liblzma/check/crc32_arm64.h b/src/liblzma/check/crc32_arm64.h new file mode 100644 index 000000000000..6cdb5dab32e6 --- /dev/null +++ b/src/liblzma/check/crc32_arm64.h @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_arm64.h +/// \brief CRC32 calculation with ARM64 optimization +// +// Authors: Chenxi Mao +// Jia Tan +// Hans Jansen +// +/////////////////////////////////////////////////////////////////////////////// + + +#ifndef LZMA_CRC32_ARM64_H +#define LZMA_CRC32_ARM64_H + +// MSVC always has the CRC intrinsics available when building for ARM64 +// there is no need to include any header files. +#ifndef _MSC_VER +# include <arm_acle.h> +#endif + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +# if defined(HAVE_GETAUXVAL) || defined(HAVE_ELF_AUX_INFO) +# include <sys/auxv.h> +# elif defined(_WIN32) +# include <processthreadsapi.h> +# elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) +# include <sys/sysctl.h> +# endif +#endif + +// Some EDG-based compilers support ARM64 and define __GNUC__ +// (such as Nvidia's nvcc), but do not support function attributes. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target \ + __attribute__((__target__("+crc"))) +#else +# define crc_attr_target +#endif + + +crc_attr_target +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ + crc = ~crc; + + // Align the input buffer because this was shown to be + // significantly faster than unaligned accesses. + const size_t align_amount = my_min(size, (8 - (uintptr_t)buf) & 7); + + for (const uint8_t *limit = buf + align_amount; buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + size -= align_amount; + + // Process 8 bytes at a time. The end point is determined by + // ignoring the least significant three bits of size to ensure + // we do not process past the bounds of the buffer. This guarantees + // that limit is a multiple of 8 and is strictly less than size. + for (const uint8_t *limit = buf + (size & ~((size_t)7)); + buf < limit; buf += 8) + crc = __crc32d(crc, aligned_read64le(buf)); + + // Process the remaining bytes that are not 8 byte aligned. + for (const uint8_t *limit = buf + (size & 7); buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + return ~crc; +} + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +static inline bool +is_arch_extension_supported(void) +{ +#if defined(HAVE_GETAUXVAL) + return (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0; + +#elif defined(HAVE_ELF_AUX_INFO) + unsigned long feature_flags; + + elf_aux_info(AT_HWCAP, &feature_flags, sizeof(feature_flags)); + return feature_flags & HWCAP_CRC32 != 0; + +#elif defined(_WIN32) + return IsProcessorFeaturePresent( + PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + +#elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) + int has_crc32 = 0; + size_t size = sizeof(has_crc32); + + // The sysctlbyname() function requires a string identifier for the + // CPU feature it tests. The Apple documentation lists the string + // "hw.optional.armv8_crc32", which can be found here: + // (https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics#3915619) + int err = sysctlbyname("hw.optional.armv8_crc32", &has_crc32, + &size, NULL, 0); + + return !err && has_crc32; + +#else + // If a runtime detection method cannot be found, then this must + // be a compile time error. The checks in crc_common.h should ensure + // a runtime detection method is always found if this function is + // built. It would be possible to just return false here, but this + // is inefficient for binary size and runtime since only the generic + // method could ever be used. +# error Runtime detection method unavailable. +#endif +} +#endif + +#endif // LZMA_CRC32_ARM64_H diff --git a/src/liblzma/check/crc32_fast.c b/src/liblzma/check/crc32_fast.c index eed7350582e4..5e26914a4d1b 100644 --- a/src/liblzma/check/crc32_fast.c +++ b/src/liblzma/check/crc32_fast.c @@ -1,30 +1,35 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32.c /// \brief CRC32 calculation -/// -/// Calculate the CRC32 using the slice-by-eight algorithm. -/// It is explained in this document: -/// http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf -/// The code in this file is not the same as in Intel's paper, but -/// the basic principle is identical. -// -// Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen // /////////////////////////////////////////////////////////////////////////////// #include "check.h" -#include "crc_macros.h" +#include "crc_common.h" +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC32_CLMUL +# include "crc_x86_clmul.h" +#elif defined(CRC32_ARM64) +# include "crc32_arm64.h" +#endif -// If you make any changes, do some benchmarking! Seemingly unrelated -// changes can very easily ruin the performance (and very probably is -// very compiler dependent). -extern LZMA_API(uint32_t) -lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) + +#ifdef CRC32_GENERIC + +/////////////////// +// Generic CRC32 // +/////////////////// + +static uint32_t +crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; @@ -80,3 +85,153 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) return ~crc; } +#endif + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + +////////////////////////// +// Function dispatching // +////////////////////////// + +// If both the generic and arch-optimized implementations are built, then +// the function to use is selected at runtime because the system running +// the binary might not have the arch-specific instruction set extension(s) +// available. The three dispatch methods in order of priority: +// +// 1. Indirect function (ifunc). This method is slightly more efficient +// than the constructor method because it will change the entry in the +// Procedure Linkage Table (PLT) for the function either at load time or +// at the first call. This avoids having to call the function through a +// function pointer and will treat the function call like a regular call +// through the PLT. ifuncs are created by using +// __attribute__((__ifunc__("resolver"))) on a function which has no +// body. The "resolver" is the name of the function that chooses at +// runtime which implementation to use. +// +// 2. Constructor. This method uses __attribute__((__constructor__)) to +// set crc32_func at load time. This avoids extra computation (and any +// unlikely threading bugs) on the first call to lzma_crc32() to decide +// which implementation should be used. +// +// 3. First Call Resolution. On the very first call to lzma_crc32(), the +// call will be directed to crc32_dispatch() instead. This will set the +// appropriate implementation function and will not be called again. +// This method does not use any kind of locking but is safe because if +// multiple threads run the dispatcher simultaneously then they will all +// set crc32_func to the same value. + +typedef uint32_t (*crc32_func_type)( + const uint8_t *buf, size_t size, uint32_t crc); + +// Clang 16.0.0 and older has a bug where it marks the ifunc resolver +// function as unused since it is static and never used outside of +// __attribute__((__ifunc__())). +#if defined(CRC_USE_IFUNC) && defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +// This resolver is shared between all three dispatch methods. It serves as +// the ifunc resolver if ifunc is supported, otherwise it is called as a +// regular function by the constructor or first call resolution methods. +static crc32_func_type +crc32_resolve(void) +{ + return is_arch_extension_supported() + ? &crc32_arch_optimized : &crc32_generic; +} + +#if defined(CRC_USE_IFUNC) && defined(__clang__) +# pragma GCC diagnostic pop +#endif + +#ifndef CRC_USE_IFUNC + +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +// Constructor method. +# define CRC32_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc32_func_type crc32_func; +#else +// First Call Resolution method. +# define CRC32_SET_FUNC_ATTR +static uint32_t crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc); +static crc32_func_type crc32_func = &crc32_dispatch; +#endif + +CRC32_SET_FUNC_ATTR +static void +crc32_set_func(void) +{ + crc32_func = crc32_resolve(); + return; +} + +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +static uint32_t +crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc) +{ + // When __attribute__((__ifunc__(...))) and + // __attribute__((__constructor__)) isn't supported, set the + // function pointer without any locking. If multiple threads run + // the detection code in parallel, they will all end up setting + // the pointer to the same value. This avoids the use of + // mythread_once() on every call to lzma_crc32() but this likely + // isn't strictly standards compliant. Let's change it if it breaks. + crc32_set_func(); + return crc32_func(buf, size, crc); +} + +#endif +#endif +#endif + + +#ifdef CRC_USE_IFUNC +extern LZMA_API(uint32_t) +lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) + __attribute__((__ifunc__("crc32_resolve"))); +#else +extern LZMA_API(uint32_t) +lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +{ +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + // On x86-64, if CLMUL is available, it is the best for non-tiny + // inputs, being over twice as fast as the generic slice-by-four + // version. However, for size <= 16 it's different. In the extreme + // case of size == 1 the generic version can be five times faster. + // At size >= 8 the CLMUL starts to become reasonable. It + // varies depending on the alignment of buf too. + // + // The above doesn't include the overhead of mythread_once(). + // At least on x86-64 GNU/Linux, pthread_once() is very fast but + // it still makes lzma_crc32(buf, 1, crc) 50-100 % slower. When + // size reaches 12-16 bytes the overhead becomes negligible. + // + // So using the generic version for size <= 16 may give better + // performance with tiny inputs but if such inputs happen rarely + // it's not so obvious because then the lookup table of the + // generic version may not be in the processor cache. +#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) + return crc32_generic(buf, size, crc); +#endif + +/* +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR + // See crc32_dispatch(). This would be the alternative which uses + // locking and doesn't use crc32_dispatch(). Note that on Windows + // this method needs Vista threads. + mythread_once(crc64_set_func); +#endif +*/ + return crc32_func(buf, size, crc); + +#elif defined(CRC32_ARCH_OPTIMIZED) + return crc32_arch_optimized(buf, size, crc); + +#else + return crc32_generic(buf, size, crc); +#endif +} +#endif diff --git a/src/liblzma/check/crc32_small.c b/src/liblzma/check/crc32_small.c index 186966e99216..6a1bd66185ea 100644 --- a/src/liblzma/check/crc32_small.c +++ b/src/liblzma/check/crc32_small.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/crc32_table.c b/src/liblzma/check/crc32_table.c index b11762ae0ac7..fb1b6585422a 100644 --- a/src/liblzma/check/crc32_table.c +++ b/src/liblzma/check/crc32_table.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_table.c @@ -5,18 +7,38 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" + +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too +// so that in 32-bit builds crc32_x86.S won't break due to a missing table. +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ + && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6)) +# define X86_CLMUL_NO_TABLE 1 +#endif + +#if defined(HAVE_ARM64_CRC32) \ + && !defined(WORDS_BIGENDIAN) \ + && defined(__ARM_FEATURE_CRC32) +# define ARM64_CRC32_NO_TABLE 1 +#endif + + +#if !defined(HAVE_ENCODERS) && (defined(X86_CLMUL_NO_TABLE) \ + || defined(ARM64_CRC32_NO_TABLE_)) +// No table needed. Use a typedef to avoid an empty translation unit. +typedef void lzma_crc32_dummy; + +#else // Having the declaration here silences clang -Wmissing-variable-declarations. extern const uint32_t lzma_crc32_table[8][256]; -#ifdef WORDS_BIGENDIAN -# include "crc32_table_be.h" -#else -# include "crc32_table_le.h" +# ifdef WORDS_BIGENDIAN +# include "crc32_table_be.h" +# else +# include "crc32_table_le.h" +# endif #endif diff --git a/src/liblzma/check/crc32_table_be.h b/src/liblzma/check/crc32_table_be.h index c483cb670dcb..505c23074c11 100644 --- a/src/liblzma/check/crc32_table_be.h +++ b/src/liblzma/check/crc32_table_be.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { { diff --git a/src/liblzma/check/crc32_table_le.h b/src/liblzma/check/crc32_table_le.h index 25f4fc443537..e89c21a7b23d 100644 --- a/src/liblzma/check/crc32_table_le.h +++ b/src/liblzma/check/crc32_table_le.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { { diff --git a/src/liblzma/check/crc32_tablegen.c b/src/liblzma/check/crc32_tablegen.c index 31a4d2751db2..01047d3eca47 100644 --- a/src/liblzma/check/crc32_tablegen.c +++ b/src/liblzma/check/crc32_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_tablegen.c @@ -9,9 +11,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -54,9 +53,11 @@ init_crc32_table(void) static void print_crc32_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); for (size_t s = 0; s < 8; ++s) { for (size_t b = 0; b < 256; ++b) { @@ -82,9 +83,11 @@ print_crc32_table(void) static void print_lz_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_lz_hash_table[256] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_lz_hash_table[256] = {"); for (size_t b = 0; b < 256; ++b) { if ((b % 4) == 0) diff --git a/src/liblzma/check/crc32_x86.S b/src/liblzma/check/crc32_x86.S index 4f395df8122a..ddc3cee6ea5b 100644 --- a/src/liblzma/check/crc32_x86.S +++ b/src/liblzma/check/crc32_x86.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC32 using slicing-by-eight algorithm * @@ -11,9 +13,6 @@ * Authors: Igor Pavlov (original version) * Lasse Collin (AT&T syntax, PIC support, better portability) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc32_table, which can be created using the * following C code: diff --git a/src/liblzma/check/crc64_fast.c b/src/liblzma/check/crc64_fast.c index 0c8622a1f367..f29fe3d3c5e6 100644 --- a/src/liblzma/check/crc64_fast.c +++ b/src/liblzma/check/crc64_fast.c @@ -1,85 +1,30 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64.c /// \brief CRC64 calculation -/// -/// There are two methods in this file. crc64_generic uses the -/// the slice-by-four algorithm. This is the same idea that is -/// used in crc32_fast.c, but for CRC64 we use only four tables -/// instead of eight to avoid increasing CPU cache usage. -/// -/// crc64_clmul uses 32/64-bit x86 SSSE3, SSE4.1, and CLMUL instructions. -/// It was derived from -/// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf -/// and the public domain code from https://github.com/rawrunprotected/crc -/// (URLs were checked on 2022-11-07). -/// -/// FIXME: Builds for 32-bit x86 use crc64_x86.S by default instead -/// of this file and thus CLMUL version isn't available on 32-bit x86 -/// unless configured with --disable-assembler. Even then the lookup table -/// isn't omitted in crc64_table.c since it doesn't know that assembly -/// code has been disabled. // // Authors: Lasse Collin // Ilya Kurdyukov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" +#include "crc_common.h" -#undef CRC_GENERIC -#undef CRC_CLMUL -#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS - -// If CLMUL cannot be used then only the generic slice-by-four is built. -#if !defined(HAVE_USABLE_CLMUL) -# define CRC_GENERIC 1 - -// If CLMUL is allowed unconditionally in the compiler options then the -// generic version can be omitted. Note that this doesn't work with MSVC -// as I don't know how to detect the features here. -// -// NOTE: Keep this this in sync with crc64_table.c. -#elif (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6) -# define CRC_CLMUL 1 - -// Otherwise build both and detect at runtime which version to use. -#else -# define CRC_GENERIC 1 -# define CRC_CLMUL 1 - -/* - // The generic code is much faster with 1-8-byte inputs and has - // similar performance up to 16 bytes at least in microbenchmarks - // (it depends on input buffer alignment too). If both versions are - // built, this #define will use the generic version for inputs up to - // 16 bytes and CLMUL for bigger inputs. It saves a little in code - // size since the special cases for 0-16-byte inputs will be omitted - // from the CLMUL code. -# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 -*/ - -# if defined(_MSC_VER) -# include <intrin.h> -# elif defined(HAVE_CPUID_H) -# include <cpuid.h> -# endif +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC64_CLMUL +# include "crc_x86_clmul.h" #endif +#ifdef CRC64_GENERIC + ///////////////////////////////// // Generic slice-by-four CRC64 // ///////////////////////////////// -#ifdef CRC_GENERIC - -#include "crc_macros.h" - - #ifdef WORDS_BIGENDIAN # define A1(x) ((x) >> 56) #else @@ -136,336 +81,51 @@ crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) #endif -///////////////////// -// x86 CLMUL CRC64 // -///////////////////// +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) -#ifdef CRC_CLMUL +////////////////////////// +// Function dispatching // +////////////////////////// -#include <immintrin.h> +// If both the generic and arch-optimized implementations are usable, then +// the function that is used is selected at runtime. See crc32_fast.c. +typedef uint64_t (*crc64_func_type)( + const uint8_t *buf, size_t size, uint64_t crc); -/* -// These functions were used to generate the constants -// at the top of crc64_clmul(). -static uint64_t -calc_lo(uint64_t poly) -{ - uint64_t a = poly; - uint64_t b = 0; - - for (unsigned i = 0; i < 64; ++i) { - b = (b >> 1) | (a << 63); - a = (a >> 1) ^ (a & 1 ? poly : 0); - } - - return b; -} - -static uint64_t -calc_hi(uint64_t poly, uint64_t a) -{ - for (unsigned i = 0; i < 64; ++i) - a = (a >> 1) ^ (a & 1 ? poly : 0); - - return a; -} -*/ - - -#define MASK_L(in, mask, r) \ - r = _mm_shuffle_epi8(in, mask) - -#define MASK_H(in, mask, r) \ - r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) - -#define MASK_LH(in, mask, low, high) \ - MASK_L(in, mask, low); \ - MASK_H(in, mask, high) - - -// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC -// code when optimizations are enabled (release build). According to the bug -// report, the ebx register is corrupted and the calculated result is wrong. -// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. -// The following pragma works and performance is still good. x86-64 builds -// aren't affected by this problem. -// -// NOTE: Another pragma after the function restores the optimizations. -// If the #if condition here is updated, the other one must be updated too. -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("g", off) -#endif - -// EDG-based compilers (Intel's classic compiler and compiler for E2K) can -// define __GNUC__ but the attribute must not be used with them. -// The new Clang-based ICX needs the attribute. -// -// NOTE: Build systems check for this too, keep them in sync with this. -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) -__attribute__((__target__("ssse3,sse4.1,pclmul"))) -#endif -// The intrinsics use 16-byte-aligned reads from buf, thus they may read -// up to 15 bytes before or after the buffer (depending on the alignment -// of the buf argument). The values of the extra bytes are ignored. -// This unavoidably trips -fsanitize=address so address sanitizier has -// to be disabled for this function. -#if lzma_has_attribute(__no_sanitize_address__) -__attribute__((__no_sanitize_address__)) -#endif -static uint64_t -crc64_clmul(const uint8_t *buf, size_t size, uint64_t crc) -{ - // The prototypes of the intrinsics use signed types while most of - // the values are treated as unsigned here. These warnings in this - // function have been checked and found to be harmless so silence them. -#if TUKLIB_GNUC_REQ(4, 6) || defined(__clang__) +#if defined(CRC_USE_IFUNC) && defined(__clang__) # pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsign-conversion" -# pragma GCC diagnostic ignored "-Wconversion" -#endif - -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - // The code assumes that there is at least one byte of input. - if (size == 0) - return crc; -#endif - - // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial - const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 - const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 - const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) - const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) - const __m128i vfold0 = _mm_set_epi64x(p, mu); - const __m128i vfold1 = _mm_set_epi64x(k2, k1); - - // Create a vector with 8-bit values 0 to 15. This is used to - // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. - const __m128i vramp = _mm_setr_epi32( - 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); - - // This is used to inverse the control mask of _mm_shuffle_epi8 - // so that bytes that wouldn't be picked with the original mask - // will be picked and vice versa. - const __m128i vsign = _mm_set1_epi8(0x80); - - // Memory addresses A to D and the distances between them: - // - // A B C D - // [skip_start][size][skip_end] - // [ size2 ] - // - // A and D are 16-byte aligned. B and C are 1-byte aligned. - // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. - // - // A = aligned_buf will initially point to this address. - // B = The address pointed by the caller-supplied buf. - // C = buf + size == aligned_buf + size2 - // D = buf + size + skip_end == aligned_buf + size2 + skip_end - const size_t skip_start = (size_t)((uintptr_t)buf & 15); - const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); - const __m128i *aligned_buf = (const __m128i *)( - (uintptr_t)buf & ~(uintptr_t)15); - - // If size2 <= 16 then the whole input fits into a single 16-byte - // vector. If size2 > 16 then at least two 16-byte vectors must - // be processed. If size2 > 16 && size <= 16 then there is only - // one 16-byte vector's worth of input but it is unaligned in memory. - // - // NOTE: There is no integer overflow here if the arguments are valid. - // If this overflowed, buf + size would too. - size_t size2 = skip_start + size; - - // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: - // The first skip_start or skip_end bytes in the vectors will have - // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 - // will produce zeros for these positions. (Bitwise-xor of these - // masks with vsign will produce the opposite behavior.) - const __m128i mask_start - = _mm_sub_epi8(vramp, _mm_set1_epi8(skip_start)); - const __m128i mask_end = _mm_sub_epi8(vramp, _mm_set1_epi8(skip_end)); - - // Get the first 1-16 bytes into data0. If loading less than 16 bytes, - // the bytes are loaded to the high bits of the vector and the least - // significant positions are filled with zeros. - const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), - _mm_setzero_si128(), mask_start); - ++aligned_buf; - -#if defined(__i386__) || defined(_M_IX86) - const __m128i initial_crc = _mm_set_epi64x(0, ~crc); -#else - // GCC and Clang would produce good code with _mm_set_epi64x - // but MSVC needs _mm_cvtsi64_si128 on x86-64. - const __m128i initial_crc = _mm_cvtsi64_si128(~crc); -#endif - - __m128i v0, v1, v2, v3; - -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - if (size <= 16) { - // Right-shift initial_crc by 1-16 bytes based on "size" - // and store the result in v1 (high bytes) and v0 (low bytes). - // - // NOTE: The highest 8 bytes of initial_crc are zeros so - // v1 will be filled with zeros if size >= 8. The highest 8 - // bytes of v1 will always become zeros. - // - // [ v1 ][ v0 ] - // [ initial_crc ] size == 1 - // [ initial_crc ] size == 2 - // [ initial_crc ] size == 15 - // [ initial_crc ] size == 16 (all in v0) - const __m128i mask_low = _mm_add_epi8( - vramp, _mm_set1_epi8(size - 16)); - MASK_LH(initial_crc, mask_low, v0, v1); - - if (size2 <= 16) { - // There are 1-16 bytes of input and it is all - // in data0. Copy the input bytes to v3. If there - // are fewer than 16 bytes, the low bytes in v3 - // will be filled with zeros. That is, the input - // bytes are stored to the same position as - // (part of) initial_crc is in v0. - MASK_L(data0, mask_end, v3); - } else { - // There are 2-16 bytes of input but not all bytes - // are in data0. - const __m128i data1 = _mm_load_si128(aligned_buf); - - // Collect the 2-16 input bytes from data0 and data1 - // to v2 and v3, and bitwise-xor them with the - // low bits of initial_crc in v0. Note that the - // the second xor is below this else-block as it - // is shared with the other branch. - MASK_H(data0, mask_end, v2); - MASK_L(data1, mask_end, v3); - v0 = _mm_xor_si128(v0, v2); - } - - v0 = _mm_xor_si128(v0, v3); - v1 = _mm_alignr_epi8(v1, v0, 8); - } else -#endif - { - const __m128i data1 = _mm_load_si128(aligned_buf); - MASK_LH(initial_crc, mask_start, v0, v1); - v0 = _mm_xor_si128(v0, data0); - v1 = _mm_xor_si128(v1, data1); - -#define FOLD \ - v1 = _mm_xor_si128(v1, _mm_clmulepi64_si128(v0, vfold1, 0x00)); \ - v0 = _mm_xor_si128(v1, _mm_clmulepi64_si128(v0, vfold1, 0x11)); - - while (size2 > 32) { - ++aligned_buf; - size2 -= 16; - FOLD - v1 = _mm_load_si128(aligned_buf); - } - - if (size2 < 32) { - MASK_H(v0, mask_end, v2); - MASK_L(v0, mask_end, v0); - MASK_L(v1, mask_end, v3); - v1 = _mm_or_si128(v2, v3); - } - - FOLD - v1 = _mm_srli_si128(v0, 8); -#undef FOLD - } - - v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold1, 0x10), v1); - v0 = _mm_clmulepi64_si128(v1, vfold0, 0x00); - v2 = _mm_clmulepi64_si128(v0, vfold0, 0x10); - v0 = _mm_xor_si128(_mm_xor_si128(v2, _mm_slli_si128(v0, 8)), v1); - -#if defined(__i386__) || defined(_M_IX86) - return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | - (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); -#else - return ~(uint64_t)_mm_extract_epi64(v0, 1); +# pragma GCC diagnostic ignored "-Wunused-function" #endif -#if TUKLIB_GNUC_REQ(4, 6) || defined(__clang__) -# pragma GCC diagnostic pop -#endif +static crc64_func_type +crc64_resolve(void) +{ + return is_arch_extension_supported() + ? &crc64_arch_optimized : &crc64_generic; } -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("", on) -#endif -#endif - - -//////////////////////// -// Detect CPU support // -//////////////////////// -#if defined(CRC_GENERIC) && defined(CRC_CLMUL) -static inline bool -is_clmul_supported(void) -{ - int success = 1; - uint32_t r[4]; // eax, ebx, ecx, edx - -#if defined(_MSC_VER) - // This needs <intrin.h> with MSVC. ICC has it as a built-in - // on all platforms. - __cpuid(r, 1); -#elif defined(HAVE_CPUID_H) - // Compared to just using __asm__ to run CPUID, this also checks - // that CPUID is supported and saves and restores ebx as that is - // needed with GCC < 5 with position-independent code (PIC). - success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); -#else - // Just a fallback that shouldn't be needed. - __asm__("cpuid\n\t" - : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) - : "a"(1), "c"(0)); +#if defined(CRC_USE_IFUNC) && defined(__clang__) +# pragma GCC diagnostic pop #endif - // Returns true if these are supported: - // CLMUL (bit 1 in ecx) - // SSSE3 (bit 9 in ecx) - // SSE4.1 (bit 19 in ecx) - const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); - return success && (r[2] & ecx_mask) == ecx_mask; - - // Alternative methods that weren't used: - // - ICC's _may_i_use_cpu_feature: the other methods should work too. - // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") - // - // CPUID decding is needed with MSVC anyway and older GCC. This keeps - // the feature checks in the build system simpler too. The nice thing - // about __builtin_cpu_supports would be that it generates very short - // code as is it only reads a variable set at startup but a few bytes - // doesn't matter here. -} - +#ifndef CRC_USE_IFUNC #ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR -# define CRC64_FUNC_INIT # define CRC64_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc64_func_type crc64_func; #else -# define CRC64_FUNC_INIT = &crc64_dispatch # define CRC64_SET_FUNC_ATTR static uint64_t crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc); +static crc64_func_type crc64_func = &crc64_dispatch; #endif -// Pointer to the the selected CRC64 method. -static uint64_t (*crc64_func)(const uint8_t *buf, size_t size, uint64_t crc) - CRC64_FUNC_INIT; - - CRC64_SET_FUNC_ATTR static void crc64_set_func(void) { - crc64_func = is_clmul_supported() ? &crc64_clmul : &crc64_generic; + crc64_func = crc64_resolve(); return; } @@ -474,65 +134,41 @@ crc64_set_func(void) static uint64_t crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc) { - // When __attribute__((__constructor__)) isn't supported, set the - // function pointer without any locking. If multiple threads run - // the detection code in parallel, they will all end up setting - // the pointer to the same value. This avoids the use of - // mythread_once() on every call to lzma_crc64() but this likely - // isn't strictly standards compliant. Let's change it if it breaks. crc64_set_func(); return crc64_func(buf, size, crc); } #endif #endif +#endif +#ifdef CRC_USE_IFUNC +extern LZMA_API(uint64_t) +lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) + __attribute__((__ifunc__("crc64_resolve"))); +#else extern LZMA_API(uint64_t) lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) { -#if defined(CRC_GENERIC) && defined(CRC_CLMUL) - // If CLMUL is available, it is the best for non-tiny inputs, - // being over twice as fast as the generic slice-by-four version. - // However, for size <= 16 it's different. In the extreme case - // of size == 1 the generic version can be five times faster. - // At size >= 8 the CLMUL starts to become reasonable. It - // varies depending on the alignment of buf too. - // - // The above doesn't include the overhead of mythread_once(). - // At least on x86-64 GNU/Linux, pthread_once() is very fast but - // it still makes lzma_crc64(buf, 1, crc) 50-100 % slower. When - // size reaches 12-16 bytes the overhead becomes negligible. - // - // So using the generic version for size <= 16 may give better - // performance with tiny inputs but if such inputs happen rarely - // it's not so obvious because then the lookup table of the - // generic version may not be in the processor cache. +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) + #ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS if (size <= 16) return crc64_generic(buf, size, crc); #endif - -/* -#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR - // See crc64_dispatch(). This would be the alternative which uses - // locking and doesn't use crc64_dispatch(). Note that on Windows - // this method needs Vista threads. - mythread_once(crc64_set_func); -#endif -*/ - return crc64_func(buf, size, crc); -#elif defined(CRC_CLMUL) - // If CLMUL is used unconditionally without runtime CPU detection - // then omitting the generic version and its 8 KiB lookup table - // makes the library smaller. +#elif defined(CRC64_ARCH_OPTIMIZED) + // If arch-optimized version is used unconditionally without runtime + // CPU detection then omitting the generic version and its 8 KiB + // lookup table makes the library smaller. // // FIXME: Lookup table isn't currently omitted on 32-bit x86, // see crc64_table.c. - return crc64_clmul(buf, size, crc); + return crc64_arch_optimized(buf, size, crc); #else return crc64_generic(buf, size, crc); #endif } +#endif diff --git a/src/liblzma/check/crc64_small.c b/src/liblzma/check/crc64_small.c index 420f7cfbb475..ee4ea26f67d0 100644 --- a/src/liblzma/check/crc64_small.c +++ b/src/liblzma/check/crc64_small.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/crc64_table.c b/src/liblzma/check/crc64_table.c index 688e527b0fd1..6dee387a1fcf 100644 --- a/src/liblzma/check/crc64_table.c +++ b/src/liblzma/check/crc64_table.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_table.c @@ -5,19 +7,21 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" -// FIXME: Compared to crc64_fast.c this has to check for __x86_64__ too +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too // so that in 32-bit builds crc64_x86.S won't break due to a missing table. -#if (defined(__x86_64__) && defined(__SSSE3__) \ +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6) + || (defined(__e2k__) && __iset__ >= 6)) +# define X86_CLMUL_NO_TABLE 1 +#endif + + +#ifdef X86_CLMUL_NO_TABLE // No table needed. Use a typedef to avoid an empty translation unit. typedef void lzma_crc64_dummy; diff --git a/src/liblzma/check/crc64_table_be.h b/src/liblzma/check/crc64_table_be.h index ea074f397a70..db76cc70e07c 100644 --- a/src/liblzma/check/crc64_table_be.h +++ b/src/liblzma/check/crc64_table_be.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { { diff --git a/src/liblzma/check/crc64_table_le.h b/src/liblzma/check/crc64_table_le.h index 1196b31e1323..e40a8c82105e 100644 --- a/src/liblzma/check/crc64_table_le.h +++ b/src/liblzma/check/crc64_table_le.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { { diff --git a/src/liblzma/check/crc64_tablegen.c b/src/liblzma/check/crc64_tablegen.c index fddaa7ed1400..af93e011ca21 100644 --- a/src/liblzma/check/crc64_tablegen.c +++ b/src/liblzma/check/crc64_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_tablegen.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -53,9 +52,11 @@ init_crc64_table(void) static void print_crc64_table(void) { - printf("/* This file has been automatically generated by " - "crc64_tablegen.c. */\n\n" - "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc64_tablegen.c.\n\n" + "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); for (size_t s = 0; s < 4; ++s) { for (size_t b = 0; b < 256; ++b) { diff --git a/src/liblzma/check/crc64_x86.S b/src/liblzma/check/crc64_x86.S index 9aecf5865455..47f608181ea8 100644 --- a/src/liblzma/check/crc64_x86.S +++ b/src/liblzma/check/crc64_x86.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC64 using slicing-by-four algorithm * @@ -7,9 +9,6 @@ * Authors: Igor Pavlov (original CRC32 assembly code) * Lasse Collin (CRC64 adaptation of the modified CRC32 code) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc64_table, which can be created using the * following C code: diff --git a/src/liblzma/check/crc_common.h b/src/liblzma/check/crc_common.h new file mode 100644 index 000000000000..856665db79a8 --- /dev/null +++ b/src/liblzma/check/crc_common.h @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_common.h +/// \brief Some functions and macros for CRC32 and CRC64 +// +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CRC_COMMON_H +#define LZMA_CRC_COMMON_H + +#include "common.h" + + +#ifdef WORDS_BIGENDIAN +# define A(x) ((x) >> 24) +# define B(x) (((x) >> 16) & 0xFF) +# define C(x) (((x) >> 8) & 0xFF) +# define D(x) ((x) & 0xFF) + +# define S8(x) ((x) << 8) +# define S32(x) ((x) << 32) + +#else +# define A(x) ((x) & 0xFF) +# define B(x) (((x) >> 8) & 0xFF) +# define C(x) (((x) >> 16) & 0xFF) +# define D(x) ((x) >> 24) + +# define S8(x) ((x) >> 8) +# define S32(x) ((x) >> 32) +#endif + + +// CRC CLMUL code needs this because accessing input buffers that aren't +// aligned to the vector size will inherently trip the address sanitizer. +#if lzma_has_attribute(__no_sanitize_address__) +# define crc_attr_no_sanitize_address \ + __attribute__((__no_sanitize_address__)) +#else +# define crc_attr_no_sanitize_address +#endif + +// Keep this in sync with changes to crc32_arm64.h +#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ + || defined(HAVE_ELF_AUX_INFO) \ + || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) +# define ARM64_RUNTIME_DETECTION 1 +#endif + + +#undef CRC32_GENERIC +#undef CRC64_GENERIC + +#undef CRC32_ARCH_OPTIMIZED +#undef CRC64_ARCH_OPTIMIZED + +// The x86 CLMUL is used for both CRC32 and CRC64. +#undef CRC_X86_CLMUL + +#undef CRC32_ARM64 +#undef CRC64_ARM64_CLMUL + +#undef CRC_USE_IFUNC + +#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS + +// ARM64 CRC32 instruction is only useful for CRC32. Currently, only +// little endian is supported since we were unable to test on a big +// endian machine. +// +// NOTE: Keep this and the next check in sync with the macro +// ARM64_CRC32_NO_TABLE in crc32_table.c +#if defined(HAVE_ARM64_CRC32) && !defined(WORDS_BIGENDIAN) +// Allow ARM64 CRC32 instruction without a runtime check if +// __ARM_FEATURE_CRC32 is defined. GCC and Clang only define this if the +// proper compiler options are used. +# if defined(__ARM_FEATURE_CRC32) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# elif defined(ARM64_RUNTIME_DETECTION) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# define CRC32_GENERIC 1 +# endif +#endif + +#if defined(HAVE_USABLE_CLMUL) +// If CLMUL is allowed unconditionally in the compiler options then the +// generic version can be omitted. Note that this doesn't work with MSVC +// as I don't know how to detect the features here. +// +// NOTE: Keep this in sync with the CLMUL_NO_TABLE macro in crc32_table.c. +# if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 +# else +# define CRC32_GENERIC 1 +# define CRC64_GENERIC 1 +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 + +# ifdef HAVE_FUNC_ATTRIBUTE_IFUNC +# define CRC_USE_IFUNC 1 +# endif +/* + // The generic code is much faster with 1-8-byte inputs and + // has similar performance up to 16 bytes at least in + // microbenchmarks (it depends on input buffer alignment + // too). If both versions are built, this #define will use + // the generic version for inputs up to 16 bytes and CLMUL + // for bigger inputs. It saves a little in code size since + // the special cases for 0-16-byte inputs will be omitted + // from the CLMUL code. +# ifndef CRC_USE_IFUNC +# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 +# endif +*/ +# endif +#endif + +// For CRC32 use the generic slice-by-eight implementation if no optimized +// version is available. +#if !defined(CRC32_ARCH_OPTIMIZED) && !defined(CRC32_GENERIC) +# define CRC32_GENERIC 1 +#endif + +// For CRC64 use the generic slice-by-four implementation if no optimized +// version is available. +#if !defined(CRC64_ARCH_OPTIMIZED) && !defined(CRC64_GENERIC) +# define CRC64_GENERIC 1 +#endif + +#endif diff --git a/src/liblzma/check/crc_macros.h b/src/liblzma/check/crc_macros.h deleted file mode 100644 index a7c21b765dca..000000000000 --- a/src/liblzma/check/crc_macros.h +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file crc_macros.h -/// \brief Some endian-dependent macros for CRC32 and CRC64 -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifdef WORDS_BIGENDIAN -# define A(x) ((x) >> 24) -# define B(x) (((x) >> 16) & 0xFF) -# define C(x) (((x) >> 8) & 0xFF) -# define D(x) ((x) & 0xFF) - -# define S8(x) ((x) << 8) -# define S32(x) ((x) << 32) - -#else -# define A(x) ((x) & 0xFF) -# define B(x) (((x) >> 8) & 0xFF) -# define C(x) (((x) >> 16) & 0xFF) -# define D(x) ((x) >> 24) - -# define S8(x) ((x) >> 8) -# define S32(x) ((x) >> 32) -#endif diff --git a/src/liblzma/check/crc_x86_clmul.h b/src/liblzma/check/crc_x86_clmul.h new file mode 100644 index 000000000000..ae66ca9f8c71 --- /dev/null +++ b/src/liblzma/check/crc_x86_clmul.h @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_x86_clmul.h +/// \brief CRC32 and CRC64 implementations using CLMUL instructions. +/// +/// The CRC32 and CRC64 implementations use 32/64-bit x86 SSSE3, SSE4.1, and +/// CLMUL instructions. This is compatible with Elbrus 2000 (E2K) too. +/// +/// They were derived from +/// https://www.researchgate.net/publication/263424619_Fast_CRC_computation +/// and the public domain code from https://github.com/rawrunprotected/crc +/// (URLs were checked on 2023-10-14). +/// +/// While this file has both CRC32 and CRC64 implementations, only one +/// should be built at a time to ensure that crc_simd_body() is inlined +/// even with compilers with which lzma_always_inline expands to plain inline. +/// The version to build is selected by defining BUILDING_CRC32_CLMUL or +/// BUILDING_CRC64_CLMUL before including this file. +/// +/// FIXME: Builds for 32-bit x86 use the assembly .S files by default +/// unless configured with --disable-assembler. Even then the lookup table +/// isn't omitted in crc64_table.c since it doesn't know that assembly +/// code has been disabled. +// +// Authors: Ilya Kurdyukov +// Hans Jansen +// Lasse Collin +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +// This file must not be included more than once. +#ifdef LZMA_CRC_X86_CLMUL_H +# error crc_x86_clmul.h was included twice. +#endif +#define LZMA_CRC_X86_CLMUL_H + +#include <immintrin.h> + +#if defined(_MSC_VER) +# include <intrin.h> +#elif defined(HAVE_CPUID_H) +# include <cpuid.h> +#endif + + +// EDG-based compilers (Intel's classic compiler and compiler for E2K) can +// define __GNUC__ but the attribute must not be used with them. +// The new Clang-based ICX needs the attribute. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target \ + __attribute__((__target__("ssse3,sse4.1,pclmul"))) +#else +# define crc_attr_target +#endif + + +#define MASK_L(in, mask, r) r = _mm_shuffle_epi8(in, mask) + +#define MASK_H(in, mask, r) \ + r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) + +#define MASK_LH(in, mask, low, high) \ + MASK_L(in, mask, low); \ + MASK_H(in, mask, high) + + +crc_attr_target +crc_attr_no_sanitize_address +static lzma_always_inline void +crc_simd_body(const uint8_t *buf, const size_t size, __m128i *v0, __m128i *v1, + const __m128i vfold16, const __m128i initial_crc) +{ + // Create a vector with 8-bit values 0 to 15. This is used to + // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. + const __m128i vramp = _mm_setr_epi32( + 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); + + // This is used to inverse the control mask of _mm_shuffle_epi8 + // so that bytes that wouldn't be picked with the original mask + // will be picked and vice versa. + const __m128i vsign = _mm_set1_epi8(-0x80); + + // Memory addresses A to D and the distances between them: + // + // A B C D + // [skip_start][size][skip_end] + // [ size2 ] + // + // A and D are 16-byte aligned. B and C are 1-byte aligned. + // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. + // + // A = aligned_buf will initially point to this address. + // B = The address pointed by the caller-supplied buf. + // C = buf + size == aligned_buf + size2 + // D = buf + size + skip_end == aligned_buf + size2 + skip_end + const size_t skip_start = (size_t)((uintptr_t)buf & 15); + const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); + const __m128i *aligned_buf = (const __m128i *)( + (uintptr_t)buf & ~(uintptr_t)15); + + // If size2 <= 16 then the whole input fits into a single 16-byte + // vector. If size2 > 16 then at least two 16-byte vectors must + // be processed. If size2 > 16 && size <= 16 then there is only + // one 16-byte vector's worth of input but it is unaligned in memory. + // + // NOTE: There is no integer overflow here if the arguments + // are valid. If this overflowed, buf + size would too. + const size_t size2 = skip_start + size; + + // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: + // The first skip_start or skip_end bytes in the vectors will have + // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 + // will produce zeros for these positions. (Bitwise-xor of these + // masks with vsign will produce the opposite behavior.) + const __m128i mask_start + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_start)); + const __m128i mask_end + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_end)); + + // Get the first 1-16 bytes into data0. If loading less than 16 + // bytes, the bytes are loaded to the high bits of the vector and + // the least significant positions are filled with zeros. + const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), + _mm_setzero_si128(), mask_start); + aligned_buf++; + + __m128i v2, v3; + +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) { + // Right-shift initial_crc by 1-16 bytes based on "size" + // and store the result in v1 (high bytes) and v0 (low bytes). + // + // NOTE: The highest 8 bytes of initial_crc are zeros so + // v1 will be filled with zeros if size >= 8. The highest + // 8 bytes of v1 will always become zeros. + // + // [ v1 ][ v0 ] + // [ initial_crc ] size == 1 + // [ initial_crc ] size == 2 + // [ initial_crc ] size == 15 + // [ initial_crc ] size == 16 (all in v0) + const __m128i mask_low = _mm_add_epi8( + vramp, _mm_set1_epi8((char)(size - 16))); + MASK_LH(initial_crc, mask_low, *v0, *v1); + + if (size2 <= 16) { + // There are 1-16 bytes of input and it is all + // in data0. Copy the input bytes to v3. If there + // are fewer than 16 bytes, the low bytes in v3 + // will be filled with zeros. That is, the input + // bytes are stored to the same position as + // (part of) initial_crc is in v0. + MASK_L(data0, mask_end, v3); + } else { + // There are 2-16 bytes of input but not all bytes + // are in data0. + const __m128i data1 = _mm_load_si128(aligned_buf); + + // Collect the 2-16 input bytes from data0 and data1 + // to v2 and v3, and bitwise-xor them with the + // low bits of initial_crc in v0. Note that the + // the second xor is below this else-block as it + // is shared with the other branch. + MASK_H(data0, mask_end, v2); + MASK_L(data1, mask_end, v3); + *v0 = _mm_xor_si128(*v0, v2); + } + + *v0 = _mm_xor_si128(*v0, v3); + *v1 = _mm_alignr_epi8(*v1, *v0, 8); + } else +#endif + { + // There is more than 16 bytes of input. + const __m128i data1 = _mm_load_si128(aligned_buf); + const __m128i *end = (const __m128i*)( + (const char *)aligned_buf - 16 + size2); + aligned_buf++; + + MASK_LH(initial_crc, mask_start, *v0, *v1); + *v0 = _mm_xor_si128(*v0, data0); + *v1 = _mm_xor_si128(*v1, data1); + + while (aligned_buf < end) { + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_load_si128(aligned_buf++); + } + + if (aligned_buf != end) { + MASK_H(*v0, mask_end, v2); + MASK_L(*v0, mask_end, *v0); + MASK_L(*v1, mask_end, v3); + *v1 = _mm_or_si128(v2, v3); + } + + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_srli_si128(*v0, 8); + } +} + + +///////////////////// +// x86 CLMUL CRC32 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc32_arch_optimized(). +static uint64_t +calc_lo(uint64_t p, uint64_t a, int n) +{ + uint64_t b = 0; int i; + for (i = 0; i < n; i++) { + b = b >> 1 | (a & 1) << (n - 1); + a = (a >> 1) ^ ((0 - (a & 1)) & p); + } + return b; +} + +// same as ~crc(&a, sizeof(a), ~0) +static uint64_t +calc_hi(uint64_t p, uint64_t a, int n) +{ + int i; + for (i = 0; i < n; i++) + a = (a >> 1) ^ ((0 - (a & 1)) & p); + return a; +} +*/ + +#ifdef BUILDING_CRC32_CLMUL + +crc_attr_target +crc_attr_no_sanitize_address +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // uint32_t poly = 0xedb88320; + const int64_t p = 0x1db710640; // p << 1 + const int64_t mu = 0x1f7011641; // calc_lo(p, p, 32) << 1 | 1 + const int64_t k5 = 0x163cd6124; // calc_hi(p, p, 32) << 1 + const int64_t k4 = 0x0ccaa009e; // calc_hi(p, p, 64) << 1 + const int64_t k3 = 0x1751997d0; // calc_hi(p, p, 128) << 1 + + const __m128i vfold4 = _mm_set_epi64x(mu, p); + const __m128i vfold8 = _mm_set_epi64x(0, k5); + const __m128i vfold16 = _mm_set_epi64x(k4, k3); + + __m128i v0, v1, v2; + + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi32_si128((int32_t)~crc)); + + v1 = _mm_xor_si128( + _mm_clmulepi64_si128(v0, vfold16, 0x10), v1); // xxx0 + v2 = _mm_shuffle_epi32(v1, 0xe7); // 0xx0 + v0 = _mm_slli_epi64(v1, 32); // [0] + v0 = _mm_clmulepi64_si128(v0, vfold8, 0x00); + v0 = _mm_xor_si128(v0, v2); // [1] [2] + v2 = _mm_clmulepi64_si128(v0, vfold4, 0x10); + v2 = _mm_clmulepi64_si128(v2, vfold4, 0x00); + v0 = _mm_xor_si128(v0, v2); // [2] + return ~(uint32_t)_mm_extract_epi32(v0, 2); +} +#endif // BUILDING_CRC32_CLMUL + + +///////////////////// +// x86 CLMUL CRC64 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc64_arch_optimized(). +static uint64_t +calc_lo(uint64_t poly) +{ + uint64_t a = poly; + uint64_t b = 0; + + for (unsigned i = 0; i < 64; ++i) { + b = (b >> 1) | (a << 63); + a = (a >> 1) ^ (a & 1 ? poly : 0); + } + + return b; +} + +static uint64_t +calc_hi(uint64_t poly, uint64_t a) +{ + for (unsigned i = 0; i < 64; ++i) + a = (a >> 1) ^ (a & 1 ? poly : 0); + + return a; +} +*/ + +#ifdef BUILDING_CRC64_CLMUL + +// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC +// code when optimizations are enabled (release build). According to the bug +// report, the ebx register is corrupted and the calculated result is wrong. +// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. +// The following pragma works and performance is still good. x86-64 builds +// and CRC32 CLMUL aren't affected by this problem. The problem does not +// happen in crc_simd_body() either (which is shared with CRC32 CLMUL anyway). +// +// NOTE: Another pragma after crc64_arch_optimized() restores +// the optimizations. If the #if condition here is updated, +// the other one must be updated too. +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("g", off) +#endif + +crc_attr_target +crc_attr_no_sanitize_address +static uint64_t +crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial + const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 + const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 + const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) + const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) + + const __m128i vfold8 = _mm_set_epi64x((int64_t)p, (int64_t)mu); + const __m128i vfold16 = _mm_set_epi64x((int64_t)k2, (int64_t)k1); + + __m128i v0, v1, v2; + +#if defined(__i386__) || defined(_M_IX86) + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_set_epi64x(0, (int64_t)~crc)); +#else + // GCC and Clang would produce good code with _mm_set_epi64x + // but MSVC needs _mm_cvtsi64_si128 on x86-64. + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi64_si128((int64_t)~crc)); +#endif + + v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold16, 0x10), v1); + v0 = _mm_clmulepi64_si128(v1, vfold8, 0x00); + v2 = _mm_clmulepi64_si128(v0, vfold8, 0x10); + v0 = _mm_xor_si128(_mm_xor_si128(v1, _mm_slli_si128(v0, 8)), v2); + +#if defined(__i386__) || defined(_M_IX86) + return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | + (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); +#else + return ~(uint64_t)_mm_extract_epi64(v0, 1); +#endif +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("", on) +#endif + +#endif // BUILDING_CRC64_CLMUL + + +// is_arch_extension_supported() must be inlined in this header file because +// the ifunc resolver function may not support calling a function in another +// translation unit. Depending on compiler-toolchain and flags, a call to +// a function defined in another translation unit could result in a +// reference to the PLT, which is unsafe to do in an ifunc resolver. The +// ifunc resolver runs very early when loading a shared library, so the PLT +// entries may not be setup at that time. Inlining this function duplicates +// the function body in crc32_resolve() and crc64_resolve(), but this is +// acceptable because the function results in very few instructions. +static inline bool +is_arch_extension_supported(void) +{ + int success = 1; + uint32_t r[4]; // eax, ebx, ecx, edx + +#if defined(_MSC_VER) + // This needs <intrin.h> with MSVC. ICC has it as a built-in + // on all platforms. + __cpuid(r, 1); +#elif defined(HAVE_CPUID_H) + // Compared to just using __asm__ to run CPUID, this also checks + // that CPUID is supported and saves and restores ebx as that is + // needed with GCC < 5 with position-independent code (PIC). + success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); +#else + // Just a fallback that shouldn't be needed. + __asm__("cpuid\n\t" + : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) + : "a"(1), "c"(0)); +#endif + + // Returns true if these are supported: + // CLMUL (bit 1 in ecx) + // SSSE3 (bit 9 in ecx) + // SSE4.1 (bit 19 in ecx) + const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); + return success && (r[2] & ecx_mask) == ecx_mask; + + // Alternative methods that weren't used: + // - ICC's _may_i_use_cpu_feature: the other methods should work too. + // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") + // + // CPUID decding is needed with MSVC anyway and older GCC. This keeps + // the feature checks in the build system simpler too. The nice thing + // about __builtin_cpu_supports would be that it generates very short + // code as is it only reads a variable set at startup but a few bytes + // doesn't matter here. +} diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c index 6feb342565fb..bd0d2806397c 100644 --- a/src/liblzma/check/sha256.c +++ b/src/liblzma/check/sha256.c @@ -1,24 +1,17 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sha256.c /// \brief SHA-256 -/// -/// \todo Crypto++ has x86 ASM optimizations. They use SSE so if they -/// are imported to liblzma, SSE instructions need to be used -/// conditionally to keep the code working on older boxes. // -// This code is based on the code found from 7-Zip, which has a modified -// version of the SHA-256 found from Crypto++ <https://www.cryptopp.com/>. -// The code was modified a little to fit into liblzma. +// The C code is based on the public domain SHA-256 code found from +// Crypto++ Library 5.5.1 released in 2007: https://www.cryptopp.com/ +// A few minor tweaks have been made in liblzma. // -// Authors: Kevin Springle -// Wei Dai -// Igor Pavlov +// Authors: Wei Dai // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c index 1dc85badf941..78af651578fc 100644 --- a/src/liblzma/common/alone_decoder.c +++ b/src/liblzma/common/alone_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "alone_decoder.h" diff --git a/src/liblzma/common/alone_decoder.h b/src/liblzma/common/alone_decoder.h index dfa031aa77dd..61ee24d97fe4 100644 --- a/src/liblzma/common/alone_decoder.h +++ b/src/liblzma/common/alone_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_ALONE_DECODER_H diff --git a/src/liblzma/common/alone_encoder.c b/src/liblzma/common/alone_encoder.c index 7d3812fa6ea4..21b039509ae5 100644 --- a/src/liblzma/common/alone_encoder.c +++ b/src/liblzma/common/alone_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c index 2a5c0894d123..fdd520f905c5 100644 --- a/src/liblzma/common/auto_decoder.c +++ b/src/liblzma/common/auto_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file auto_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/block_buffer_decoder.c b/src/liblzma/common/block_buffer_decoder.c index b0ded90ddc3e..55566cd2f2b0 100644 --- a/src/liblzma/common/block_buffer_decoder.c +++ b/src/liblzma/common/block_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h" diff --git a/src/liblzma/common/block_buffer_encoder.c b/src/liblzma/common/block_buffer_encoder.c index fdef02de8955..df3b90e8a186 100644 --- a/src/liblzma/common/block_buffer_encoder.c +++ b/src/liblzma/common/block_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_buffer_encoder.h" diff --git a/src/liblzma/common/block_buffer_encoder.h b/src/liblzma/common/block_buffer_encoder.h index 653207f73498..5274ac40d3aa 100644 --- a/src/liblzma/common/block_buffer_encoder.h +++ b/src/liblzma/common/block_buffer_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_BUFFER_ENCODER_H diff --git a/src/liblzma/common/block_decoder.c b/src/liblzma/common/block_decoder.c index be647d4855d4..2e369d316bdf 100644 --- a/src/liblzma/common/block_decoder.c +++ b/src/liblzma/common/block_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h" diff --git a/src/liblzma/common/block_decoder.h b/src/liblzma/common/block_decoder.h index 718c5ced886c..2cbf9ba6db83 100644 --- a/src/liblzma/common/block_decoder.h +++ b/src/liblzma/common/block_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_DECODER_H diff --git a/src/liblzma/common/block_encoder.c b/src/liblzma/common/block_encoder.c index 4a136ef65e33..ce8c1de69442 100644 --- a/src/liblzma/common/block_encoder.c +++ b/src/liblzma/common/block_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" diff --git a/src/liblzma/common/block_encoder.h b/src/liblzma/common/block_encoder.h index bd97c186e503..b7dfe9a08417 100644 --- a/src/liblzma/common/block_encoder.h +++ b/src/liblzma/common/block_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_ENCODER_H diff --git a/src/liblzma/common/block_header_decoder.c b/src/liblzma/common/block_header_decoder.c index c4935dcf46c1..f0b2fbe54d8d 100644 --- a/src/liblzma/common/block_header_decoder.c +++ b/src/liblzma/common/block_header_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c index 160425d27a35..45e57a26aba8 100644 --- a/src/liblzma/common/block_header_encoder.c +++ b/src/liblzma/common/block_header_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/block_util.c b/src/liblzma/common/block_util.c index acb311142c21..191f6d444aa6 100644 --- a/src/liblzma/common/block_util.c +++ b/src/liblzma/common/block_util.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_util.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c index adb50d785d52..cc0e06a51bee 100644 --- a/src/liblzma/common/common.c +++ b/src/liblzma/common/common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h index 378923e4012b..20af32f6d6cd 100644 --- a/src/liblzma/common/common.h +++ b/src/liblzma/common/common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_COMMON_H diff --git a/src/liblzma/common/easy_buffer_encoder.c b/src/liblzma/common/easy_buffer_encoder.c index 48eb56f5cc91..da610cea6bfa 100644 --- a/src/liblzma/common/easy_buffer_encoder.c +++ b/src/liblzma/common/easy_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_decoder_memusage.c b/src/liblzma/common/easy_decoder_memusage.c index 20bcd5b71758..0c76f10033b6 100644 --- a/src/liblzma/common/easy_decoder_memusage.c +++ b/src/liblzma/common/easy_decoder_memusage.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_decoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_encoder.c b/src/liblzma/common/easy_encoder.c index 5cb492dd0681..8dfe29610f79 100644 --- a/src/liblzma/common/easy_encoder.c +++ b/src/liblzma/common/easy_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_encoder_memusage.c b/src/liblzma/common/easy_encoder_memusage.c index e91057584233..1184ac665425 100644 --- a/src/liblzma/common/easy_encoder_memusage.c +++ b/src/liblzma/common/easy_encoder_memusage.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_preset.c b/src/liblzma/common/easy_preset.c index 2f9859860ad7..7908a2bb73c8 100644 --- a/src/liblzma/common/easy_preset.c +++ b/src/liblzma/common/easy_preset.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_preset.h b/src/liblzma/common/easy_preset.h index 382ade894066..d0ca1a6234a1 100644 --- a/src/liblzma/common/easy_preset.h +++ b/src/liblzma/common/easy_preset.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/file_info.c b/src/liblzma/common/file_info.c index 799bb024fe1f..7c85084a706e 100644 --- a/src/liblzma/common/file_info.c +++ b/src/liblzma/common/file_info.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_info.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_decoder.h" diff --git a/src/liblzma/common/filter_buffer_decoder.c b/src/liblzma/common/filter_buffer_decoder.c index 6620986eea8a..e80c582c916f 100644 --- a/src/liblzma/common/filter_buffer_decoder.c +++ b/src/liblzma/common/filter_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" diff --git a/src/liblzma/common/filter_buffer_encoder.c b/src/liblzma/common/filter_buffer_encoder.c index dda18e3d8e5e..7fb8922ae90e 100644 --- a/src/liblzma/common/filter_buffer_encoder.c +++ b/src/liblzma/common/filter_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" diff --git a/src/liblzma/common/filter_common.c b/src/liblzma/common/filter_common.c index fa0927cf9b9f..2da6bd9c7781 100644 --- a/src/liblzma/common/filter_common.c +++ b/src/liblzma/common/filter_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_common.h" @@ -122,6 +121,15 @@ static const struct { .changes_size = false, }, #endif +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { + .id = LZMA_FILTER_RISCV, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { .id = LZMA_FILTER_DELTA, diff --git a/src/liblzma/common/filter_common.h b/src/liblzma/common/filter_common.h index 2e47bb69f7f6..95f9fe27017b 100644 --- a/src/liblzma/common/filter_common.h +++ b/src/liblzma/common/filter_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_COMMON_H diff --git a/src/liblzma/common/filter_decoder.c b/src/liblzma/common/filter_decoder.c index fa53f5bdbad0..77441e5449c3 100644 --- a/src/liblzma/common/filter_decoder.c +++ b/src/liblzma/common/filter_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" @@ -121,6 +120,14 @@ static const lzma_filter_decoder decoders[] = { .props_decode = &lzma_simple_props_decode, }, #endif +#ifdef HAVE_DECODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif #ifdef HAVE_DECODER_DELTA { .id = LZMA_FILTER_DELTA, diff --git a/src/liblzma/common/filter_decoder.h b/src/liblzma/common/filter_decoder.h index 2dac60282826..e610bc1f44ec 100644 --- a/src/liblzma/common/filter_decoder.h +++ b/src/liblzma/common/filter_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_DECODER_H diff --git a/src/liblzma/common/filter_encoder.c b/src/liblzma/common/filter_encoder.c index 46fe8af1c153..523d37310010 100644 --- a/src/liblzma/common/filter_encoder.c +++ b/src/liblzma/common/filter_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -33,7 +32,8 @@ typedef struct { /// Calculates the recommended Uncompressed Size for .xz Blocks to /// which the input data can be split to make multithreaded /// encoding possible. If this is NULL, it is assumed that - /// the encoder is fast enough with single thread. + /// the encoder is fast enough with single thread. If the options + /// are invalid, UINT64_MAX is returned. uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are @@ -158,6 +158,16 @@ static const lzma_filter_encoder encoders[] = { .props_encode = &lzma_simple_props_encode, }, #endif +#ifdef HAVE_ENCODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif #ifdef HAVE_ENCODER_DELTA { .id = LZMA_FILTER_DELTA, @@ -219,17 +229,17 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - const lzma_filter *options) + const lzma_filter *filters) { return lzma_raw_coder_init(next, allocator, - options, (lzma_filter_find)(&encoder_find), true); + filters, (lzma_filter_find)(&encoder_find), true); } extern LZMA_API(lzma_ret) -lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters) { - lzma_next_strm_init(lzma_raw_coder_init, strm, options, + lzma_next_strm_init(lzma_raw_coder_init, strm, filters, (lzma_filter_find)(&encoder_find), true); strm->internal->supported_actions[LZMA_RUN] = true; @@ -248,26 +258,29 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -extern uint64_t +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) { + if (filters == NULL) + return UINT64_MAX; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); + if (fe == NULL) + return UINT64_MAX; + if (fe->block_size != NULL) { const uint64_t size = fe->block_size(filters[i].options); - if (size == 0) - return 0; - if (size > max) max = size; } } - return max; + return max == 0 ? UINT64_MAX : max; } diff --git a/src/liblzma/common/filter_encoder.h b/src/liblzma/common/filter_encoder.h index f1d5683fe793..88f2dafa43b0 100644 --- a/src/liblzma/common/filter_encoder.h +++ b/src/liblzma/common/filter_encoder.h @@ -1,13 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // -/// \file filter_encoder.c +/// \file filter_encoder.h /// \brief Filter ID mapping to filter-specific functions // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_ENCODER_H @@ -16,10 +15,6 @@ #include "common.h" -// FIXME: Might become a part of the public API. -extern uint64_t lzma_mt_block_size(const lzma_filter *filters); - - extern lzma_ret lzma_raw_encoder_init( lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); diff --git a/src/liblzma/common/filter_flags_decoder.c b/src/liblzma/common/filter_flags_decoder.c index ddfb085943d0..0f5d204d474e 100644 --- a/src/liblzma/common/filter_flags_decoder.c +++ b/src/liblzma/common/filter_flags_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" diff --git a/src/liblzma/common/filter_flags_encoder.c b/src/liblzma/common/filter_flags_encoder.c index b57b9fd80b06..e1d65884fb0b 100644 --- a/src/liblzma/common/filter_flags_encoder.c +++ b/src/liblzma/common/filter_flags_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" diff --git a/src/liblzma/common/hardware_cputhreads.c b/src/liblzma/common/hardware_cputhreads.c index 5d246d2cc083..4ce852b42c3d 100644 --- a/src/liblzma/common/hardware_cputhreads.c +++ b/src/liblzma/common/hardware_cputhreads.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_cputhreads.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/hardware_physmem.c b/src/liblzma/common/hardware_physmem.c index a2bbbe29d4b4..1bc34864e849 100644 --- a/src/liblzma/common/hardware_physmem.c +++ b/src/liblzma/common/hardware_physmem.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_physmem.c @@ -5,9 +7,6 @@ // // Author: Jonathan Nieder // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/index.c b/src/liblzma/common/index.c index 8a35f4398dbe..6add6a683502 100644 --- a/src/liblzma/common/index.c +++ b/src/liblzma/common/index.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/index.h b/src/liblzma/common/index.h index 7b27d7004cfa..007e1188f259 100644 --- a/src/liblzma/common/index.h +++ b/src/liblzma/common/index.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.h @@ -12,9 +14,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_H diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c index 19a31b3e944c..f105ff04e8a1 100644 --- a/src/liblzma/common/index_decoder.c +++ b/src/liblzma/common/index_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_decoder.h" diff --git a/src/liblzma/common/index_decoder.h b/src/liblzma/common/index_decoder.h index 3fec483331a6..5351d2f0dfa4 100644 --- a/src/liblzma/common/index_decoder.h +++ b/src/liblzma/common/index_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_DECODER_H diff --git a/src/liblzma/common/index_encoder.c b/src/liblzma/common/index_encoder.c index 204490cc19d4..ecc299c0159f 100644 --- a/src/liblzma/common/index_encoder.c +++ b/src/liblzma/common/index_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_encoder.h" diff --git a/src/liblzma/common/index_encoder.h b/src/liblzma/common/index_encoder.h index 4d55cd104785..29ba11066963 100644 --- a/src/liblzma/common/index_encoder.h +++ b/src/liblzma/common/index_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_ENCODER_H diff --git a/src/liblzma/common/index_hash.c b/src/liblzma/common/index_hash.c index 52c3d6507771..caa5967ca496 100644 --- a/src/liblzma/common/index_hash.c +++ b/src/liblzma/common/index_hash.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_hash.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/lzip_decoder.c b/src/liblzma/common/lzip_decoder.c index 88cc7ffd236b..651a0ae712c8 100644 --- a/src/liblzma/common/lzip_decoder.c +++ b/src/liblzma/common/lzip_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzip_decoder.c @@ -6,9 +8,6 @@ // Author: Michał Górny // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzip_decoder.h" diff --git a/src/liblzma/common/lzip_decoder.h b/src/liblzma/common/lzip_decoder.h index 33a01c352ce3..0e1f7bebd45b 100644 --- a/src/liblzma/common/lzip_decoder.h +++ b/src/liblzma/common/lzip_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzip_decoder.h @@ -5,9 +7,6 @@ // // Author: Michał Górny // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZIP_DECODER_H diff --git a/src/liblzma/common/memcmplen.h b/src/liblzma/common/memcmplen.h index 99d9c519cc0d..d8c42040d368 100644 --- a/src/liblzma/common/memcmplen.h +++ b/src/liblzma/common/memcmplen.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file memcmplen.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_MEMCMPLEN_H @@ -24,7 +23,8 @@ // can use the intrinsics without the header file. #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ && defined(_MSC_VER) \ - && defined(_M_X64) \ + && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)) \ && !defined(__INTEL_COMPILER) # include <intrin.h> #endif @@ -57,20 +57,22 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, assert(limit <= UINT32_MAX / 2); #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + && (((TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) \ + && (defined(__x86_64__) \ + || defined(__aarch64__))) \ || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ - || (defined(_MSC_VER) && defined(_M_X64))) - // I keep this x86-64 only for now since that's where I know this - // to be a good method. This may be fine on other 64-bit CPUs too. - // On big endian one should use xor instead of subtraction and switch - // to __builtin_clzll(). + || (defined(_MSC_VER) && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)))) + // This is only for x86-64 and ARM64 for now. This might be fine on + // other 64-bit processors too. On big endian one should use xor + // instead of subtraction and switch to __builtin_clzll(). #define LZMA_MEMCMPLEN_EXTRA 8 while (len < limit) { const uint64_t x = read64ne(buf1 + len) - read64ne(buf2 + len); if (x != 0) { // MSVC or Intel C compiler on Windows -# if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && defined(_M_X64) +# if defined(_MSC_VER) || defined(__INTEL_COMPILER) unsigned long tmp; _BitScanForward64(&tmp, x); len += (uint32_t)tmp >> 3; diff --git a/src/liblzma/common/microlzma_decoder.c b/src/liblzma/common/microlzma_decoder.c index e473373daaae..882cb2c808d1 100644 --- a/src/liblzma/common/microlzma_decoder.c +++ b/src/liblzma/common/microlzma_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file microlzma_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_decoder.h" diff --git a/src/liblzma/common/microlzma_encoder.c b/src/liblzma/common/microlzma_encoder.c index a787ca25b839..45ec0b12f45d 100644 --- a/src/liblzma/common/microlzma_encoder.c +++ b/src/liblzma/common/microlzma_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file microlzma_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder.h" diff --git a/src/liblzma/common/outqueue.c b/src/liblzma/common/outqueue.c index 71e8648a294d..eb018eb42b26 100644 --- a/src/liblzma/common/outqueue.c +++ b/src/liblzma/common/outqueue.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "outqueue.h" diff --git a/src/liblzma/common/outqueue.h b/src/liblzma/common/outqueue.h index 596911e95ee1..ae56f636e8b1 100644 --- a/src/liblzma/common/outqueue.h +++ b/src/liblzma/common/outqueue.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/stream_buffer_decoder.c b/src/liblzma/common/stream_buffer_decoder.c index b9745b5dbe18..c4f91fb49839 100644 --- a/src/liblzma/common/stream_buffer_decoder.c +++ b/src/liblzma/common/stream_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/stream_buffer_encoder.c b/src/liblzma/common/stream_buffer_encoder.c index 73157590e6f4..04d586959469 100644 --- a/src/liblzma/common/stream_buffer_encoder.c +++ b/src/liblzma/common/stream_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c index 64283812f29a..7f426841366a 100644 --- a/src/liblzma/common/stream_decoder.c +++ b/src/liblzma/common/stream_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/stream_decoder.h b/src/liblzma/common/stream_decoder.h index c13c6ba12706..5803715374d6 100644 --- a/src/liblzma/common/stream_decoder.h +++ b/src/liblzma/common/stream_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_DECODER_H diff --git a/src/liblzma/common/stream_decoder_mt.c b/src/liblzma/common/stream_decoder_mt.c index 76212b46da32..244624a47900 100644 --- a/src/liblzma/common/stream_decoder_mt.c +++ b/src/liblzma/common/stream_decoder_mt.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder_mt.c @@ -6,9 +8,6 @@ // Authors: Sebastian Andrzej Siewior // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/stream_encoder.c b/src/liblzma/common/stream_encoder.c index ee9204601856..e7e5b3fce7e0 100644 --- a/src/liblzma/common/stream_encoder.c +++ b/src/liblzma/common/stream_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" diff --git a/src/liblzma/common/stream_encoder_mt.c b/src/liblzma/common/stream_encoder_mt.c index f64de9bdbc57..f0fef1523318 100644 --- a/src/liblzma/common/stream_encoder_mt.c +++ b/src/liblzma/common/stream_encoder_mt.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder_mt.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -979,20 +978,18 @@ get_options(const lzma_mt *options, lzma_options_easy *opt_easy, *filters = opt_easy->filters; } - // Block size - if (options->block_size > 0) { - if (options->block_size > BLOCK_SIZE_MAX) - return LZMA_OPTIONS_ERROR; - + // If the Block size is not set, determine it from the filter chain. + if (options->block_size > 0) *block_size = options->block_size; - } else { - // Determine the Block size from the filter chain. + else *block_size = lzma_mt_block_size(*filters); - if (*block_size == 0) - return LZMA_OPTIONS_ERROR; - assert(*block_size <= BLOCK_SIZE_MAX); - } + // UINT64_MAX > BLOCK_SIZE_MAX, so the second condition + // should be optimized out by any reasonable compiler. + // The second condition should be there in the unlikely event that + // the macros change and UINT64_MAX < BLOCK_SIZE_MAX. + if (*block_size > BLOCK_SIZE_MAX || *block_size == UINT64_MAX) + return LZMA_OPTIONS_ERROR; // Calculate the maximum amount output that a single output buffer // may need to hold. This is the same as the maximum total size of diff --git a/src/liblzma/common/stream_flags_common.c b/src/liblzma/common/stream_flags_common.c index fbe8eb8abda2..41b8dcb70d74 100644 --- a/src/liblzma/common/stream_flags_common.c +++ b/src/liblzma/common/stream_flags_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/stream_flags_common.h b/src/liblzma/common/stream_flags_common.h index 84e96ba1ff66..28729dbcb6f2 100644 --- a/src/liblzma/common/stream_flags_common.h +++ b/src/liblzma/common/stream_flags_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_FLAGS_COMMON_H diff --git a/src/liblzma/common/stream_flags_decoder.c b/src/liblzma/common/stream_flags_decoder.c index b8d263ba4429..522c98b6fd5c 100644 --- a/src/liblzma/common/stream_flags_decoder.c +++ b/src/liblzma/common/stream_flags_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/stream_flags_encoder.c b/src/liblzma/common/stream_flags_encoder.c index b98ab17c456c..f94b5cd0a237 100644 --- a/src/liblzma/common/stream_flags_encoder.c +++ b/src/liblzma/common/stream_flags_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/string_conversion.c b/src/liblzma/common/string_conversion.c index d2c1e80936b2..92d9032bdd18 100644 --- a/src/liblzma/common/string_conversion.c +++ b/src/liblzma/common/string_conversion.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file string_conversion.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_common.h" @@ -250,7 +249,9 @@ static const char *parse_options(const char **const str, const char *str_end, || defined(HAVE_ENCODER_IA64) \ || defined(HAVE_DECODER_IA64) \ || defined(HAVE_ENCODER_SPARC) \ - || defined(HAVE_DECODER_SPARC) + || defined(HAVE_DECODER_SPARC) \ + || defined(HAVE_ENCODER_RISCV) \ + || defined(HAVE_DECODER_RISCV) static const option_map bcj_optmap[] = { { .name = "start", @@ -509,6 +510,11 @@ static const struct { &parse_bcj, bcj_optmap, 1, 1, true }, #endif +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { "riscv", sizeof(lzma_options_bcj), LZMA_FILTER_RISCV, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { "powerpc", sizeof(lzma_options_bcj), LZMA_FILTER_POWERPC, &parse_bcj, bcj_optmap, 1, 1, true }, diff --git a/src/liblzma/common/vli_decoder.c b/src/liblzma/common/vli_decoder.c index af2799d1fb90..3254ccc35bde 100644 --- a/src/liblzma/common/vli_decoder.c +++ b/src/liblzma/common/vli_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c index f8642694e291..3859006a94f1 100644 --- a/src/liblzma/common/vli_encoder.c +++ b/src/liblzma/common/vli_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/vli_size.c b/src/liblzma/common/vli_size.c index ec1b4fa488b6..c8cb2ec10ade 100644 --- a/src/liblzma/common/vli_size.c +++ b/src/liblzma/common/vli_size.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_size.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/delta/delta_common.c b/src/liblzma/delta/delta_common.c index 4768201d1a9f..5dbe253b4b3a 100644 --- a/src/liblzma/delta/delta_common.c +++ b/src/liblzma/delta/delta_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_common.h" diff --git a/src/liblzma/delta/delta_common.h b/src/liblzma/delta/delta_common.h index 7e7e1baaf680..bd0912769724 100644 --- a/src/liblzma/delta/delta_common.h +++ b/src/liblzma/delta/delta_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_COMMON_H diff --git a/src/liblzma/delta/delta_decoder.c b/src/liblzma/delta/delta_decoder.c index 77cf65cc76d8..10d53687894e 100644 --- a/src/liblzma/delta/delta_decoder.c +++ b/src/liblzma/delta/delta_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_decoder.h" diff --git a/src/liblzma/delta/delta_decoder.h b/src/liblzma/delta/delta_decoder.h index ad89cc659764..e2268ed44e72 100644 --- a/src/liblzma/delta/delta_decoder.h +++ b/src/liblzma/delta/delta_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_DECODER_H diff --git a/src/liblzma/delta/delta_encoder.c b/src/liblzma/delta/delta_encoder.c index 056bf7468ea3..ba4a50b1f42d 100644 --- a/src/liblzma/delta/delta_encoder.c +++ b/src/liblzma/delta/delta_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_encoder.h" diff --git a/src/liblzma/delta/delta_encoder.h b/src/liblzma/delta/delta_encoder.h index 4ab984785171..735f0ed0091b 100644 --- a/src/liblzma/delta/delta_encoder.h +++ b/src/liblzma/delta/delta_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_ENCODER_H diff --git a/src/liblzma/delta/delta_private.h b/src/liblzma/delta/delta_private.h index 0d6cb3866115..e54721a84665 100644 --- a/src/liblzma/delta/delta_private.h +++ b/src/liblzma/delta/delta_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_PRIVATE_H diff --git a/src/liblzma/liblzma.pc.in b/src/liblzma/liblzma.pc.in index d077cb77135a..a432992b7072 100644 --- a/src/liblzma/liblzma.pc.in +++ b/src/liblzma/liblzma.pc.in @@ -1,9 +1,5 @@ -# +# SPDX-License-Identifier: 0BSD # Author: Lasse Collin -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# prefix=@prefix@ exec_prefix=@exec_prefix@ diff --git a/src/liblzma/liblzma_generic.map b/src/liblzma/liblzma_generic.map index bb82167ed57a..f74c15484559 100644 --- a/src/liblzma/liblzma_generic.map +++ b/src/liblzma/liblzma_generic.map @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + XZ_5.0 { global: lzma_alone_decoder; @@ -119,3 +121,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/src/liblzma/liblzma_linux.map b/src/liblzma/liblzma_linux.map index 449f5fd682db..7e4b25e17620 100644 --- a/src/liblzma/liblzma_linux.map +++ b/src/liblzma/liblzma_linux.map @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + XZ_5.0 { global: lzma_alone_decoder; @@ -134,3 +136,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/src/liblzma/lz/lz_decoder.c b/src/liblzma/lz/lz_decoder.c index 06c95c1137df..92913f225a0d 100644 --- a/src/liblzma/lz/lz_decoder.c +++ b/src/liblzma/lz/lz_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// // liblzma supports multiple LZ77-based filters. The LZ part is shared @@ -54,9 +53,10 @@ typedef struct { static void lz_decoder_reset(lzma_coder *coder) { - coder->dict.pos = 0; + coder->dict.pos = 2 * LZ_DICT_REPEAT_MAX; coder->dict.full = 0; - coder->dict.buf[coder->dict.size - 1] = '\0'; + coder->dict.buf[2 * LZ_DICT_REPEAT_MAX - 1] = '\0'; + coder->dict.has_wrapped = false; coder->dict.need_reset = false; return; } @@ -70,8 +70,15 @@ decode_buffer(lzma_coder *coder, { while (true) { // Wrap the dictionary if needed. - if (coder->dict.pos == coder->dict.size) - coder->dict.pos = 0; + if (coder->dict.pos == coder->dict.size) { + // See the comment of #define LZ_DICT_REPEAT_MAX. + coder->dict.pos = LZ_DICT_REPEAT_MAX; + coder->dict.has_wrapped = true; + memcpy(coder->dict.buf, coder->dict.buf + + coder->dict.size + - LZ_DICT_REPEAT_MAX, + LZ_DICT_REPEAT_MAX); + } // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. @@ -253,21 +260,31 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, // dictionary to the output buffer, since applications are // recommended to give aligned buffers to liblzma. // + // Reserve 2 * LZ_DICT_REPEAT_MAX bytes of extra space which is + // needed for alloc_size. + // // Avoid integer overflow. - if (lz_options.dict_size > SIZE_MAX - 15) + if (lz_options.dict_size > SIZE_MAX - 15 - 2 * LZ_DICT_REPEAT_MAX) return LZMA_MEM_ERROR; lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); + // Reserve extra space as explained in the comment + // of #define LZ_DICT_REPEAT_MAX. + const size_t alloc_size + = lz_options.dict_size + 2 * LZ_DICT_REPEAT_MAX; + // Allocate and initialize the dictionary. - if (coder->dict.size != lz_options.dict_size) { + if (coder->dict.size != alloc_size) { lzma_free(coder->dict.buf, allocator); - coder->dict.buf - = lzma_alloc(lz_options.dict_size, allocator); + coder->dict.buf = lzma_alloc(alloc_size, allocator); if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - coder->dict.size = lz_options.dict_size; + // NOTE: Yes, alloc_size, not lz_options.dict_size. The way + // coder->dict.full is updated will take care that we will + // still reject distances larger than lz_options.dict_size. + coder->dict.size = alloc_size; } lz_decoder_reset(next->coder); @@ -280,9 +297,12 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf + coder->dict.pos, + lz_options.preset_dict + offset, copy_size); - coder->dict.pos = copy_size; + + // dict.pos isn't zero after lz_decoder_reset(). + coder->dict.pos += copy_size; coder->dict.full = copy_size; } diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h index ad80d4dd0d14..cb61b6e24c78 100644 --- a/src/liblzma/lz/lz_decoder.h +++ b/src/liblzma/lz/lz_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_DECODER_H @@ -17,10 +16,28 @@ #include "common.h" +/// Maximum length of a match rounded up to a nice power of 2 which is +/// a good size for aligned memcpy(). The allocated dictionary buffer will +/// be 2 * LZ_DICT_REPEAT_MAX bytes larger than the actual dictionary size: +/// +/// (1) Every time the decoder reaches the end of the dictionary buffer, +/// the last LZ_DICT_REPEAT_MAX bytes will be copied to the beginning. +/// This way dict_repeat() will only need to copy from one place, +/// never from both the end and beginning of the buffer. +/// +/// (2) The other LZ_DICT_REPEAT_MAX bytes is kept as a buffer between +/// the oldest byte still in the dictionary and the current write +/// position. This way dict_repeat(dict, dict->size - 1, &len) +/// won't need memmove() as the copying cannot overlap. +/// +/// Note that memcpy() still cannot be used if distance < len. +/// +/// LZMA's longest match length is 273 so pick a multiple of 16 above that. +#define LZ_DICT_REPEAT_MAX 288 + + typedef struct { - /// Pointer to the dictionary buffer. It can be an allocated buffer - /// internal to liblzma, or it can a be a buffer given by the - /// application when in single-call mode (not implemented yet). + /// Pointer to the dictionary buffer. uint8_t *buf; /// Write position in dictionary. The next byte will be written to @@ -35,9 +52,16 @@ typedef struct { /// Write limit size_t limit; - /// Size of the dictionary + /// Allocated size of buf. This is 2 * LZ_DICT_REPEAT_MAX bytes + /// larger than the actual dictionary size. This is enforced by + /// how the value for "full" is set; it can be at most + /// "size - 2 * LZ_DICT_REPEAT_MAX". size_t size; + /// True once the dictionary has become full and the writing position + /// has been wrapped in decode_buffer() in lz_decoder.c. + bool has_wrapped; + /// True when dictionary should be reset before decoding more data. bool need_reset; @@ -103,7 +127,16 @@ static inline uint8_t dict_get(const lzma_dict *const dict, const uint32_t distance) { return dict->buf[dict->pos - distance - 1 - + (distance < dict->pos ? 0 : dict->size)]; + + (distance < dict->pos + ? 0 : dict->size - LZ_DICT_REPEAT_MAX)]; +} + + +/// Optimized version of dict_get(dict, 0) +static inline uint8_t +dict_get0(const lzma_dict *const dict) +{ + return dict->buf[dict->pos - 1]; } @@ -132,68 +165,51 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) uint32_t left = my_min(dict_avail, *len); *len -= left; + size_t back = dict->pos - distance - 1; + if (distance >= dict->pos) + back += dict->size - LZ_DICT_REPEAT_MAX; + // Repeat a block of data from the history. Because memcpy() is faster // than copying byte by byte in a loop, the copying process gets split - // into three cases. + // into two cases. if (distance < left) { // Source and target areas overlap, thus we can't use // memcpy() nor even memmove() safely. do { - dict->buf[dict->pos] = dict_get(dict, distance); - ++dict->pos; + dict->buf[dict->pos++] = dict->buf[back++]; } while (--left > 0); - - } else if (distance < dict->pos) { - // The easiest and fastest case - memcpy(dict->buf + dict->pos, - dict->buf + dict->pos - distance - 1, - left); - dict->pos += left; - } else { - // The bigger the dictionary, the more rare this - // case occurs. We need to "wrap" the dict, thus - // we might need two memcpy() to copy all the data. - assert(dict->full == dict->size); - const uint32_t copy_pos - = dict->pos - distance - 1 + dict->size; - uint32_t copy_size = dict->size - copy_pos; - - if (copy_size < left) { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - copy_size); - dict->pos += copy_size; - copy_size = left - copy_size; - memcpy(dict->buf + dict->pos, dict->buf, copy_size); - dict->pos += copy_size; - } else { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - left); - dict->pos += left; - } + memcpy(dict->buf + dict->pos, dict->buf + back, left); + dict->pos += left; } // Update how full the dictionary is. - if (dict->full < dict->pos) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; - return unlikely(*len != 0); + return *len != 0; +} + + +static inline void +dict_put(lzma_dict *dict, uint8_t byte) +{ + dict->buf[dict->pos++] = byte; + + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; } /// Puts one byte into the dictionary. Returns true if the dictionary was /// already full and the byte couldn't be added. static inline bool -dict_put(lzma_dict *dict, uint8_t byte) +dict_put_safe(lzma_dict *dict, uint8_t byte) { if (unlikely(dict->pos == dict->limit)) return true; - dict->buf[dict->pos++] = byte; - - if (dict->pos > dict->full) - dict->full = dict->pos; - + dict_put(dict, byte); return false; } @@ -217,8 +233,8 @@ dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, *left -= lzma_bufcpy(in, in_pos, in_size, dict->buf, &dict->pos, dict->limit); - if (dict->pos > dict->full) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; return; } diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c index 5489085a0860..4af23e14c423 100644 --- a/src/liblzma/lz/lz_encoder.c +++ b/src/liblzma/lz/lz_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -196,9 +195,7 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. - if (lz_options->dict_size < LZMA_DICT_SIZE_MIN - || lz_options->dict_size - > (UINT32_C(1) << 30) + (UINT32_C(1) << 29) + if (!IS_ENC_DICT_SIZE_VALID(lz_options->dict_size) || lz_options->nice_len > lz_options->match_len_max) return true; @@ -549,7 +546,7 @@ lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_lz_options *lz_options)) { #if defined(HAVE_SMALL) && !defined(HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR) - // We need that the CRC32 table has been initialized. + // The CRC32 table must be initialized. lzma_crc32_init(); #endif @@ -569,6 +566,8 @@ lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, coder->lz.coder = NULL; coder->lz.code = NULL; coder->lz.end = NULL; + coder->lz.options_update = NULL; + coder->lz.set_out_limit = NULL; // mf.size is initialized to silence Valgrind // when used on optimized binaries (GCC may reorder diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h index ffcba02ce931..429836c8bc4e 100644 --- a/src/liblzma/lz/lz_encoder.h +++ b/src/liblzma/lz/lz_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_H @@ -17,6 +16,14 @@ #include "common.h" +// For now, the dictionary size is limited to 1.5 GiB. This may grow +// in the future if needed, but it needs a little more work than just +// changing this check. +#define IS_ENC_DICT_SIZE_VALID(size) \ + ((size) >= LZMA_DICT_SIZE_MIN \ + && (size) <= (UINT32_C(1) << 30) + (UINT32_C(1) << 29)) + + /// A table of these is used by the LZ-based encoder to hold /// the length-distance pairs found by the match finder. typedef struct { @@ -153,9 +160,13 @@ typedef struct { /// Maximum search depth uint32_t depth; - /// TODO: Comment + /// Initial dictionary for the match finder to search. const uint8_t *preset_dict; + /// If the preset dictionary is NULL, this value is ignored. + /// Otherwise this member must indicate the preset dictionary's + /// buffer size. If this size is larger than dict_size, then only + /// the dict_size sized tail of the preset_dict will be used. uint32_t preset_dict_size; } lzma_lz_options; @@ -217,7 +228,7 @@ typedef struct { // 3. The literals and matches are encoded using e.g. LZMA. // // The bytes that have been ran through the match finder, but not encoded yet, -// are called `read ahead'. +// are called 'read ahead'. /// Get how many bytes the match finder hashes in its initial step. diff --git a/src/liblzma/lz/lz_encoder_hash.h b/src/liblzma/lz/lz_encoder_hash.h index 4d9971ae6a5d..8ace82b04c51 100644 --- a/src/liblzma/lz/lz_encoder_hash.h +++ b/src/liblzma/lz/lz_encoder_hash.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_hash.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_HASH_H diff --git a/src/liblzma/lz/lz_encoder_hash_table.h b/src/liblzma/lz/lz_encoder_hash_table.h index 8c51717d704f..2b3a60e43e80 100644 --- a/src/liblzma/lz/lz_encoder_hash_table.h +++ b/src/liblzma/lz/lz_encoder_hash_table.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_lz_hash_table[256] = { 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, diff --git a/src/liblzma/lz/lz_encoder_mf.c b/src/liblzma/lz/lz_encoder_mf.c index 1fdc2d794909..557c2612f2a2 100644 --- a/src/liblzma/lz/lz_encoder_mf.c +++ b/src/liblzma/lz/lz_encoder_mf.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_mf.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" diff --git a/src/liblzma/lzma/fastpos.h b/src/liblzma/lzma/fastpos.h index dbeb16f7e31a..d3969a753fac 100644 --- a/src/liblzma/lzma/fastpos.h +++ b/src/liblzma/lzma/fastpos.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FASTPOS_H diff --git a/src/liblzma/lzma/fastpos_table.c b/src/liblzma/lzma/fastpos_table.c index 6a3ceac0e90a..4e10e3795e29 100644 --- a/src/liblzma/lzma/fastpos_table.c +++ b/src/liblzma/lzma/fastpos_table.c @@ -1,4 +1,6 @@ -/* This file has been automatically generated by fastpos_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by fastpos_tablegen.c. #include "common.h" #include "fastpos.h" diff --git a/src/liblzma/lzma/fastpos_tablegen.c b/src/liblzma/lzma/fastpos_tablegen.c index 57ed15039b27..957ccb7a6436 100644 --- a/src/liblzma/lzma/fastpos_tablegen.c +++ b/src/liblzma/lzma/fastpos_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos_tablegen.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> @@ -35,11 +34,13 @@ main(void) fastpos[c] = slot_fast; } - printf("/* This file has been automatically generated " - "by fastpos_tablegen.c. */\n\n" - "#include \"common.h\"\n" - "#include \"fastpos.h\"\n\n" - "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by fastpos_tablegen.c.\n\n" + "#include \"common.h\"\n" + "#include \"fastpos.h\"\n\n" + "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); for (size_t i = 0; i < (1 << FASTPOS_BITS); ++i) { if (i % 16 == 0) diff --git a/src/liblzma/lzma/lzma2_decoder.c b/src/liblzma/lzma/lzma2_decoder.c index 567df490ca5b..37ab253f5b0a 100644 --- a/src/liblzma/lzma/lzma2_decoder.c +++ b/src/liblzma/lzma/lzma2_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_decoder.h" diff --git a/src/liblzma/lzma/lzma2_decoder.h b/src/liblzma/lzma/lzma2_decoder.h index ef2dcbfa76f0..cdd8b463abfd 100644 --- a/src/liblzma/lzma/lzma2_decoder.h +++ b/src/liblzma/lzma/lzma2_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_DECODER_H diff --git a/src/liblzma/lzma/lzma2_encoder.c b/src/liblzma/lzma/lzma2_encoder.c index 4b6b23118d70..e20b75b30037 100644 --- a/src/liblzma/lzma/lzma2_encoder.c +++ b/src/liblzma/lzma/lzma2_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -409,6 +408,9 @@ lzma_lzma2_block_size(const void *options) { const lzma_options_lzma *const opt = options; + if (!IS_ENC_DICT_SIZE_VALID(opt->dict_size)) + return UINT64_MAX; + // Use at least 1 MiB to keep compression ratio better. return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); } diff --git a/src/liblzma/lzma/lzma2_encoder.h b/src/liblzma/lzma/lzma2_encoder.h index 515f1839347a..29966a66d237 100644 --- a/src/liblzma/lzma/lzma2_encoder.h +++ b/src/liblzma/lzma/lzma2_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_ENCODER_H diff --git a/src/liblzma/lzma/lzma_common.h b/src/liblzma/lzma/lzma_common.h index 9d040d95bb2f..c3c587f090ec 100644 --- a/src/liblzma/lzma/lzma_common.h +++ b/src/liblzma/lzma/lzma_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_common.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_COMMON_H @@ -84,6 +83,20 @@ typedef enum { ? (state) - 3 \ : (state) - 6)) +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is true. +#define update_literal_normal(state) \ + state = ((state) <= STATE_SHORTREP_LIT_LIT \ + ? STATE_LIT_LIT \ + : (state) - 3); + +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is false. +#define update_literal_matched(state) \ + state = ((state) <= STATE_LIT_SHORTREP \ + ? (state) - 3 \ + : (state) - 6); + /// Indicate that the latest state was a match. #define update_match(state) \ state = ((state) < LIT_STATES ? STATE_LIT_MATCH : STATE_NONLIT_MATCH) @@ -112,31 +125,33 @@ typedef enum { /// /// Match byte is used when the previous LZMA symbol was something else than /// a literal (that is, it was some kind of match). -#define LITERAL_CODER_SIZE 0x300 +#define LITERAL_CODER_SIZE UINT32_C(0x300) /// Maximum number of literal coders #define LITERAL_CODERS_MAX (1 << LZMA_LCLP_MAX) +/// Calculates the literal_mask that literal_subcoder() needs. +#define literal_mask_calc(lc, lp) \ + ((UINT32_C(0x100) << (lp)) - (UINT32_C(0x100) >> (lc))) + /// Locate the literal coder for the next literal byte. The choice depends on /// - the lowest literal_pos_bits bits of the position of the current /// byte; and /// - the highest literal_context_bits bits of the previous byte. -#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \ - ((probs)[(((pos) & (lp_mask)) << (lc)) \ - + ((uint32_t)(prev_byte) >> (8U - (lc)))]) +#define literal_subcoder(probs, lc, literal_mask, pos, prev_byte) \ + ((probs) + UINT32_C(3) * \ + (((((pos) << 8) + (prev_byte)) & (literal_mask)) << (lc))) static inline void -literal_init(probability (*probs)[LITERAL_CODER_SIZE], - uint32_t lc, uint32_t lp) +literal_init(probability *probs, uint32_t lc, uint32_t lp) { assert(lc + lp <= LZMA_LCLP_MAX); - const uint32_t coders = 1U << (lc + lp); + const size_t coders = LITERAL_CODER_SIZE << (lc + lp); - for (uint32_t i = 0; i < coders; ++i) - for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) - bit_reset(probs[i][j]); + for (size_t i = 0; i < coders; ++i) + bit_reset(probs[i]); return; } diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c index 26c148a95e25..0abed02b8154 100644 --- a/src/liblzma/lzma/lzma_decoder.c +++ b/src/liblzma/lzma/lzma_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.c @@ -5,9 +7,7 @@ /// // Authors: Igor Pavlov // Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -22,25 +22,20 @@ # pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #endif +// Minimum number of input bytes to safely decode one LZMA symbol. +// The worst case is that we decode 22 bits using probabilities and 26 +// direct bits. This may decode at maximum 20 bytes of input. +#define LZMA_IN_REQUIRED 20 -#ifdef HAVE_SMALL // Macros for (somewhat) size-optimized code. -#define seq_4(seq) seq - -#define seq_6(seq) seq - -#define seq_8(seq) seq - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _CHOICE2, \ - seq ## _BITTREE - +// This is used to decode the match length (how many bytes must be repeated +// from the dictionary). This version is used in the Resumable mode and +// does not unroll any loops. #define len_decode(target, ld, pos_state, seq) \ do { \ case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0_safe(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ probs = ld.low[pos_state];\ limit = LEN_LOW_SYMBOLS; \ @@ -48,7 +43,7 @@ case seq ## _CHOICE: \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0_safe(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ probs = ld.mid[pos_state]; \ limit = LEN_MID_SYMBOLS; \ @@ -64,98 +59,39 @@ case seq ## _CHOICE2: \ symbol = 1; \ case seq ## _BITTREE: \ do { \ - rc_bit(probs[symbol], , , seq ## _BITTREE); \ + rc_bit_safe(probs[symbol], , , seq ## _BITTREE); \ } while (symbol < limit); \ target += symbol - limit; \ } while (0) -#else // HAVE_SMALL - -// Unrolled versions -#define seq_4(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3 - -#define seq_6(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5 - -#define seq_8(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5, \ - seq ## 6, \ - seq ## 7 - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _LOW0, \ - seq ## _LOW1, \ - seq ## _LOW2, \ - seq ## _CHOICE2, \ - seq ## _MID0, \ - seq ## _MID1, \ - seq ## _MID2, \ - seq ## _HIGH0, \ - seq ## _HIGH1, \ - seq ## _HIGH2, \ - seq ## _HIGH3, \ - seq ## _HIGH4, \ - seq ## _HIGH5, \ - seq ## _HIGH6, \ - seq ## _HIGH7 -#define len_decode(target, ld, pos_state, seq) \ +// This is the faster version of the match length decoder that does not +// worry about being resumable. It unrolls the bittree decoding loop. +#define len_decode_fast(target, ld, pos_state) \ do { \ symbol = 1; \ -case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0(ld.choice) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ - target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ + rc_bittree3(ld.low[pos_state], \ + -LEN_LOW_SYMBOLS + MATCH_LEN_MIN); \ + target = symbol; \ } else { \ rc_update_1(ld.choice); \ -case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0(ld.choice2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID2); \ - target = symbol - LEN_MID_SYMBOLS \ - + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ + rc_bittree3(ld.mid[pos_state], -LEN_MID_SYMBOLS \ + + MATCH_LEN_MIN + LEN_LOW_SYMBOLS); \ + target = symbol; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ - target = symbol - LEN_HIGH_SYMBOLS \ + rc_bittree8(ld.high, -LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ - + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ + + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS); \ + target = symbol; \ } \ } \ } while (0) -#endif // HAVE_SMALL - /// Length decoder probabilities; see comments in lzma_common.h. typedef struct { @@ -173,7 +109,7 @@ typedef struct { /////////////////// /// Literals; see comments in lzma_common.h. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; /// If 1, it's a match. Otherwise it's a single 8-bit literal. probability is_match[STATES][POS_STATES_MAX]; @@ -232,7 +168,7 @@ typedef struct { uint32_t pos_mask; // (1U << pb) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; /// Uncompressed size as bytes, or LZMA_VLI_UNKNOWN if end of /// payload marker is expected. @@ -251,22 +187,26 @@ typedef struct { enum { SEQ_NORMALIZE, SEQ_IS_MATCH, - seq_8(SEQ_LITERAL), - seq_8(SEQ_LITERAL_MATCHED), + SEQ_LITERAL, + SEQ_LITERAL_MATCHED, SEQ_LITERAL_WRITE, SEQ_IS_REP, - seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_DIST_SLOT), + SEQ_MATCH_LEN_CHOICE, + SEQ_MATCH_LEN_CHOICE2, + SEQ_MATCH_LEN_BITTREE, + SEQ_DIST_SLOT, SEQ_DIST_MODEL, SEQ_DIRECT, - seq_4(SEQ_ALIGN), + SEQ_ALIGN, SEQ_EOPM, SEQ_IS_REP0, SEQ_SHORTREP, SEQ_IS_REP0_LONG, SEQ_IS_REP1, SEQ_IS_REP2, - seq_len(SEQ_REP_LEN), + SEQ_REP_LEN_CHOICE, + SEQ_REP_LEN_CHOICE2, + SEQ_REP_LEN_BITTREE, SEQ_COPY, } sequence; @@ -321,7 +261,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, const size_t dict_start = dict.pos; // Range decoder - rc_to_local(coder->rc, *in_pos); + rc_to_local(coder->rc, *in_pos, LZMA_IN_REQUIRED); // State uint32_t state = coder->state; @@ -340,7 +280,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, uint32_t offset = coder->offset; uint32_t len = coder->len; - const uint32_t literal_pos_mask = coder->literal_pos_mask; + const uint32_t literal_mask = coder->literal_mask; const uint32_t literal_context_bits = coder->literal_context_bits; // Temporary variables @@ -367,8 +307,24 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, might_finish_without_eopm = true; } - // The main decoder loop. The "switch" is used to restart the decoder at - // correct location. Once restarted, the "switch" is no longer used. + // The main decoder loop. The "switch" is used to resume the decoder at + // correct location. Once resumed, the "switch" is no longer used. + // The decoder loops is split into two modes: + // + // 1 - Non-resumable mode (fast). This is used when it is guaranteed + // there is enough input to decode the next symbol. If the output + // limit is reached, then the decoder loop will save the place + // for the resumable mode to continue. This mode is not used if + // HAVE_SMALL is defined. This is faster than Resumable mode + // because it reduces the number of branches needed and allows + // for more compiler optimizations. + // + // 2 - Resumable mode (slow). This is used when a previous decoder + // loop did not have enough space in the input or output buffers + // to complete. It uses sequence enum values to set remind + // coder->sequence where to resume in the decoder loop. This + // is the only mode used when HAVE_SMALL is defined. + switch (coder->sequence) while (true) { // Calculate new pos_state. This is skipped on the first loop @@ -376,13 +332,339 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // variables. pos_state = dict.pos & pos_mask; +#ifndef HAVE_SMALL + + /////////////////////////////// + // Non-resumable Mode (fast) // + /////////////////////////////// + + // Go to Resumable mode (1) if there is not enough input to + // safely decode any possible LZMA symbol or (2) if the + // dictionary is full, which may need special checks that + // are only done in the Resumable mode. + if (unlikely(!rc_is_fast_allowed() + || dict.pos == dict.limit)) + goto slow; + + // Decode the first bit from the next LZMA symbol. + // If the bit is a 0, then we handle it as a literal. + // If the bit is a 1, then it is a match of previously + // decoded data. + rc_if_0(coder->is_match[state][pos_state]) { + ///////////////////// + // Decode literal. // + ///////////////////// + + // Update the RC that we have decoded a 0. + rc_update_0(coder->is_match[state][pos_state]); + + // Get the correct probability array from lp and + // lc params. + probs = literal_subcoder(coder->literal, + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); + + if (is_literal_state(state)) { + update_literal_normal(state); + + // Decode literal without match byte. + rc_bittree8(probs, 0); + } else { + update_literal_matched(state); + + // Decode literal with match byte. + rc_matched_literal(probs, + dict_get(&dict, rep0)); + } + + // Write decoded literal to dictionary + dict_put(&dict, symbol); + continue; + } + + /////////////////// + // Decode match. // + /////////////////// + + // Instead of a new byte we are going to decode a + // distance-length pair. The distance represents how far + // back in the dictionary to begin copying. The length + // represents how many bytes to copy. + + rc_update_1(coder->is_match[state][pos_state]); + + rc_if_0(coder->is_rep[state]) { + /////////////////// + // Simple match. // + /////////////////// + + // Not a repeated match. In this case, + // the length (how many bytes to copy) must be + // decoded first. Then, the distance (where to + // start copying) is decoded. + // + // This is also how we know when we are done + // decoding. If the distance decodes to UINT32_MAX, + // then we know to stop decoding (end of payload + // marker). + + rc_update_0(coder->is_rep[state]); + update_match(state); + + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + + // Decode the length of the match. + len_decode_fast(len, coder->match_len_decoder, + pos_state); + + // Next, decode the distance into rep0. + + // The next 6 bits determine how to decode the + // rest of the distance. + probs = coder->dist_slot[get_dist_state(len)]; + + rc_bittree6(probs, -DIST_SLOTS); + assert(symbol <= 63); + + if (symbol < DIST_MODEL_START) { + // If the decoded symbol is < DIST_MODEL_START + // then we use its value directly as the + // match distance. No other bits are needed. + // The only possible distance values + // are [0, 3]. + rep0 = symbol; + } else { + // Use the first two bits of symbol as the + // highest bits of the match distance. + + // "limit" represents the number of low bits + // to decode. + limit = (symbol >> 1) - 1; + assert(limit >= 1 && limit <= 30); + rep0 = 2 + (symbol & 1); + + if (symbol < DIST_MODEL_END) { + // When symbol is > DIST_MODEL_START, + // but symbol < DIST_MODEL_END, then + // it can decode distances between + // [4, 127]. + assert(limit <= 5); + rep0 <<= limit; + assert(rep0 <= 96); + + // -1 is fine, because we start + // decoding at probs[1], not probs[0]. + // NOTE: This violates the C standard, + // since we are doing pointer + // arithmetic past the beginning of + // the array. + assert((int32_t)(rep0 - symbol - 1) + >= -1); + assert((int32_t)(rep0 - symbol - 1) + <= 82); + probs = coder->pos_special + rep0 + - symbol - 1; + symbol = 1; + offset = 1; + + // Variable number (1-5) of bits + // from a reverse bittree. This + // isn't worth manual unrolling. + // + // NOTE: Making one or many of the + // variables (probs, symbol, offset, + // or limit) local here (instead of + // using those declared outside the + // main loop) can affect code size + // and performance which isn't a + // surprise but it's not so clear + // what is the best. + do { + rc_bit_add_if_1(probs, + rep0, offset); + offset <<= 1; + } while (--limit > 0); + } else { + // The distance is >= 128. Decode the + // lower bits without probabilities + // except the lowest four bits. + assert(symbol >= 14); + assert(limit >= 6); + + limit -= ALIGN_BITS; + assert(limit >= 2); + + rc_direct(rep0, limit); + + // Decode the lowest four bits using + // probabilities. + rep0 <<= ALIGN_BITS; + rc_bittree_rev4(coder->pos_align); + rep0 += symbol; + + // If the end of payload marker (EOPM) + // is detected, jump to the safe code. + // The EOPM handling isn't speed + // critical at all. + // + // A final normalization is needed + // after the EOPM (there can be a + // dummy byte to read in some cases). + // If the normalization was done here + // in the fast code, it would need to + // be taken into account in the value + // of LZMA_IN_REQUIRED. Using the + // safe code allows keeping + // LZMA_IN_REQUIRED as 20 instead of + // 21. + if (rep0 == UINT32_MAX) + goto eopm; + } + } + + // Validate the distance we just decoded. + if (unlikely(!dict_is_distance_valid(&dict, rep0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + } else { + rc_update_1(coder->is_rep[state]); + + ///////////////////// + // Repeated match. // + ///////////////////// + + // The match distance is a value that we have decoded + // recently. The latest four match distances are + // available as rep0, rep1, rep2 and rep3. We will + // now decode which of them is the new distance. + // + // There cannot be a match if we haven't produced + // any output, so check that first. + if (unlikely(!dict_is_distance_valid(&dict, 0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + rc_if_0(coder->is_rep0[state]) { + rc_update_0(coder->is_rep0[state]); + // The distance is rep0. + + // Decode the next bit to determine if 1 byte + // should be copied from rep0 distance or + // if the number of bytes needs to be decoded. + + // If the next bit is 0, then it is a + // "Short Rep Match" and only 1 bit is copied. + // Otherwise, the length of the match is + // decoded after the "else" statement. + rc_if_0(coder->is_rep0_long[state][pos_state]) { + rc_update_0(coder->is_rep0_long[ + state][pos_state]); + + update_short_rep(state); + dict_put(&dict, dict_get(&dict, rep0)); + continue; + } + + // Repeating more than one byte at + // distance of rep0. + rc_update_1(coder->is_rep0_long[ + state][pos_state]); + + } else { + rc_update_1(coder->is_rep0[state]); + + // The distance is rep1, rep2 or rep3. Once + // we find out which one of these three, it + // is stored to rep0 and rep1, rep2 and rep3 + // are updated accordingly. There is no + // "Short Rep Match" option, so the length + // of the match must always be decoded next. + rc_if_0(coder->is_rep1[state]) { + // The distance is rep1. + rc_update_0(coder->is_rep1[state]); + + const uint32_t distance = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + rc_update_1(coder->is_rep1[state]); + + rc_if_0(coder->is_rep2[state]) { + // The distance is rep2. + rc_update_0(coder->is_rep2[ + state]); + + const uint32_t distance = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + // The distance is rep3. + rc_update_1(coder->is_rep2[ + state]); + + const uint32_t distance = rep3; + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + } + } + } + + update_long_rep(state); + + // Decode the length of the repeated match. + len_decode_fast(len, coder->rep_len_decoder, + pos_state); + } + + ///////////////////////////////// + // Repeat from history buffer. // + ///////////////////////////////// + + // The length is always between these limits. There is no way + // to trigger the algorithm to set len outside this range. + assert(len >= MATCH_LEN_MIN); + assert(len <= MATCH_LEN_MAX); + + // Repeat len bytes from distance of rep0. + if (unlikely(dict_repeat(&dict, rep0, &len))) { + coder->sequence = SEQ_COPY; + goto out; + } + + continue; + +slow: +#endif + /////////////////////////// + // Resumable Mode (slow) // + /////////////////////////// + + // This is very similar to Non-resumable Mode, so most of the + // comments are not repeated. The main differences are: + // - case labels are used to resume at the correct location. + // - Loops are not unrolled. + // - Range coder macros take an extra sequence argument + // so they can save to coder->sequence the location to + // resume in case there is not enough input. case SEQ_NORMALIZE: case SEQ_IS_MATCH: if (unlikely(might_finish_without_eopm && dict.pos == dict.limit)) { // In rare cases there is a useless byte that needs // to be read anyway. - rc_normalize(SEQ_NORMALIZE); + rc_normalize_safe(SEQ_NORMALIZE); // If the range decoder state is such that we can // be at the end of the LZMA stream, then the @@ -405,49 +687,37 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, eopm_is_valid = true; } - rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - rc_update_0(coder->is_match[state][pos_state]); + rc_if_0_safe(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + ///////////////////// + // Decode literal. // + ///////////////////// - // It's a literal i.e. a single 8-bit byte. + rc_update_0(coder->is_match[state][pos_state]); probs = literal_subcoder(coder->literal, - literal_context_bits, literal_pos_mask, - dict.pos, dict_get(&dict, 0)); + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); symbol = 1; if (is_literal_state(state)) { + update_literal_normal(state); + // Decode literal without match byte. -#ifdef HAVE_SMALL + // The "slow" version does not unroll + // the loop. case SEQ_LITERAL: do { - rc_bit(probs[symbol], , , SEQ_LITERAL); + rc_bit_safe(probs[symbol], , , + SEQ_LITERAL); } while (symbol < (1 << 8)); -#else - rc_bit_case(probs[symbol], , , SEQ_LITERAL0); - rc_bit_case(probs[symbol], , , SEQ_LITERAL1); - rc_bit_case(probs[symbol], , , SEQ_LITERAL2); - rc_bit_case(probs[symbol], , , SEQ_LITERAL3); - rc_bit_case(probs[symbol], , , SEQ_LITERAL4); - rc_bit_case(probs[symbol], , , SEQ_LITERAL5); - rc_bit_case(probs[symbol], , , SEQ_LITERAL6); - rc_bit_case(probs[symbol], , , SEQ_LITERAL7); -#endif } else { + update_literal_matched(state); + // Decode literal with match byte. - // - // We store the byte we compare against - // ("match byte") to "len" to minimize the - // number of variables we need to store - // between decoder calls. len = (uint32_t)(dict_get(&dict, rep0)) << 1; - // The usage of "offset" allows omitting some - // branches, which should give tiny speed - // improvement on some CPUs. "offset" gets - // set to zero if match_bit didn't match. offset = 0x100; -#ifdef HAVE_SMALL case SEQ_LITERAL_MATCHED: do { const uint32_t match_bit @@ -456,7 +726,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, = offset + match_bit + symbol; - rc_bit(probs[subcoder_index], + rc_bit_safe(probs[subcoder_index], offset &= ~match_bit, offset &= match_bit, SEQ_LITERAL_MATCHED); @@ -469,61 +739,10 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, len <<= 1; } while (symbol < (1 << 8)); -#else - // Unroll the loop. - uint32_t match_bit; - uint32_t subcoder_index; - -# define d(seq) \ - case seq: \ - match_bit = len & offset; \ - subcoder_index = offset + match_bit + symbol; \ - rc_bit(probs[subcoder_index], \ - offset &= ~match_bit, \ - offset &= match_bit, \ - seq) - - d(SEQ_LITERAL_MATCHED0); - len <<= 1; - d(SEQ_LITERAL_MATCHED1); - len <<= 1; - d(SEQ_LITERAL_MATCHED2); - len <<= 1; - d(SEQ_LITERAL_MATCHED3); - len <<= 1; - d(SEQ_LITERAL_MATCHED4); - len <<= 1; - d(SEQ_LITERAL_MATCHED5); - len <<= 1; - d(SEQ_LITERAL_MATCHED6); - len <<= 1; - d(SEQ_LITERAL_MATCHED7); -# undef d -#endif } - //update_literal(state); - // Use a lookup table to update to literal state, - // since compared to other state updates, this would - // need two branches. - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - state = next_state[state]; - case SEQ_LITERAL_WRITE: - if (unlikely(dict_put(&dict, symbol))) { + if (dict_put_safe(&dict, symbol)) { coder->sequence = SEQ_LITERAL_WRITE; goto out; } @@ -531,64 +750,47 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, continue; } - // Instead of a new byte we are going to get a byte range - // (distance and length) which will be repeated from our - // output history. + /////////////////// + // Decode match. // + /////////////////// rc_update_1(coder->is_match[state][pos_state]); case SEQ_IS_REP: - rc_if_0(coder->is_rep[state], SEQ_IS_REP) { - // Not a repeated match + rc_if_0_safe(coder->is_rep[state], SEQ_IS_REP) { + /////////////////// + // Simple match. // + /////////////////// + rc_update_0(coder->is_rep[state]); update_match(state); - // The latest three match distances are kept in - // memory in case there are repeated matches. rep3 = rep2; rep2 = rep1; rep1 = rep0; - // Decode the length of the match. len_decode(len, coder->match_len_decoder, pos_state, SEQ_MATCH_LEN); - // Prepare to decode the highest two bits of the - // match distance. probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; -#ifdef HAVE_SMALL case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + rc_bit_safe(probs[symbol], , , SEQ_DIST_SLOT); } while (symbol < DIST_SLOTS); -#else - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); -#endif - // Get rid of the highest bit that was needed for - // indexing of the probability array. + symbol -= DIST_SLOTS; assert(symbol <= 63); if (symbol < DIST_MODEL_START) { - // Match distances [0, 3] have only two bits. rep0 = symbol; } else { - // Decode the lowest [1, 29] bits of - // the match distance. limit = (symbol >> 1) - 1; assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); if (symbol < DIST_MODEL_END) { - // Prepare to decode the low bits for - // a distance of [4, 127]. assert(limit <= 5); rep0 <<= limit; assert(rep0 <= 96); @@ -607,95 +809,36 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, symbol = 1; offset = 0; case SEQ_DIST_MODEL: -#ifdef HAVE_SMALL do { - rc_bit(probs[symbol], , + rc_bit_safe(probs[symbol], , rep0 += 1U << offset, SEQ_DIST_MODEL); } while (++offset < limit); -#else - switch (limit) { - case 5: - assert(offset == 0); - rc_bit(probs[symbol], , - rep0 += 1U, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 4: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 3: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 2: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 1: - // We need "symbol" only for - // indexing the probability - // array, thus we can use - // rc_bit_last() here to omit - // the unneeded updating of - // "symbol". - rc_bit_last(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - } -#endif } else { - // The distance is >= 128. Decode the - // lower bits without probabilities - // except the lowest four bits. assert(symbol >= 14); assert(limit >= 6); limit -= ALIGN_BITS; assert(limit >= 2); case SEQ_DIRECT: - // Not worth manual unrolling - do { - rc_direct(rep0, SEQ_DIRECT); - } while (--limit > 0); + rc_direct_safe(rep0, limit, + SEQ_DIRECT); - // Decode the lowest four bits using - // probabilities. rep0 <<= ALIGN_BITS; - symbol = 1; -#ifdef HAVE_SMALL - offset = 0; + symbol = 0; + offset = 1; case SEQ_ALIGN: do { - rc_bit(coder->pos_align[ - symbol], , - rep0 += 1U << offset, + rc_bit_last_safe( + coder->pos_align[ + offset + + symbol], + , + symbol += offset, SEQ_ALIGN); - } while (++offset < ALIGN_BITS); -#else - case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], , - rep0 += 1, SEQ_ALIGN0); - case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], , - rep0 += 2, SEQ_ALIGN1); - case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], , - rep0 += 4, SEQ_ALIGN2); - case SEQ_ALIGN3: - // Like in SEQ_DIST_MODEL, we don't - // need "symbol" for anything else - // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], , - rep0 += 8, SEQ_ALIGN3); -#endif + offset <<= 1; + } while (offset < ALIGN_SIZE); + + rep0 += symbol; if (rep0 == UINT32_MAX) { // End of payload marker was @@ -710,6 +853,9 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // that EOPM might be used // (it's not allowed in // LZMA2). +#ifndef HAVE_SMALL +eopm: +#endif if (!eopm_is_valid) { ret = LZMA_DATA_ERROR; goto out; @@ -718,7 +864,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, case SEQ_EOPM: // LZMA1 stream with // end-of-payload marker. - rc_normalize(SEQ_EOPM); + rc_normalize_safe(SEQ_EOPM); ret = rc_is_finished(rc) ? LZMA_STREAM_END : LZMA_DATA_ERROR; @@ -727,36 +873,30 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, } } - // Validate the distance we just decoded. if (unlikely(!dict_is_distance_valid(&dict, rep0))) { ret = LZMA_DATA_ERROR; goto out; } } else { + ///////////////////// + // Repeated match. // + ///////////////////// + rc_update_1(coder->is_rep[state]); - // Repeated match - // - // The match distance is a value that we have had - // earlier. The latest four match distances are - // available as rep0, rep1, rep2 and rep3. We will - // now decode which of them is the new distance. - // - // There cannot be a match if we haven't produced - // any output, so check that first. if (unlikely(!dict_is_distance_valid(&dict, 0))) { ret = LZMA_DATA_ERROR; goto out; } case SEQ_IS_REP0: - rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) { + rc_if_0_safe(coder->is_rep0[state], SEQ_IS_REP0) { rc_update_0(coder->is_rep0[state]); - // The distance is rep0. case SEQ_IS_REP0_LONG: - rc_if_0(coder->is_rep0_long[state][pos_state], + rc_if_0_safe(coder->is_rep0_long + [state][pos_state], SEQ_IS_REP0_LONG) { rc_update_0(coder->is_rep0_long[ state][pos_state]); @@ -764,8 +904,9 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, update_short_rep(state); case SEQ_SHORTREP: - if (unlikely(dict_put(&dict, dict_get( - &dict, rep0)))) { + if (dict_put_safe(&dict, + dict_get(&dict, + rep0))) { coder->sequence = SEQ_SHORTREP; goto out; } @@ -773,8 +914,6 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, continue; } - // Repeating more than one byte at - // distance of rep0. rc_update_1(coder->is_rep0_long[ state][pos_state]); @@ -782,11 +921,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, rc_update_1(coder->is_rep0[state]); case SEQ_IS_REP1: - // The distance is rep1, rep2 or rep3. Once - // we find out which one of these three, it - // is stored to rep0 and rep1, rep2 and rep3 - // are updated accordingly. - rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { + rc_if_0_safe(coder->is_rep1[state], SEQ_IS_REP1) { rc_update_0(coder->is_rep1[state]); const uint32_t distance = rep1; @@ -796,7 +931,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, } else { rc_update_1(coder->is_rep1[state]); case SEQ_IS_REP2: - rc_if_0(coder->is_rep2[state], + rc_if_0_safe(coder->is_rep2[state], SEQ_IS_REP2) { rc_update_0(coder->is_rep2[ state]); @@ -821,7 +956,6 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, update_long_rep(state); - // Decode the length of the repeated match. len_decode(len, coder->rep_len_decoder, pos_state, SEQ_REP_LEN); } @@ -830,13 +964,10 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // Repeat from history buffer. // ///////////////////////////////// - // The length is always between these limits. There is no way - // to trigger the algorithm to set len outside this range. assert(len >= MATCH_LEN_MIN); assert(len <= MATCH_LEN_MAX); case SEQ_COPY: - // Repeat len bytes from distance of rep0. if (unlikely(dict_repeat(&dict, rep0, &len))) { coder->sequence = SEQ_COPY; goto out; @@ -890,7 +1021,6 @@ out: } - static void lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size, bool allow_eopm) @@ -917,7 +1047,7 @@ lzma_decoder_reset(void *coder_ptr, const void *opt) literal_init(coder->literal, options->lc, options->lp); coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // State coder->state = STATE_LIT_LIT; diff --git a/src/liblzma/lzma/lzma_decoder.h b/src/liblzma/lzma/lzma_decoder.h index 1427bc2461f4..9730f56fc268 100644 --- a/src/liblzma/lzma/lzma_decoder.h +++ b/src/liblzma/lzma/lzma_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_DECODER_H diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c index 559c63eda1d2..543ca321c3c2 100644 --- a/src/liblzma/lzma/lzma_encoder.c +++ b/src/liblzma/lzma/lzma_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_encoder.h" @@ -49,24 +48,24 @@ literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) const uint8_t cur_byte = mf->buffer[ mf->read_pos - mf->read_ahead]; probability *subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); if (is_literal_state(coder->state)) { // Previous LZMA-symbol was a literal. Encode a normal // literal without a match byte. + update_literal_normal(coder->state); rc_bittree(&coder->rc, subcoder, 8, cur_byte); } else { // Previous LZMA-symbol was a match. Use the last byte of // the match as a "match byte". That is, compare the bits // of the current literal and the match byte. + update_literal_matched(coder->state); const uint8_t match_byte = mf->buffer[ mf->read_pos - coder->reps[0] - 1 - mf->read_ahead]; literal_matched(&coder->rc, subcoder, match_byte, cur_byte); } - - update_literal(coder->state); } @@ -283,7 +282,7 @@ encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) mf_skip(mf, 1); mf->read_ahead = 0; rc_bit(&coder->rc, &coder->is_match[0][0], 0); - rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]); + rc_bittree(&coder->rc, coder->literal + 0, 8, mf->buffer[0]); ++coder->uncomp_size; } @@ -535,7 +534,7 @@ lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, coder->pos_mask = (1U << options->pb) - 1; coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // Range coder rc_reset(&coder->rc); @@ -712,6 +711,9 @@ static lzma_ret lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, lzma_vli id, const void *options, lzma_lz_options *lz_options) { + if (options == NULL) + return LZMA_PROG_ERROR; + lz->code = &lzma_encode; lz->set_out_limit = &lzma_lzma_set_out_limit; return lzma_lzma_encoder_create( diff --git a/src/liblzma/lzma/lzma_encoder.h b/src/liblzma/lzma/lzma_encoder.h index 84d8c9163f2d..e8ae8079306c 100644 --- a/src/liblzma/lzma/lzma_encoder.h +++ b/src/liblzma/lzma/lzma_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_H diff --git a/src/liblzma/lzma/lzma_encoder_optimum_fast.c b/src/liblzma/lzma/lzma_encoder_optimum_fast.c index 6c53d2bd0082..0f063d5be7a5 100644 --- a/src/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/src/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_fast.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" diff --git a/src/liblzma/lzma/lzma_encoder_optimum_normal.c b/src/liblzma/lzma/lzma_encoder_optimum_normal.c index 101c8d479008..a6c0398f3af3 100644 --- a/src/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/src/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_normal.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" @@ -24,7 +23,7 @@ get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, uint32_t match_byte, uint32_t symbol) { const probability *const subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, pos, prev_byte); uint32_t price = 0; diff --git a/src/liblzma/lzma/lzma_encoder_presets.c b/src/liblzma/lzma/lzma_encoder_presets.c index 711df0255296..e53483f99582 100644 --- a/src/liblzma/lzma/lzma_encoder_presets.c +++ b/src/liblzma/lzma/lzma_encoder_presets.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_presets.c @@ -6,9 +8,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/lzma/lzma_encoder_private.h b/src/liblzma/lzma/lzma_encoder_private.h index b228c5776173..eeea5e9c1289 100644 --- a/src/liblzma/lzma/lzma_encoder_private.h +++ b/src/liblzma/lzma/lzma_encoder_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_private.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_PRIVATE_H @@ -116,10 +115,10 @@ struct lzma_lzma1_encoder_s { uint32_t pos_mask; ///< (1 << pos_bits) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; // These are the same as in lzma_decoder.c. See comments there. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; probability is_match[STATES][POS_STATES_MAX]; probability is_rep[STATES]; probability is_rep0[STATES]; diff --git a/src/liblzma/rangecoder/price.h b/src/liblzma/rangecoder/price.h index 45dbbbb20cef..cce6bdae5f93 100644 --- a/src/liblzma/rangecoder/price.h +++ b/src/liblzma/rangecoder/price.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_PRICE_H diff --git a/src/liblzma/rangecoder/price_table.c b/src/liblzma/rangecoder/price_table.c index ac64bf62c767..c33433f718ca 100644 --- a/src/liblzma/rangecoder/price_table.c +++ b/src/liblzma/rangecoder/price_table.c @@ -1,4 +1,6 @@ -/* This file has been automatically generated by price_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by price_tablegen.c. #include "range_encoder.h" diff --git a/src/liblzma/rangecoder/price_tablegen.c b/src/liblzma/rangecoder/price_tablegen.c index bf08ce39d7e5..4b6ca37efadf 100644 --- a/src/liblzma/rangecoder/price_tablegen.c +++ b/src/liblzma/rangecoder/price_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price_tablegen.c @@ -8,13 +10,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> #include <stdio.h> + +// Make it compile without common.h. +#define BUILDING_PRICE_TABLEGEN +#define lzma_attr_visibility_hidden + #include "range_common.h" #include "price.h" @@ -54,11 +58,13 @@ init_price_table(void) static void print_price_table(void) { - printf("/* This file has been automatically generated by " - "price_tablegen.c. */\n\n" - "#include \"range_encoder.h\"\n\n" - "const uint8_t lzma_rc_prices[" - "RC_PRICE_TABLE_SIZE] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by price_tablegen.c.\n\n" + "#include \"range_encoder.h\"\n\n" + "const uint8_t lzma_rc_prices[" + "RC_PRICE_TABLE_SIZE] = {"); const size_t array_size = sizeof(lzma_rc_prices) / sizeof(lzma_rc_prices[0]); diff --git a/src/liblzma/rangecoder/range_common.h b/src/liblzma/rangecoder/range_common.h index 2c74dc1537c8..ac4dbe196f50 100644 --- a/src/liblzma/rangecoder/range_common.h +++ b/src/liblzma/rangecoder/range_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_common.h @@ -6,15 +8,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_COMMON_H #define LZMA_RANGE_COMMON_H -#include "common.h" +// Skip common.h if building price_tablegen.c. +#ifndef BUILDING_PRICE_TABLEGEN +# include "common.h" +#endif /////////////// @@ -66,6 +68,10 @@ /// /// I will be sticking to uint16_t unless some specific architectures /// are *much* faster (20-50 %) with uint32_t. +/// +/// Update in 2024: The branchless C and x86-64 assembly was written so that +/// probability is assumed to be uint16_t. (In contrast, LZMA SDK 23.01 +/// assembly supports both types.) typedef uint16_t probability; #endif diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h index e0b051fac2d2..b6422247f3c3 100644 --- a/src/liblzma/rangecoder/range_decoder.h +++ b/src/liblzma/rangecoder/range_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_DECODER_H @@ -17,6 +16,54 @@ #include "range_common.h" +// Choose the range decoder variants to use using a bitmask. +// If no bits are set, only the basic version is used. +// If more than one version is selected for the same feature, +// the last one on the list below is used. +// +// Bitwise-or of the following enable branchless C versions: +// 0x01 normal bittrees +// 0x02 fixed-sized reverse bittrees +// 0x04 variable-sized reverse bittrees (not faster) +// 0x08 matched literal (not faster) +// +// GCC & Clang compatible x86-64 inline assembly: +// 0x010 normal bittrees +// 0x020 fixed-sized reverse bittrees +// 0x040 variable-sized reverse bittrees +// 0x080 matched literal +// 0x100 direct bits +// +// The default can be overridden at build time by defining +// LZMA_RANGE_DECODER_CONFIG to the desired mask. +// +// 2024-02-22: Feedback from benchmarks: +// - Brancless C (0x003) can be better than basic on x86-64 but often it's +// slightly worse on other archs. Since asm is much better on x86-64, +// branchless C is not used at all. +// - With x86-64 asm, there are slight differences between GCC and Clang +// and different processors. Overall 0x1F0 seems to be the best choice. +#ifndef LZMA_RANGE_DECODER_CONFIG +# if defined(__x86_64__) && !defined(__ILP32__) \ + && (defined(__GNUC__) || defined(__clang__)) +# define LZMA_RANGE_DECODER_CONFIG 0x1F0 +# else +# define LZMA_RANGE_DECODER_CONFIG 0 +# endif +#endif + + +// Negative RC_BIT_MODEL_TOTAL but the lowest RC_MOVE_BITS are flipped. +// This is useful for updating probability variables in branchless decoding: +// +// uint32_t decoded_bit = ...; +// probability tmp = RC_BIT_MODEL_OFFSET; +// tmp &= decoded_bit - 1; +// prob -= (prob + tmp) >> RC_MOVE_BITS; +#define RC_BIT_MODEL_OFFSET \ + ((UINT32_C(1) << RC_MOVE_BITS) - 1 - RC_BIT_MODEL_TOTAL) + + typedef struct { uint32_t range; uint32_t code; @@ -50,18 +97,28 @@ rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, /// Makes local copies of range decoder and *in_pos variables. Doing this /// improves speed significantly. The range decoder macros expect also -/// variables `in' and `in_size' to be defined. -#define rc_to_local(range_decoder, in_pos) \ +/// variables 'in' and 'in_size' to be defined. +#define rc_to_local(range_decoder, in_pos, fast_mode_in_required) \ lzma_range_decoder rc = range_decoder; \ - size_t rc_in_pos = (in_pos); \ + const uint8_t *rc_in_ptr = in + (in_pos); \ + const uint8_t *rc_in_end = in + in_size; \ + const uint8_t *rc_in_fast_end \ + = (rc_in_end - rc_in_ptr) <= (fast_mode_in_required) \ + ? rc_in_ptr \ + : rc_in_end - (fast_mode_in_required); \ + (void)rc_in_fast_end; /* Silence a warning with HAVE_SMALL. */ \ uint32_t rc_bound +/// Evaluates to true if there is enough input remaining to use fast mode. +#define rc_is_fast_allowed() (rc_in_ptr < rc_in_fast_end) + + /// Stores the local copes back to the range decoder structure. #define rc_from_local(range_decoder, in_pos) \ do { \ range_decoder = rc; \ - in_pos = rc_in_pos; \ + in_pos = (size_t)(rc_in_ptr - in); \ } while (0) @@ -81,18 +138,30 @@ do { \ ((range_decoder).code == 0) -/// Read the next input byte if needed. If more input is needed but there is +// Read the next input byte if needed. +#define rc_normalize() \ +do { \ + if (rc.range < RC_TOP_VALUE) { \ + rc.range <<= RC_SHIFT_BITS; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ + } \ +} while (0) + + +/// If more input is needed but there is /// no more input available, "goto out" is used to jump out of the main -/// decoder loop. -#define rc_normalize(seq) \ +/// decoder loop. The "_safe" macros are used in the Resumable decoder +/// mode in order to save the sequence to continue decoding from that +/// point later. +#define rc_normalize_safe(seq) \ do { \ if (rc.range < RC_TOP_VALUE) { \ - if (unlikely(rc_in_pos == in_size)) { \ + if (rc_in_ptr == rc_in_end) { \ coder->sequence = seq; \ goto out; \ } \ rc.range <<= RC_SHIFT_BITS; \ - rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ } \ } while (0) @@ -100,7 +169,7 @@ do { \ /// Start decoding a bit. This must be used together with rc_update_0() /// and rc_update_1(): /// -/// rc_if_0(prob, seq) { +/// rc_if_0(prob) { /// rc_update_0(prob); /// // Do something /// } else { @@ -108,18 +177,28 @@ do { \ /// // Do something else /// } /// -#define rc_if_0(prob, seq) \ - rc_normalize(seq); \ +#define rc_if_0(prob) \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ + if (rc.code < rc_bound) + + +#define rc_if_0_safe(prob, seq) \ + rc_normalize_safe(seq); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) /// Update the range decoder state and the used probability variable to /// match a decoded bit of 0. +/// +/// The x86-64 assembly uses the commented method but it seems that, +/// at least on x86-64, the first version is slightly faster as C code. #define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ + /* prob -= ((prob) + RC_BIT_MODEL_OFFSET) >> RC_MOVE_BITS; */ \ } while (0) @@ -137,9 +216,21 @@ do { \ /// This macro is used as the last step in bittree reverse decoders since /// those don't use "symbol" for anything else than indexing the probability /// arrays. -#define rc_bit_last(prob, action0, action1, seq) \ +#define rc_bit_last(prob, action0, action1) \ +do { \ + rc_if_0(prob) { \ + rc_update_0(prob); \ + action0; \ + } else { \ + rc_update_1(prob); \ + action1; \ + } \ +} while (0) + + +#define rc_bit_last_safe(prob, action0, action1, seq) \ do { \ - rc_if_0(prob, seq) { \ + rc_if_0_safe(prob, seq) { \ rc_update_0(prob); \ action0; \ } else { \ @@ -151,35 +242,724 @@ do { \ /// Decodes one bit, updates "symbol", and runs action0 or action1 depending /// on the decoded bit. -#define rc_bit(prob, action0, action1, seq) \ +#define rc_bit(prob, action0, action1) \ rc_bit_last(prob, \ symbol <<= 1; action0, \ + symbol = (symbol << 1) + 1; action1); + + +#define rc_bit_safe(prob, action0, action1, seq) \ + rc_bit_last_safe(prob, \ + symbol <<= 1; action0, \ symbol = (symbol << 1) + 1; action1, \ seq); +// Unroll fixed-sized bittree decoding. +// +// A compile-time constant in final_add can be used to get rid of the high bit +// from symbol that is used for the array indexing (1U << bittree_bits). +// final_add may also be used to add offset to the result (LZMA length +// decoder does that). +// +// The reason to have final_add here is that in the asm code the addition +// can be done for free: in x86-64 there is SBB instruction with -1 as +// the immediate value, and final_add is combined with that value. +#define rc_bittree_bit(prob) \ + rc_bit(prob, , ) + +#define rc_bittree3(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) -/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled -/// loops more readable because the code isn't littered with "case" -/// statements. On the other hand this also makes it less readable, since -/// spotting the places where the decoder loop may be restarted is less -/// obvious. -#define rc_bit_case(prob, action0, action1, seq) \ - case seq: rc_bit(prob, action0, action1, seq) +#define rc_bittree6(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + +#define rc_bittree8(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + + +// Fixed-sized reverse bittree +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_bit_last(probs[symbol + 1], , symbol += 1); \ + rc_bit_last(probs[symbol + 2], , symbol += 2); \ + rc_bit_last(probs[symbol + 4], , symbol += 4); \ + rc_bit_last(probs[symbol + 8], , symbol += 8); \ +} while (0) + + +// Decode one bit from variable-sized reverse bittree. The loop is done +// in the code that uses this macro. This could be changed if the assembly +// version benefited from having the loop done in assembly but it didn't +// seem so in early 2024. +// +// Also, if the loop was done here, the loop counter would likely be local +// to the macro so that it wouldn't modify yet another input variable. +// If a _safe version of a macro with a loop was done then a modifiable +// input variable couldn't be avoided though. +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_bit(probs[symbol], \ + , \ + dest += value_to_add_if_1); + + +// Matched literal +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_bit(probs[t_subcoder_index], \ + t_offset &= ~t_match_bit, \ + t_offset &= t_match_bit) + +#define rc_matched_literal(probs_base_var, match_byte) \ +do { \ + uint32_t t_match_byte = (match_byte); \ + uint32_t t_match_bit; \ + uint32_t t_subcoder_index; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ +} while (0) /// Decode a bit without using a probability. -#define rc_direct(dest, seq) \ +// +// NOTE: GCC 13 and Clang/LLVM 16 can, at least on x86-64, optimize the bound +// calculation to use an arithmetic right shift so there's no need to provide +// the alternative code which, according to C99/C11/C23 6.3.1.3-p3 isn't +// perfectly portable: rc_bound = (uint32_t)((int32_t)rc.code >> 31); +#define rc_direct(dest, count_var) \ do { \ - rc_normalize(seq); \ + dest = (dest << 1) + 1; \ + rc_normalize(); \ + rc.range >>= 1; \ + rc.code -= rc.range; \ + rc_bound = UINT32_C(0) - (rc.code >> 31); \ + dest += rc_bound; \ + rc.code += rc.range & rc_bound; \ +} while (--count_var > 0) + + + +#define rc_direct_safe(dest, count_var, seq) \ +do { \ + rc_normalize_safe(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ rc.code += rc.range & rc_bound; \ dest = (dest << 1) + (rc_bound + 1); \ +} while (--count_var > 0) + + +////////////////// +// Branchless C // +////////////////// + +/// Decode a bit using a branchless method. This reduces the number of +/// mispredicted branches and thus can improve speed. +#define rc_c_bit(prob, action_bit, action_neg) \ +do { \ + probability *p = &(prob); \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * *p; \ + uint32_t rc_mask = rc.code >= rc_bound; /* rc_mask = decoded bit */ \ + action_bit; /* action when rc_mask is 0 or 1 */ \ + /* rc_mask becomes 0 if bit is 0 and 0xFFFFFFFF if bit is 1: */ \ + rc_mask = 0U - rc_mask; \ + rc.range &= rc_mask; /* If bit 0: set rc.range = 0 */ \ + rc_bound ^= rc_mask; \ + rc_bound -= rc_mask; /* If bit 1: rc_bound = 0U - rc_bound */ \ + rc.range += rc_bound; \ + rc_bound &= rc_mask; \ + rc.code += rc_bound; \ + action_neg; /* action when rc_mask is 0 or 0xFFFFFFFF */ \ + rc_mask = ~rc_mask; /* If bit 0: all bits are set in rc_mask */ \ + rc_mask &= RC_BIT_MODEL_OFFSET; \ + *p -= (*p + rc_mask) >> RC_MOVE_BITS; \ } while (0) -// NOTE: No macros are provided for bittree decoding. It seems to be simpler -// to just write them open in the code. +// Testing on x86-64 give an impression that only the normal bittrees and +// the fixed-sized reverse bittrees are worth the branchless C code. +// It should be tested on other archs for which there isn't assembly code +// in this file. + +// Using addition in "(symbol << 1) + rc_mask" allows use of x86 LEA +// or RISC-V SH1ADD instructions. Compilers might infer it from +// "(symbol << 1) | rc_mask" too if they see that mask is 0 or 1 but +// the use of addition doesn't require such analysis from compilers. +#if LZMA_RANGE_DECODER_CONFIG & 0x01 +#undef rc_bittree_bit +#define rc_bittree_bit(prob) \ + rc_c_bit(prob, \ + symbol = (symbol << 1) + rc_mask, \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x01 + +#if LZMA_RANGE_DECODER_CONFIG & 0x02 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_c_bit(probs[symbol + 1], symbol += rc_mask, ); \ + rc_c_bit(probs[symbol + 2], symbol += rc_mask << 1, ); \ + rc_c_bit(probs[symbol + 4], symbol += rc_mask << 2, ); \ + rc_c_bit(probs[symbol + 8], symbol += rc_mask << 3, ); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x02 + +#if LZMA_RANGE_DECODER_CONFIG & 0x04 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_c_bit(probs[symbol], \ + symbol = (symbol << 1) + rc_mask, \ + dest += (value_to_add_if_1) & rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x04 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x08 +#undef decode_with_match_bit +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_c_bit(probs[t_subcoder_index], \ + symbol = (symbol << 1) + rc_mask, \ + t_offset &= ~t_match_bit ^ rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x08 + + +//////////// +// x86-64 // +//////////// + +#if LZMA_RANGE_DECODER_CONFIG & 0x1F0 + +// rc_asm_y and rc_asm_n are used as arguments to macros to control which +// strings to include or omit. +#define rc_asm_y(str) str +#define rc_asm_n(str) + +// There are a few possible variations for normalization. +// This is the smallest variant which is also used by LZMA SDK. +// +// - This has partial register write (the MOV from (%[in_ptr])). +// +// - INC saves one byte in code size over ADD. False dependency on +// partial flags from INC shouldn't become a problem on any processor +// because the instructions after normalization don't read the flags +// until SUB which sets all flags. +// +#define rc_asm_normalize \ + "cmp %[top_value], %[range]\n\t" \ + "jae 1f\n\t" \ + "shl %[shift_bits], %[code]\n\t" \ + "mov (%[in_ptr]), %b[code]\n\t" \ + "shl %[shift_bits], %[range]\n\t" \ + "inc %[in_ptr]\n" \ + "1:\n" + +// rc_asm_calc(prob) is roughly equivalent to the C version of rc_if_0(prob)... +// +// rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// if (rc.code < rc_bound) +// +// ...but the bound is stored in "range": +// +// t0 = range; +// range = (range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// t0 -= range; +// t1 = code; +// code -= range; +// +// The carry flag (CF) from the last subtraction holds the negation of +// the decoded bit (if CF==0 then the decoded bit is 1). +// The values in t0 and t1 are needed for rc_update_0(prob) and +// rc_update_1(prob). If the bit is 0, rc_update_0(prob)... +// +// rc.range = rc_bound; +// +// ...has already been done but the "code -= range" has to be reverted using +// the old value stored in t1. (Also, prob needs to be updated.) +// +// If the bit is 1, rc_update_1(prob)... +// +// rc.range -= rc_bound; +// rc.code -= rc_bound; +// +// ...is already done for "code" but the value for "range" needs to be taken +// from t0. (Also, prob needs to be updated here as well.) +// +// The assignments from t0 and t1 can be done in a branchless manner with CMOV +// after the instructions from this macro. The CF from SUB tells which moves +// are needed. +#define rc_asm_calc(prob) \ + "mov %[range], %[t0]\n\t" \ + "shr %[bit_model_total_bits], %[range]\n\t" \ + "imul %[" prob "], %[range]\n\t" \ + "sub %[range], %[t0]\n\t" \ + "mov %[code], %[t1]\n\t" \ + "sub %[range], %[code]\n\t" + +// Also, prob needs to be updated: The update math depends on the decoded bit. +// It can be expressed in a few slightly different ways but this is fairly +// convenient here: +// +// prob -= (prob + (bit ? 0 : RC_BIT_MODEL_OFFSET)) >> RC_MOVE_BITS; +// +// To do it in branchless way when the negation of the decoded bit is in CF, +// both "prob" and "prob + RC_BIT_MODEL_OFFSET" are needed. Then the desired +// value can be picked with CMOV. The addition can be done using LEA without +// affecting CF. +// +// (This prob update method is a tiny bit different from LZMA SDK 23.01. +// In the LZMA SDK a single register is reserved solely for a constant to +// be used with CMOV when updating prob. That is fine since there are enough +// free registers to do so. The method used here uses one fewer register, +// which is valuable with inline assembly.) +// +// * * * +// +// In bittree decoding, each (unrolled) loop iteration decodes one bit +// and needs one prob variable. To make it faster, the prob variable of +// the iteration N+1 is loaded during iteration N. There are two possible +// prob variables to choose from for N+1. Both are loaded from memory and +// the correct one is chosen with CMOV using the same CF as is used for +// other things described above. +// +// This preloading/prefetching requires an extra register. To avoid +// useless moves from "preloaded prob register" to "current prob register", +// the macros swap between the two registers for odd and even iterations. +// +// * * * +// +// Finally, the decoded bit has to be stored in "symbol". Since the negation +// of the bit is in CF, this can be done with SBB: symbol -= CF - 1. That is, +// if the decoded bit is 0 (CF==1) the operation is a no-op "symbol -= 0" +// and when bit is 1 (CF==0) the operation is "symbol -= 0 - 1" which is +// the same as "symbol += 1". +// +// The instructions for all things are intertwined for a few reasons: +// - freeing temporary registers for new use +// - not modifying CF too early +// - instruction scheduling +// +// The first and last iterations can cheat a little. For example, +// on the first iteration "symbol" is known to start from 1 so it +// doesn't need to be read; it can even be immediately initialized +// to 2 to prepare for the second iteration of the loop. +// +// * * * +// +// a = number of the current prob variable (0 or 1) +// b = number of the next prob variable (1 or 0) +// *_only = rc_asm_y or _n to include or exclude code marked with them +#define rc_asm_bittree(a, b, first_only, middle_only, last_only) \ + first_only( \ + "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "mov $2, %[symbol]\n\t" \ + "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + /* Note the scaling of 4 instead of 2: */ \ + "movzw (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \ + ) \ + last_only( \ + "add %[symbol], %[symbol]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzw 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \ + "lea (%q[symbol], %q[symbol]), %[symbol]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + first_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + middle_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + last_only( \ + "sbb %[last_sbb], %[symbol]\n\t" \ + ) \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob" #a "], (%[probs_base], %q[t1], 1)\n\t" + +// NOTE: The order of variables in __asm__ can affect speed and code size. +#define rc_asm_bittree_n(probs_base_var, final_add, asm_str) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob0; \ + uint32_t t_prob1; \ + \ + __asm__( \ + asm_str \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob0] "=&r"(t_prob0), \ + [prob1] "=&r"(t_prob1), \ + [symbol] "=&r"(symbol), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [last_sbb] "n"(-1 - (final_add)), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) + + +#if LZMA_RANGE_DECODER_CONFIG & 0x010 +#undef rc_bittree3 +#define rc_bittree3(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree6 +#define rc_bittree6(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree8 +#define rc_bittree8(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x010 + + +// Fixed-sized reverse bittree +// +// This uses the indexing that constructs the final value in symbol directly. +// add = 1, 2, 4, 8 +// dcur = -, 4, 8, 16 +// dnext0 = 4, 8, 16, - +// dnext0 = 6, 12, 24, - +#define rc_asm_bittree_rev(a, b, add, dcur, dnext0, dnext1, \ + first_only, middle_only, last_only) \ + first_only( \ + "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "xor %[symbol], %[symbol]\n\t" \ + "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw " #dnext0 "(%[probs_base], %q[symbol], 2), " \ + "%[prob" #b "]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzw 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea " #add "(%q[symbol]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + middle_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + last_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + "cmovae %[t0], %[symbol]\n\t" \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + first_only( \ + "mov %w[prob" #a "], 2(%[probs_base])\n\t" \ + ) \ + middle_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) \ + last_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) + +#if LZMA_RANGE_DECODER_CONFIG & 0x020 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs_base_var) \ +rc_asm_bittree_n(probs_base_var, 4, \ + rc_asm_bittree_rev(0, 1, 1, -, 4, 6, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 2, 4, 8, 12, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(0, 1, 4, 8, 16, 24, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 8, 16, -, -, rc_asm_n, rc_asm_n, rc_asm_y) \ +) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x020 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x040 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs_base_var, dest_var, value_to_add_if_1) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t2 = (value_to_add_if_1); \ + uint32_t t_prob; \ + uint32_t t_index; \ + \ + __asm__( \ + "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + "mov %[symbol], %[index]\n\t" \ + \ + "add %[dest], %[t2]\n\t" \ + "add %[symbol], %[symbol]\n\t" \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + "cmovae %[t2], %[dest]\n\t" \ + "sbb $-1, %[symbol]\n\t" \ + \ + "sar %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + "mov %w[prob], (%[probs_base], %q[index], 2)" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [index] "=&r"(t_index), \ + [symbol] "+&r"(symbol), \ + [t2] "+&r"(t2), \ + [dest] "+&r"(dest_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x040 + + +// Literal decoding uses a normal 8-bit bittree but literal with match byte +// is more complex in picking the probability variable from the correct +// subtree. This doesn't use preloading/prefetching of the next prob because +// there are four choices instead of two. +// +// FIXME? The first iteration starts with symbol = 1 so it could be optimized +// by a tiny amount. +#define rc_asm_matched_literal(nonlast_only) \ + "add %[offset], %[symbol]\n\t" \ + "and %[offset], %[match_bit]\n\t" \ + "add %[match_bit], %[symbol]\n\t" \ + \ + "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + \ + "add %[symbol], %[symbol]\n\t" \ + \ + nonlast_only( \ + "xor %[match_bit], %[offset]\n\t" \ + "add %[match_byte], %[match_byte]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + nonlast_only( \ + "cmovae %[match_bit], %[offset]\n\t" \ + "mov %[match_byte], %[match_bit]\n\t" \ + ) \ + \ + "sbb $-1, %[symbol]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + /* Undo symbol += match_bit + offset: */ \ + "and $0x1FF, %[symbol]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob], (%[probs_base], %q[t1], 1)\n\t" + + +#if LZMA_RANGE_DECODER_CONFIG & 0x080 +#undef rc_matched_literal +#define rc_matched_literal(probs_base_var, match_byte_value) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob; \ + uint32_t t_match_byte = (uint32_t)(match_byte_value) << 1; \ + uint32_t t_match_bit = t_match_byte; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + \ + __asm__( \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_n) \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [match_bit] "+&r"(t_match_bit), \ + [symbol] "+&r"(symbol), \ + [match_byte] "+&r"(t_match_byte), \ + [offset] "+&r"(t_offset), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x080 + + +// Doing the loop in asm instead of C seems to help a little. +#if LZMA_RANGE_DECODER_CONFIG & 0x100 +#undef rc_direct +#define rc_direct(dest_var, count_var) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + \ + __asm__( \ + "2:\n\t" \ + "add %[dest], %[dest]\n\t" \ + "lea 1(%q[dest]), %[t1]\n\t" \ + \ + rc_asm_normalize \ + \ + "shr $1, %[range]\n\t" \ + "mov %[code], %[t0]\n\t" \ + "sub %[range], %[code]\n\t" \ + "cmovns %[t1], %[dest]\n\t" \ + "cmovs %[t0], %[code]\n\t" \ + "dec %[count]\n\t" \ + "jnz 2b\n\t" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [dest] "+&r"(dest_var), \ + [count] "+&r"(count_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x100 + +#endif // x86_64 #endif diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h index d794eabbccea..8f62a47ac0a6 100644 --- a/src/liblzma/rangecoder/range_encoder.h +++ b/src/liblzma/rangecoder/range_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_ENCODER_H diff --git a/src/liblzma/simple/arm.c b/src/liblzma/simple/arm.c index 6e53970d2f27..58acb2d11adf 100644 --- a/src/liblzma/simple/arm.c +++ b/src/liblzma/simple/arm.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file arm.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/arm64.c b/src/liblzma/simple/arm64.c index 0fe0824eb931..0a73f6c8bf2d 100644 --- a/src/liblzma/simple/arm64.c +++ b/src/liblzma/simple/arm64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file arm64.c @@ -16,9 +18,6 @@ // Jia Tan // Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/armthumb.c b/src/liblzma/simple/armthumb.c index 25d8dbd4f36e..f1eeca9b80f1 100644 --- a/src/liblzma/simple/armthumb.c +++ b/src/liblzma/simple/armthumb.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file armthumb.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/ia64.c b/src/liblzma/simple/ia64.c index 692b0a295ef2..502501409977 100644 --- a/src/liblzma/simple/ia64.c +++ b/src/liblzma/simple/ia64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file ia64.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/powerpc.c b/src/liblzma/simple/powerpc.c index 3a340fd171a5..ba6cfbef3ab6 100644 --- a/src/liblzma/simple/powerpc.c +++ b/src/liblzma/simple/powerpc.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file powerpc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/riscv.c b/src/liblzma/simple/riscv.c new file mode 100644 index 000000000000..aabbb0520577 --- /dev/null +++ b/src/liblzma/simple/riscv.c @@ -0,0 +1,755 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file riscv.c +/// \brief Filter for 32-bit/64-bit little/big endian RISC-V binaries +/// +/// This converts program counter relative addresses in function calls +/// (JAL, AUIPC+JALR), address calculation of functions and global +/// variables (AUIPC+ADDI), loads (AUIPC+load), and stores (AUIPC+store). +/// +/// For AUIPC+inst2 pairs, the paired instruction checking is fairly relaxed. +/// The paired instruction opcode must only have its lowest two bits set, +/// meaning it will convert any paired instruction that is not a 16-bit +/// compressed instruction. This was shown to be enough to keep the number +/// of false matches low while improving code size and speed. +// +// Authors: Lasse Collin +// Jia Tan +// +// Special thanks: +// +// - Chien Wong <m@xv97.com> provided a few early versions of RISC-V +// filter variants along with test files and benchmark results. +// +// - Igor Pavlov helped a lot in the filter design, getting it both +// faster and smaller. The implementation here is still independently +// written, not based on LZMA SDK. +// +/////////////////////////////////////////////////////////////////////////////// + +/* + +RISC-V filtering +================ + + RV32I and RV64I, possibly combined with extensions C, Zfh, F, D, + and Q, are identical enough that the same filter works for both. + + The instruction encoding is always little endian, even on systems + with big endian data access. Thus the same filter works for both + endiannesses. + + The following instructions have program counter relative + (pc-relative) behavior: + +JAL +--- + + JAL is used for function calls (including tail calls) and + unconditional jumps within functions. Jumps within functions + aren't useful to filter because the absolute addresses often + appear only once or at most a few times. Tail calls and jumps + within functions look the same to a simple filter so neither + are filtered, that is, JAL x0 is ignored (the ABI name of the + register x0 is "zero"). + + Almost all calls store the return address to register x1 (ra) + or x5 (t0). To reduce false matches when the filter is applied + to non-code data, only the JAL instructions that use x1 or x5 + are converted. JAL has pc-relative range of +/-1 MiB so longer + calls and jumps need another method (AUIPC+JALR). + +C.J and C.JAL +------------- + + C.J and C.JAL have pc-relative range of +/-2 KiB. + + C.J is for tail calls and jumps within functions and isn't + filtered for the reasons mentioned for JAL x0. + + C.JAL is an RV32C-only instruction. Its encoding overlaps with + RV64C-only C.ADDIW which is a common instruction. So if filtering + C.JAL was useful (it wasn't tested) then a separate filter would + be needed for RV32 and RV64. Also, false positives would be a + significant problem when the filter is applied to non-code data + because C.JAL needs only five bits to match. Thus, this filter + doesn't modify C.JAL instructions. + +BEQ, BNE, BLT, BGE, BLTU, BGEU, C.BEQZ, and C.BNEZ +-------------------------------------------------- + + These are conditional branches with pc-relative range + of +/-4 KiB (+/-256 B for C.*). The absolute addresses often + appear only once and very short distances are the most common, + so filtering these instructions would make compression worse. + +AUIPC with rd != x0 +------------------- + + AUIPC is paired with a second instruction (inst2) to do + pc-relative jumps, calls, loads, stores, and for taking + an address of a symbol. AUIPC has a 20-bit immediate and + the possible inst2 choices have a 12-bit immediate. + + AUIPC stores pc + 20-bit signed immediate to a register. + The immediate encodes a multiple of 4 KiB so AUIPC itself + has a pc-relative range of +/-2 GiB. AUIPC does *NOT* set + the lowest 12 bits of the result to zero! This means that + the 12-bit immediate in inst2 cannot just include the lowest + 12 bits of the absolute address as is; the immediate has to + compensate for the lowest 12 bits that AUIPC copies from the + program counter. This means that a good filter has to convert + not only AUIPC but also the paired inst2. + + A strict filter would focus on filtering the following + AUIPC+inst2 pairs: + + - AUIPC+JALR: Function calls, including tail calls. + + - AUIPC+ADDI: Calculating the address of a function + or a global variable. + + - AUIPC+load/store from the base instruction sets + (RV32I, RV64I) or from the floating point extensions + Zfh, F, D, and Q: + * RV32I: LB, LH, LW, LBU, LHU, SB, SH, SW + * RV64I has also: LD, LWU, SD + * Zhf: FLH, FSH + * F: FLW, FSW + * D: FLD, FSD + * Q: FLQ, FSQ + + NOTE: AUIPC+inst2 can only be a pair if AUIPC's rd specifies + the same register as inst2's rs1. + + Instead of strictly accepting only the above instructions as inst2, + this filter uses a much simpler condition: the lowest two bits of + inst2 must be set, that is, inst2 must not be a 16-bit compressed + instruction. So this will accept all 32-bit and possible future + extended instructions as a pair to AUIPC if the bits in AUIPC's + rd [11:7] match the bits [19:15] in inst2 (the bits that I-type and + S-type instructions use for rs1). Testing showed that this relaxed + condition for inst2 did not consistently or significantly affect + compression ratio but it reduced code size and improved speed. + + Additionally, the paired instruction is always treated as an I-type + instruction. The S-type instructions used by stores (SB, SH, SW, + etc.) place the lowest 5 bits of the immediate in a different + location than I-type instructions. AUIPC+store pairs are less + common than other pairs, and testing showed that the extra + code required to handle S-type instructions was not worth the + compression ratio gained. + + AUIPC+inst2 don't necessarily appear sequentially next to each + other although very often they do. Especially AUIPC+JALR are + sequential as that may allow instruction fusion in processors + (and perhaps help branch prediction as a fused AUIPC+JALR is + a direct branch while JALR alone is an indirect branch). + + Clang 16 can generate code where AUIPC+inst2 is split: + + - AUIPC is outside a loop and inst2 (load/store) is inside + the loop. This way the AUIPC instruction needs to be + executed only once. + + - Load-modify-store may have AUIPC for the load and the same + AUIPC-result is used for the store too. This may get combined + with AUIPC being outside the loop. + + - AUIPC is before a conditional branch and inst2 is hundreds + of bytes away at the branch target. + + - Inner and outer pair: + + auipc a1,0x2f + auipc a2,0x3d + ld a2,-500(a2) + addi a1,a1,-233 + + - Many split pairs with an untaken conditional branch between: + + auipc s9,0x1613 # Pair 1 + auipc s4,0x1613 # Pair 2 + auipc s6,0x1613 # Pair 3 + auipc s10,0x1613 # Pair 4 + beqz a5,a3baae + ld a0,0(a6) + ld a6,246(s9) # Pair 1 + ld a1,250(s4) # Pair 2 + ld a3,254(s6) # Pair 3 + ld a4,258(s10) # Pair 4 + + It's not possible to find all split pairs in a filter like this. + At least in 2024, simple sequential pairs are 99 % of AUIPC uses + so filtering only such pairs gives good results and makes the + filter simpler. However, it's possible that future compilers will + produce different code where sequential pairs aren't as common. + + This filter doesn't convert AUIPC instructions alone because: + + (1) The conversion would be off-by-one (or off-by-4096) half the + time because the lowest 12 bits from inst2 (inst2_imm12) + aren't known. We only know that the absolute address is + pc + AUIPC_imm20 + [-2048, +2047] but there is no way to + know the exact 4096-byte multiple (or 4096 * n + 2048): + there are always two possibilities because AUIPC copies + the 12 lowest bits from pc instead of zeroing them. + + NOTE: The sign-extension of inst2_imm12 adds a tiny bit + of extra complexity to AUIPC math in general but it's not + the reason for this problem. The sign-extension only changes + the relative position of the pc-relative 4096-byte window. + + (2) Matching AUIPC instruction alone requires only seven bits. + When the filter is applied to non-code data, that leads + to many false positives which make compression worse. + As long as most AUIPC+inst2 pairs appear as two consecutive + instructions, converting only such pairs gives better results. + + In assembly, AUIPC+inst2 tend to look like this: + + # Call: + auipc ra, 0x12345 + jalr ra, -42(ra) + + # Tail call: + auipc t1, 0x12345 + jalr zero, -42(t1) + + # Getting the absolute address: + auipc a0, 0x12345 + addi a0, a0, -42 + + # rd of inst2 isn't necessarily the same as rs1 even + # in cases where there is no reason to preserve rs1. + auipc a0, 0x12345 + addi a1, a0, -42 + + As of 2024, 16-bit instructions from the C extension don't + appear as inst2. The RISC-V psABI doesn't list AUIPC+C.* as + a linker relaxation type explicitly but it's not disallowed + either. Usefulness is limited as most of the time the lowest + 12 bits won't fit in a C instruction. This filter doesn't + support AUIPC+C.* combinations because this makes the filter + simpler, there are no test files, and it hopefully will never + be needed anyway. + + (Compare AUIPC to ARM64 where ADRP does set the lowest 12 bits + to zero. The paired instruction has the lowest 12 bits of the + absolute address as is in a zero-extended immediate. Thus the + ARM64 filter doesn't need to care about the instructions that + are paired with ADRP. An off-by-4096 issue can still occur if + the code section isn't aligned with the filter's start offset. + It's not a problem with standalone ELF files but Windows PE + files need start_offset=3072 for best results. Also, a .tar + stores files with 512-byte alignment so most of the time it + won't be the best for ARM64.) + +AUIPC with rd == x0 +------------------- + + AUIPC instructions with rd=x0 are reserved for HINTs in the base + instruction set. Such AUIPC instructions are never filtered. + + As of January 2024, it seems likely that AUIPC with rd=x0 will + be used for landing pads (pseudoinstruction LPAD). LPAD is used + to mark valid targets for indirect jumps (for JALR), for example, + beginnings of functions. The 20-bit immediate in LPAD instruction + is a label, not a pc-relative address. Thus it would be + counterproductive to convert AUIPC instructions with rd=x0. + + Often the next instruction after LPAD won't have rs1=x0 and thus + the filtering would be skipped for that reason alone. However, + it's not good to rely on this. For example, consider a function + that begins like this: + + int foo(int i) + { + if (i <= 234) { + ... + } + + A compiler may generate something like this: + + lpad 0x54321 + li a5, 234 + bgt a0, a5, .L2 + + Converting the pseudoinstructions to raw instructions: + + auipc x0, 0x54321 + addi x15, x0, 234 + blt x15, x10, .L2 + + In this case the filter would undesirably convert the AUIPC+ADDI + pair if the filter didn't explicitly skip AUIPC instructions + that have rd=x0. + +*/ + + +#include "simple_private.h" + + +// This checks two conditions at once: +// - AUIPC rd == inst2 rs1. +// - inst2 opcode has the lowest two bits set. +// +// The 8 bit left shift aligns the rd of AUIPC with the rs1 of inst2. +// By XORing the registers, any non-zero value in those bits indicates the +// registers are not equal and thus not an AUIPC pair. Subtracting 3 from +// inst2 will zero out the first two opcode bits only when they are set. +// The mask tests if any of the register or opcode bits are set (and thus +// not an AUIPC pair). +// +// Alternative expression: (((((auipc) << 8) ^ (inst2)) & 0xF8003) != 3) +#define NOT_AUIPC_PAIR(auipc, inst2) \ + ((((auipc) << 8) ^ ((inst2) - 3)) & 0xF8003) + +// This macro checks multiple conditions: +// (1) AUIPC rd [11:7] == x2 (special rd value). +// (2) AUIPC bits 12 and 13 set (the lowest two opcode bits of packed inst2). +// (3) inst2_rs1 doesn't equal x0 or x2 because the opposite +// conversion is only done when +// auipc_rd != x0 && +// auipc_rd != x2 && +// auipc_rd == inst2_rs1. +// +// The left-hand side takes care of (1) and (2). +// (a) The lowest 7 bits are already known to be AUIPC so subtracting 0x17 +// makes those bits zeros. +// (b) If AUIPC rd equals x2, subtracting 0x10 makes bits [11:7] zeros. +// If rd doesn't equal x2, then there will be at least one non-zero bit +// and the next step (c) is irrelevant. +// (c) If the lowest two opcode bits of the packed inst2 are set in [13:12], +// then subtracting 0x300 will make those bits zeros. Otherwise there +// will be at least one non-zero bit. +// +// The shift by 18 removes the high bits from the final '>=' comparison and +// ensures that any non-zero result will be larger than any possible result +// from the right-hand side of the comparison. The cast ensures that the +// left-hand side didn't get promoted to a larger type than uint32_t. +// +// On the right-hand side, inst2_rs1 & 0x1D will be non-zero as long as +// inst2_rs1 is not x0 or x2. +// +// The final '>=' comparison will make the expression true if: +// - The subtraction caused any bits to be set (special AUIPC rd value not +// used or inst2 opcode bits not set). (non-zero >= non-zero or 0) +// - The subtraction did not cause any bits to be set but inst2_rs1 was +// x0 or x2. (0 >= 0) +#define NOT_SPECIAL_AUIPC(auipc, inst2_rs1) \ + ((uint32_t)(((auipc) - 0x3117) << 18) >= ((inst2_rs1) & 0x1D)) + + +// The encode and decode functions are split for this filter because of the +// AUIPC+inst2 filtering. This filter design allows a decoder-only +// implementation to be smaller than alternative designs. + +#ifdef HAVE_ENCODER_RISCV +static size_t +riscv_encode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + // Avoid using i + 8 <= size in the loop condition. + // + // NOTE: If there is a JAL in the last six bytes of the stream, it + // won't be converted. This is intentional to keep the code simpler. + if (size < 8) + return 0; + + size -= 8; + + size_t i; + + // The loop is advanced by 2 bytes every iteration since the + // instruction stream may include 16-bit instructions (C extension). + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + // The 20-bit immediate is in four pieces. + // The encoder stores it in big endian form + // since it improves compression slightly. + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// The following chart shows the highest three bytes of JAL, focusing on +// the 20-bit immediate field [31:12]. The first row of numbers is the +// bit position in a 32-bit little endian instruction. The second row of +// numbers shows the order of the immediate field in a J-type instruction. +// The last row is the bit number in each byte. +// +// To determine the amount to shift each bit, subtract the value in +// the last row from the value in the second last row. If the number +// is positive, shift left. If negative, shift right. +// +// For example, at the rightmost side of the chart, the bit 4 in b1 is +// the bit 12 of the address. Thus that bit needs to be shifted left +// by 12 - 4 = 8 bits to put it in the right place in the addr variable. +// +// NOTE: The immediate of a J-type instruction holds bits [20:1] of +// the address. The bit [0] is always 0 and not part of the immediate. +// +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 8) + | ((b2 & 0x0F) << 16) + | ((b2 & 0x10) << 7) + | ((b2 & 0xE0) >> 4) + | ((b3 & 0x7F) << 4) + | ((b3 & 0x80) << 13); + + addr += pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 13) & 0xF0)); + + buffer[i + 2] = (uint8_t)(addr >> 9); + buffer[i + 3] = (uint8_t)(addr >> 1); + + // The "-2" is included because the for-loop will + // always increment by 2. In this case, we want to + // skip an extra 2 bytes since we used 4 bytes + // of input. + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + // Branch based on AUIPC's rd. The bitmask test does + // the same thing as this: + // + // const uint32_t auipc_rd = (inst >> 7) & 0x1F; + // if (auipc_rd != 0 && auipc_rd != 2) { + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if AUIPC+inst2 are a pair. + uint32_t inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + // The NOT_AUIPC_PAIR macro allows + // a false AUIPC+AUIPC pair if the + // bits [19:15] (where rs1 would be) + // in the second AUIPC match the rd + // of the first AUIPC. + // + // We must skip enough forward so + // that the first two bytes of the + // second AUIPC cannot get converted. + // Such a conversion could make the + // current pair become a valid pair + // which would desync the decoder. + // + // Skipping six bytes is enough even + // though the above condition looks + // at the lowest four bits of the + // buffer[i + 6] too. This is safe + // because this filter never changes + // those bits if a conversion at + // that position is done. + i += 6 - 2; + continue; + } + + // Convert AUIPC+inst2 to a special format: + // + // - The lowest 7 bits [6:0] retain the + // AUIPC opcode. + // + // - The rd [11:7] is set to x2(sp). x2 is + // used as the stack pointer so AUIPC with + // rd=x2 should be very rare in real-world + // executables. + // + // - The remaining 20 bits [31:12] (that + // normally hold the pc-relative immediate) + // are used to store the lowest 20 bits of + // inst2. That is, the 12-bit immediate of + // inst2 is not included. + // + // - The location of the original inst2 is + // used to store the 32-bit absolute + // address in big endian format. Compared + // to the 20+12-bit split encoding, this + // results in a longer uninterrupted + // sequence of identical common bytes + // when the same address is referred + // with different instruction pairs + // (like AUIPC+LD vs. AUIPC+ADDI) or + // when the occurrences of the same + // pair use different registers. When + // referring to adjacent memory locations + // (like function calls that go via the + // ELF PLT), in big endian order only the + // last 1-2 bytes differ; in little endian + // the differing 1-2 bytes would be in the + // middle of the 8-byte sequence. + // + // When reversing the transformation, the + // original rd of AUIPC can be restored + // from inst2's rs1 as they are required to + // be the same. + + // Arithmetic right shift makes sign extension + // trivial but (1) it's implementation-defined + // behavior (C99/C11/C23 6.5.7-p5) and so is + // (2) casting unsigned to signed (6.3.1.3-p3). + // + // One can check for (1) with + // + // if ((-1 >> 1) == -1) ... + // + // but (2) has to be checked from the + // compiler docs. GCC promises that (1) + // and (2) behave in the common expected + // way and thus + // + // addr += (uint32_t)( + // (int32_t)inst2 >> 20); + // + // does the same as the code below. But since + // the 100 % portable way is only a few bytes + // bigger code and there is no real speed + // difference, let's just use that, especially + // since the decoder doesn't need this at all. + uint32_t addr = inst & 0xFFFFF000; + addr += (inst2 >> 20) + - ((inst2 >> 19) & 0x1000); + + addr += now_pos + (uint32_t)i; + + // Construct the first 32 bits: + // [6:0] AUIPC opcode + // [11:7] Special AUIPC rd = x2 + // [31:12] The lowest 20 bits of inst2 + inst = 0x17 | (2 << 7) | (inst2 << 12); + + write32le(buffer + i, inst); + + // The second 32 bits store the absolute + // address in big endian order. + write32be(buffer + i + 4, addr); + } else { + // AUIPC's rd equals x0 or x2. + // + // x0 indicates a landing pad (LPAD). + // It's always skipped. + // + // AUIPC with rd == x2 is used for the special + // format as explained above. When the input + // contains a byte sequence that matches the + // special format, "fake" decoding must be + // done to keep the filter bijective (that + // is, safe to apply on arbitrary data). + // + // See the "x0 or x2" section in riscv_decode() + // for how the "real" decoding is done. The + // "fake" decoding is a simplified version + // of "real" decoding with the following + // differences (these reduce code size of + // the decoder): + // (1) The lowest 12 bits aren't sign-extended. + // (2) No address conversion is done. + // (3) Big endian format isn't used (the fake + // address is in little endian order). + + // Check if inst matches the special format. + const uint32_t fake_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, fake_rs1)) { + i += 4 - 2; + continue; + } + + const uint32_t fake_addr = + read32le(buffer + i + 4); + + // Construct the second 32 bits: + // [19:0] Upper 20 bits from AUIPC + // [31:20] The lowest 12 bits of fake_addr + const uint32_t fake_inst2 = (inst >> 12) + | (fake_addr << 20); + + // Construct new first 32 bits from: + // [6:0] AUIPC opcode + // [11:7] Fake AUIPC rd = fake_rs1 + // [31:12] The highest 20 bits of fake_addr + inst = 0x17 | (fake_rs1 << 7) + | (fake_addr & 0xFFFFF000); + + write32le(buffer + i, inst); + write32le(buffer + i + 4, fake_inst2); + } + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_encode, 0, 8, 2, true); +} +#endif + + +#ifdef HAVE_DECODER_RISCV +static size_t +riscv_decode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + if (size < 8) + return 0; + + size -= 8; + + size_t i; + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 13) + | (b2 << 9) | (b3 << 1); + + addr -= pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 8) & 0xF0)); + + buffer[i + 2] = (uint8_t)(((addr >> 16) & 0x0F) + | ((addr >> 7) & 0x10) + | ((addr << 4) & 0xE0)); + + buffer[i + 3] = (uint8_t)(((addr >> 4) & 0x7F) + | ((addr >> 13) & 0x80)); + + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + uint32_t inst2; + + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if it is a "fake" AUIPC+inst2 pair. + inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + i += 6 - 2; + continue; + } + + // Decode (or more like re-encode) the "fake" + // pair. The "fake" format doesn't do + // sign-extension, address conversion, or + // use big endian. (The use of little endian + // allows sharing the write32le() calls in + // the decoder to reduce code size when + // unaligned access isn't supported.) + uint32_t addr = inst & 0xFFFFF000; + addr += inst2 >> 20; + + inst = 0x17 | (2 << 7) | (inst2 << 12); + inst2 = addr; + } else { + // AUIPC's rd equals x0 or x2. + + // Check if inst matches the special format + // used by the encoder. + const uint32_t inst2_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, inst2_rs1)) { + i += 4 - 2; + continue; + } + + // Decode the "real" pair. + uint32_t addr = read32be(buffer + i + 4); + + addr -= now_pos + (uint32_t)i; + + // The second instruction: + // - Get the lowest 20 bits from inst. + // - Add the lowest 12 bits of the address + // as the immediate field. + inst2 = (inst >> 12) | (addr << 20); + + // AUIPC: + // - rd is the same as inst2_rs1. + // - The sign extension of the lowest 12 bits + // must be taken into account. + inst = 0x17 | (inst2_rs1 << 7) + | ((addr + 0x800) & 0xFFFFF000); + } + + // Both decoder branches write in little endian order. + write32le(buffer + i, inst); + write32le(buffer + i + 4, inst2); + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_decode, 0, 8, 2, false); +} +#endif diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c index ed2d7fb02cca..5cbfa8227047 100644 --- a/src/liblzma/simple/simple_coder.c +++ b/src/liblzma/simple/simple_coder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/simple_coder.h b/src/liblzma/simple/simple_coder.h index 668a5092ad5e..c9ccc3f1e634 100644 --- a/src/liblzma/simple/simple_coder.h +++ b/src/liblzma/simple/simple_coder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_CODER_H @@ -78,4 +77,13 @@ extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + #endif diff --git a/src/liblzma/simple/simple_decoder.c b/src/liblzma/simple/simple_decoder.c index dc4d24151101..d9820ee8ed23 100644 --- a/src/liblzma/simple/simple_decoder.c +++ b/src/liblzma/simple/simple_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_decoder.h" diff --git a/src/liblzma/simple/simple_decoder.h b/src/liblzma/simple/simple_decoder.h index bed8d37a9653..2ae87bb86328 100644 --- a/src/liblzma/simple/simple_decoder.h +++ b/src/liblzma/simple/simple_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_DECODER_H diff --git a/src/liblzma/simple/simple_encoder.c b/src/liblzma/simple/simple_encoder.c index d2cc03e58b81..d1f35096e2ae 100644 --- a/src/liblzma/simple/simple_encoder.c +++ b/src/liblzma/simple/simple_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_encoder.h" diff --git a/src/liblzma/simple/simple_encoder.h b/src/liblzma/simple/simple_encoder.h index 1cee4823a4ed..bf5edbb1c3f4 100644 --- a/src/liblzma/simple/simple_encoder.h +++ b/src/liblzma/simple/simple_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_ENCODER_H diff --git a/src/liblzma/simple/simple_private.h b/src/liblzma/simple/simple_private.h index 9d2c0fdd7618..7aa360ff49e8 100644 --- a/src/liblzma/simple/simple_private.h +++ b/src/liblzma/simple/simple_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_PRIVATE_H diff --git a/src/liblzma/simple/sparc.c b/src/liblzma/simple/sparc.c index bad8492ebc06..e8ad285a1927 100644 --- a/src/liblzma/simple/sparc.c +++ b/src/liblzma/simple/sparc.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sparc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/x86.c b/src/liblzma/simple/x86.c index 232b29542e66..10d70e91697b 100644 --- a/src/liblzma/simple/x86.c +++ b/src/liblzma/simple/x86.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file x86.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/validate_map.sh b/src/liblzma/validate_map.sh index 2bf6f8b98cbb..dd1589d236e7 100644 --- a/src/liblzma/validate_map.sh +++ b/src/liblzma/validate_map.sh @@ -1,4 +1,5 @@ #!/bin/sh +# SPDX-License-Identifier: 0BSD ############################################################################### # @@ -78,9 +79,6 @@ # # Author: Lasse Collin # -# This file has been put into the public domain. -# You can do whatever you want with this file. -# ############################################################################### LC_ALL=C @@ -124,7 +122,7 @@ DUPS=$(sort liblzma_generic.map | sed '/^$/d;/^global:$/d' | uniq -d) # ignored (@XZ_5.1.2alpha or @XZ_5.2.2 won't be added at all when # the #define HAVE_SYMBOL_VERSIONS_LINUX isn't used). IN_SYNC= -if ! sed '109,123d' liblzma_linux.map \ +if ! sed '111,125d' liblzma_linux.map \ | cmp -s - liblzma_generic.map; then IN_SYNC=no fi |